Browse Source

Scrappe expanded

master
diligent 4 years ago
parent
commit
2b3b0b719c
2 changed files with 7 additions and 1 deletions
  1. 1
    0
      apartments.js
  2. 6
    1
      app.js

+ 1
- 0
apartments.js View File

phone: $('.contactInfo .phoneNumber span').html(), phone: $('.contactInfo .phoneNumber span').html(),
language: $('.contactInfo .languages span').html(), language: $('.contactInfo .languages span').html(),
url: $('.contactInfo .mortar-wrapper a').attr('href'), url: $('.contactInfo .mortar-wrapper a').attr('href'),
todayHours: $('.todaysHoursContactContainer .todaysHours>span').text(),
officeHours: $('.daysHoursContainer').map(function () { officeHours: $('.daysHoursContainer').map(function () {
return { return {
days: $(this).find('.days').text(), days: $(this).find('.days').text(),

+ 6
- 1
app.js View File

let collection = dbo.collection('scrapes'); let collection = dbo.collection('scrapes');
let scrape = await collection.findOne({ _id: _id }); let scrape = await collection.findOne({ _id: _id });


for (var page = 1; page <= scrape.pageCount; page++) {
// for (var page = 1; page <= scrape.pageCount; page++) {
for (var page = 1; page <= 1; page++) {
console.log("scrapping page " + page) console.log("scrapping page " + page)


const filterPage = await axios(scrape.sourceUrl + `/${page}`); const filterPage = await axios(scrape.sourceUrl + `/${page}`);
}).get(); }).get();


var properties = []; var properties = [];
// var limit = 0;
for (const link of propertyLinks) { for (const link of propertyLinks) {
// if(limit == 1) break;
try { try {
var response = await axios(link); var response = await axios(link);


var property = apartments.apartment(cheerio.load(response.data)); var property = apartments.apartment(cheerio.load(response.data));
property.url = link;
properties.push(property); properties.push(property);
console.log(`${link} scraped.`); console.log(`${link} scraped.`);
limit++;
}catch(err){ }catch(err){
console.error(`${link} scrape failed.`); console.error(`${link} scrape failed.`);
} }

Loading…
Cancel
Save