Parcourir la source

Scrape estimate endpoint added

master^2
diligent il y a 4 ans
Parent
révision
ed10972619
1 fichiers modifiés avec 72 ajouts et 26 suppressions
  1. 72
    26
      app.js

+ 72
- 26
app.js Voir le fichier

@@ -172,32 +172,7 @@ app.post("/scrapes/", async (req, res) => {
const baths = req.body.baths;

// query builder
var query = `https://www.apartments.com`;
if (type) {
query += `/${type}`;
}
if (location) {
var locationQuery = location.replace(", ", "-").replace(" ", "-").toLowerCase();
query += `/${locationQuery}`;
}
if (beds) {
query += `/${beds}-bedrooms`;
}

if (baths) {
query += `${beds ? '-' : '/'}${baths}-bathrooms`;
}

if (price) {
if (beds) {
query += `-over-${price}`;
} else {
query += `/over-${price}`;
}
}
if (lifestyle) {
query += `/${lifestyle}`;
}
var query = buildQuery(type, location, beds, baths, price, lifestyle);

console.log(query);

@@ -250,6 +225,47 @@ app.post("/scrapes/", async (req, res) => {
}
return res.json();
});
app.post("/scrapes/estimate", async (req, res) => {
const location = req.body.location;
const description = req.body.description;
const price = req.body.price;
const beds = req.body.beds;
const type = req.body.type;
const lifestyle = req.body.lifestyle;
const baths = req.body.baths;

var query = buildQuery(type, location, beds, baths, price, lifestyle);

console.log(query);

const filterPage = await axios(query);
const html = filterPage.data;
const $ = cheerio.load(html);
var $pageRange = $(".pageRange");
var pagesCount = 0;
var resultCount = 0;
if (!$pageRange.length) {
let propertyLinks = $('#placardContainer .property-link').map(function () {
return $(this).attr('href');
}).get();
if (!propertyLinks.length) {
console.error("No results");
return res.status(404).json();
}
resultCount = propertyLinks.length;
} else {
pagesCount = $pageRange.text().slice($pageRange.text().lastIndexOf("of ") + 3);
resultCount = pagesCount * 25;
}
const dt = new Date();
dt.setSeconds(dt.getSeconds() + resultCount);

return res.json({
count: resultCount,
pageCount: pagesCount,
estimate: dt,
});
});
app.patch("/scrapes/:id/execute", async (req, res) => {
const id = req.params.id;

@@ -273,6 +289,36 @@ app.listen(port, () => {
console.log(`Example app listening at http://localhost:${port}`)
});

function buildQuery(type, location, beds, baths, price, lifestyle) {
var query = `https://www.apartments.com`;
if (type) {
query += `/${type}`;
}
if (location) {
var locationQuery = location.replace(", ", "-").replace(" ", "-").toLowerCase();
query += `/${locationQuery}`;
}
if (beds) {
query += `/${beds}-bedrooms`;
}

if (baths) {
query += `${beds ? '-' : '/'}${baths}-bathrooms`;
}

if (price) {
if (beds) {
query += `-over-${price}`;
} else {
query += `/over-${price}`;
}
}
if (lifestyle) {
query += `/${lifestyle}`;
}
return query;
}

// Handles graceful stopping of jobs
function graceful() {
agenda.stop(function () {

Chargement…
Annuler
Enregistrer