| @@ -20,7 +20,6 @@ agenda.define('scrape', async function (job, done) { | |||
| const dbo = client.db(database); | |||
| let collection = dbo.collection('scrapes'); | |||
| let scrape = await collection.findOne({ _id: _id }); | |||
| scrape.sourceUrl = "https://www.apartments.com/new-york-ny/"; | |||
| const filterPage = await axios(scrape.sourceUrl); | |||
| const html = filterPage.data; | |||
| @@ -174,7 +173,30 @@ app.post("/scrapes/", async (req, res) => { | |||
| const lifestyle = req.body.lifestyle; | |||
| // query builder | |||
| var query = `https://www.apartments.com`; | |||
| if(type){ | |||
| query += `/${type}`; | |||
| } | |||
| if(location){ | |||
| var locationQuery = location.replace(", ", "-").replace(" ", "-").toLowerCase(); | |||
| query += `/${locationQuery}`; | |||
| } | |||
| if(beds){ | |||
| query += `/${beds}-bedrooms`; | |||
| } | |||
| if(price){ | |||
| if(beds){ | |||
| query += `-over-${price}`; | |||
| }else{ | |||
| query += `/over-${price}`; | |||
| } | |||
| } | |||
| if(lifestyle){ | |||
| query += `/${lifestyle}`; | |||
| } | |||
| console.log(query); | |||
| //todo: save data into the database | |||
| try { | |||
| const dbo = client.db(database); | |||
| @@ -183,7 +205,7 @@ app.post("/scrapes/", async (req, res) => { | |||
| let res = await collection.insertOne({ | |||
| count: 21, | |||
| estimate: Date.now(), | |||
| sourceUrl: "https://www.apartments.com", | |||
| sourceUrl: query, | |||
| location: location, | |||
| filters: [ | |||
| { name: 'price', value: price }, | |||