scrapper-api/app.js

var config = require('config');
const express = require('express');
const cors = require('cors');
const MongoClient = require('mongodb').MongoClient;
const ObjectID = require('mongodb').ObjectID;

// express application
const app = express();

app.use(express.json());
app.use(cors());

// database setup
var mongoUrl = config.get("mongo");
var database = config.get("database");

var client = undefined;
MongoClient.connect(mongoUrl, function (err, db) {
  if (err) throw err;
  console.log("Database created!");
  console.log(mongoUrl);
  client = db;

  var dbo = db.db(database);
  dbo.createCollection("scrapes", function (err, res) {
    if (err) {
      console.log("Collection already created!");
      return;
    }
    console.log("Collection created!");
  });
});

// const cron = require('node-cron');

// var apartments = require('./apartments.js');
// var houses = require('./houses.js');

// app.set('json spaces', 2);

// const axios = require('axios');
// const cheerio = require('cheerio');


// const url = 'https://www.apartments.com/two-west-chicago-il/jqn1nf6/';
// app.get('/', (req, res) => {
//   axios(url).then(response => {
//     const html = response.data;
//     const $ = cheerio.load(html);

//     var data = apartments.apartment($);
//     res.json(data);
//   });
// });
// app.get('/houses/*', (req, res) => {
//   var url = req.params[0];
//   axios(url).then(response => {
//     const html = response.data;
//     const $ = cheerio.load(html);

//     var data = houses.house($);
//     res.json(data);
//   });
// });
// app.get('/filters/*', async (req, res) => {
//   var url = req.params[0];
//   const filterPage = await axios(url);
//   const html = filterPage.data;
//   const $ = cheerio.load(html);

//   const propertyLins = $('#placardContainer .property-link').map(function () {
//     return $(this).attr('href');
//   }).get();

//   var properties = [];
//   for (const link of propertyLins){
//     var response = await axios(link);

//     var property = apartments.apartment(cheerio.load(response.data));
//     properties.push(property);
//   }

//   res.json(properties);
// });
// app.get('/apartments/*', (req, res) => {
//   var url = req.params[0];
//   axios(url).then(response => {
//     const html = response.data;
//     const $ = cheerio.load(html);

//     var data = apartments.apartment($);

//     createListing(client, data);
//     res.json(data);


//   });
// });

app.get("/scrapes", async (req, res) => {
  try {
    const dbo = client.db(database);
    let collection = dbo.collection('scrapes');
    let data = await collection.find({}).toArray();
    return res.json(data);
  } catch (err) {
    console.log(err);
    return res.status(500).json();
  }
});
app.get("/scrapes/:id", async (req, res) => {
  const id = req.params.id;
  try {
    const dbo = client.db(database);
    let collection = dbo.collection('scrapes');

    var o_id = new ObjectID(id);
    let data = await collection.findOne({ _id: o_id});
    return res.json(data);
  } catch (err) {
    console.log(err);
    res.status(500).json();
  }
});
app.post("/scrapes/", async (req, res) => {
  const location = req.body.location;
  const price = req.body.price;
  const beds = req.body.beds;
  const type = req.body.type;
  const lifestyle = req.body.lifestyle;

  // query builder

  //todo: save data into the database
  try {
    const dbo = client.db(database);
    let collection = dbo.collection('scrapes');

    let res = await collection.insertOne({
      count: 21,
      estimate: Date.now(),
      sourceUrl: "https://www.apartments.com",
      location: location,
      filters: [
        { name: 'price', value: price },
        { name: 'beds', value: beds },
        { name: 'type', value: type },
        { name: 'lifestyle', value: lifestyle },
      ],
      status: "requested"
    });
    console.log(res);
  } catch (err) {
    console.log(err);
    return res.status(500).json();
  }
  return res.json();
});
app.patch("/scrapes/:id/execute", async (req, res) => {
  const id = req.params.id;

  try {
    const dbo = client.db(database);
    let collection = dbo.collection('scrapes');

    var o_id = new ObjectID(id);
    var newvalues = { $set: {status: "pending"} };
    let data = await collection.updateOne({ _id: o_id}, newvalues);
    return res.status(204).json();
  } catch (err) {
    console.log(err);
    res.status(500).json();
  }
});

const port = 3333;
// var task = cron.schedule('* * * * *', function() {
//   console.log(`Runned job...`)
// });
//   var options = {
//     host: 'http://localhost',
//     port:port,
//     path: '/apartments/https://www.apartments.com/essex-on-the-park-chicago-il/begd58b/',
//     method: 'GET'
//   };


// task.start()

// task.stop();

app.listen(port, () => {
  console.log(`Example app listening at http://localhost:${port}`)
});
process.on('exit', function () {
  client.close();
});