{"size": 6029, "ext": "js", "max_stars_repo_path": "server.js", "max_stars_repo_name": "mattsears18/articlesearch", "max_stars_repo_stars_event_min_datetime": null, "max_stars_repo_stars_event_max_datetime": null, "max_issues_repo_path": "server.js", "max_issues_repo_name": "mattsears18/articlesearch", "max_issues_repo_issues_event_min_datetime": null, "max_issues_repo_issues_event_max_datetime": null, "max_forks_repo_path": "server.js", "max_forks_repo_name": "mattsears18/articlesearch", "max_forks_repo_forks_event_min_datetime": null, "max_forks_repo_forks_event_max_datetime": null}
{"max_stars_count": null, "max_issues_count": null, "max_forks_count": null, "avg_line_length": 23.8300395257, "max_line_length": 97, "alphanum_fraction": 0.5984408691}
'use strict'
var express = require('express');
var bodyParser = require('body-parser');
var path = require('path');
var MongoClient = require('mongodb').MongoClient;
var mongoose = require('mongoose');
var pdftotext = require('pdftotextjs');
var fs = require('fs');
var csv = require('csvtojson');

var multer = require('multer');

var upload = multer({
  dest: __dirname + '/public/uploads/',
  fileFilter: function (req, file, cb) {
    if (path.extname(file.originalname) !== '.pdf') {
      return cb(null, false)
    }

    cb(null, true)
  }
});

var app = express();

var Quote = require('./models/Quote');
var Article = require('./models/Article');
var Search = require('./models/Search');

app.use(bodyParser.urlencoded({extended: true}));
app.use(express.static('public'));
app.set('view engine', 'pug');

var options = { server: { socketOptions: { keepAlive: 3700000, connectTimeoutMS: 3600000 } },
                replset: { socketOptions: { keepAlive: 3700000, connectTimeoutMS : 3600000 } } };

var mongodbUri = process.env.MONGODB_URI;

mongoose.connect(mongodbUri, options);
var db = mongoose.connection;
mongoose.Promise = require('bluebird');

db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', function() {
  var port = process.env.PORT || 3000;
  app.listen(port, () => {
    console.log('listening on ' + port);
  });
});

/**
 * GET /
 * Get index of searches
 */
app.get('/', (req, res) => {
  Article.collection.count((err, articleCount) => {

    Article.find({processed: true}, (err, textArticles) => {
      var processedCount = textArticles.length;

      Search.find(function (err, searches) {
        if (err) return console.error(err);
        res.render('index', {
          searches: searches,
          articleCount: articleCount,
          processedCount: processedCount
        });
      }).sort('-createdAt');
    }).select('originalname');
  });
});

/**
 * GET /searches/:id
 * Get index of searches
 */
app.get('/searches/:id', (req, res) => {
  Search.findById(req.params.id, function(err, search) {
    res.render('search', { search: search });
  }).populate('articles');
});

/**
 * GET /searches/delete/:id
 * Get index of searches
 */
app.get('/searches/delete/:id', (req, res) => {
  Search.findByIdAndRemove(req.params.id, function(err, search) {
    res.redirect('/');
  });
});

/**
 * POST /searches
 * Adds new search to the database.
 */
app.post('/searches', (req, res) => {
  var data = req.body;
  data.createdAt = Date();

  var newSearch = new Search(data);

  Article.find({text: new RegExp("\\b" + newSearch.term + "\\b", 'i')}, (err, articles) => {
    console.log(articles);

    newSearch.articles = articles;

    newSearch.save(function (err, quote) {
      if (err) console.log(err);
      console.log('search saved to databse');

      res.redirect('/');
    });

  }).select('text').sort('normalizedName');
});

/**
* GET /
* Get index of searches
*/
app.get('/articles', (req, res) => {
  Article.find(function (err, articles) {
    if (err) return console.error(err);
    res.render('articles', {
      articles: articles
    });
  }).sort('normalizedName').select('originalname filename processed uri');
});

/**
 * GET /articles/:id
 * Get an article
 */
app.get('/articles/:id', (req, res) => {
  Article.findById(req.params.id, function(err, article) {
    res.render('articles/article', { article: article });
  });
});

/**
 * GET /articles/delete/:id
 * Delete and article
 */
app.get('/articles/delete/:id', (req, res) => {
  Article.findByIdAndRemove(req.params.id, function(err, article) {
    res.redirect('/articles');
  });
});

/**
 * POST /articles
 * Adds new article to the database.
 */
app.post('/articles', upload.array('pdfs', 15000), (req, res) => {
  makeArticles(req.files);
  res.redirect('/');
});

/**
 * GET /csv
 * Get an article
 */
app.get('/csv', (req, res) => {
  res.render('csv');
});

var uploadCsv = multer({
  dest: __dirname + '/public/uploads/'
});

/**
 * POST /csv
 * Adds new articles to the database based upon CSV file.
 */
app.post('/csv', uploadCsv.array('csv', 1), (req, res) => {
  var file = req.files[0];

  csv()
  .fromFile(__dirname + '/public/uploads/' + file.filename)
  .on('json', (jsonObj) => {
    console.log(jsonObj);
  })
  .on('end_parsed', (jsonObjs) => {
    makeArticles(jsonObjs);
    res.redirect('/');
  })
});

/**
* GET /
* Get index of searches
*/
app.get('/processArticles', (req, res) => {
  processArticles();
});

function processArticles(err, articles) {
  Article.find({ processed: { $exists: false } }, (err, articles) => {

    for (var i=0; i < articles.length; i++) {
      var article = articles[i];
      if(!article.processed) {
        var filePath = path.join(__dirname + "/public/uploads/" + article.filename);

        if (fs.existsSync(filePath)) {
          var pdf = new pdftotext(filePath);
          var text = pdf.getTextSync();
          article.text = text;
          article.processed = true;

          article.save(function (err, article) {
            if (err) {
              console.log(err);
              res.send(err);
            }

            console.log('Text parsed: ' + article.originalname);
          });
        } else {
          console.log('File does not exist');
        }
      }
    }

    Article.find({ processed: { $exists: false } }, (err, articles) => {
      if(articles.length > 1) {
        processArticles();
      }
    }).select('filename');
  }).select('originalname filename');
}

function makeArticles(files) {
  var counter = 0;
  files.forEach(function(file) {
    file.normalizedName = file.originalname.toLowerCase();
    file.createdAt = Date();
    var newArticle = new Article(file);
    newArticle.save(function (err, article) {
      if (err) res.send(err);

      console.log(article.originalname);
      counter++;
      console.log(counter + ' : ' + files.length);

      if(counter === files.length) {
        console.log('FINISHED UPLOADING ALL FILES!');
      }
    });
  });
}
