{"size": 1270, "ext": "js", "max_stars_repo_path": "scripts/prepare-data/util.js", "max_stars_repo_name": "leettran/CovidDash", "max_stars_repo_stars_event_min_datetime": null, "max_stars_repo_stars_event_max_datetime": null, "max_issues_repo_path": "scripts/prepare-data/util.js", "max_issues_repo_name": "leettran/CovidDash", "max_issues_repo_issues_event_min_datetime": null, "max_issues_repo_issues_event_max_datetime": null, "max_forks_repo_path": "scripts/prepare-data/util.js", "max_forks_repo_name": "leettran/CovidDash", "max_forks_repo_forks_event_min_datetime": null, "max_forks_repo_forks_event_max_datetime": null}
{"max_stars_count": null, "max_issues_count": null, "max_forks_count": null, "avg_line_length": 25.9183673469, "max_line_length": 88, "alphanum_fraction": 0.6708661417}
const {Transform} = require('stream')
const stripBomStream = require('strip-bom-stream')
const getStream = require('get-stream')
const csvParse = require('csv-parser')
const {readJson} = require('fs-extra')
const got = require('got')
const Airtable = require('airtable')

async function fetchCsv(url, options = {}) {
  const rows = await getStream.array(
    got.stream(url)
      .pipe(stripBomStream())
      .pipe(csvParse(options))
      .pipe(new Transform({
        transform(row, enc, cb) {
          if (!options.filter || options.filter(row)) {
            return cb(null, row)
          }

          cb()
        },
        objectMode: true
      }))
  )
  return rows
}

async function fetchJson(url) {
  const response = await got(url, {responseType: 'json'})
  return response.body
}

async function loadJson(dataSource) {
  if (dataSource.startsWith('http')) {
    return fetchJson(dataSource)
  }

  return readJson(dataSource)
}

async function extractFromAirtable(databaseId, tabName) {
  const airtable = new Airtable({apiKey: process.env.AIRTABLE_API_KEY}).base(databaseId)
  const records = await airtable(tabName).select().all()

  return records.map(record => record.fields)
}

module.exports = {loadJson, fetchJson, fetchCsv, extractFromAirtable}
