{"size": 19602, "ext": "js", "max_stars_repo_path": "build.js", "max_stars_repo_name": "meringlab/paxdb-api", "max_stars_repo_stars_event_min_datetime": "2020-07-26T21:26:10.000Z", "max_stars_repo_stars_event_max_datetime": "2020-07-26T21:26:10.000Z", "max_issues_repo_path": "build.js", "max_issues_repo_name": "meringlab/paxdb-api", "max_issues_repo_issues_event_min_datetime": "2019-02-11T12:44:29.000Z", "max_issues_repo_issues_event_max_datetime": "2021-05-07T12:53:31.000Z", "max_forks_repo_path": "build.js", "max_forks_repo_name": "meringlab/paxdb-api", "max_forks_repo_forks_event_min_datetime": null, "max_forks_repo_forks_event_max_datetime": null}
{"max_stars_count": 1, "max_issues_count": 2, "max_forks_count": null, "avg_line_length": 36.5027932961, "max_line_length": 356, "alphanum_fraction": 0.6240179574}
/**
 * Created by milans on 9/8/16.
 */

const PAYLOAD_VERSION = 17;
const uniprotMappingFile = './data/paxdb_uniprot_linkins_ids.tsv';
const PAXDB_URL = 'https://pax-db.org/';
const fs = require('fs');
const async = require('async');
const pg = require('pg');
const readline = require('readline');

const speciesIds = [882,1148,3055,3702,4081,4577,4896,4932,5061,5691,5833,6239,7165,7227,7460,7955,8364,9031,9598,9606,9615,9796,9823,9913,10090,10116,39947,44689,64091,83332,85962,99287,122586,158878,160490,169963,192222,198214,208964,211586,214092,214684,224308,226186,243159,260799,267671,272623,272624,283166,353153,449447,511145,546414,593117,722438];

const connectionString = process.env.DATABASE_URL || 'postgres://postgres@atlas.meringlab.org:5432/string_10_5';
const client = new pg.Client(connectionString);
client.connect();

function loadSpeciesInfo(callback) {
  console.log(`loading species info`);
  const sqlSpeciesInfo = `select species_id,official_name,compact_name from items.species where species_id in (${speciesIds.join(',')})`;
  const sqlNumProteins = `select species_id,count(protein_id) as c from items.proteins where species_id in (${speciesIds.join(',')}) group by species_id; `;
  const species = {};
  client.query(sqlSpeciesInfo).then(res => {
    res.rows.forEach(function(r) {
      species[r.species_id] = { id: r.species_id, name: r.official_name, compact_name: r.compact_name };
    });
    client.query(sqlNumProteins).then(npres => {
      npres.rows.forEach(function(r) {
        species[r.species_id]['num_proteins'] = parseInt(r.c);
      });
      console.log(`loading species info DONE`);
      callback(species);
    });
  });
}

function parseOrthgroups(contents, familySet) {
  contents.split('\n').forEach(function(line) {
    if (line.trim() == 0) {
      return
    }
    var rec = line.split('\t');
    //{"id": 9443, "name": "NOG21051", "clade": "PRIMATES", "members": [1803841, 1854701]},
    rec.slice(1, rec.length).forEach(function(el) {
      familySet.add(parseInt(el));
    });
  });
}

function loadUniprotMapping() {
  "use strict";
  console.log(`loadUniprotMapping from ${uniprotMappingFile}`);
  const paxdbUniprotIdMap = {};

  const uniprotLinkins = fs.readFileSync(uniprotMappingFile, 'utf-8');
  uniprotLinkins.split("\n").forEach((line) => {
    if (line) {
      let rec = line.trim().split('\t');
      paxdbUniprotIdMap[rec[0]] = rec[1];
    }
  });
  return paxdbUniprotIdMap;
}

function loadProteins(cb, createProteinModules = false) {
  console.log(`loading proteins`);
  console.log("loading orthgroups");
  const familySet = new Set();
  fs.readdirSync('./data/orthgroups').forEach(function(file) {
    parseOrthgroups(fs.readFileSync(`./data/orthgroups/${file}`, { 'encoding': 'utf8' }), familySet);
  });

  const paxdbUniprotIdsMap = loadUniprotMapping();
  const uniprotPaxdbIdsMap = {};
  const speciesForProtein = {};

  async.eachSeries(speciesIds, function(speciesId, callback) {
    console.log(`loading proteins for ${speciesId}`);
    const proteins = {};
    const sql = `select protein_id, protein_external_id, preferred_name, annotation ` +
      ` from items.proteins where species_id = ${speciesId}`;
    client.query(sql).then(res => {
      res.rows.forEach(function(r) {
        proteins[r.protein_id] = {
          id: r.protein_id,
          externalId: r.protein_external_id,
          name: r.preferred_name,
          hasFamily: familySet.has(r.protein_id),
          annotation: r.annotation
        };
        if (Object.prototype.hasOwnProperty.call(paxdbUniprotIdsMap, r.protein_external_id)) {
          const ac = paxdbUniprotIdsMap[r.protein_external_id];
          proteins[r.protein_id].uniprotId = ac;
          if (! Object.prototype.hasOwnProperty.call(uniprotPaxdbIdsMap, ac)) {
            uniprotPaxdbIdsMap[ac] = r.protein_id;
          } else {
            let prev = uniprotPaxdbIdsMap[ac];
            if (!Array.isArray(prev)) {
              console.log(`multiple paxdb ids for ${ac}, must to use an array`);
              uniprotPaxdbIdsMap[ac] = [prev];
            }
            uniprotPaxdbIdsMap[ac].push(r.protein_id);
          }
        }
        speciesForProtein[r.protein_id] = speciesId;
      });
      console.log(`loading proteins for ${speciesId} DONE`);
      if (createProteinModules) {
        var writeStream = fs.createWriteStream(`./lib/proteins/${speciesId}.js`);
        writeStream.write(`//FILE GENERATED BY build.js on ${new Date()}, DO NOT MODIFY!\n`);
        writeStream.write("const proteins = ");
        writeStream.write(JSON.stringify(proteins));
        writeStream.write(';\n');
        writeStream.write('module.exports = proteins;');
        writeStream.end();
      }
      callback();
    });
  }, function(err) {
    console.log(`loading proteins DONE`);
    if (err) throw err;
    cb(speciesForProtein, uniprotPaxdbIdsMap);
  });
}


function loadDatasetInfo(cb) {
  console.log(`loading dataset info`);
  const datasets = {};
  const abundances_asc = {};
  const abundances_desc = {};
  const proteinsCovered = {}
  async.eachSeries(fs.readdirSync('./data/abundances'), function(d, callback) {
    const dataset = {};
    const abundances = {};
    const peptideCounts = {};
    const species = parseInt(d.split('-')[0])
    if (!(species in datasets)) {
      datasets[species] = []
    }
    if (!(species in proteinsCovered)) {
      proteinsCovered[species] = new Set();
    }
    datasets[species].push(dataset);
    const input = fs.createReadStream(`./data/abundances/${d}`);
    const rl = readline.createInterface({ input })
    rl.on('close', function() {

      //add ranks
      var abundancesSorted = []
      for (var p in abundances) {
        abundancesSorted.push({ id: p, abundance: abundances[p] })
      }
      abundancesSorted = abundancesSorted.sort((a, b) => {
        return b.abundance - a.abundance
      });
      dataset.abundances = {};
      const asc = [];
      const desc = [];
      abundancesSorted.forEach((a, rank) => {
        dataset.abundances[a.id] = { a: a.abundance, r: rank }
        var intId = parseInt(a.id, 10);
        desc.push(intId);
        asc.push(intId);
      });
      asc.reverse();
      abundances_asc[dataset.id] = asc;
      abundances_desc[dataset.id] = desc;

      dataset.num_abundances = abundancesSorted.length;
      if (Object.keys(peptideCounts).length === 0) {
        dataset.hasPeptideCounts = false;
      } else {
        dataset.hasPeptideCounts = true;
        dataset.peptideCounts = peptideCounts;
      }
      dataset.filename = d;

      callback(null);
    });

    rl.on('line', function(line) {
      if (!line.startsWith("#")) {
        var rec = line.split('\t');
        if (rec.length > 1) {
          if (!dataset.integrated) {
            proteinsCovered[species].add(rec[0]);
          }
          abundances[rec[0]] = parseFloat(rec[2]);
        }
        if (rec.length == 4) {
          peptideCounts[rec[0]] = parseInt(rec[3], 10);
        }
      } else {
        switch (line.split(':')[0].trim()) {
          case '#id':
            dataset.id = parseInt(line.split(':')[1].trim());
            break;
          case '#name':
            dataset.name = line.split(':')[1].trim();
            break;
          case '#score':
            dataset.score = parseFloat(line.split(':')[1].trim());
            break;
          case '#weight':
            dataset.weight = line.split(':')[1].trim();
            break;
          case '#description':
            dataset.description = line.replace(/#\W*description:\W*/, '').trim();
            break;
          case '#organ':
            dataset.organ = line.split(':')[1].trim();
            break;
          case '#integrated':
            dataset.integrated = line.split(':')[1].trim() === 'true';
            break;
          case '#coverage':
            dataset.coverage = parseFloat(line.split(':')[1].trim());
            break;
          case '#publication_year':
            dataset.publication_year = parseInt(line.split(':')[1].trim());
            break;
        }
      }
    })
  }, function(err) {
    console.log(`loading dataset info DONE`);
    if (err) throw err;
    speciesIds.forEach(function(id) {
      proteinsCovered[id] = proteinsCovered[id].size
    });

    cb(datasets, proteinsCovered, abundances_asc, abundances_desc);
  });
}

function loadGenomeSources(callback) {
  const input = fs.createReadStream('./data/eggnog4_genome_linkout.txt');
  const rl = readline.createInterface({ input })
  const sources = {};
  const versions = {};

  rl.on('close', function() {
    callback(sources, versions);
  });

  rl.on('line', function(line) {
    const rec = line.split('\t');
    if (rec.length > 4) {
      sources[parseInt(rec[1])] = `<a href='${rec[4]}'>${rec[2]}</a>`;
      versions[parseInt(rec[1])] = rec[3];
    }
  });
}

function buildSpecies() {
  loadGenomeSources(function(sources, sourcesVersions) {

    loadSpeciesInfo(function(species) {
      for (var s in species) {
        species[s].genome_source = sources[s];
        species[s].genome_source_version = sourcesVersions[s];
      }

      loadDatasetInfo(function(datasets, proteinsCovered) {
        for (var id in species) {
          var s = species[id];
          s.datasets = datasets[s.id];
          s.datasets.forEach(function(d) {
            delete d.abundances;
            delete d.peptideCounts;
          });
          s.total_coverage = parseInt((100 * proteinsCovered[s.id]) / s.num_proteins, 10);
          if (s.datasets.length == 1) {
            s.average_coverage = s.datasets[0].coverage;
          } else {
            var numDatasets = 0;
            var sum = 0;
            s.datasets.forEach(function(d) {
              if (!d.integrated) {
                numDatasets++;
                sum += d.coverage;
              }
            });
            s.average_coverage = sum / numDatasets; //round?
          }
        }

        var writeStream = fs.createWriteStream('./lib/species.js');
        writeStream.write(`//FILE GENERATED BY build.js on ${new Date()}, DO NOT MODIFY!\n`);
        writeStream.write("const species = ");
        writeStream.write(JSON.stringify(species));
        writeStream.write(';\n');
        writeStream.write('module.exports = species;');

        writeStream.end();
        console.log('./lib/species.js generated');
        client.on('drain', client.end.bind(client));
        client.end();
      });
    });
  });
}

function buildProteins() {
  loadProteins(function(speciesForProtein, uniprotIdsMap) {
    var writeStream = fs.createWriteStream('./lib/proteins_index.js');
    writeStream.write(`//FILE GENERATED BY build.js on ${new Date()}, DO NOT MODIFY!\n`);
    writeStream.write("const speciesForProtein = ");
    writeStream.write(JSON.stringify(speciesForProtein));
    writeStream.write(';\n');
    writeStream.write(';\n');
    writeStream.write("const uniprotIdsMap = ");
    writeStream.write(JSON.stringify(uniprotIdsMap));
    writeStream.write(';\n');
    writeStream.write('module.exports.speciesForProtein = speciesForProtein;\n');
    writeStream.write('module.exports.uniprotIdsMap = uniprotIdsMap;\n');
    writeStream.end();
    console.log('./lib/proteins_index.js generated');
    client.on('drain', client.end.bind(client));
    client.end();
  }, true);
}

function buildDatasets() {

  loadDatasetInfo(function(datasets, proteinsCovered, abundances_asc, abundances_desc) {
    speciesIds.forEach(function(id) {
      datasets[parseInt(id)].forEach(function(d) {
        var abundances = d.abundances;
        var peptideCounts = d.peptideCounts;
        delete d.abundances;
        if (d.hasPeptideCounts) {
          delete d.peptideCounts;
        }
        d.species_id = parseInt(id);

        const generated = `./lib/dataset/${d.id}.js`;
        var writeStream = fs.createWriteStream(generated);
        writeStream.write(`//FILE GENERATED BY build.js on ${new Date()}, DO NOT MODIFY!\n`);
        writeStream.write("const info = ");
        writeStream.write(JSON.stringify(d));
        writeStream.write(';\n');
        writeStream.write(';\n');

        writeStream.write("const abundances = ");
        writeStream.write(JSON.stringify(abundances));
        writeStream.write(';\n');
        writeStream.write(';\n');

        writeStream.write("const abundances_asc = ");
        writeStream.write(JSON.stringify(abundances_asc[d.id]));
        writeStream.write(';\n');
        writeStream.write(';\n');

        writeStream.write("const abundances_desc = ");
        writeStream.write(JSON.stringify(abundances_desc[d.id]));
        writeStream.write(';\n');
        writeStream.write(';\n');
        writeStream.write("var peptideCounts;");
        writeStream.write(';\n');
        if (peptideCounts) {
          writeStream.write("peptideCounts =");
          writeStream.write(JSON.stringify(peptideCounts));
          writeStream.write(';\n');
          writeStream.write('\n');
        }

        writeStream.write('module.exports.info = info;\n');
        writeStream.write('module.exports.abundances = abundances;\n');
        writeStream.write('module.exports.abundances_asc = abundances_asc;\n');
        writeStream.write('module.exports.abundances_desc = abundances_desc;\n');
        writeStream.write('module.exports.peptideCounts = peptideCounts;\n');
        writeStream.end();
        console.log(`${generated} generated`);
      })
    });

    client.on('drain', client.end.bind(client));
    client.end();
  });
}

/**
 * depends on proteins data!
 */
function build_proteins_index(){
  "use strict";

  let speciesForProtein = {};
  let externalToInternalMap = {};
  let uniprotIdsMap = {};
  console.log('reading proteins and uniprot mapping');

  //IMPORTANT: this assumes the linkins file has all the paxdb species!

  let exportedSpecies = new Set();
  let currentSpecies = undefined, proteins = undefined;
  const uniprotLinkins = fs.readFileSync('./data/paxdb_uniprot_linkins_ids.tsv', 'utf-8');
  uniprotLinkins.split("\n").forEach((line) => {
    if (line) {
      let rec = line.trim().split('\t');
      let speciesId = rec[0].split('.')[0];

      if (currentSpecies != speciesId) {
        console.log(`found species ${speciesId}: ${species[speciesId].name}`);
        exportedSpecies.add(speciesId);
        currentSpecies = speciesId;
        proteins = require(`./lib/proteins/${speciesId}.js`);
        externalToInternalMap = {};
        for (let proteinId in proteins) {
          if (proteins.hasOwnProperty(proteinId)) {
            let p = proteins[proteinId];
            speciesForProtein[proteinId] = speciesId;
            externalToInternalMap[p.externalId] = p.id;
          }
        }
      }
      uniprotIdsMap[rec[1]] = externalToInternalMap[rec[0]];
      //append linkout ids as well:
      let protein = proteins[externalToInternalMap[rec[0]]];
      if (protein.uniprotId && !(protein.uniprotId in uniprotIdsMap) ) {
        uniprotIdsMap[protein.uniprotId] = protein.id;
      }
    }
  });
  if (exportedSpecies.size != speciesIds.length) {
    let missing = [...speciesIds].filter(s => !exportedSpecies.has(s));
    throw Error(`failed to find all species, missing: ${missing}`);
  }

  console.log('writing proteins_index.js');
  let writeStream = fs  .createWriteStream('./lib/proteins_index.js');
  writeStream.write(`//FILE GENERATED BY build.js on ${new Date()}, DO NOT MODIFY!\n`);
  writeStream.write("const speciesForProtein = ");
  writeStream.write(JSON.stringify(speciesForProtein));
  writeStream.write(';\n');
  writeStream.write(';\n');
  writeStream.write("const uniprotIdsMap = ");
  writeStream.write(JSON.stringify(uniprotIdsMap));
  writeStream.write(';\n');
  writeStream.write('module.exports.speciesForProtein = speciesForProtein;');
  writeStream.write('module.exports.uniprotIdsMap = uniprotIdsMap;');
  writeStream.end();
  console.log('proteins_index.js written');

}



/**
 * depends on dataset_data!
 */
function buildHistograms() {
  var makeHistogram = require('./lib/histo');
  //TODO list ./lib/dataset folder and load each module
  var glob = require("glob");

  glob('./lib/dataset/*js', (er, files) => {
    files.forEach(f => {
      var dataset = require(f);
      var abundancesMap = dataset.abundances; //map proteinId -> {a : , r: , ..}
      var abundances = [];
      for (var proteinId in abundancesMap) {
        var a = abundancesMap[proteinId].a;
        if (a >= 0.01) { //otherwise it cannot be plotted
          abundances.push(a);
        }
      }
      var d3n = makeHistogram(abundances);
      fs.writeFile(`./public/images/datasets/${dataset.info.id}.svg`, d3n.svgString(), (err) => {
        if (err) console.log(`${dataset.info.id} failed: ${err.message}`);
      });
      var d3n = makeHistogram(abundances, undefined, true);
      fs.writeFile(`./public/images/datasets/${dataset.info.id}-thumb.svg`, d3n.svgString(), (err) => {
        if (err) console.log(`${dataset.info.id} failed: ${err.message}`);
      });
    });
  });
  console.log('histograms generated');
}


function buildPayload() {
  const defaultDataset = require('./lib/speciesDefaultDataset');
  const datasetLib = require('./lib/dataset');
  const speciesRepo = require('./lib/species');

  // for (var speciesId = 882; speciesId == 882; speciesId = undefined) {
  for (let speciesId in speciesRepo) {
    console.log(`writing payload for ${speciesId}`);
    const species = speciesRepo[speciesId];
    const payloadStream = fs.createWriteStream(`./public/payload/${species.id}-payload-v${PAYLOAD_VERSION}.json`);
    payloadStream.write(`{
      "nodes_file": "${PAXDB_URL}payload/${species.id}-payload-nodes-v${PAYLOAD_VERSION}.txt",
      "edges_file" : "",
      "logo_file"  : "${PAXDB_URL}images/paxdb_logo.png",
      "legend_file": "${PAXDB_URL}images/payload_legend.png",
      "name"       : "PaxDB"
    }`);
    payloadStream.end(e=> {
      if (e) console.log(`error writing ${speciesId} payload: ${e.message}`); else console.log(`${speciesId} payload written`);
    });

    const d = defaultDataset(species.datasets);
    const dataset = require(`./lib/dataset/${d.id}.js`);
    const proteins = require(`./lib/proteins/${speciesId}.js`);
    const ranking = new datasetLib.Ranking(dataset.info.num_abundances);

    const nodesStream = fs.createWriteStream(`./public/payload/${species.id}-payload-nodes-v${PAYLOAD_VERSION}.txt`);
    Object.keys(dataset.abundances).forEach((proteinId) => {
      //  3702.AT5G51880.1    #B95050 Abundance: 3.96 ppm, rank: 7077. out of 20185   http://archive.pax-db.org/v4.0/#!protein/3702.AT5G51880.1   http://archive.pax-db.org/v4.0/images/proteins_histograms/323_140812-small.png
      const abundance = dataset.abundances[proteinId];
      const hexColor = ranking.toRGB(abundance.r);
      nodesStream.write(`${proteins[proteinId].externalId}\t${hexColor}\tAbundance: ${datasetLib.formattedAbundance(abundance.a)}, rank: ${ranking.formatRank(abundance.r)}\t`);
      nodesStream.write(`${PAXDB_URL}protein/${proteinId}/${proteins[proteinId].name}\t`);
      nodesStream.write(`${PAXDB_URL}dataset/${d.id}/histogram?hightlightProteinId=${proteinId}\n`);

    });

    nodesStream.end(e=> {
      if (e) console.log(`error writing ${speciesId} payload nodes: ${e.message}`); else console.log(`${speciesId} payload nodes written`)
    });

  }
}

buildSpecies();
buildDatasets();
buildProteins();
//TODO FIXME writing streams is async, so lib/species.js won' show up before buildPayload is called
// buildPayload();
// buildHistograms();


