{"size": 2257, "ext": "js", "max_stars_repo_path": "src/main.js", "max_stars_repo_name": "TiZho/article-parser", "max_stars_repo_stars_event_min_datetime": "2015-12-08T22:16:27.000Z", "max_stars_repo_stars_event_max_datetime": "2022-03-31T20:08:55.000Z", "max_issues_repo_path": "src/main.js", "max_issues_repo_name": "TiZho/article-parser", "max_issues_repo_issues_event_min_datetime": "2016-09-21T05:18:15.000Z", "max_issues_repo_issues_event_max_datetime": "2022-03-28T02:31:30.000Z", "max_forks_repo_path": "src/main.js", "max_forks_repo_name": "TiZho/article-parser", "max_forks_repo_forks_event_min_datetime": "2016-10-27T17:50:23.000Z", "max_forks_repo_forks_event_max_datetime": "2022-03-28T21:21:29.000Z"}
{"max_stars_count": 313, "max_issues_count": 238, "max_forks_count": 60, "avg_line_length": 21.9126213592, "max_line_length": 76, "alphanum_fraction": 0.6118741693}
/**
 * Article parser
 * @ndaidong
 **/

const {
  isString,
  unique,
} = require('bellajs');

const {hasProvider, extract: extractOembed} = require('oembed-parser');

const retrieve = require('./utils/retrieve');
const isValidUrl = require('./utils/isValidUrl');
const normalizeUrl = require('./utils/normalizeUrl');
const parseFromHtml = require('./utils/parseFromHtml');

const {
  info,
} = require('./utils/logger');

const {
  setParserOptions,
  setNodeFetchOptions,
  setSanitizeHtmlOptions,
  getParserOptions,
  getNodeFetchOptions,
  getSanitizeHtmlOptions,
} = require('./config');


const extract = async (input) => {
  if (!isString(input)) {
    throw new Error('Input must be a string');
  }
  const article = {
    url: '',
    links: [],
    title: '',
    description: '',
    image: '',
    author: '',
    content: '',
    source: '',
    published: '',
    ttr: 0,
  };

  if (!isValidUrl(input)) {
    return parseFromHtml(input, []);
  }

  const trimmedUrl = input.trim();

  if (isValidUrl(trimmedUrl)) {
    const normalizedUrl = normalizeUrl(trimmedUrl);
    const links = [trimmedUrl, normalizedUrl];
    article.url = normalizedUrl;
    if (hasProvider(normalizedUrl)) {
      info('Provider found, loading as oEmbed data...');
      const json = await extractOembed(normalizedUrl);
      if (json) {
        article.title = json.title || '';
        article.content = json.html || '';
        article.author = json.author_name || '';
        article.image = json.thumbnail_url || '';
        article.source = json.provider_name || '';
        if (json.url) {
          article.url = json.url;
          links.push(json.url);
        }
        article.links = unique(links);
        return article;
      }
    }
    const res = await retrieve(normalizedUrl);
    if (!res) {
      throw new Error(`Could not retrieve content from "${normalizedUrl}"`);
    }
    const {
      html,
      url,
      resUrl,
    } = res;

    const result = parseFromHtml(html, [...links, url, resUrl]);
    if (result) {
      return result;
    }
  }
  return null;
};

module.exports = {
  setParserOptions,
  setNodeFetchOptions,
  setSanitizeHtmlOptions,
  getParserOptions,
  getNodeFetchOptions,
  getSanitizeHtmlOptions,
  extract,
};
