/**
 * This file is part of Eclipse Steady.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * SPDX-License-Identifier: Apache-2.0
 * SPDX-FileCopyrightText: Copyright (c) 2018-2020 SAP SE or an SAP affiliate company and Eclipse Steady contributors
 */
package org.eclipse.steady.python.pip;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.logging.log4j.Logger;
import org.eclipse.steady.python.ProcessWrapper;
import org.eclipse.steady.python.ProcessWrapperException;
import org.eclipse.steady.python.utils.PythonConfiguration;
import org.eclipse.steady.shared.json.JacksonUtil;
import org.eclipse.steady.shared.util.FileUtil;
import org.eclipse.steady.shared.util.StopWatch;
import org.eclipse.steady.shared.util.StringList;
import org.eclipse.steady.shared.util.ThreadUtil;
import org.eclipse.steady.shared.util.VulasConfiguration;
import org.eclipse.steady.shared.util.StringList.CaseSensitivity;
import org.eclipse.steady.shared.util.StringList.ComparisonMode;

/**
 * <p>PipWrapper class.</p>
 */
public class PipWrapper {

  private static final Logger log = org.apache.logging.log4j.LogManager.getLogger();

  static final Pattern DOWNLOAD_PATTERN = Pattern.compile("^\\s*Downloading\\s*(http\\S*).*$");

  static final Pattern SAVED_PATTERN_1 =
      Pattern.compile("^\\s*File was already downloaded\\s*(.*)$");
  static final Pattern SAVED_PATTERN_2 = Pattern.compile("^\\s*Saved\\s*(.*)$");

  static final String PACK_PATTERN_REGEX = ".*<pack>-<version>.*";

  private Path pathToPip = null;

  private Path logDir = null;

  private StringList ignorePacks = new StringList();

  /**
   * Assumes that the pip executable is part of the PATH environment variable.
   *
   * @throws org.eclipse.steady.python.ProcessWrapperException if any.
   */
  public PipWrapper() throws ProcessWrapperException {
    this(Paths.get("pip"), null);
  }

  /**
   * Creates a new wrapper for the pip executable at the given path.
   *
   * @param _path_to_pip a {@link java.nio.file.Path} object.
   * @param _log_dir a {@link java.nio.file.Path} object.
   * @throws org.eclipse.steady.python.ProcessWrapperException if any.
   */
  public PipWrapper(Path _path_to_pip, Path _log_dir) throws ProcessWrapperException {
    this.pathToPip = _path_to_pip;
    if (_log_dir != null) this.logDir = _log_dir;
    else {
      try {
        this.logDir = FileUtil.createTmpDir("vulas-pip-");
        log.info("Created tmp directory [" + this.logDir + "]");
      } catch (IOException e) {
        throw new ProcessWrapperException("Cannot create tmp directory: " + e.getMessage());
      }
    }

    // Not to be added as dependencies
    final String[] ignore_packs =
        VulasConfiguration.getGlobal()
            .getStringArray(PythonConfiguration.PY_BOM_IGNORE_PACKS, new String[] {});
    this.ignorePacks.addAll(ignore_packs, true);
  }

  private boolean ignorePackage(String _p) {
    return this.ignorePacks.contains(_p, ComparisonMode.EQUALS, CaseSensitivity.CASE_SENSITIVE);
  }

  /**
   * <p>isAvailable.</p>
   *
   * @return a boolean.
   */
  public boolean isAvailable() {
    boolean exists = false;
    try {
      final Process p = new ProcessBuilder(this.pathToPip.toString()).start();
      final int exit_code = p.waitFor();
      exists = exit_code == 0;
    } catch (IOException ioe) {
      log.error("Error calling [pip]: " + ioe.getMessage(), ioe);
    } catch (InterruptedException ie) {
      log.error("Error calling [pip]: " + ie.getMessage(), ie);
    }
    return exists;
  }

  /**
   * Calls pip install on the given project {@link Path} and the returns a list of all installed packages (including the dependencies).
   *
   * @param _project a {@link java.nio.file.Path} object.
   * @return a {@link java.util.Set} object.
   */
  public Set<PipInstalledPackage> installPackages(Path _project) {
    Set<PipInstalledPackage> packages = null;
    try {
      // Make download dir
      final Path download_dir = Paths.get(logDir.toString(), "pip-download");
      FileUtil.createDirectory(download_dir);

      // Download all deps
      ProcessWrapper pw = new ProcessWrapper();
      pw.setCommand(
          this.pathToPip,
          "download",
          "-d",
          download_dir.toString(),
          "--no-cache-dir",
          _project.toString());
      pw.setPath(logDir);
      Thread t = new Thread(pw);
      t.start();
      t.join();
      final Path download_info = pw.getOutFile();

      // Install all deps
      pw = new ProcessWrapper();
      pw.setCommand(this.pathToPip, "install", _project.toString());
      pw.setPath(logDir);
      t = new Thread(pw);
      t.start();
      t.join();

      // Get all deps
      packages = this.getListPackages();

      // Enrich with download info
      this.searchDownloadInfo(packages, FileUtil.readFile(download_info));
    } catch (ProcessWrapperException e) {
      log.error("Error calling installing packages: " + e.getMessage(), e);
    } catch (IOException e) {
      log.error("Error calling installing packages: " + e.getMessage(), e);
    } catch (InterruptedException e) {
      log.error("Error calling installing packages: " + e.getMessage(), e);
    }
    return packages;
  }

  /**
   * Calls pip freeze and pip show &lt;package&gt; in order to create and return all {@link PipInstalledPackage} of the Python environment.
   *
   * @return a {@link java.util.Set} object.
   * @throws org.eclipse.steady.python.ProcessWrapperException if any.
   * @throws java.io.IOException if any.
   * @throws java.lang.InterruptedException if any.
   */
  public Set<PipInstalledPackage> getFreezePackages()
      throws ProcessWrapperException, IOException, InterruptedException {
    final StopWatch sw = new StopWatch("pip freeze").start();
    Set<PipInstalledPackage> packages = null;
    ProcessWrapper pw = new ProcessWrapper().setCommand(this.pathToPip, "freeze").setPath(logDir);
    final Thread t = new Thread(pw, "pip");
    t.start();
    t.join();
    if (pw.terminatedWithSuccess()) {
      // Create packages
      packages = this.parsePipFreezeOutput(pw.getOutFile());

      // Log
      log.info("Found [" + packages.size() + "] pip packages:");
      for (PipInstalledPackage pack : packages) log.info("    " + pack);

      // Call pip show and pip download in separate threads
      final ExecutorService pool =
          Executors.newFixedThreadPool(ThreadUtil.getNoThreads(2)); // newSingleThreadExecutor();
      final Set<Future<PipInstalledPackage>> futures = new HashSet<Future<PipInstalledPackage>>();
      for (PipInstalledPackage pack : packages) {
        futures.add(pool.submit(new PipShow(pack)));
        futures.add(pool.submit(new PipDownload(pack)));
      }
      pool.shutdown();
      try {
        while (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
          int done = 0;
          for (Future<PipInstalledPackage> f : futures) {
            if (f.isDone()) done++;
          }
          sw.lap("[" + done + "/" + futures.size() + "] pip jobs are done");
        }
      } catch (InterruptedException e) {
        PipWrapper.log.error("Interrupt exception");
      }
    }
    sw.stop();
    return packages;
  }

  /**
   * Parses the output of pip list, and instantiates {@link PipInstalledPackage} for every installed pip package.
   * @param _file
   * @return
   * @throws IOException
   */
  private Set<PipInstalledPackage> parsePipFreezeOutput(Path _file) throws IOException {
    final Set<PipInstalledPackage> packs = new TreeSet<PipInstalledPackage>();

    // The pattern to search for
    final Pattern pattern = Pattern.compile("^(.*)==(.*)$");

    // Read line by line
    final BufferedReader reader = new BufferedReader(new FileReader(_file.toFile()));
    String line;
    while ((line = reader.readLine()) != null) {
      final Matcher m = pattern.matcher(line);
      if (m.matches()) {
        if (this.ignorePackage(m.group(1))) {
          log.warn("Package [" + m.group(1) + "] not added as installed package");
        } else {
          packs.add(new PipInstalledPackage(m.group(1), m.group(2)));
        }
      }
    }
    reader.close();

    return packs;
  }

  /**
   * Calls pip list and pip show &lt;package&gt; in order to create and return all {@link PipInstalledPackage}s of the Python environment.
   *
   * @return a {@link java.util.Set} object.
   * @throws org.eclipse.steady.python.ProcessWrapperException if any.
   * @throws java.io.IOException if any.
   * @throws java.lang.InterruptedException if any.
   */
  public Set<PipInstalledPackage> getListPackages()
      throws ProcessWrapperException, IOException, InterruptedException {
    final StopWatch sw = new StopWatch("pip list").start();

    // Try legacy format
    ProcessWrapper pw =
        new ProcessWrapper()
            .setCommand(this.pathToPip, "list", "--format", "legacy")
            .setPath(logDir);
    Thread t = new Thread(pw, "pip");
    t.start();
    t.join();

    Set<PipInstalledPackage> packages = null;
    if (pw.terminatedWithSuccess()) {
      packages = this.parsePipListOutput(pw.getOutFile());
    }
    // Try JSON format
    else {
      log.info("Legacy format did not work, trying JSON format...");
      pw =
          new ProcessWrapper()
              .setCommand(this.pathToPip, "list", "--format", "json")
              .setPath(logDir);
      t = new Thread(pw, "pip");
      t.start();
      t.join();
      if (pw.terminatedWithSuccess()) packages = this.deserializePipListOutput(pw.getOutFile());
    }

    // Collect package details
    if (packages != null) {
      log.info("Found [" + packages.size() + "] pip packages:");
      for (PipInstalledPackage pack : packages) log.info("    " + pack);

      // Call pip show in separate threads
      final ExecutorService pool =
          Executors.newFixedThreadPool(ThreadUtil.getNoThreads(2)); // newSingleThreadExecutor();
      final Set<Future<PipInstalledPackage>> futures = new HashSet<Future<PipInstalledPackage>>();
      for (PipInstalledPackage pack : packages) {
        futures.add(pool.submit(new PipShow(pack)));
      }
      pool.shutdown();
      try {
        while (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
          int done = 0;
          for (Future<PipInstalledPackage> f : futures) {
            if (f.isDone()) done++;
          }
          sw.lap("[" + done + "/" + futures.size() + "] pip jobs are done");
        }
      } catch (InterruptedException e) {
        PipWrapper.log.error("Interrupt exception");
      }
    } else {
      log.warn("No pip packages found with pip list");
    }

    sw.stop();
    return packages;
  }

  /**
   * Parses the output of pip list, and instantiates {@link PipInstalledPackage} for every installed pip package.
   * @param _file
   * @return
   * @throws IOException
   */
  private Set<PipInstalledPackage> parsePipListOutput(Path _file) throws IOException {
    final Set<PipInstalledPackage> packs = new TreeSet<PipInstalledPackage>();

    // The pattern to search for
    // HP, 17.04.2018: There seem to be versions of PIP with and without using brackets
    final Pattern pattern_brackets = Pattern.compile("^(.*)\\s+\\((.*)\\)$");
    // final Pattern pattern = Pattern.compile("^(.*)\\s+(.*)$");

    // Read line by line
    final BufferedReader reader = new BufferedReader(new FileReader(_file.toFile()));
    String line;
    while ((line = reader.readLine()) != null) {
      final Matcher mb = pattern_brackets.matcher(line);
      if (mb.matches()) {
        if (this.ignorePackage(mb.group(1))) {
          log.warn("Package [" + mb.group(1) + "] not added as installed package");
        } else {
          packs.add(new PipInstalledPackage(mb.group(1), mb.group(2)));
        }
      }
    }
    reader.close();

    return packs;
  }

  /**
   * Parses the output of pip list, and instantiates {@link PipInstalledPackage} for every installed pip package.
   * @param _file
   * @return
   * @throws IOException
   */
  private Set<PipInstalledPackage> deserializePipListOutput(Path _file) throws IOException {
    // Deserialize
    final String json = FileUtil.readFile(_file);
    final PipPackageJson[] packs =
        (PipPackageJson[]) JacksonUtil.asObject(json, PipPackageJson[].class);
    // Create set
    final Set<PipInstalledPackage> set = new HashSet<PipInstalledPackage>();
    for (PipPackageJson p : packs) {
      if (this.ignorePackage(p.getName())) {
        log.warn("Package [" + p.getName() + "] not added as installed package");
      } else {
        set.add(new PipInstalledPackage(p.getName(), p.getVersion()));
      }
    }
    return set;
  }

  /**
   * Calls pip show <package> in order to set the properties of the given {@link PipInstalledPackage}.
   * @param _p
   * @param _pool TODO
   */
  final class PipShow implements Callable<PipInstalledPackage> {
    private PipInstalledPackage pack = null;

    PipShow(PipInstalledPackage _pack) {
      this.pack = _pack;
    }

    public PipInstalledPackage call() throws ProcessWrapperException, IOException {
      final ProcessWrapper pw = new ProcessWrapper(this.pack.getName());
      pw.setCommand(pathToPip, "show", this.pack.getName()).setPath(logDir);
      pw.run();
      if (pw.terminatedWithSuccess()) {
        this.pack.addProperties(parsePipShowOutput(pw.getOutFile()));
        log.info("Added properties to " + this.pack);
      } else {
        log.info("Properties for " + this.pack + " are missing");
      }
      return this.pack;
    }
  }

  private Map<String, String> parsePipShowOutput(Path _file) throws IOException {
    final Map<String, String> props = new HashMap<String, String>();

    // The pattern to search for
    final Pattern pattern = Pattern.compile("^([^:]*):(.*)$");

    // Read line by line
    final BufferedReader reader = new BufferedReader(new FileReader(_file.toFile()));
    String line;
    while ((line = reader.readLine()) != null) {
      final Matcher m = pattern.matcher(line);
      if (m.matches()) props.put(m.group(1).trim(), m.group(2).trim());
    }
    reader.close();

    return props;
  }

  /**
   * Calls pip download <package> in order to add download path and url of the given {@link PipInstalledPackage}.
   * @param _p
   */
  final class PipDownload implements Callable<PipInstalledPackage> {
    private PipInstalledPackage pack = null;

    PipDownload(PipInstalledPackage _pack) {
      this.pack = _pack;
    }

    public PipInstalledPackage call() throws ProcessWrapperException, IOException {
      // Make download dir
      // final Path download_dir = Paths.get(logDir.toString(), "pip-download");
      final Path download_dir = FileUtil.createTmpDir(this.pack.getName() + "-");

      // Download all deps
      ProcessWrapper pw = new ProcessWrapper();
      pw.setCommand(
          pathToPip,
          "download",
          "-d",
          download_dir.toString(),
          "--no-cache-dir",
          this.pack.getName() + "==" + this.pack.getVersion());
      pw.setPath(logDir);
      pw.run();
      final Path download_info = pw.getOutFile();

      // Enrich with download info
      searchDownloadInfo(this.pack, FileUtil.readFile(download_info));
      return this.pack;
    }
  }

  /**
   * Searches the provided output of 'pip download' for download URL and download path of the given packages.
   * @param _packs
   * @param _out standard out of pip download
   */
  void searchDownloadInfo(Set<PipInstalledPackage> _packs, String _out) throws IOException {
    if (_packs != null) for (PipInstalledPackage pack : _packs) this.searchDownloadInfo(pack, _out);
  }

  /**
   * Searches the provided output of 'pip download' for download URL and download path of the given package.
   * @param _p
   * @param _out standard out of pip download
   */
  private void searchDownloadInfo(PipInstalledPackage _p, String _out) throws IOException {
    // Find all download URLs and files in pip output
    final Set<String> urls = new HashSet<String>();
    final Set<String> files = new HashSet<String>();
    final BufferedReader r =
        new BufferedReader(new InputStreamReader(new ByteArrayInputStream(_out.getBytes())));
    String line = null;
    while ((line = r.readLine()) != null) {
      final Matcher d = PipWrapper.DOWNLOAD_PATTERN.matcher(line);
      final Matcher s1 = PipWrapper.SAVED_PATTERN_1.matcher(line);
      final Matcher s2 = PipWrapper.SAVED_PATTERN_2.matcher(line);
      if (d.matches()) urls.add(d.group(1));
      else if (s1.matches()) files.add(s1.group(1));
      else if (s2.matches()) files.add(s2.group(1));
    }

    // Regex to search for the file name
    String regex = PACK_PATTERN_REGEX.replace("<pack>", _p.getName().toLowerCase());
    regex = regex.replace("<version>", _p.getVersion().toLowerCase());
    regex =
        regex.replace(
            "-",
            "[_-]{1,1}"); // For some reason, the file names of some packages use _ instead of -

    final Pattern p = Pattern.compile(regex);
    for (String url : urls) {
      final Matcher m = p.matcher(url.toLowerCase());
      if (m.matches()) {
        _p.setDownloadUrl(url);
        break;
      }
    }
    for (String file : files) {
      final Matcher m = p.matcher(file.toLowerCase());
      if (m.matches()) {
        _p.setDownloadPath(Paths.get(file));
        break;
      }
    }

    // Warn if no info could be found
    if (_p.getDownloadPath() == null && _p.getDownloadUrl() == null)
      log.warn("Download path and URL for " + _p + " missing");
    else if (_p.getDownloadPath() == null) log.warn("Download path for " + _p + " missing");
    else if (_p.getDownloadUrl() == null) log.warn("Download URL for " + _p + " missing");
    else log.info("Found download info for " + _p);
  }

  /**
   * Helper class for deserializing the output of pip list --format json.
   */
  static class PipPackageJson {
    String name;
    String version;
    String installer;
    String location;

    public String getName() {
      return name;
    }

    public void setName(String name) {
      this.name = name;
    }

    public String getVersion() {
      return version;
    }

    public void setVersion(String version) {
      this.version = version;
    }

    public String getInstaller() {
      return installer;
    }

    public void setInstaller(String installer) {
      this.installer = installer;
    }

    public String getLocation() {
      return location;
    }

    public void setLocation(String location) {
      this.location = location;
    }
  }
}
