text: #!/usr/bin/env python3
  import glob
  import os
  import datetime
  import argparse
  import json
  import csv
  import re
  import pdb
  import copy
  
  
  def read_file(filename):
  
      if not os.path.isfile(filename):
          print("File not found {}".format(filename))
          return ""
  
      with open(filename, 'r', encoding='utf_8') as f:
          return f.read()
  
  
  def read_go_mod(vendor_dir):
      lines = []
      with open(os.path.join(vendor_dir, "modules.txt"), encoding="utf_8") as f:
          lines = f.readlines()
  
      deps = []
      for line in lines:
          if line.startswith("# "):
              line = line[2:]
              elems = line.split(" ")
              if len(elems) == 2:
                  data = _get_version_info(elems)
                  deps.append(data)
                  continue
  
              if " => " in line:
                  data = _get_replaced_dep_data(line)
                  deps.append(data)
  
      return deps
  
  
  def _get_version_info(elems):
      data = {}
      data["path"] = elems[0]
  
      version_info = elems[1].rstrip("\n")
      if version_info.endswith("+incompatible"):
          version_info = version_info[:-len("+incompatible")]
  
      if len(version_info) < 30:
          data["version"] = version_info
          return data
  
      revision_elems = version_info.split("-")
      if len(revision_elems) != 3:
          raise ValueError("unexpected number of elements")
  
      if revision_elems[0] != "v0.0.0":
          data["version"] = revision_elems[0]
      data["revision"] = revision_elems[2]
  
      return data
  
  
  def _get_replaced_dep_data(line):
      original, fork = line.split(" => ")
      elems = fork.split(" ")
      fork_data = _get_version_info(elems)
      elems = original.split(" ")
      data = _get_version_info(elems)
  
      if "path" in fork_data:
          data["overwrite-path"] = fork_data["path"]
      if "version" in fork_data:
          data["overwrite-version"] = fork_data["version"]
      if "revision" in fork_data:
          data["overwrite-revision"] = fork_data["revision"]
  
      if "revision" in data and data["revision"] == "000000000000":
          del(data["revision"])
  
      return data
  
  
  def get_library_path(license):
      """
      Get the contents up to the vendor folder.
      """
      split = license.split(os.sep)
      for i, word in reversed(list(enumerate(split))):
          if word == "vendor":
              return "/".join(split[i + 1:])
      return "/".join(split)
  
  
  def gather_dependencies(vendor_dir, overrides=None):
      dependencies = {}   # lib_path -> [array of lib]
      libs = read_go_mod(vendor_dir)
  
      # walk looking for LICENSE files
      for root, dirs, filenames in os.walk("./vendor"):
          licenses = get_licenses(root)
          for filename in licenses:
              lib_path = get_library_path(root)
              lib_search = [l for l in libs if l["path"].startswith(lib_path)]
              if len(lib_search) == 0:
                  print("WARNING: No version information found for: {}".format(lib_path))
                  lib = {"path": lib_path}
              else:
                  lib = copy.deepcopy(lib_search[0])
  
              lib["license_file"] = os.path.join(root, filename)
  
              lib["license_contents"] = read_file(lib["license_file"])
              lib["license_summary"] = detect_license_summary(lib["license_contents"])
              if lib["license_summary"] == "UNKNOWN":
                  print("WARNING: Unknown license for: {}".format(lib_path))
  
              revision = overrides.get(lib_path, {}).get("revision")
              if revision:
                  lib["revision"] = revision
  
              if lib_path not in dependencies:
                  dependencies[lib_path] = [lib]
              else:
                  dependencies[lib_path].append(lib)
  
          # don't walk down into another vendor dir
          if "vendor" in dirs:
              dirs.remove("vendor")
  
      return dependencies
  
  
  # Allow to skip files that could match the `LICENSE` pattern but does not have any license information.
  SKIP_FILES = [
      # AWS lambda go defines that some part of the code is APLv2 and other on a MIT Modified license.
      "./vendor/github.com/aws/aws-lambda-go/LICENSE-SUMMARY"
  ]
  
  
  def get_licenses(folder):
      """
      Get a list of license files from a given directory.
      """
      licenses = []
      for filename in sorted(os.listdir(folder)):
          if filename.startswith("LICENSE") and "docs" not in filename and os.path.join(folder, filename) not in SKIP_FILES:
              licenses.append(filename)
          elif filename.startswith("APLv2"):  # gorhill/cronexpr
              licenses.append(filename)
          elif filename in ("COPYING",):  # BurntSushi/toml
              licenses.append(filename)
      return licenses
  
  
  def has_license(folder):
      """
      Checks if a particular repo has a license files.
  
      There are two cases accepted:
          * The folder contains a LICENSE
          * The parent folder contains a LICENSE
          * The folder only contains subdirectories AND all these
            subdirectories contain a LICENSE
          * The folder only contains subdirectories AND all these
            subdirectories contain  subdirectories which contain a LICENSE (ex Azure folder)
      """
  
      if len(get_licenses(folder)) > 0:
          return True, ""
      elif len(get_licenses(os.path.join(folder, os.pardir))) > 0:  # For go.opencensus.io.
          return True, ""
  
      for subdir in os.listdir(folder):
          if not os.path.isdir(os.path.join(folder, subdir)):
              return False, folder
          if len(get_licenses(os.path.join(folder, subdir))) > 0:
              continue
          for dir in os.listdir(os.path.join(folder, subdir)):
              if not os.path.isdir(os.path.join(folder, subdir, dir)):
                  return False, subdir
              if len(get_licenses(os.path.join(folder, subdir, dir))) == 0:
                  return False, os.path.join(folder, subdir, dir)
  
      return True, ""
  
  
  def check_all_have_license_files(vendor_dir):
      """
      Checks that everything in the vendor folders has a license one way
      or the other. This doesn't collect the licenses, because the code that
      collects the licenses needs to walk the full tree. This one makes sure
      that every folder in the `vendor` directories has at least one license.
      """
      issues = []
      for root, dirs, filenames in os.walk(vendor_dir):
          depth = 2
          if root.count(os.sep) - vendor_dir.count(os.sep) == depth:  # two levels deep
              # Two level deep means folders like `github.com/elastic`.
              # look for the license in root but also one level up
              ok, issue = has_license(root)
              if not ok:
                  depth += 1
  
                  if depth > 5:
                      print("No license in: {}".format(issue))
                      issues.append(issue)
  
      if len(issues) > 0:
          raise Exception("I have found licensing issues in the following folders: {}"
                          .format(issues))
  
  
  def write_notice_file(f, beat, copyright, dependencies):
  
      now = datetime.datetime.now()
  
      # Add header
      f.write("{}\n".format(beat))
      f.write("Copyright 2014-{0} {1}\n".format(now.year, copyright))
      f.write("\n")
      f.write("This product includes software developed by The Apache Software \n" +
              "Foundation (http://www.apache.org/).\n\n")
  
      # Add licenses for 3rd party libraries
      f.write("==========================================================================\n")
      f.write("Third party libraries used by the {} project:\n".format(beat))
      f.write("==========================================================================\n\n")
  
      # Sort licenses by package path, ignore upper / lower case
      for key in sorted(dependencies, key=str.lower):
          for lib in dependencies[key]:
              f.write("\n--------------------------------------------------------------------\n")
              f.write("Dependency: {}\n".format(key))
              if "version" in lib:
                  f.write("Version: {}\n".format(lib["version"]))
              if "revision" in lib:
                  f.write("Revision: {}\n".format(lib["revision"]))
              if "overwrite-path" in lib:
                  f.write("Overwrite: {}\n".format(lib["overwrite-path"]))
              if "overwrite-version" in lib:
                  f.write("Overwrite-Version: {}\n".format(lib["overwrite-version"]))
              if "overwrite-revision" in lib:
                  f.write("Overwrite-Revision: {}\n".format(lib["overwrite-revision"]))
              f.write("License type (autodetected): {}\n".format(lib["license_summary"]))
              f.write("{}:\n".format(lib["license_file"]))
              f.write("--------------------------------------------------------------------\n")
              if lib["license_summary"] != "Apache-2.0":
                  f.write(lib["license_contents"])
              else:
                  # it's an Apache License, so include only the NOTICE file
                  f.write("Apache License 2.0\n\n")
  
                  # Skip NOTICE files which are not needed
                  if os.path.join(os.path.dirname(lib["license_file"])) in SKIP_NOTICE:
                      continue
  
                  for notice_file in glob.glob(os.path.join(os.path.dirname(lib["license_file"]), "NOTICE*")):
                      notice_file_hdr = "-------{}-----\n".format(os.path.basename(notice_file))
                      f.write(notice_file_hdr)
                      f.write(read_file(notice_file))
  
  
  def write_csv_file(csvwriter, dependencies):
      csvwriter.writerow(["name", "url", "version", "revision", "license"])
      for key in sorted(dependencies, key=str.lower):
          for lib in dependencies[key]:
              csvwriter.writerow([key, get_url(key), lib.get("version", ""), lib.get("revision", ""),
                                  lib["license_summary"]])
  
  
  def get_url(repo):
      words = repo.split("/")
      if words[0] != "github.com":
          return repo
      return "https://github.com/{}/{}".format(words[1], words[2])
  
  
  def create_notice(filename, beat, copyright, vendor_dir, csvfile, overrides=None):
      dependencies = gather_dependencies(vendor_dir, overrides=overrides)
      if not csvfile:
          with open(filename, "w+", encoding='utf_8') as f:
              write_notice_file(f, beat, copyright, dependencies)
              print("Available at {}".format(filename))
      else:
          with open(csvfile, "w") as f:
              csvwriter = csv.writer(f)
              write_csv_file(csvwriter, dependencies)
              print("Available at {}".format(csvfile))
      return dependencies
  
  
  APACHE2_LICENSE_TITLES = [
      "Apache License 2.0",
      "Apache License Version 2.0",
      "Apache License, Version 2.0",
      "licensed under the Apache 2.0 license",  # github.com/zmap/zcrypto
      re.sub(r"\s+", " ", """Apache License
      ==============
  
      _Version 2.0, January 2004_"""),
  ]
  
  MIT_LICENSES = [
      re.sub(r"\s+", " ", """Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files (the "Software"), to deal
  in the Software without restriction, including without limitation the rights
  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  copies of the Software, and to permit persons to whom the Software is
  furnished to do so, subject to the following conditions:
  
  The above copyright notice and this permission notice shall be included in all
  copies or substantial portions of the Software.
      """),
      re.sub(r"\s+", " ", """Permission to use, copy, modify, and distribute this software for any
  purpose with or without fee is hereby granted, provided that the above
  copyright notice and this permission notice appear in all copies."""),
      re.sub(r"\s+", " ", """Permission is hereby granted, free of charge, to any person obtaining
  a copy of this software and associated documentation files (the
  'Software'), to deal in the Software without restriction, including
  without limitation the rights to use, copy, modify, merge, publish,
  distribute, sublicense, and/or sell copies of the Software, and to
  permit persons to whom the Software is furnished to do so, subject to
  the following conditions:
      """),
      re.sub(r"\s+", " ", """Permission is hereby granted, free of charge, to any person obtaining a copy of this
  software and associated documentation files (the "Software"), to deal in the Software
  without restriction, including without limitation the rights to use, copy, modify,
  merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
  permit persons to whom the Software is furnished to do so.
  
  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
  INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
  PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
  HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
      """),
  ]
  
  BSD_LICENSE_CONTENTS = [
      re.sub(r"\s+", " ", """Redistribution and use in source and binary forms, with or without modification,
  are permitted provided that the following conditions are met:"""),
      re.sub(r"\s+", " ", """Redistributions of source code must retain the above copyright notice, this
     list of conditions and the following disclaimer."""),
      re.sub(r"\s+", " ", """Redistributions in binary form must reproduce the above copyright notice,
     this list of conditions and the following disclaimer in the documentation
     and/or other materials provided with the distribution.
  """)]
  
  # This is a temporary workaround to solve UNKNOWN licence issue
  # for `evanphx/json-patch`: https://github.com/evanphx/json-patch/pull/105
  BSD_LICENSE_CONTENTS_COMMA_MISSING = [
      re.sub(r"\s+", " ", """Redistribution and use in source and binary forms, with or without modification,
  are permitted provided that the following conditions are met:"""),
      re.sub(r"\s+", " ", """Redistributions of source code must retain the above copyright notice, this
     list of conditions and the following disclaimer."""),
      re.sub(r"\s+", " ", """Redistributions in binary form must reproduce the above copyright notice
     this list of conditions and the following disclaimer in the documentation
     and/or other materials provided with the distribution.
  """)]
  
  BSD_LICENSE_3_CLAUSE = [
      re.sub(r"\s+", " ", """Neither the name of"""),
      re.sub(r"\s+", " ", """nor the
        names of its contributors may be used to endorse or promote products
        derived from this software without specific prior written permission.""")
  ]
  
  BSD_LICENSE_4_CLAUSE = [
      re.sub(r"\s+", " ", """All advertising materials mentioning features or use of this software
     must display the following acknowledgement"""),
  ]
  
  CC_SA_4_LICENSE_TITLE = [
      "Creative Commons Attribution-ShareAlike 4.0 International"
  ]
  
  ECLIPSE_PUBLIC_LICENSE_TITLES = [
      "Eclipse Public License - v 1.0"
  ]
  
  LGPL_3_LICENSE_TITLE = [
      "GNU LESSER GENERAL PUBLIC LICENSE Version 3"
  ]
  
  MPL_LICENSE_TITLES = [
      "Mozilla Public License Version 2.0",
      "Mozilla Public License, version 2.0"
  ]
  
  UNIVERSAL_PERMISSIVE_LICENSE_TITLES = [
      "The Universal Permissive License (UPL), Version 1.0"
  ]
  
  ISC_LICENSE_TITLE = [
      "ISC License",
  ]
  
  ELASTIC_LICENSE_TITLE = [
      "ELASTIC LICENSE AGREEMENT",
  ]
  
  
  # return SPDX identifiers from https://spdx.org/licenses/
  def detect_license_summary(content):
      # replace all white spaces with a single space
      content = re.sub(r"\s+", ' ', content)
      # replace smart quotes with less intelligent ones
      content = content.replace(bytes(b'\xe2\x80\x9c').decode(), '"').replace(bytes(b'\xe2\x80\x9d').decode(), '"')
      if any(sentence in content[0:1000] for sentence in APACHE2_LICENSE_TITLES):
          return "Apache-2.0"
      if any(sentence in content[0:1000] for sentence in MIT_LICENSES):
          return "MIT"
      if all(sentence in content[0:1000] for sentence in BSD_LICENSE_CONTENTS) or \
              all(sentence in content[0:1000] for sentence in BSD_LICENSE_CONTENTS_COMMA_MISSING):
          if all(sentence in content[0:1000] for sentence in BSD_LICENSE_3_CLAUSE):
              if all(sentence in content[0:1000] for sentence in BSD_LICENSE_4_CLAUSE):
                  return "BSD-4-Clause"
              return "BSD-3-Clause"
          else:
              return "BSD-2-Clause"
      if any(sentence in content[0:300] for sentence in MPL_LICENSE_TITLES):
          return "MPL-2.0"
      if any(sentence in content[0:3000] for sentence in CC_SA_4_LICENSE_TITLE):
          return "CC-BY-SA-4.0"
      if any(sentence in content[0:3000] for sentence in LGPL_3_LICENSE_TITLE):
          return "LGPL-3.0"
      if any(sentence in content[0:1500] for sentence in UNIVERSAL_PERMISSIVE_LICENSE_TITLES):
          return "UPL-1.0"
      if any(sentence in content[0:1500] for sentence in ECLIPSE_PUBLIC_LICENSE_TITLES):
          return "EPL-1.0"
      if any(sentence in content[0:1500] for sentence in ISC_LICENSE_TITLE):
          return "ISC"
      if any(sentence in content[0:1500] for sentence in ECLIPSE_PUBLIC_LICENSE_TITLES):
          return "EPL-1.0"
      if any(sentence in content[0:1500] for sentence in ELASTIC_LICENSE_TITLE):
          return "ELASTIC"
      return "UNKNOWN"
  
  
  ACCEPTED_LICENSES = [
      "Apache-2.0",
      "BSD-4-Clause",
      "BSD-3-Clause",
      "BSD-2-Clause",
      "EPL-1.0",
      "MIT",
      "MPL-2.0",
      "UPL-1.0",
      "ISC",
      "ELASTIC",
  ]
  SKIP_NOTICE = []
  
  if __name__ == "__main__":
  
      parser = argparse.ArgumentParser(
          description="Generate the NOTICE file from all vendor directories available in a given directory")
      parser.add_argument("vendor",
                          help="directory where to search for vendor directories")
      parser.add_argument("-b", "--beat", default="Elastic Beats",
                          help="Beat name")
      parser.add_argument("-c", "--copyright", default="Elasticsearch BV",
                          help="copyright owner")
      parser.add_argument("--csv", dest="csvfile",
                          help="Output to a csv file")
      parser.add_argument("-e", "--excludes", default=["dev-tools", "build"],
                          help="List of top directories to exclude")
      # no need to be generic for now, no other transitive dependency information available
      parser.add_argument("--beats-origin", type=argparse.FileType('r'),
                          help="path to beats vendor.json")
      parser.add_argument("-s", "--skip-notice", default=[],
                          help="List of NOTICE files to skip")
      args = parser.parse_args()
  
      cwd = os.getcwd()
      notice = os.path.join(cwd, "NOTICE.txt")
      vendor_dir = "./vendor"
  
      excludes = args.excludes
      if not isinstance(excludes, list):
          excludes = [excludes]
      SKIP_NOTICE = args.skip_notice
  
      overrides = {}  # revision overrides only for now
      if args.beats_origin:
          govendor = json.load(args.beats_origin)
          overrides = {package['path']: package for package in govendor["package"]}
  
      print("Get the licenses available from {}".format(vendor_dir))
      check_all_have_license_files(vendor_dir)
      dependencies = create_notice(notice, args.beat, args.copyright, vendor_dir, args.csvfile, overrides=overrides)
  
      # check that all licenses are accepted
      for _, deps in dependencies.items():
          for dep in deps:
              if dep["license_summary"] not in ACCEPTED_LICENSES:
                  raise Exception("Dependency {} has invalid license {}"
                                  .format(dep["path"], dep["license_summary"]))
  