text: ##    MIT License
  ##
  ##    Copyright (c) 2018-2020 Fredrik Mellbin
  ##
  ##    Permission is hereby granted, free of charge, to any person obtaining a copy
  ##    of this software and associated documentation files (the "Software"), to deal
  ##    in the Software without restriction, including without limitation the rights
  ##    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  ##    copies of the Software, and to permit persons to whom the Software is
  ##    furnished to do so, subject to the following conditions:
  ##
  ##    The above copyright notice and this permission notice shall be included in all
  ##    copies or substantial portions of the Software.
  ##
  ##    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  ##    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  ##    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  ##    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  ##    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  ##    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  ##    SOFTWARE.
  
  import urllib.request
  import json
  import sys
  import os
  import os.path
  import argparse
  import hashlib
  import subprocess
  import difflib
  import tempfile
  import ftplib
  import zipfile
  
  try:
      import winreg
  except ImportError:
      print('{} is only supported on Windows.'.format(__file__))
      sys.exit(1)
  
  try:
      import tqdm
  except ImportError:
      pass
  
  parser = argparse.ArgumentParser(description='Package list generator for VSRepo')
  parser.add_argument('operation', choices=['compile', 'update-local', 'upload', 'create-package'])
  parser.add_argument('-g', dest='git_token', nargs=1, help='OAuth access token for github')
  parser.add_argument('-p', dest='package', nargs=1, help='Package to update')
  parser.add_argument('-o', action='store_true', dest='overwrite', help='Overwrite existing package file')
  parser.add_argument('-host', dest='host', nargs=1, help='FTP Host')
  parser.add_argument('-user', dest='user', nargs=1, help='FTP User')
  parser.add_argument('-passwd', dest='passwd', nargs=1, help='FTP Password')
  parser.add_argument('-dir', dest='dir', nargs=1, help='FTP dir')
  parser.add_argument('-url', dest='packageurl', nargs=1, help='URL of the archive from which a package is to be created')
  parser.add_argument('-pname', dest='packagename', nargs=1, help='Filename or namespace of your package')
  parser.add_argument('-script', action='store_true', dest='packagescript', help='Type of the package is script. Otherwise a package of type plugin is created')
  parser.add_argument('-types', dest='packagefiletypes', nargs='+', help='Which file types should be included. default is .dll')
  parser.add_argument('-kf', dest='keepfolder', type=int, default=-1, nargs='?', help='Keep the folder structure')
  
  args = parser.parse_args()
  
  cmd7zip_path = '7z.exe'
  try:
      with winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\7-Zip', reserved=0, access=winreg.KEY_READ) as regkey:
          cmd7zip_path = winreg.QueryValueEx(regkey, 'Path')[0] + '7z.exe'
  except:
      pass
  
  def similarity(a, b):
      return difflib.SequenceMatcher(None, a, b).ratio()
  
  def get_most_similar(a, b):
      res = (0, '')
      for s in b:
          v = similarity(a, s)
          if v >= res[0]:
              res = (v, s)
      return res[1]
  
  def get_git_api_url(url):
      if url.startswith('https://github.com/'):
          s = url.rsplit('/', 3)
          return 'https://api.github.com/repos/' + s[-2] + '/' + s[-1] + '/releases'
      else:
          return None
          
  def get_pypi_api_url(name):
      return 'https://pypi.org/pypi/' + name + '/json'
  
  def fetch_url(url, desc = None, token = None):
      req = urllib.request.Request(url, headers={'Authorization': 'token ' + token}) if token is not None else urllib.request.Request(url)
      with urllib.request.urlopen(req) as urlreq:
          if ('tqdm' in sys.modules) and (urlreq.headers['content-length'] is not None):
              size = int(urlreq.headers['content-length'])
              remaining = size
              data = bytearray()
              with tqdm.tqdm(total=size, unit='B', unit_scale=True, unit_divisor=1024, desc=desc) as t:
                  while remaining > 0:
                      blocksize = min(remaining, 1024*128)
                      data.extend(urlreq.read(blocksize))
                      remaining = remaining - blocksize
                      t.update(blocksize)
              return data
          else:
              print('Fetching: ' + url)
              return urlreq.read()
  
  def fetch_url_to_cache(url, name, tag_name, desc = None):
      cache_path = os.path.join('dlcache', name + '_' + tag_name, os.path.basename(url))
      if not os.path.isfile(cache_path):
          os.makedirs(os.path.split(cache_path)[0], exist_ok=True)
          with urllib.request.urlopen(urllib.request.Request(url, method='HEAD')) as urlreq:
              if not os.path.isfile(cache_path):
                  data = fetch_url(url, desc)
                  with open(cache_path, 'wb') as pl:
                      pl.write(data)
      return cache_path
  
  def list_archive_files(fn):
      result = subprocess.run([cmd7zip_path, "l", "-ba", fn], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
      result.check_returncode()
      l = {}
      lines = result.stdout.decode('utf-8').splitlines()
      for line in lines:
          t = line[53:].replace('\\', '/')
          l[t.lower()] = t
      return l
  
  def generate_fn_candidates(fn, insttype):
      tmp_fn = fn.lower()
      fn_guesses = [
          tmp_fn,
          tmp_fn.replace('x64', 'win64'),
          tmp_fn.replace('win64', 'x64'),
          tmp_fn.replace('x86', 'win32'),
          tmp_fn.replace('win32', 'x86')]
      if insttype == 'win32':
          return list(filter(lambda x: (x.find('64') == -1) and (x.find('x64') == -1) , fn_guesses))
      elif insttype == 'win64':
          return list(filter(lambda x: (x.find('32') == -1) and (x.find('x86') == -1) , fn_guesses))
      else:
          return fn_guesses;
  
  def decompress_and_hash(archivefn, fn, insttype):
      existing_files = list_archive_files(archivefn)
      for fn_guess in generate_fn_candidates(fn, insttype):
          if fn_guess in existing_files:  
              result = subprocess.run([cmd7zip_path, "e", "-so", archivefn, fn_guess], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
              result.check_returncode()
              return (existing_files[fn_guess], hashlib.sha256(result.stdout).hexdigest())
      base_dirs = []
      for f in existing_files:
          bn = f.split('/')[0]
          if bn not in base_dirs:
              base_dirs.append(bn)
      if len(base_dirs) == 1:
          sfn = fn.split('/')
          if len(sfn) > 1:
              sfn[0] = base_dirs[0]
              mfn = '/'.join(sfn)
              for fn_guess in generate_fn_candidates(mfn, insttype):
                  if fn_guess in existing_files:  
                      result = subprocess.run([cmd7zip_path, "e", "-so", archivefn, fn_guess], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                      result.check_returncode()
                      return (existing_files[fn_guess], hashlib.sha256(result.stdout).hexdigest())
      raise Exception('No file match found')
  
  def get_latest_installable_release(p, bin_name):
      for rel in p['releases']:
          if bin_name in rel:
              return rel
      return None
  
  def get_python_package_name(pkg):
      if "wheelname" in pkg:
          return pkg["wheelname"].replace(".", "_").replace(" ", "_")
      else:
          return pkg["name"].replace(".", "_").replace(" ", "_")
  
  def write_new_releses(name, pfile, new_rels, rel_order):
      has_new_releases = bool(new_rels)
      if has_new_releases:
          for rel in pfile['releases']:
              new_rels[rel['version']] = rel
          rel_list = []
          for rel_ver in rel_order:
              rel_list.append(new_rels[rel_ver])
          pfile['releases'] = rel_list
          pfile['releases'].sort(key=lambda r: r['published'], reverse=True)
  
          fnext = '.json' if args.overwrite else '.new.json'          
          with open('local/' + name + fnext, 'w', encoding='utf-8') as pl:
              json.dump(fp=pl, obj=pfile, ensure_ascii=False, indent='\t')
          print('Release file updated')
          return 1
      else:
          print('Release file already up to date')
          return 0
          
  def update_package(name):
      pfile = None
      with open('local/' + name + '.json', 'r', encoding='utf-8') as ml:
          pfile = json.load(ml)
  
      existing_rel_list = []
      for rel in pfile['releases']:
          existing_rel_list.append(rel['version'])
      rel_order = list(existing_rel_list)
      
      use_pypi = (pfile['type'] == 'PyWheel')
      if ('source' in pfile) and (pfile['source'] != 'pypi'):
          use_pypi = False
      
      if pfile['type'] == 'PyWheel':
          if use_pypi:
              new_rels = {}
              apifile = json.loads(fetch_url(get_pypi_api_url(get_python_package_name(pfile)), pfile['name']))
              for version in apifile['releases']:
                  for rel in apifile['releases'][version]:
                      if rel['yanked'] or rel['packagetype'] != 'bdist_wheel':
                          continue
                      new_rel_entry = { 'version': version, 'published': rel['upload_time_iso_8601'], 'wheel': { 'url': rel['url'], 'hash': rel['digests']['sha256'] } }
                      new_rels[version] = new_rel_entry
                      if version not in rel_order:
                          rel_order.insert(0, version)
              write_new_releses(name, pfile, new_rels, rel_order)
          else:
              print('PyWheel can only be scanned from pypi')
              return -1
      elif 'github' in pfile:
          new_rels = {}
          apifile = json.loads(fetch_url(get_git_api_url(pfile['github']), pfile['name'], token=args.git_token[0]))
          is_plugin = (pfile['type'] == 'VSPlugin')
          is_pyscript = (pfile['type'] == 'PyScript')
          is_pywheel = (pfile['type'] == 'PyWheel')
          for rel in apifile:
              if rel['prerelease']:
                  continue
              if rel['tag_name'] in pfile.get('ignore', []):
                  continue
              if rel['tag_name'] not in rel_order:
                  rel_order.insert(0, rel['tag_name'])
              if rel['tag_name'] not in existing_rel_list:
                  print(rel['tag_name'] + ' (new)')
                  zipball = rel['zipball_url']
                  dl_files = []
                  for asset in rel['assets']:
                      dl_files.append(asset['browser_download_url'])
                  
                  #ugly copypaste here because I'm lazy
                  if is_plugin:
                      new_rel_entry = { 'version': rel['tag_name'], 'published': rel['published_at'] }
                      try:
                          latest_rel = get_latest_installable_release(pfile, 'win32')
                          if latest_rel is not None:
                              new_url = get_most_similar(latest_rel['win32']['url'], dl_files)
                              temp_fn = fetch_url_to_cache(new_url, name, rel['tag_name'], pfile['name'] + ' ' +rel['tag_name'] + ' win32')
                              new_rel_entry['win32'] = { 'url': new_url, 'files': {}}
                              for fn in latest_rel['win32']['files']:
                                  new_fn, digest = decompress_and_hash(temp_fn, latest_rel['win32']['files'][fn][0], 'win32')
                                  new_rel_entry['win32']['files'][fn] = [new_fn, digest]
                      except:
                          new_rel_entry.pop('win32', None)
                          print('No win32 binary found')
                      try:
                          latest_rel = get_latest_installable_release(pfile, 'win64')
                          if latest_rel is not None:
                              new_url = get_most_similar(latest_rel['win64']['url'], dl_files)
                              temp_fn = fetch_url_to_cache(new_url, name, rel['tag_name'], pfile['name'] + ' ' +rel['tag_name'] + ' win64')
                              new_rel_entry['win64'] = { 'url': new_url, 'files': {} }
                              for fn in latest_rel['win64']['files']:
                                  new_fn, digest = decompress_and_hash(temp_fn, latest_rel['win64']['files'][fn][0], 'win64')
                                  new_rel_entry['win64']['files'][fn] = [new_fn, digest]
                      except:
                          new_rel_entry.pop('win64', None)
                          print('No win64 binary found')
                  else:
                      new_rel_entry = { 'version': rel['tag_name'], 'published': rel['published_at'] }
                      try:
                          latest_rel = get_latest_installable_release(pfile, 'script')
                          new_url = None
                          if ('/archive/' in latest_rel['script']['url']) or ('/zipball/' in latest_rel['script']['url']):
                              new_url = zipball
                          else:
                              new_url = get_most_similar(latest_rel['script']['url'], dl_files)
                          temp_fn = fetch_url_to_cache(new_url, name, rel['tag_name'], pfile['name'] + ' ' +rel['tag_name'] + ' script')
                          new_rel_entry['script'] = { 'url': new_url, 'files': {} }
                          for fn in latest_rel['script']['files']:
                              new_fn, digest = decompress_and_hash(temp_fn, latest_rel['script']['files'][fn][0], 'script')
                              new_rel_entry['script']['files'][fn] = [new_fn, digest]
                      except:
                          new_rel_entry.pop('script', None)
                          print('No script found')
                  new_rels[new_rel_entry['version']] = new_rel_entry
          return write_new_releses(name, pfile, new_rels, rel_order)
      else:
          print('Only github and pypi projects supported, ' + name + ' not scanned')
          return -1
  
  def verify_package(pfile, existing_identifiers):
      name = pfile['name']
      for key in pfile.keys():
          if key not in ('name', 'type', 'description', 'website', 'category', 'identifier', 'modulename', 'wheelname', 'namespace', 'github', 'doom9', 'dependencies', 'ignore', 'releases'):
              raise Exception('Unkown key: ' + key + ' in ' + name)
      if pfile['type'] not in ('VSPlugin', 'PyScript', 'PyWheel'):
          raise Exception('Invalid type in ' + name)
      if (pfile['type'] == 'VSPlugin') and ('modulename' in pfile):
          raise Exception('VSPlugin can\'t have modulenames: ' + name)
      if (pfile['type'] == 'VSPlugin') and (('modulename' in pfile) or ('namespace' not in pfile)):
          raise Exception('VSPlugin must have namespace, not modulename: ' + name)
      if (pfile['type'] == 'PyScript' or pfile['type'] == 'PyWheel') and (('namespace' in pfile) or ('modulename' not in pfile)):
          raise Exception('PyScript must have modulename, not namespace: ' + name)
      if ((pfile['type'] == 'PyScript' or pfile['type'] == 'VSPlugin') and 'wheelname' in pfile):
          raise Exception('Only PyWheel type can have wheelname: ' + name)
      allowed_categories = ('Scripts', 'Plugin Dependency', 'Resizing and Format Conversion', 'Other', 'Dot Crawl and Rainbows', 'Sharpening', 'Denoising', 'Deinterlacing', 'Inverse Telecine', 'Source/Output', 'Subtitles', 'Color/Levels')
      if pfile['category'] not in allowed_categories:
          raise Exception('Not allowed catogry in ' + name + ': ' + pfile['category'] + ' not in ' + repr(allowed_categories))
      if 'dependencies' in pfile:
          for dep in pfile['dependencies']:
              if dep not in existing_identifiers:
                  raise Exception('Referenced unknown identifier ' + dep + ' in ' + name)
  
  def compile_packages():
      combined = []
      existing_identifiers = []
      for f in os.scandir('local'):
          if f.is_file() and f.path.endswith('.json'):
              with open(f.path, 'r', encoding='utf-8') as ml:
                  pfile = json.load(ml)
                  if pfile['identifier'] in existing_identifiers:
                      raise Exception('Duplicate identifier: ' + pfile['identifier'])
                  existing_identifiers.append(pfile['identifier'])
  
      for f in os.scandir('local'):
          if f.is_file() and f.path.endswith('.json') and not f.path.endswith('.new.json'):
              with open(f.path, 'r', encoding='utf-8') as ml:
                  print('Combining: ' + f.path)
                  pfile = json.load(ml)
                  verify_package(pfile, existing_identifiers)
                  pfile.pop('ignore', None)
                  combined.append(pfile)
  
      data = json.dumps(obj={ 'file-format': 3, 'packages': combined}, ensure_ascii=False, indent=2)
  
      try:
          os.remove('vspackages3.zip')
      except:
          pass
          
      with zipfile.ZipFile('vspackages3.zip', mode='w', compression=zipfile.ZIP_DEFLATED, compresslevel=9) as zf:
          zf.writestr('vspackages3.json', data)
  
  
  def getBinaryArch(bin):
  	if b"PE\x00\x00d\x86" in bin: 	# hex: 50 45 00 00 64 86 | PE..d† 
  		return 64
  	if b"PE\x00\x00L" in bin: 		# hex: 50 45 00 00 4c	 | PE..L 
  		return 32
  	return None
  
  def decompress_hash_simple(archive, file):
  	result = subprocess.run([cmd7zip_path, "e", "-so", archive, file], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  	result.check_returncode()
  	return (file, hashlib.sha256(result.stdout).hexdigest(), getBinaryArch(result.stdout))
  
  def extract_git_repo(url):
      if url.startswith('https://github.com/'):
          return '/'.join(url.split('/', 5)[:-1])
      else:
          return None
  
  def keep_folder_structure(path, level = 0):
  	folder = path.split('/', level)
  	return folder[-1]
  
  def blank_package(name = "", is_script = False, is_wheel = False, url = ""):
  	p = { 	'name': '',
  			'type': 'PyScript' if is_script else ('PyWheel' if is_wheel else 'VSPlugin'),
  			'category': '',
  			'description': '',
  			'doom9': '',
  			'website': '',
  			'github': extract_git_repo(url) if extract_git_repo(url) else '',
  			'identifier': name if is_script or is_wheel else '',
  			'modulename' if is_script or is_wheel else 'namespace': name,
  			'wheelname': name,
  			'releases': ''
  		}
  	if not is_wheel:
  		del p['wheelname']
  	return p
  
  if args.operation == 'compile':
      compile_packages()
      print('Packages successfully compiled')
  elif args.operation == 'update-local':
      if args.package is None:
          num_skipped = 0
          num_nochange = 0
          num_updated = 0
          for f in os.scandir('local'):
              if f.is_file() and f.path.endswith('.json'):         
                  result = update_package(os.path.splitext(os.path.basename(f))[0])
                  if result == -1:
                      num_skipped = num_skipped + 1
                  elif result == 1:
                      num_updated = num_updated + 1
                  elif result == 0:
                      num_nochange = num_nochange + 1
          print('Summary:\nUpdated: {} \nNo change: {} \nSkipped: {}\n'.format(num_updated, num_nochange, num_skipped))
      else:
          update_package(args.package[0])
  elif args.operation == 'create-package':
      
  	import pathlib
  
  	if not args.packageurl:
  		print('-url parameter is missing')
  		sys.exit(1)
  	if not args.packagename:
  		print('-pname parameter is missing')
  		sys.exit(1)
  		
  	url = args.packageurl[0]
  	filetypes = ['.dll', '.py']
  	
  	if args.packagefiletypes:
  		filetypes = args.packagefiletypes
  	
  	is_wheel = True if (pathlib.Path(url).suffix.lower() == '.whl') else False
  
  	print("fetching remote url")
  	dlfile = fetch_url_to_cache(url, "package", "creator", "")
  	
  	print("creating package")
  	new_rel_entry = { 'version': 'create-package', 'published': '' }
  	
  	
  	if is_wheel:
  		new_rel_entry['wheel'] = { 'url': url }
  		new_rel_entry['wheel']['hash'] = hashlib.sha256(open(dlfile,'rb').read()).hexdigest()
  		
  	else: # is plugin or script
  		listzip = list_archive_files(dlfile)
  		files_to_hash = []
  		for f in listzip.values():
  			if pathlib.Path(f).suffix: # simple folder filter
  				if "*" in filetypes:
  					files_to_hash.append(f)
  				else:
  					if pathlib.Path(f).suffix in filetypes:
  						files_to_hash.append(f)
  
  		files_to_hash = sorted(files_to_hash)
  
  		if not args.packagescript: # is plugin
  			new_rel_entry['win32'] = { 'url': url, 'files': {} }
  			new_rel_entry['win64'] = { 'url': url, 'files': {} }
  			for f in files_to_hash:
  				fullpath, hash, arch = decompress_hash_simple(dlfile, f)
  				file = keep_folder_structure(fullpath, args.keepfolder) if args.keepfolder >= 0 else os.path.basename(fullpath)
  				if arch == 32:
  					new_rel_entry['win32']['files'][file] = [fullpath, hash]
  				if arch == 64:
  					new_rel_entry['win64']['files'][file] = [fullpath, hash]
  				if arch == None:
  					new_rel_entry['win32']['files'][file] = [fullpath, hash]
  					new_rel_entry['win64']['files'][file] = [fullpath, hash]
  			
  			# remove 32/64 entry if no files are present
  			if not new_rel_entry['win32']['files']:
  				new_rel_entry.pop('win32', None)
  			if not new_rel_entry['win64']['files']:
  				new_rel_entry.pop('win64', None)
  				
  		else: # is script
  			new_rel_entry['script'] = { 'url': url, 'files': {} }
  			for f in files_to_hash:
  				fullpath, hash, arch = decompress_hash_simple(dlfile, f)
  				file = keep_folder_structure(fullpath, args.keepfolder) if args.keepfolder >= 0 else os.path.basename(fullpath)
  				new_rel_entry['script']['files'][file] = [fullpath, hash]
  		
  	if not args.packagescript:
  		if is_wheel:
  			final_package = blank_package(name = args.packagename[0], url = url, is_wheel = True)
  		else:
  			final_package = blank_package(name = args.packagename[0], url = url)
  	else:
  		final_package = blank_package(name = args.packagename[0], url = url, is_script = True)
  	final_package['releases'] = [ new_rel_entry ]
  	
  	
  	print(json.dumps(final_package, indent=4))
  	if not os.path.exists('local/' + args.packagename[0] + '.json'):
  		with open('local/' + args.packagename[0] + '.json', 'x', encoding='utf-8') as pl:
  			json.dump(fp=pl, obj=final_package, ensure_ascii=False, indent='\t')
  		
  		print("package created")
  		
  		if extract_git_repo(url) and not is_wheel:
  			print("Is hosted on GitHub")
  			if args.git_token:
  				print("Auto updating package")
  				args.overwrite = True
  				update_package(args.packagename[0])
  			else:
  				print("No git token ( -g ) was set, skipping auto update")
  
  		if is_wheel and url.startswith('https://files.pythonhosted.org'):
  			print("Auto updating wheel package")
  			args.overwrite = True
  			update_package(args.packagename[0])
  	else:
  		print("package file '{}'.json already exists. Skipping writing file.".format(args.packagename[0])) 
  	
  	
  elif args.operation == 'upload':
      compile_packages()
      print('Packages successfully compiled')
      with open('vspackages3.zip', 'rb') as pl:
          with ftplib.FTP_TLS(host=args.host[0], user=args.user[0], passwd=args.passwd[0]) as ftp:
              if args.dir is not None:
                  ftp.cwd(args.dir[0])
              try:
                  ftp.delete('vspackages3.zip')
              except:
                  print('Failed to delete vspackages3.zip')
              ftp.storbinary('STOR vspackages3.zip', pl)
      print('Upload done')
  
  def noop():
      pass
  