text: #! /usr/bin/env python2
  # -*- coding: utf-8 -*-
  import argparse
  import json
  import os
  import re
  import sys
  import tarfile
  import urllib
  from urlparse import urljoin
  
  try:
      from bs4 import BeautifulSoup
  except ImportError:
      raise ImportError("Please install BeautifulSoup (apt-get install python-bs4 should do it)")
  
  parser = argparse.ArgumentParser(description='Docenizes HTML version of the official Intel Asm PDFs')
  parser.add_argument('-i', '--inputfolder', type=str,
                      help='Folder where the input files reside as .html. Default is ./asm-docs/',
                      default='asm-docs')
  parser.add_argument('-o', '--outputpath', type=str, help='Final path of the .js file. Default is ./asm-docs.js',
                      default='./asm-docs.js')
  parser.add_argument('-d', '--downloadfolder', type=str,
                      help='Folder where the archive will be downloaded and extracted', default='asm-docs')
  
  # The maximum number of paragraphs from the description to copy.
  MAX_DESC_PARAS = 5
  STRIP_PREFIX = re.compile(r'^(([0-9a-fA-F]{2}|m64|NP|(REX|E?VEX\.)[.0-9A-Z]*|/[0-9a-z]+|[a-z]+)\b\s*)*')
  INSTRUCTION_RE = re.compile(r'^([A-Z][A-Z0-9]+)\*?(\s+|$)')
  # Some instructions are so broken we just take their names from the filename
  UNPARSEABLE_INSTR_NAMES = ['PSRLW:PSRLD:PSRLQ', 'PSLLW:PSLLD:PSLLQ']
  # Some files contain instructions which cannot be parsed and which compilers are unlikely to emit
  IGNORED_FILE_NAMES = [
      # SGX pseudo-instructions
      "EADD",
      "EACCEPT",
      "EAUG",
      "EACCEPTCOPY",
      "EDECVIRTCHILD",
      "EINCVIRTCHILD",
      "EINIT",
      "ELDB:ELDU:ELDBC:ELBUC",
      "EMODPE",
      "EMODPR",
      "EMODT",
      "ERDINFO",
      "ESETCONTEXT",
      "ETRACKC",
      "EBLOCK",
      "ECREATE",
      "EDBGRD",
      "EDBGWR",
      "EENTER",
      "EEXIT",
      "EEXTEND",
      "EGETKEY",
      "ELDB",
      "ELDU",
      "ENCLS",
      "ENCLU",
      "EPA",
      "EREMOVE",
      "EREPORT",
      "ERESUME",
      "ETRACK",
      "EWB",
      # VMX instructions
      "INVEPT",
      "INVVPID",
      "VMCALL",
      "VMCLEAR",
      "VMFUNC",
      "VMLAUNCH",
      "VMLAUNCH:VMRESUME",
      "VMPTRLD",
      "VMPTRST",
      "VMREAD",
      "VMRESUME",
      "VMWRITE",
      "VMXOFF",
      "VMXON",
      # Other instructions
      "INVLPG",
      "LAHF",
      "RDMSR",
      "SGDT",
      # Unparsable instructions
      # These instructions should be supported in the future
      "MONITOR",
      "MOVDQ2Q",
      "MOVBE",
      "MFENCE",
  ]
  # Some instructions are defined in multiple files. We ignore a specific set of the
  # duplicates here.
  IGNORED_DUPLICATES = [
      'MOV-1',  # move to control reg
      'MOV-2',  # move to debug reg
      'CMPSD',  # compare doubleword (defined in CMPS:CMPSB:CMPSW:CMPSD:CMPSQ)
      'MOVQ',  # defined in MOVD:MOVQ
      'MOVSD',  # defined in MOVS:MOVSB:MOVSW:MOVSD:MOVSQ
      'VPBROADCASTB:VPBROADCASTW:VPBROADCASTD:VPBROADCASTQ',  # defined in VPBROADCAST
      "VGATHERDPS:VGATHERDPD",
      "VGATHERQPS:VGATHERQPD",
      "VPGATHERDD:VPGATHERQD",
      "VPGATHERDQ:VPGATHERQQ",
  ]
  # Where to extract the asmdoc archive.
  ASMDOC_DIR = "asm-docs"
  ARCHIVE_URL = "http://www.felixcloutier.com/x86/x86.tbz2"
  ARCHIVE_NAME = "x86.tbz2"
  
  
  class Instruction(object):
      def __init__(self, name, names, tooltip, body):
          self.name = name
          self.names = names
          self.tooltip = tooltip.rstrip(': ,')
          self.body = body
  
      def __str__(self):
          return "{} = {}\n{}".format(self.names, self.tooltip, self.body)
  
  
  def get_url_for_instruction(instr):
      return "http://www.felixcloutier.com/x86/{}.html".format(urllib.quote(instr.name))
  
  
  def download_asm_doc_archive(downloadfolder):
      if not os.path.exists(downloadfolder):
          print "Creating {} as download folder".format(downloadfolder)
          os.makedirs(downloadfolder)
      elif not os.path.isdir(downloadfolder):
          print "Error: download folder {} is not a directory".format(downloadfolder)
          sys.exit(1)
      archive_name = os.path.join(downloadfolder, ARCHIVE_NAME)
      print("Downloading archive...")
      urllib.urlretrieve(ARCHIVE_URL, archive_name)
  
  
  def extract_asm_doc_archive(downloadfolder, inputfolder):
      print "Extracting file..."
      if os.path.isdir(os.path.join(inputfolder, "html")):
          for root, dirs, files in os.walk(os.path.join(inputfolder, "html")):
              for file in files:
                  if os.path.splitext(file)[1] == ".html":
                      os.remove(os.path.join(root, file))
      tar = tarfile.open(os.path.join(downloadfolder, ARCHIVE_NAME))
      tar.extractall(path=inputfolder)
  
  
  def strip_non_instr(i):
      # removes junk from encodings where the opcode is in the middle
      # of prefix stuff. e.g.
      # 66 0f 38 30 /r PMOVZXBW xmm1, xmm2/m64
      return STRIP_PREFIX.sub('', i)
  
  
  def instr_name(i):
      match = INSTRUCTION_RE.match(strip_non_instr(i))
      if match:
          return match.group(1)
  
  
  def get_description_paragraphs(document_soup):
      description_header_node = document_soup.find(id="description")
      i = 0
      description_paragraph_node = description_header_node.next_sibling.next_sibling
      description_paragraphs = []
      while i < MAX_DESC_PARAS and len(description_paragraph_node.text) > 20:
          if description_paragraph_node.name == "p":
              description_paragraphs.append(description_paragraph_node)
              i = i + 1
              # Move two siblings forward. Next sibling is the line feed.
          description_paragraph_node = description_paragraph_node.next_sibling.next_sibling
      return description_paragraphs
  
  
  def parse(filename, f):
      doc = BeautifulSoup(f, 'html.parser')
      if doc.table is None:
          print filename + ": Failed to find table"
          return None
      table = read_table(doc.table)
      names = set()
  
      def add_all(instrs):
          for i in instrs:
              instruction_name = instr_name(i)
              if instruction_name:
                  names.add(instruction_name)
  
      for inst in table:
          if 'Opcode/Instruction' in inst:
              add_all(inst['Opcode/Instruction'].split("\n"))
          elif 'OpcodeInstruction' in inst:
              add_all(inst['OpcodeInstruction'].split("\n"))
          elif 'Opcode Instruction' in inst:
              add_all(inst['Opcode Instruction'].split("\n"))
          elif 'Opcode*/Instruction' in inst:
              add_all(inst['Opcode*/Instruction'].split("\n"))
          elif 'Opcode / Instruction' in inst:
              add_all(inst['Opcode / Instruction'].split("\n"))
          elif 'Instruction' in inst:
              instruction_name = instr_name(inst['Instruction'])
              if not instruction_name:
                  print "Unable to get instruction from:", inst['Instruction']
              else:
                  names.add(instruction_name)
          # else, skip the line
      if not names:
          if filename in UNPARSEABLE_INSTR_NAMES:
              for inst in filename.split(":"):
                  names.add(inst)
          else:
              print filename + ": Failed to read instruction table"
              return None
      sections = {}
      for section_header in doc.find_all("h2"):
          children = []
          first = section_header.next_sibling
          while first and first.name != 'h2':
              if str(first).strip():
                  children.append(first)
              first = first.next_sibling
          sections[section_header.text] = children
  
      description_paragraphs = get_description_paragraphs(doc)
  
      for para in description_paragraphs:
          for link in para.find_all('a'):
              # this urljoin will only ensure relative urls are prefixed
              # if a url is already absolute it does nothing
              link['href'] = urljoin('http://www.felixcloutier.com/x86/', link['href'])
              link['target'] = '_blank'
              link['rel'] = 'noreferrer noopener'
  
  
      return Instruction(
          filename,
          names,
          description_paragraphs[0].text.strip(),
          ''.join(map(lambda x: str(x), description_paragraphs)).strip())
  
  
  def read_table(table):
      # Finding all 'th' is not enough, since some headers are 'td'.
      # Instead, walk through all children of the first 'tr', filter out those
      # that are only whitespace, keep `get_text()` on the others.
      headers = list(
          map(lambda th: th.get_text(),
              filter(lambda th: unicode(th).strip(), table.tr.children)))
  
      result = []
      if headers:
          # common case
          for row in table.find_all('tr'):
              obj = {}
              for column, name in zip(row.find_all('td'), headers):
                  # Remove '\n's in names that contain it.
                  obj[name.replace('\n', '')] = column.get_text()
              if obj:
                  result.append(obj)
      else:
          # Cases like BEXTR and BZHI
          rows = table.find_all('tr')
          if len(rows) != 1:
              return []
          obj = {}
          for td in rows[0].find_all('td'):
              header = td.p.strong.get_text()
              td.p.strong.decompose()
              obj[header] = td.get_text()
          result.append(obj)
  
      return result
  
  
  def parse_html(directory):
      print "Parsing instructions..."
      instructions = []
      for root, dirs, files in os.walk(directory):
          for file in files:
              if file.endswith(".html") and file != 'index.html':
                  with open(os.path.join(root, file)) as f2:
                      name = os.path.splitext(file)[0]
                      if name in IGNORED_DUPLICATES or name in IGNORED_FILE_NAMES:
                          continue
                      instruction = parse(name, f2)
                      if not instruction:
                          continue
                      instructions.append(instruction)
      return instructions
  
  
  def self_test(instructions, directory):
      # For each generated instruction, check that there is a path to a file in
      # the documentation.
      directory = os.path.join(directory, "html")
      ok = True
      for inst in instructions:
          if not os.path.isfile(os.path.join(directory, inst.name + ".html")):
              print("Warning: {} has not file associated".format(inst.name))
              ok = False
      return ok
  
  
  def docenizer():
      args = parser.parse_args()
      print "Called with: {}".format(args)
      # If we don't have the html folder already...
      if not os.path.isdir(os.path.join(args.inputfolder, 'html')):
          # We don't, try with the compressed file
          if not os.path.isfile(os.path.join(args.downloadfolder, "x86.tbz2")):
              # We can't find that either. Download it
              try:
                  download_asm_doc_archive(args.downloadfolder)
                  extract_asm_doc_archive(args.downloadfolder, args.inputfolder)
              except IOError as e:
                  print("Error when downloading archive:")
                  print(e)
                  sys.exit(1)
          else:
              # We have a file already downloaded
              extract_asm_doc_archive(args.downloadfolder, args.inputfolder)
      instructions = parse_html(args.inputfolder)
      instructions.sort(lambda x, y: cmp(x.name, y.name))
      self_test(instructions, args.inputfolder)
      all_inst = set()
      for inst in instructions:
          if not all_inst.isdisjoint(inst.names):
              print "Overlap in instruction names: {} for {}".format(
                  inst.names.intersection(all_inst), inst.name)
          all_inst = all_inst.union(inst.names)
      if not self_test(instructions, args.inputfolder):
          print("Tests do not pass. Not writing output file. Aborting.")
          sys.exit(3)
      print "Writing {} instructions".format(len(instructions))
      with open(args.outputpath, 'w') as f:
          f.write("""
  export function getAsmOpcode(opcode) {
      if (!opcode) return;
      switch (opcode.toUpperCase()) {
  """)
          for inst in instructions:
              for name in inst.names:
                  f.write('        case "{}":\n'.format(name))
              f.write('            return {}'.format(json.dumps({
                  "tooltip": inst.tooltip,
                  "html": inst.body,
                  "url": get_url_for_instruction(inst)
                  }, indent=16, separators=(',', ': ')))[:-1] + '            };\n\n')
          f.write("""
      }
  }
  """)
      print "REMINDER: Check if https://github.com/compiler-explorer/compiler-explorer/issues/2380 is still relevant"
  
  if __name__ == '__main__':
      docenizer()
  