code: import numpy as np
  from bs4 import BeautifulSoup
  from sklearn.cluster import KMeans
  from math import sqrt
  from statistics import mean
  from PIL import Image, ImageOps, ImageDraw
  import re
  import json
  import argparse
  import os
  import uuid
  from cdparser import Classifier, Features, LabeledEntry, Utils
  import sys
  
  
  def build_manifest(main_path, entries_json):
      directory_uuid = entries_json[0]['directory_uuid']
      page_uuid = entries_json[0]['page_uuid']
      num_columns = max([int(entries_json[i]['col']) for i in entries_json])
      num_skipped_lines = len(
          [i for i in entries_json if entries_json[i]['skipped_line_after'] == '1'])
      if not os.path.exists(os.path.join(main_path, 'manifest')):
          os.makedirs(os.path.join(main_path, 'manifest'))
      manifest_path = os.path.join(
          main_path, 'manifest', directory_uuid + '_manifest.txt')
      with open(manifest_path, 'a') as f:
          f.write('directory_uuid:' + directory_uuid + '\n')
          f.write('page_uuid:' + page_uuid + '\n')
          f.write('number_extracted_entries:' + str(len(entries_json)) + '\n')
          f.write('total_number_lines_hocr:' +
                  entries_json[0]['total_lines_from_hocr'] + '\n')
          f.write('number_extracted_columns:' + str(num_columns) + '\n')
          f.write('number_skipped_lines:' + str(num_skipped_lines))
          f.write('\n\n')
      f.close()
  
  
  def make_tsv(filepath, type):
      with open(filepath, 'w+') as f:
          f.write('\t'.join(['directory_uuid', 'page_uuid',
                  'entry_uuid', type + '_count', 'offset_count', 'token']))
          f.write('\n')
          f.close()
  
  
  def build_entries_tsv(entries_json, dir_tsv, directory_uuid):
      if not os.path.exists(os.path.join(dir_tsv, directory_uuid + '_subjects.tsv')):
          make_tsv(os.path.join(dir_tsv, directory_uuid +
                   '_subjects.tsv'), 'subject')
      with open(os.path.join(dir_tsv, directory_uuid + '_subjects.tsv'), 'a') as f:
          for rec in entries_json:
              subject_count = 0
              for subject in entries_json[rec]['labeled_entry']['subjects']:
                  offset_count = 0
                  for sub_token in subject.split():
                      f.write(entries_json[rec]['directory_uuid'] + '\t'
                              + entries_json[rec]['page_uuid'] + '\t'
                              + entries_json[rec]['entry_uuid'] + '\t')
                      f.write(str(subject_count) + '\t')
                      f.write(str(offset_count) + '\t')
                      f.write(sub_token + '\n')
                      offset_count += 1
                  subject_count += 1
      f.close()
      if not os.path.exists(os.path.join(dir_tsv, directory_uuid + '_occupations.tsv')):
          make_tsv(os.path.join(dir_tsv, directory_uuid +
                   '_occupations.tsv'), 'occupation')
      with open(os.path.join(dir_tsv, directory_uuid + '_occupations.tsv'), 'a') as f:
          for rec in entries_json:
              occupation_count = 0
              for occupation in entries_json[rec]['labeled_entry']['occupations']:
                  offset_count = 0
                  for occ_token in occupation.split():
                      f.write(entries_json[rec]['directory_uuid'] + '\t'
                              + entries_json[rec]['page_uuid'] + '\t'
                              + entries_json[rec]['entry_uuid'] + '\t')
                      f.write(str(occupation_count) + '\t')
                      f.write(str(offset_count) + '\t')
                      f.write(occ_token + '\n')
                      offset_count += 1
                  occupation_count += 1
      f.close()
      if not os.path.exists(os.path.join(dir_tsv, directory_uuid + '_locations.tsv')):
          make_tsv(os.path.join(dir_tsv, directory_uuid +
                   '_locations.tsv'), 'location')
      with open(os.path.join(dir_tsv, directory_uuid + '_locations.tsv'), 'a') as f:
          for rec in entries_json:
              location_count = 0
              for location in entries_json[rec]['labeled_entry']['locations']:
                  offset_count = 0
                  for loc_token in location['value'].split():
                      f.write(entries_json[rec]['directory_uuid'] + '\t'
                              + entries_json[rec]['page_uuid'] + '\t'
                              + entries_json[rec]['entry_uuid'] + '\t')
                      f.write(str(location_count) + '\t')
                      f.write(str(offset_count) + '\t')
                      f.write(loc_token + '\n')
                      offset_count += 1
                  location_count += 1
      f.close()
  
  
  def imagebuilder(r, col_locations, image_filename, std1, gap_locations, page_uuid, output_directory):
  
      # It would appear that the incoming cropped jpegs are grayscale, necessitating a conversion to RGB to move forward
      # For the overlay, we use RGBA to enable opacity settings.
  
      pageimg = Image.open(image_filename).convert('RGB')
      overlay = ImageDraw.Draw(pageimg, 'RGBA')
  
      color = {0: (255, 0, 0, 90), 1: (0, 0, 255, 90), 2: (0, 255, 0, 90)}
  
      # Draw the hocr boxes
      for i in range(len(r)):
          overlay.polygon([(r[i:i+1, 1], r[i:i+1, 4]),
                           (r[i:i+1, 1], r[i:i+1, 2]),
                           (r[i:i+1, 3], r[i:i+1, 2]),
                           (r[i:i+1, 3], r[i:i+1, 4])],
                          fill=color[int(r[i:i+1, 5])], outline=color[int(r[i:i+1, 5])])
  
      # Draw the column bounds
  
      for loc in col_locations:
  
          overlay.polygon([(loc - std1, 0), (loc + std1, 0),
                           (loc + std1, pageimg.size[1]),
                           (loc - std1, pageimg.size[1])],
                          fill=(255, 1, 255, 100))
          overlay.line([(loc, 0), (loc, pageimg.size[1])],
                       fill=(0, 0, 0, 127), width=4)
  
      # Draw the gap lines
  
      for gap_location in gap_locations:
          overlay.line([(0, gap_location), (pageimg.size[0], gap_location)],
                       fill=(0, 0, 0, 127), width=4)
      pageimg.save(os.path.join(output_directory, page_uuid + '.jpeg'), 'JPEG')
  
  
  def json_from_hocr(line_array, page_html, page_uuid, directory_uuid):
      hocr_entries = {}
      entries_json = {}
      total_page_line_count = 0
      for line in page_html.html.body.div.find_all('span'):
          if line['class'][0] == 'ocr_line':
              total_page_line_count += 1
              id_num = int(line['id'].split('_')[2])
              words = ' '.join([word.string.replace('\n', '').strip()
                               for word in line.children])
              hocr_entries[id_num] = normalize_entry(words)
      entry_id = 0
      for keep_line in line_array:
          entry_uuid = uuid.uuid1()
          if keep_line[5] == 0:
              entries_json[entry_id] = {
                  'directory_uuid': directory_uuid,
                  'page_uuid': page_uuid,
                  'entry_uuid': entry_uuid.hex,
                  'total_lines_from_hocr': str(total_page_line_count),
                  'original_hocr_line_number': str(keep_line[0]),
                  'bbox': ' '.join([str(val) for val in keep_line[1:5]]),
                  'col': str(keep_line[6]),
                  'appended': 'no',
                  'skipped_line_after': str(keep_line[7]),
                  'complete_entry': hocr_entries[keep_line[0]]
              }
              entry_id += 1
  
              # Indents
          else:
              try:
                  if entries_json[entry_id - 1]['skipped_line_after'] != "1":
                      if entries_json[entry_id - 1]['complete_entry'][-1] == '-':
                          entries_json[entry_id -
                                       1]['complete_entry'] += hocr_entries[keep_line[0]]
                      else:
                          entries_json[entry_id - 1]['complete_entry'] += ' ' + \
                              hocr_entries[keep_line[0]]
                      entries_json[entry_id - 1]['appended'] = 'yes'
                  else:
                      entries_json[entry_id] = {
                          'directory_uuid': directory_uuid,
                          'page_uuid': page_uuid,
                          'entry_uuid': entry_uuid.hex,
                          'total_lines_from_hocr': str(total_page_line_count),
                          'original_hocr_line_number': str(keep_line[0]),
                          'bbox': ' '.join([str(val) for val in keep_line[1:5]]),
                          'col': str(keep_line[6]),
                          'appended': 'no',
                          'skipped_line_after': str(keep_line[7]),
                          'complete_entry': hocr_entries[keep_line[0]]
                      }
                      entry_id += 1
              except:
  
                  # Cases where an indent is the first line in page array and there is no preceding entry
  
                  entries_json[entry_id] = {
                      'directory_uuid': directory_uuid,
                      'page_uuid': page_uuid,
                      'entry_uuid': entry_uuid.hex,
                      'total_lines_from_hocr': str(total_page_line_count),
                      'original_hocr_line_number': str(keep_line[0]),
                      'bbox': ' '.join([str(val) for val in keep_line[1:5]]),
                      'col': str(keep_line[6]),
                      'appended': 'no',
                      'skipped_line_after': str(keep_line[7]),
                      'complete_entry': hocr_entries[keep_line[0]]
                  }
                  entry_id += 1
  
      return entries_json
  
  
  def load_hocr_lines(filepath):
      page_array = []
      rawhtml = BeautifulSoup(open(filepath, encoding='utf-8'), "lxml")
      for line in rawhtml.html.body.div.find_all('span'):
          line_list = []
          if line['class'][0] == 'ocr_line':
              line_list.append(int(line['id'].split('_')[2]))
              line_list += [int(i)
                            for i in line['title'].split(';')[0].split(' ')[1:]]
              line_list += [0, 0, 0]
              page_array.append(line_list)
      return np.array(page_array), rawhtml
  
  
  def normalize_entry(entry):
      replacements = [("‘", "'"), ("’", "'"), (" ay.", " av."),
                      (" ay,", " av,"), ("- ", "-"), (" -", "-"), ("\t", ' ')]
      for swap in replacements:
          entry = entry.replace(swap[0], swap[1])
      return ' '.join(entry.split())
  
  
  def remove_precede_space(entry):
      if re.search(r'\s\.|\s\,', entry):
          entry = entry.replace(' .', '.').replace(' ,', ',')
      return entry
  
  
  def normalize_labeled_entry(labeled_entry_dict):
      new_subjects = []
      for subject in labeled_entry_dict['subjects']:
          new_subjects.append(remove_precede_space(subject))
      labeled_entry_dict['subjects'] = new_subjects
      new_occs = []
      for occ in labeled_entry_dict['occupations']:
          new_occs.append(remove_precede_space(occ))
      labeled_entry_dict['occupations'] = new_occs
      new_locations = []
      for loc_dict in labeled_entry_dict['locations']:
          new_loc_dict = {}
          new_loc_dict['value'] = remove_precede_space(loc_dict['value'])
          try:
              new_loc_dict['labels'] = loc_dict['labels']
          except:
              pass
          new_locations.append(new_loc_dict)
      labeled_entry_dict['locations'] = new_locations
      return labeled_entry_dict
  
  
  def build_entries(args):
      """
      Page Array Structure
      col 0 = ID number of line
      col 1-4 = bbbox x1, y1, x2, y2
      col 5 = 0:col line; 1: indent line; 2:kill line
      col 6 = 1:col 1; 2: col2 (column assignment)
      col 7 = 0:ok to append, 1: do not append (likely gap)"""
  
      root = '/'.join(args.path.split('/')[:-1])
      directory_uuid = root.split('/')[-1]
      hocr_files = [file for file in os.listdir(
          args.path) if file.endswith('.hocr')]
  
      print("Processing: ", directory_uuid)
  
      for hocr_file in hocr_files:
  
          try:
              page_uuid = hocr_file.replace('_rotated', '').replace(
                  '_cropped', '').replace('.hocr', '')
              # try:
              raw_hocr_array, page_html = load_hocr_lines(
                  os.path.join(args.path, hocr_file))
  
              # jpeg_path = os.path.join(
              #     root, args.jpeg_directory, hocr_file.replace('.hocr', '.jpeg'))
  
              ##
              #   Find our likely column locations
              ##
  
              if raw_hocr_array.size == 0:
                  print("Empty HOCR file: " + hocr_file)
                  print("Passing: ----------------------------------")
                  continue
  
              kmeans = KMeans(n_clusters=8).fit(
                  raw_hocr_array[:, 1].reshape(-1, 1))
              centroids = kmeans.cluster_centers_
              cands_cols = {}
  
              for j in range(len(centroids)):
                  cands_cols[centroids[j, 0]] = 0
                  std = sqrt(mean([((i - centroids[j, 0])**2)
                                  for i in raw_hocr_array[:, 1]]))
                  for i in range(len(raw_hocr_array)):
                      if abs(raw_hocr_array[i:i+1, 1] - centroids[j, 0]) > (std/16):
                          pass
                      else:
                          cands_cols[centroids[j, 0]] += 1
  
              # We have our dict of possible column location along with the number of entries proximate to each
              # But in case our k-means clustering was compromised by a slanted column line (yielding double col locations)
              # we need to check for that and take an average of the resultant double col1 and double col2 locations
  
              halfway_page_rough = (max(raw_hocr_array[:, 1])/2)*.95
              lefthand_cands = [
                  i for i in cands_cols.items() if i[0] < halfway_page_rough]
              righthand_cands = [
                  i for i in cands_cols.items()if i[0] > halfway_page_rough]
              col1_xval_cands = sorted(
                  lefthand_cands, key=lambda x: x[1], reverse=True)
              col2_xval_cands = sorted(
                  righthand_cands, key=lambda x: x[1], reverse=True)
  
              # Now that we've split our candidate locations into what we think is roughly the col1 and col2 areas
              # we look for two closely proximate top candidates. If they exist, we average them out
  
              if abs(col1_xval_cands[0][0] - col1_xval_cands[1][0]) < 50:
                  col1_xval = mean(
                      [col1_xval_cands[0][0], col1_xval_cands[1][0]])
              else:
                  col1_xval = col1_xval_cands[0][0]
              try:
                  if abs(col2_xval_cands[0][0] - col2_xval_cands[1][0]) < 50:
                      col2_xval = mean(
                          [col2_xval_cands[0][0], col2_xval_cands[1][0]])
                  else:
                      col2_xval = col2_xval_cands[0][0]
  
              # For cases where we don't have multiple candidates for a second column,
              # possibly because Tesseract missed second column entirely:
              except:
                  col2_xval = col2_xval_cands[0][0]
  
              ##
              #  Pass to identify and keep lines that are at x-val of column edges
              ##
  
              std1 = sqrt(mean([((i - col1_xval)**2)
                          for i in raw_hocr_array[:, 1]]))/16
              std2 = sqrt(mean([((i - col2_xval)**2)
                          for i in raw_hocr_array[:, 1]]))/16
              for i in range(len(raw_hocr_array)):
                  if abs(raw_hocr_array[i:i + 1, 1] - col1_xval) < std1 or abs(raw_hocr_array[i:i + 1, 1] - col2_xval) < std2:
                      # Col 1 identification:
                      if abs(raw_hocr_array[i:i + 1, 1] - col1_xval) < std1:
                          raw_hocr_array[i:i + 1, 6] = 1
                      # Col 2 identification:
                      elif abs(raw_hocr_array[i:i + 1, 1] - col2_xval) < std2:
                          raw_hocr_array[i:i + 1, 6] = 2
                  # Id of indents and any potential chopped off lines to right of columns
                  elif raw_hocr_array[i:i + 1, 1] - (col1_xval + std1) > 0 and raw_hocr_array[i:i + 1, 1] < col2_xval:
                      # Col 1 indents identification
                      raw_hocr_array[i:i + 1, 5] = 1
                      raw_hocr_array[i:i + 1, 6] = 1
                  elif raw_hocr_array[i:i + 1, 1] - (col2_xval + std2) > 0:
                      # Col 2 indents identification
                      raw_hocr_array[i:i + 1, 5] = 1
                      raw_hocr_array[i:i + 1, 6] = 2
                  # Eliminating anything to left of column 1
                  if raw_hocr_array[i:i+1, 1] < (col1_xval - std1):
                      raw_hocr_array[i:i + 1, 5] = 2
                      # Eliminating anything to right of column 2 right edge
                  if raw_hocr_array[i:i + 1, 1] > (col2_xval + (col2_xval - col1_xval)):
                      raw_hocr_array[i:i + 1, 5] = 2
  
              ##
              #   Pass to find lines flush with a column whose y vals make them unlikely to be in the page block for entries
              ##
  
              reduced_array = raw_hocr_array[raw_hocr_array[:, 5] != 2]
              sorted_y_array = np.sort(reduced_array.view(
                  'i8,i8,i8,i8,i8,i8,i8,i8'), order=['f2'], axis=0).view(np.int)
  
              # To find an appropriate vertical line density of all likely lines, we grab a sample at a point roughly 1/4
              # the way through our entries, then gather the line density at that point within a gap around that point
              # The gap is calculated at 0.5% of the highest yval (roughly 0.5% of the yval height of the page)
  
              quarter_page = len(sorted_y_array)//4
              gap = float(max(raw_hocr_array[:, 2]))*.05
              entry_density = len([i for i in sorted_y_array[:, 2] if abs(
                  i - sorted_y_array[quarter_page:quarter_page+1, 2]) < gap/2])
  
              # We now examine the line density around every line in the page; if the density is low, we do a second check to make
              # sure the reason isn't that it is a first or last line; in those cases we check for gap density after/before the line
              # Anything that still fails we cut
  
              for i in range(len(sorted_y_array)):
                  proximate_lines = [yval for yval in sorted_y_array[:, 2] if abs(
                      yval - sorted_y_array[i:i+1, 2]) < gap/2]
                  if len(proximate_lines) - 1 < entry_density/2:
                      top_line_proximate_lines = [yval for yval in sorted_y_array[:, 2] if yval - sorted_y_array[i:i + 1, 2] < gap
                                                  and yval - sorted_y_array[i:i + 1, 2] > 0]
                      bottom_line_proximate_lines = [yval for yval in sorted_y_array[:, 2] if
                                                     sorted_y_array[i:i + 1, 2] - yval < gap and sorted_y_array[i:i + 1, 2] - yval > 0]
                      if len(top_line_proximate_lines) > entry_density or len(bottom_line_proximate_lines) > entry_density:
                          pass
                      else:
                          sorted_y_array[i:i+1, 5] = 2
  
              ##
              #   Pass to look for missing lines by finding no line with a yval that is 1.95% of the expected space
              #   between lines; if a missing line is found, we mark the previous line to make sure an indent isn't appended to that line
              #   Those indents following a gap will become a standalone line because their head-entry is missing
              ##
  
              line_only_array = sorted_y_array[sorted_y_array[:, 5] != 2]
              sorted_line_only_array = np.sort(line_only_array.view(
                  'i8,i8,i8,i8,i8,i8,i8,i8'), order=['f6', 'f2'], axis=0).view(np.int)
              sample_lines = sorted_line_only_array[sorted_line_only_array[:, 6] == 1]
              gaps = []
  
              for i in range(len(sample_lines)):
                  try:
                      gaps.append(
                          int(sample_lines[i+1:i+2, 2] - sample_lines[i:i+1, 2]))
                  except:
                      pass
  
              average_line_gap = sum(gaps) // len(sample_lines)
              gap_locations = []
              for i in range(len(sorted_line_only_array)):
                  try:
                      if int(sorted_line_only_array[i + 1:i + 2, 2] - sorted_line_only_array[i:i + 1, 2]) > average_line_gap*1.95:
                          sorted_line_only_array[i:i+1, 7] = 1
                          gap_locations.append(
                              sorted_line_only_array[i:i + 1, 2] + average_line_gap*1.5)
                  except:
                      pass
  
              # We can either build the image or return the json
  
              if args.make_image == 'True':
                  imagebuilder(sorted_line_only_array, [
                      col1_xval, col2_xval], jpeg_path, std1, gap_locations, page_uuid, os.path.join(root, args.bbox_location))
              entries_json = json_from_hocr(
                  sorted_line_only_array, page_html, page_uuid, directory_uuid)
              build_manifest(root, entries_json)
              if args.mode == 'P':
                  print(entries_json)
                  outpath = os.path.join(
                      args.path_out, os.path.splitext(hocr_file)[0] + ".json")
                  f = open(outpath, "w")
                  f.write(json.dumps(entries_json))
                  f.close()
                  print("----------------------------------------")
              else:
                  classifier = Classifier.Classifier()
                  classifier.load_training(args.crf_training_path)
                  classifier.train()
                  for rec in entries_json:
                      entry = LabeledEntry.LabeledEntry(
                          entries_json[rec]['complete_entry'])
                      classifier.label(entry)
                      final_entries = normalize_labeled_entry(entry.categories)
                      entries_json[rec]['labeled_entry'] = final_entries
                      if args.mode == 'CRF-print':
                          print(entries_json[rec])
                  if args.mode == 'CRF':
                      with open(os.path.join(root, 'final-entries', page_uuid + '_labeled.json'), 'w') as f:
                          for rec in sorted(entries_json.keys()):
                              f.write(json.dumps(entries_json[rec]) + '\n')
                      f.close()
                  if args.tsv_path != "False":
                      build_entries_tsv(
                          entries_json, args.tsv_path, directory_uuid)
              print("Completed processing of ", page_uuid)
  
          except Exception as exception:
              print(exception.__traceback__)
              print("Likely ad or problematic hocr in :", hocr_file, ". Skipped.")
  
  
  def main():
      parser = argparse.ArgumentParser(
          description="Parse hocr files and return entries")
      parser.add_argument("-in", help="Full-path directory containing hocr files",
                          dest="path", type=str, required=True)
      parser.add_argument("-build-image", help="Set whether to make images (True/False)",
                          dest="make_image", default="False", type=str, required=True)
      parser.add_argument("-jpegs", help="Name of directory (not path) containing jpegs",
                          dest="jpeg_directory", type=str, required=False)
      parser.add_argument("-bbox-out", help="Full path to directory to place output bbox images",
                          dest="bbox_location", type=str, required=False)
      parser.add_argument("-mode", help="Either (P)rint out extracted entries, apply (CRF-print) and print out entries, or (CRF) and save JSON entries in labeled-json directory",
                          dest="mode", type=str, required=True)
      parser.add_argument("-path-training", help="Path to the training files for CRF classifer",
                          dest="crf_training_path", type=str, required=False)
      parser.add_argument("-path-out", help="Path to the training files for CRF classifer",
                          dest="path_out", type=str, required=False)
      parser.add_argument("-build-tsv", help="(False) or path to directory where tsv will be made",
                          dest="tsv_path", type=str, required=False)
      parser.set_defaults(func=build_entries)
      args = parser.parse_args()
      args.func(args)
  
  
  if __name__ == "__main__":
      main()
  
apis: numpy.array sklearn.cluster.KMeans