text: #!/usr/bin/python
  # -*- coding: utf-8 -*-
  """
  Script to run analysis on lots of files and show statistics and so on about it.
  This tries to add all possible analysis about the file and is a very
  developmental and possible messy script.
  
  Syntax:
  
      wikibot-filemeta-log [-arguments ...]
  
  Arguments can be:
  
  -showinfo      Show either "cats", "info" or if left empty defaults to "all".
  
  -limitsize     The maximum file size to download and parse in mega bytes.
                 Default is 100MB.
                 Example: "-limitsize:10" only analyzes files less than 10MB.
  
  -cachefiles    If used, the files are cached into a folder called "cache" in
                 current directory. The directory can be given by using
                 "-cachefiles:/place/to/cache". It should already exist.
  
  -dry           If used, the output is printed on the terminal and not to the
                 User page.
  
  -skip          If used, all the images in the pagegenerator is not processed.
                 After every image it processes, it skips the number of images
                 provided as the argument.
                 Example: "-skip:1" can be used to only run on alternate images.
  """
  
  from __future__ import (division, absolute_import, unicode_literals,
                          print_function)
  
  import logging
  import os
  import re
  import sys
  import tempfile
  from collections import Counter
  from datetime import datetime
  
  import numpy
  from six import string_types
  from six.moves.urllib.error import URLError
  
  from file_metadata.generic_file import GenericFile
  from file_metadata.wikibot.utilities import (pywikibot, download_page,
                                               stringify)
  
  from pywikibot import pagegenerators
  
  
  def make_link(val):
      if val.lower().startswith('category:'):
          return '[[:' + val + ']]'
      elif val.lower().startswith('{{'):
          template = re.sub('(^{{|}}$)', '', val)
          tlx = re.sub('=', '&#61;', template)
          return '{{tlx|' + tlx + '}}'
      else:
          return '[[:' + val + ']]'
  
  
  def dump_log(data, logname, append=False):
      if isinstance(data, (tuple, list)):
          data = "\n".join(data)
      if not isinstance(data, string_types):
          raise ValueError('Unexpected output got. Expected str, tuple, '
                           'list but got {0}.'.format(type(data)))
      page = pywikibot.Page(
          pywikibot.Site(),
          'User:{user}/logs/{name}'.format(user=pywikibot.Site().username(),
                                           name=logname))
      if append:
          data = page.text + '\n' + data
      if options.get('dry'):
          pywikibot.output(data)
      else:
          page.put(data, "Logged using file-metadata")
  
  
  def handle_bulk_pages(gen):
      parsed_pages = set()
      log, categories, category_buckets = [], [], []
      count, exception_count = 0, 0
      total_start_time = datetime.now()
  
      mime_cats = {
          "JPEG files": {'image/jpeg'},
          "GIF files": {'image/gif'},
          "PNG files": {'image/png'},
          "TIFF files": {'image/tiff'},
          "XCF files": {'image/x-xcf', 'application/x-xcf'},
          "FLAC files": {'audio/x-flac'},
          "WAV files": {'audio/x-wav'},
          "MIDI files": {'audio/midi'},
          "DjVu files": {'image/vnd-djvu', 'image/vnd.djvu'},
          "PDF files": {'application/pdf'},
          "SVG files": {'svg'},
          "Ogg sound files": {'ogg'},
          "Ogv videos": {'ogv'},
          'WebM videos': {'video/webm'},
      }
      for ipage, page in enumerate(gen):
          if not (page.exists() and
                  not page.isRedirectPage() and
                  page.namespace() == "File" and
                  page.title() not in parsed_pages and
                  (page.latest_file_info['size'] / 1024 / 1024 <
                   options.get('limitsize', float("inf"))) and
                  ipage % (options.get('skip', 0) + 1) == 0):
              continue
          parsed_pages.add(page.title())
          try:
              page_path = download_page(
                  page, timeout=15 * 60,
                  cache_dir=options.get('cachefiles', tempfile.gettempdir()))
          except URLError:  # Download timed out
              continue
  
          pywikibot.output(str(count + 1) + '. Analyzing ' +
                           page.title(underscore=False))
  
          start_time = datetime.now()
          _file = GenericFile.create(page_path)
          try:
              meta = _file.analyze()
          except Exception as err:
              logging.exception(err)
              exception_count += 1
              continue
          finally:
              _file.close()
  
          info, cats, cat_buckets, img = [], set(), set(), []
          # info - Information analyzed from the file
          # cats - The suggested categories to add to the file
          # im - Image preview with bounding boxes (if applicable)
  
          #################################################################
          # Mime analysis
          mime = meta.get('File:MIMEType', "ERROR")
          info.append("* '''Mime Type''': " + mime)
          if mime == 'application/ogg':
              _type = meta.get('File:FileType', 'ERROR')
              info.append("* '''File type''': " + _type)
  
          for cat, mimeset in mime_cats.items():
              if mime in mimeset or meta.get('Composite:FileFormat') in mimeset:
                  cats.add('Category:' + cat)
                  cat_buckets.add('Type')
                  break
  
          #################################################################
          # Software analysis
          # Find more files at https://commons.wikimedia.org/wiki/
          # Category:Created_with_..._templates
          if (meta.get('SVG:Output_extension', '') ==
                  'org.inkscape.output.svg.inkscape'):
              # Example: File:Db-omega.svg , Joetsu_Shinkansen_icon.png
              cats.add('{{Created with Inkscape}}')
  
          for sw_key in ('PNG:Software', 'EXIF:Software'):
              sw = stringify(meta.get(sw_key, ''))
              if re.match('MATLAB', sw, re.I):
                  # Example: File:Fat_absoprtion.png
                  cats.add('{{Created with MATLAB}}')
              elif re.match('ImageMagick', sw, re.I):
                  # Example: File:Groz-01.PNG
                  match = re.match(r'ImageMagick (?P<ver>[\d\.]+)', sw, re.I)
                  ver = '|version=' + match.groupdict()['ver'] if match else ''
                  cats.add('{{Created with ImageMagick' + ver + '}}')
              elif re.match('Adobe ImageReady', sw, re.I):
                  # Example: File:Holtz.png
                  cats.add('{{Created with Adobe ImageReady}}')
              elif re.match('Adobe Photoshop', sw, re.I):
                  if re.match('Elements', sw, re.I):
                      # Example: File:1010_Bazylika_archikatedralna_%C5%9
                      #               Bw_Jakuba_Szczecin_sygnaturka_0.jpg
                      cats.add('{{Created with Adobe Photoshop Elements}}')
                  elif re.match('Express', sw, re.I):
                      # Example: File:Politecnico_di_Milano_Bovisa_4.jpg
                      cats.add('{{Created with Adobe Photoshop Express}}')
                  elif re.match(r'Photoshop CS\d?', sw, re.I):
                      match = re.match(r'Photoshop CS\d?', sw, re.I)
                      if match:
                          # Example: File:Cervicomanubriotomie.jpg
                          cats.add('{{Created with Adobe ' +
                                   match.group().strip() + '}}')
                  else:
                      # Example: File:Cervicomanubriotomie.jpg
                      cats.add('{{Created with Adobe Photoshop}}')
                  # Check if photomerge was used
                  if (stringify(meta.get('Photoshop:HasRealMergedData',
                                         '')).lower()
                          in ('1', 'yes')):
                      # Example:01-118_Koenigstein_Panorama.jpg
                      cats.add('{{Created with Photoshop Photomerge}}')
                      cats.add('Category:Panoramics')
              elif re.match('Picasa', sw, re.I):
                  # Example: File:08_Ny_Alesund_prn.JPG
                  match = re.match(r'Picasa (?P<ver>[\d\.]+)', sw, re.I)
                  ver = '|version=' + match.groupdict()['ver'] if match else ''
                  cats.add('{{Created with Picasa' + ver + '}}')
              elif re.match('GIMP', sw, re.I):
                  # Example: File:2013-04-25_21-09-18-ecl-lune-mosaic.jpg
                  match = re.match(r'GIMP (?P<ver>[\d\.]+)', sw, re.I)
                  ver = '|version=' + match.groupdict()['ver'] if match else ''
                  cats.add('{{Created with GIMP' + ver + '}}')
              elif re.match('Microsoft ICE', sw, re.I):
                  # Example: File:Bochnia_kopalnia_kaplica_2.jpg
                  match = re.match(r'ICE v(?P<ver>[\d\.]+)', sw, re.I)
                  ver = '|version=' + match.groupdict()['ver'] if match else ''
                  cats.add('{{Created with Microsoft Image Composite '
                           'Editor' + ver + '}}')
              elif re.match(r'Paint\.NET', sw, re.I):
                  # Example: File:
                  match = re.match(r'Paint\.NET v(?P<ver>[\d\.]+)', sw, re.I)
                  ver = '|version=' + match.groupdict()['ver'] if match else ''
                  cats.add('{{Created with Paint.NET' + ver + '}}')
              elif re.match('gnome-screenshot', sw, re.I):
                  # Example: File:LibreOfficePresentationTeluguExample1.png
                  cats.add('Category:Screenshots')
                  cat_buckets.add('Content')
  
          for desc_key in ('SVG:Desc',):
              desc = stringify(meta.get(desc_key, '')).lower()
              if re.match('GNUPLOT', desc, re.I):
                  # Example: File:Beta_versus_rapidity.svg
                  match = re.match(r'GNUPLOT (?P<ver>[\d\.]+)', sw, re.I)
                  ver = '|version=' + match.groupdict()['ver'] if match else ''
                  cats.add('{{Created with GNU Plot' + ver + '}}')
              elif re.match('Chemtool', desc, re.I):
                  # Example: File:Chitobiose_glucosamine.svg
                  match = re.match(r'Chemtool (?P<ver>[\d\.]+)', sw, re.I)
                  ver = '|version=' + match.groupdict()['ver'] if match else ''
                  cats.add('{{Created with Chemtool' + ver + '}}')
                  cats.add('Category:Chemical compounds')
                  cats.add('Category:Graphics')
                  cat_buckets.add('Content')
              elif re.match('VectorFieldPlot', desc, re.I):
                  # Example: File:VFPt_minus.svg
                  match = re.match(r'VectorFieldPlot (?P<ver>[\d\.]+)', sw, re.I)
                  ver = '|version=' + match.groupdict()['ver'] if match else ''
                  cats.add('{{Created with VectorFieldPlot' + ver + '}}')
                  cats.add('Category:Field diagrams')
                  cats.add('Category:Graphics')
                  cat_buckets.add('Content')
  
          for comment_key in ('PNG:Comment', 'File:Comment'):
              comment = stringify(meta.get(comment_key, '')).lower()
              if re.match('Stella4D', comment, re.I):
                  # Example: File:10-3_deltohedron.png
                  match = re.match(r'Stella4D (?P<ver>[\d\.]+)', sw, re.I)
                  ver = '|version=' + match.groupdict()['ver'] if match else ''
                  cats.add('{{Created with Stella' + ver + '}}')
              elif re.match('GIMP', comment, re.I):
                  # Example: File:105_H_61-37.jpeg
                  cats.add('{{Created with GIMP}}')
  
          for prod_key in ('PDF:Producer',):
              prod = stringify(meta.get(prod_key, ''))
              if re.match(r'Libre ?Office', prod, re.I):
                  match = re.match(r'Libre ?Office (?P<ver>[\d\.]+)', prod, re.I)
                  ver = '|version=' + match.groupdict()['ver'] if match else ''
                  if meta.get('PDF:Creator'):
                      cats.add('{{Created with LibreOffice ' +
                               meta['PDF:Creator'] + ver + '}}')
                  else:
                      cats.add('{{Created with LibreOffice' + ver + '}}')
              elif re.match(r'doPDF', prod, re.I):
                  cats.add('{{Created with doPDF}}')
              elif re.match(r'ACDSee', prod, re.I):
                  cats.add('{{Created with ACDSee}}')
              elif re.match(r'iText', prod, re.I):
                  cats.add('{{Created with iText}}')
  
          #################################################################
          # Device analysis
          model = stringify(meta.get('EXIF:Model', ''))
          make = stringify(meta.get('EXIF:Make', ''))
          if model:
              # Modify the model so that it conforms to the category names
              for _make in ('NIKON', 'FUJIFILM', 'Xiaomi', 'Panasonic', 'SONY',
                            'Nokia'):
                  if make == _make and not re.match(_make, model, re.I):
                      # Category is named as make + model
                      model = make + ' ' + model
              if make == 'i2S DigiBook Scanner':
                  model = 'i2s Digibook ' + model
              # Simple replacements
              model = model.replace('NIKON', 'Nikon')
              model = model.replace('PENTAX', 'Pentax')
              model = model.replace('FUJIFILM', 'Fujifilm')
              model = model.replace('SONY', 'Sony')
              if 'nikon' in make.lower():
                  model = model.replace('COOLPIX', 'Coolpix')
              if 'xiaomi' in make.lower():
                  model = model.replace('Note3', 'Note 3')
              if 'samsung' in make.lower():
                  # In samsung, the cameras have multiple names like:
                  # "<Digimax S600 / Kenox S600 / Digimax Cyber 630>" or
                  # "Digimax A4/Kenox D4". The categories are only named as
                  # "Taken with Samsung S600", "Taken with Samsung A4" though.
                  samsung_model = ''
                  possible_models = re.sub('[<>]', '', model).split('/')
                  for imod in possible_models:
                      imod = imod.strip()
                      if (imod.lower().startswith('digimax') and
                              not imod.lower().startswith('digimax cyber')):
                          samsung_model = imod
                          break
                  if not samsung_model and len(possible_models) > 0:
                      samsung_model = possible_models[0].strip()
                  # Remove digimax if present
                  samsung_model = re.sub('Digimax', '', samsung_model, count=1,
                                         flags=re.IGNORECASE).strip()
  
                  model = 'Samsung ' + samsung_model
              if 'olympus' in make.lower():
                  model = 'Olympus ' + model
              if 'motorola' in make.lower():
                  model = model.replace('MotoG2', 'Moto G2')
                  model = model.replace('MotoG3', 'Moto G3')
                  model = model.replace('MotoG4', 'Moto G4')
              if 'sanyo' in make.lower():
                  model = 'Sanyo ' + model
              if 'canon' in make.lower():
                  model = model.replace('EOS REBEL', 'EOS')
              if 'casio' in make.lower():
                  model = 'Casio ' + model
  
              for prefix in ['Sanned with ', 'Taken with ',
                             'Taken or Scanned with ']:
                  possible_cat = pywikibot.Page(pywikibot.Site(),
                                                "Category:" + prefix + model)
                  if (possible_cat.exists() or
                          prefix == 'Taken or Scanned with '):
                      info.append("* '''Model''': " +
                                  possible_cat.title(underscore=False,
                                                     textlink=True, asLink=True))
                      cats.add(possible_cat.title(underscore=False))
                      break
  
          #################################################################
          # Color analysis
          for key in ['AverageRGB', 'ClosestLabeledColor', 'EdgeRatio',
                      'NumberOfGreyShades', 'PercentFrequentColors',
                      'MeanSquareErrorFromGrey', 'Monochrome', 'UsesAlpha',
                      'IT8TopBar', 'IT8BottomBar', 'IT8TopBarGreyMSE',
                      'IT8BottomBarGreyMSE']:
              if 'Color:' + key in meta:
                  if isinstance(meta['Color:' + key], (list, tuple)):
                      val = ', '.join(map(str, meta['Color:' + key]))
                  elif isinstance(meta['Color:' + key], float):
                      val = str(round(meta['Color:' + key], 5))
                  else:
                      val = stringify(meta['Color:' + key])
                  info.append("* '''" + key + "''': " + val)
          greys = meta.get('Color:NumberOfGreyShades')
          edges = meta.get('Color:EdgeRatio')
          if ((greys is not None and greys < 2) or
                  (edges is not None and edges < 0.13) or
                  meta.get('Composite:FileFormat') == 'svg'):
              if meta.get('EXIF:Make') is None:
                  cats.add('Category:Graphics')
          if meta.get('Color:MeanSquareErrorFromGrey', 999) < 20:
              if meta.get('EXIF:Make') is None:
                  cats.add('Category:Black and white')
              else:  # If it's scanned or from a camera and black-white
                  cats.add('Category:Black and white photographs')
          if meta.get('Color:MeanSquareErrorFromGrey', 999) == 0:
              cats.add('Category:Grayscale')
          if meta.get('Color:UsesAlpha') is True:
              cats.add('Category:Transparent background')
  
          for spike, greymse in [('Color:IT8TopBar',
                                  'Color:IT8TopBarGreyMSE'),
                                 ('Color:IT8BottomBar',
                                  'Color:IT8BottomBarGreyMSE')]:
              if (18 <= meta.get(spike, -1) <= 22 and
                      meta.get(greymse, 999) < 15):
                  cats.add('Category:Scans with IT8 target')
                  break
  
          if meta.get('Misc:StereoCardMSE'):
              info.append("* '''Stereo Card MSE''': " +
                          str(meta.get('Misc:StereoCardMSE')))
          if meta.get('Misc:StereoCardHistogramMSE'):
              info.append("* '''Stereo Card Histogram MSE''': " +
                          str(meta.get('Misc:StereoCardHistogramMSE')))
  
          #################################################################
          # Location analysis
          for key in ['GPSCity', 'GPSState', 'GPSCountry']:
              val = meta.get('Composite:' + key)
              if val is not None:
                  info.append("* '''" + key + "''': " + val)
          for val in [(meta.get('Composite:GPSCity', '') + ', ' +
                       meta.get('Composite:GPSState', '')),
                      (meta.get('Composite:GPSCity', '') + ', ' +
                       meta.get('Composite:GPSCountry', '')),
                      meta.get('Composite:GPSCity', ''),
                      (meta.get('Composite:GPSState', '') + ', ' +
                       meta.get('Composite:GPSCountry', '')),
                      meta.get('Composite:GPSState', ''),
                      meta.get('Composite:GPSCountry', '')]:
              if val:
                  if pywikibot.Page(
                          pywikibot.Site(), "Category:" + val).exists():
                      cats.add('Category:' + val)
                      break
  
          if (meta.get('Composite:GPSLatitude') and
                  meta.get('Composite:GPSLongitude')):
              cats.add('{{GPS EXIF}}')
              cat_buckets.add('Location')
              info.append("* '''GPS Coordinates''': {0}, {1}"
                          .format(meta.get('Composite:GPSLatitude'),
                                  meta.get('Composite:GPSLongitude')))
  
          #################################################################
          # Author analysis
          if meta.get('EXIF:Artist') == "New York Public Library":
              cats.add('Category:Images from the New York Public Library')
  
          #################################################################
          # Image analysis
          # Fill second column (image cell) for ImageFiles (Only 2 dim images)
          if (isinstance(_file.fetch('ndarray'), numpy.ndarray) and
                  _file.fetch('ndarray').ndim in (2, 3)):
              height, width = _file.fetch('ndarray').shape[:2]
              max_dim = max(width, height)
              scale = min(200, max_dim) / max_dim
  
              img.append('<div style="position:relative;">')
              img.append('{0}|{1}x{2}px]]'.format(
                         page.title(underscore=False, asLink=True)[:-2],
                         int(width * scale), int(height * scale)))
  
              img_bbox = ('<div class="position-marker file-meta-{css_class}" '
                          'style="position:absolute; left:{left}px; '
                          'top:{top}px; width:{width}px; height:{height}px; '
                          'border:2px solid #{color};"></div>')
  
              def _str_bbox(bbox):
                  return ("Left:" + str(bbox['left']) +
                          ", Top:" + str(bbox['top']) +
                          ", Width:" + str(bbox['width']) +
                          ", Height:" + str(bbox['height']))
  
              def _mean_bbox(bbox):
                  return (bbox['left'] + bbox['width'] / 2,
                          bbox['top'] + bbox['height'] / 2)
  
              def _area_bbox(bbox):
                  return bbox['width'] * bbox['height']
  
              #################################################################
              # Analysis for very specific images: Icons, Football kits, etc
              if height == width and height in (16, 32, 48, 96):
                  cats.add('Category:Icons')
              elif width == 100 and height == 36:
                  cats.add('Category:Football kit shorts')
                  cat_buckets.add('Content')
              elif width == 100 and height == 25:
                  cats.add('Category:Football kit socks')
                  cat_buckets.add('Content')
              elif width == 38 and height == 59:
                  cats.add('Category:Football kit body')
                  cat_buckets.add('Content')
              elif width == 31 and height == 59:
                  cats.add('Category:Football kit sleeves')
                  cat_buckets.add('Content')
  
              #################################################################
              # Barcode analysis
              bar_cats = {
                  "Code 39": ('code_39', 'code39'),
                  "Code 93": ('code_93', 'code93'),
                  "Code 128": ('code_128', 'code128'),
                  "Data Matrix": ('data_matrix',),
                  "Quick Response Codes": ('qr_code', 'qrcode')}
  
              def print_barcode_data(bar):
                  info.append("** Data: " + bar['data'])
                  info.append("** Format: " + str(bar['format']))
                  info.append("** Position : " + _str_bbox(bar['bounding box']))
                  box_kwargs = {k: int(v * scale)
                                for k, v in bar['bounding box'].items()}
                  box_kwargs["css_class"] = "barcode"
                  if (bar['bounding box']['height'] < 5 and
                          bar['bounding box']['width'] < 5):
                      box_kwargs["color"] = "ff0000"
                      img.append(img_bbox.format(**box_kwargs))
                      return set()
                  box_kwargs["color"] = "00ff00"
                  img.append(img_bbox.format(**box_kwargs))
                  for cat, formats in bar_cats.items():
                      if bar['format'].lower() in formats:
                          return {'Category:' + cat, 'Category:Barcode'}
                  return {'Category:Barcode'}
  
              # Barcodes from zxing:
              for i, bar in enumerate(meta.get('zxing:Barcodes', [])):
                  info.append("* '''Barcode''' (zxing) #" + str(i))
                  barcode_cats = print_barcode_data(bar)
                  cats = cats.union(barcode_cats)
                  if len(barcode_cats) > 0:
                      cat_buckets.add('Content')
  
              # Barcodes from zbar:
              for i, bar in enumerate(meta.get('zbar:Barcodes', [])):
                  info.append("* '''Barcode''' (zbar) #" + str(i))
                  barcode_cats = print_barcode_data(bar)
                  cats = cats.union(barcode_cats)
                  if len(barcode_cats) > 0:
                      cat_buckets.add('Content')
  
              #################################################################
              # Face analysis
              def print_face_data(face, _type):
                  _cats = set()  # These may or may not be added ...
                  feats = set()
                  _cats.add('Category:Human faces')
                  _cats.add('Category:Unidentified people')
                  if _type == 'opencv':  # Dlib always finds all features
                      if len(face.get('eyes', ())) > 0:
                          # _cats.add('Category:Human eyes')
                          feats.add('Eyes (' + str(len(face['eyes'])) + ')')
                      if face.get('ears') is not None:
                          # _cats.add('Category:Human ears')
                          feats.add('Ears')
                      if face.get('nose') is not None:
                          # _cats.add('Category:Human noses')
                          feats.add('Nose')
                      if face.get('mouth') is not None:
                          # _cats.add('Category:Human mouths')
                          feats.add('Mouth')
                      if face.get('glasses') is not None:
                          feats.add('Glasses')
  
                  if (_area_bbox(face['position']) > 0.55 * height * width):
                      # If face is very large...
                      _cats.add('Category:Portrait')
  
                  if _type == 'dlib':
                      info.append("** Score: " + str(round(face['score'], 3)))
                  info.append("** Bounding Box: " + _str_bbox(face['position']))
                  if len(feats) > 0:
                      info.append("** Other features: " + ", ".join(feats))
                  box_kwargs = {k: int(v * scale)
                                for k, v in face['position'].items()}
                  box_kwargs["css_class"] = "barcode"
                  if (((_type == 'opencv' and len(feats) > 2) or
                          (_type == 'dlib' and face['score'] > 0.045)) and
                          _area_bbox(face['position']) > width * height * 0):
                      box_kwargs["color"] = "00ff00"
                      img.append(img_bbox.format(**box_kwargs))
                      return _cats
                  box_kwargs["color"] = "ff0000"
                  img.append(img_bbox.format(**box_kwargs))
                  return set()
  
              # Save all face categories in this set, ad add it later if it
              # seems appropirate. This is because we need to delay the writing
              # of these categories based on some logic.
              valid_faces = []
              face_cats = set()
  
              # Faces with dlib:
              for iface, face in enumerate(meta.get('dlib:Faces', [])):
                  info.append("* '''Face''' (dlib) #" + str(iface + 1))
                  icats = print_face_data(face, 'dlib')
                  face_cats = face_cats.union(icats)
                  if 'Category:Human faces' in icats:
                      valid_faces.append(face)
  
              # Faces with opencv's haarcascades:
              for iface, face in enumerate(meta.get('OpenCV:Faces', [])):
                  info.append("* '''Face''' (haarcascade) #" + str(iface + 1))
                  icats = print_face_data(face, 'opencv')
                  face_cats = face_cats.union(icats)
                  if 'Category:Human faces' in icats:
                      valid_faces.append(face)
  
              if len(valid_faces) >= 3:
                  face_cats.add('Category:Groups of people')
  
              if len(face_cats) > 0:
                  cat_buckets.add('Content')
              cats = cats.union(face_cats)
              img.append('</div>')  # Close the image's div
  
          #################################################################
          # Leaf cats using Location analysis
          if 'Category:Unidentified people' in cats:
              for key in ['GPSState', 'GPSCountry']:
                  val = meta.get('Composite:' + key)
                  if val is None:
                      continue
                  cat_val = "Category:Unidentified people in " + val
                  if (pywikibot.Page(pywikibot.Site(), cat_val).exists() or
                          key == 'GPSCountry'):
                      cats.add(cat_val)
                      cats.remove('Category:Unidentified people')
                      break
  
          if 'Category:Groups of people' in cats:
              for key in ['GPSState', 'GPSCountry']:
                  val = meta.get('Composite:' + key)
                  if val is None:
                      continue
                  cat_val = "Category:Groups of people in " + val
                  if (pywikibot.Page(pywikibot.Site(), cat_val).exists() or
                          key == 'GPSCountry'):
                      cats.add(cat_val)
                      cats.remove('Category:Groups of people')
                      break
  
          categories.append(cats)
          category_buckets.append(cat_buckets)
  
          info.append("* '''Time taken''': {0} sec"
                      .format((datetime.now() - start_time).total_seconds()))
  
          count += 1
          log.append("\n==== {0} ====" .format(page.title(asLink=True,
                                                          textlink=True)))
          # Make a table for text and image
  
          log += ['{| class="wikitable"', '|']
          if options.get('showinfo', "all") in ('all', 'info'):
              log += info
          if options.get('showinfo', "all") in ('all', 'cats'):
              log.append("* '''Categories''' ({0}): {1}"
                         .format(len(cats),
                                 ", ".join(make_link(c) for c in cats)))
          log += ['|'] + img + ['|}']
          # Clean up the downloaded file if no need to cache
          if options.get('cachefiles') is None:
              os.remove(page_path)
  
      if count + exception_count == 0:
          # Nothing happened, don't go on.
          pywikibot.output('Analysis was not run on any files.')
          return
  
      stats = ["* '''Time taken''': {0} sec"
               .format((datetime.now() - total_start_time).total_seconds())]
      stats += ["* '''Number of files successfully analyzed''': "
                "{0} ({1:0.4f} %)".format(
                    count, 100 * count / (count + exception_count))]
      stats += ["* '''Number of files with exceptions''': "
                "{0} ({1:0.4f} %)".format(
                    exception_count,
                    100 * exception_count / (count + exception_count))]
      cat_counter = Counter(cat for fcats in categories for cat in fcats)
      stats += ["* '''Number of distinct categories used''': {0}"
                .format(len(cat_counter))]
  
      stats += ['{| class="wikitable sortable plainrowheaders"',
                '|+ Data of files categorized by category name',
                '! Category name',
                '! Number of files',
                '! colspan=2 | Percent of files in the category',
                ]
      for icat, (catname, numfiles) in enumerate(
              sorted(cat_counter.items(), key=lambda x: x[1], reverse=True)):
          stats.append('|-')
          stats.append('! scope=row | ' + make_link(catname))
          stats.append('| ' + str(numfiles))
          stats.append('| {{bartable| ' +
                       str(round(100 * numfiles / count, 2)) + '}}')
      stats.append('|}')
  
      stats += ['{{User:AbdealiJK/Templates/Bar chart',
                '| title = Histogram of number of categories found per file',
                '| label_type = Number of categories found',
                '| data_type = Number of files with that many categories',
                '| data_max = ' + str(count)]
      for icat, (numcats, numfiles) in \
              enumerate(Counter(len(cats) for cats in categories).items()):
          stats += ['| label{0} = {1} ({2:.4f} %)'.format(
                    icat + 1, numcats, 100 * numfiles / count),
                    '| data{0} = {1}'.format(icat + 1, numfiles)]
      stats.append('}}')
  
      stats += ['{{User:AbdealiJK/Templates/Bar chart',
                '| title = Histogram of number of categories found per file '
                '(without file-type categories)',
                '| label_type = Number of categories found',
                '| data_type = Number of files with that many categories',
                '| data_max = ' + str(count)]
      file_type_cats = set('Category:' + c for c in mime_cats.keys())
      non_file_type_counter = Counter(
          len(cats - file_type_cats) for cats in categories)
      for icat, (numcats, numfiles) in enumerate(non_file_type_counter.items()):
          stats += ['| label{0} = {1} ({2:.4f} %)'.format(
                    icat + 1, numcats, 100 * numfiles / count),
                    '| data{0} = {1}'.format(icat + 1, numfiles)]
      stats.append('}}')
  
      stats += ['{{User:AbdealiJK/Templates/Bar chart',
                '| title = Histogram of buckets',
                '| label_type = Bucket name',
                '| data_type = Number of files in bucket',
                '| data_max = ' + str(count)]
      bucket_counter = Counter(
          b for fbuckets in category_buckets for b in fbuckets)
      for ibucket, (bucketname, numfiles) in enumerate(
              sorted(bucket_counter.items(), key=lambda x: x[1], reverse=True)):
          stats += ['| label{0} = {1} ({2:.4f} %)'.format(
                    ibucket + 1, bucketname, 100 * numfiles / count),
                    '| data{0} = {1}'.format(ibucket + 1, numfiles)]
      stats.append('}}')
  
      log = stats + ['\n'] + log
  
      # Dump all logs to the log page
      dump_log(log, logname=options.get('logname'))
  
  
  options = {}
  
  
  def main(*args):
      local_args = pywikibot.handle_args(args)
      gen_factory = pagegenerators.GeneratorFactory()
  
      for local_arg in local_args:
          if gen_factory.handleArg(local_arg):
              continue
          arg, sep, value = local_arg.partition(':')
          if arg == '-showinfo':
              options[arg[1:]] = value or "all"
              if value not in ("cats", "info", "all"):
                  pywikibot.error("Invalid value for -showinfo. It can only be "
                                  "cats, info, or all.\nUse -help for further "
                                  "information.")
                  sys.exit(1)
          elif arg == '-limitsize':
              options[arg[1:]] = int(value) if value != "" else 100
          elif arg == '-cachefiles':
              options[arg[1:]] = value or 'cache'
          elif arg == '-logname':
              options[arg[1:]] = value
          elif arg == '-dry':
              options[arg[1:]] = True
          elif arg == '-skip':
              options[arg[1:]] = int(value) or 0
  
      if not options.get('dry') and not options.get('logname'):
          pywikibot.error('-logname is required to decide the page to write '
                          'to.\nUse -help for further information.')
          sys.exit(2)
  
      gen = gen_factory.getCombinedGenerator()
      if not gen:
          pywikibot.bot.suggest_help(missing_generator=True)
          return False
      else:
          pywikibot.Site().login()
          pregenerator = pagegenerators.PreloadingGenerator(gen)
          handle_bulk_pages(pregenerator)
          return True
  
  
  if __name__ == "__main__":
      main()
  