text: """Run a simple web server which responds to requests to monitor a directory
  tree and identify files over a certain size threshold. The directory is
  scanned to pick up the current situation and then monitored for new / removed
  files. By default, the page auto-refreshes every 60 seconds and shows the
  top 50 files ordered by size.
  """
  from __future__ import with_statement
  import os, sys
  import cgi
  import datetime
  import operator
  import socket
  import threading
  import time
  import traceback
  import Queue
  import urllib
  import urlparse
  from wsgiref.simple_server import make_server
  from wsgiref.util import shift_path_info
  import win32timezone
  
  import error_handler
  from winsys import core, fs, misc
  print("Logging to %s" % core.log_filepath)
  
  def deltastamp(delta):
  
      def pluralise(base, n):
          if n > 1:
              return "%d %ss" % (n, base)
          else:
              return "%d %s" % (n, base)
  
      if delta > datetime.timedelta(0):
          output_format = "%s ago"
      else:
          output_format = "in %s"
  
      days = delta.days
      if days != 0:
          wks, days = divmod(days, 7)
          if wks > 0:
              if wks < 9:
                  output = pluralise("wk", wks)
              else:
                  output = pluralse("mth", int(round(1.0 * wks / 4.125)))
          else:
              output = pluralise("day", days)
      else:
          mins, secs = divmod(delta.seconds, 60)
          hrs, mins = divmod(mins, 60)
          if hrs > 0:
              output = pluralise("hr", hrs)
          elif mins > 0:
              output = pluralise("min", mins)
          else:
              output = pluralise("sec", secs)
  
      return output_format % output
  
  class x_stop_exception(Exception):
      pass
  
  def get_files(path, size_threshold_mb, results, stop_event):
      """Intended to run inside a thread: scan the contents of
      a tree recursively, pushing every file which is at least
      as big as the size threshold onto a results queue. Stop
      if the stop_event is set.
      """
      size_threshold = size_threshold_mb * 1024 * 1024
      root = fs.dir(path)
      top_level_folders = sorted(root.dirs(), key=operator.attrgetter("written_at"), reverse=True)
      try:
          for tlf in top_level_folders:
              for f in tlf.flat(ignore_access_errors=True):
                  if stop_event.isSet():
                      print("stop event set")
                      raise x_stop_exception
                  try:
                      if f.size > size_threshold:
                          results.put(f)
                  except fs.exc.x_winsys:
                      continue
      except x_stop_exception:
          return
  
  def watch_files(path, size_threshold_mb, results, stop_event):
      """Intended to run inside a thread: monitor a directory tree
      for file changes. Convert the changed files to fs.File objects
      and push then onto a results queue. Stop if the stop_event is set.
      """
      size_threshold = size_threshold_mb * 1024 * 1024
      BUFFER_SIZE = 8192
      MAX_BUFFER_SIZE = 1024 * 1024
  
      #
      # The double loop is because the watcher process
      # can fall over with an internal which is (I think)
      # related to a small buffer size. If that happens,
      # restart the process with a bigger buffer up to a
      # maximum size.
      #
      buffer_size = BUFFER_SIZE
      while True:
          watcher = fs.watch(path, True, buffer_size=buffer_size)
  
          while True:
              if stop_event.isSet(): break
              try:
                  action, old_file, new_file = watcher.next()
                  core.warn("Monitored: %s - %s => %s" % (action, old_file, new_file))
                  if old_file is not None:
                      if (not old_file) or (old_file and old_file.size > size_threshold):
                          results.put(old_file)
                  if new_file is not None and new_file != old_file:
                      if new_file and new_file.size > size_threshold:
                          results.put(new_file)
              except fs.exc.x_winsys:
                  pass
              except RuntimeError:
                  try:
                      watcher.stop()
                  except:
                      pass
                  buffer_size = min(2 * buffer_size, MAX_BUFFER_SIZE)
                  print("Tripped up on a RuntimeError. Trying with buffer of", buffer_size)
  
  class Path(object):
      """Keep track of the files and changes under a particular
      path tree. No attempt is made to optimise the cases where
      one tree is contained within another.
  
      When the Path is started, it kicks of two threads: one to
      do a complete scan; the other to monitor changes. Both
      write back to the same results queue which is the basis
      for the set of files which will be sorted and presented
      on the webpage.
  
      For manageability, the files are pulled off the queue a
      chunk at a time (by default 1000).
      """
  
      def __init__(self, path, size_threshold_mb, n_files_at_a_time):
          self._path = path
          self._size_threshold_mb = size_threshold_mb
          self._n_files_at_a_time = n_files_at_a_time
          self._changes = Queue.Queue()
          self._stop_event = threading.Event()
          self._files = set()
  
          self.file_getter = threading.Thread(
              target=get_files,
              args=(path, size_threshold_mb, self._changes, self._stop_event)
          )
          self.file_getter.setDaemon(1)
          self.file_getter.start()
  
          self.file_watcher = threading.Thread(
              target=watch_files,
              args=(path, size_threshold_mb, self._changes, self._stop_event)
          )
          self.file_watcher.setDaemon(1)
          self.file_watcher.start()
  
      def __str__(self):
          return "<Path: %s (%d files above %d Mb)>" % (self._path, len(self._files), self._size_threshold_mb)
      __repr__ = __str__
  
      def updated(self):
          """Pull at most _n_files_at_a_time files from the queue. If the
          file exists, add it to the set (which will, of course, ignore
          duplicates). If it doesn't exist, remove it from the set, ignoring
          the case where it isn't there to start with.
          """
          for i in range(self._n_files_at_a_time):
              try:
                  f = self._changes.get_nowait()
                  if f:
                      self._files.add(f)
                  else:
                      self._files.discard(f)
              except Queue.Empty:
                  break
          return self._files
  
      def finish(self):
          self._stop_event.set()
  
      def status(self):
          status = []
          if self.file_getter.isAlive():
              status.append("Scanning")
          if self.file_watcher.isAlive():
              status.append("Monitoring")
          return " & ".join(status)
  
  class App(object):
      """The controlling WSGI app. On each request, it looks up the
      path handler which corresponds to the path form variable. It then
      pulls any new entries and displays them according to the user's
      parameters.
      """
  
      PATH = ""
      N_FILES_AT_A_TIME = 1000
      SIZE_THRESHOLD_MB = 100
      TOP_N_FILES = 50
      REFRESH_SECS = 60
      HIGHLIGHT_DAYS = 0
      HIGHLIGHT_HRS = 12
      HIGHLIGHT_MINS = 0
  
      def __init__(self):
          self._paths_lock = threading.Lock()
          self.paths = {}
          self._paths_accessed = {}
  
      def doc(self, files, status, form):
          path = form.get("path", self.PATH)
          top_n_files = int(form.get("top_n_files", self.TOP_N_FILES) or 0)
          size_threshold_mb = int(form.get("size_threshold_mb", self.SIZE_THRESHOLD_MB) or 0)
          refresh_secs = int(form.get("refresh_secs", self.REFRESH_SECS) or 0)
          highlight_days = int(form.get("highlight_days", self.HIGHLIGHT_DAYS) or 0)
          highlight_hrs = int(form.get("highlight_hrs", self.HIGHLIGHT_HRS) or 0)
          highlight_mins = int(form.get("highlight_mins", self.HIGHLIGHT_MINS) or 0)
          highlight_delta = datetime.timedelta(days=highlight_days, hours=highlight_hrs, minutes=highlight_mins)
          highlight_deltastamp = deltastamp(highlight_delta)
          if files:
              title = cgi.escape("Top %d files on %s over %dMb - %s" % (min(len(files), self.TOP_N_FILES), path, size_threshold_mb, status))
          else:
              title = cgi.escape("Top files on %s over %dMb - %s" % (path, size_threshold_mb, status))
  
          doc = []
          doc.append("<html><head><title>%s</title>" % title)
          doc.append("""<style>
          body {font-family : calibri, verdana, sans-serif;}
          h1 {font-size : 120%;}
          form#params {font-size : 120%;}
          form#params input {font-family : calibri, verdana, sans-serif;}
          form#params span.label {font-weight : bold;}
          p.updated {margin-bottom : 1em; font-style : italic;}
          table {width : 100%;}
          thead tr {background-color : black; color : white; font-weight : bold;}
          table tr.odd {background-color : #ddd;}
          table tr.highlight td {background-color : #ffff80;}
          table td {padding-right : 0.5em;}
          table td.filename {width : 72%;}
          </style>""")
          doc.append("""<style media="print">
          form#params {display : none;}
          </style>""")
          doc.append("</head><body>")
          doc.append("""<form id="params" action="/" method="GET">
          <span class="label">Scan</span>&nbsp;<input type="text" name="path" value="%(path)s" size="20" maxlength="20" />&nbsp;
          <span class="label">for files over</span>&nbsp;<input type="text" name="size_threshold_mb" value="%(size_threshold_mb)s" size="5" maxlength="5" />Mb
          <span class="label">showing the top</span>&nbsp;<input type="text" name="top_n_files" value="%(top_n_files)s" size="3" maxlength="3" /> files
          <span class="label">refreshing every</span>&nbsp;<input type="text" name="refresh_secs" value="%(refresh_secs)s" size="3" maxlength="3" /> secs
          <span class="label">highlighting the last&nbsp;</span>&nbsp;<input type="text" name="highlight_days" value="%(highlight_days)s" size="3" maxlength="3" /> days
              </span>&nbsp;<input type="text" name="highlight_hrs" value="%(highlight_hrs)s" size="3" maxlength="3" /> hrs
              </span>&nbsp;<input type="text" name="highlight_mins" value="%(highlight_mins)s" size="3" maxlength="3" /> mins
          <input type="submit" value="Refresh" />
          </form><hr>""" % locals())
  
          now = win32timezone.utcnow()
          if path:
              doc.append("<h1>%s</h1>" % title)
              latest_filename = "\\".join(files[-1].parts[1:]) if files else "(no file yet)"
              doc.append(u'<p class="updated">Last updated %s</p>' % time.asctime())
              doc.append(u'<table><thead><tr><td class="filename">Filename</td><td class="size">Size (Mb)</td><td class="updated">Updated</td></tr></thead>')
              for i, f in enumerate(files[:top_n_files]):
                  try:
                      doc.append(
                          u'<tr class="%s %s"><td class="filename">%s</td><td class="size">%5.2f</td><td class="updated">%s</td>' % (
                              "odd" if i % 2 else "even",
                              "highlight" if ((now - max(f.written_at, f.created_at)) <= highlight_delta) else "",
                              f.relative_to(path).lstrip(fs.seps),
                              f.size / 1024.0 / 1024.0,
                              max(f.written_at, f.created_at)
                          )
                      )
                  except fs.exc.x_winsys:
                      pass
              doc.append("</table>")
  
          doc.append("</body></html>")
          return doc
  
      def handler(self, form):
          path = form.get("path", self.PATH)
          size_threshold_mb = int(form.get("size_threshold_mb", self.SIZE_THRESHOLD_MB) or 0)
          refresh_secs = int(form.get("refresh_secs", self.REFRESH_SECS) or 0)
          status = "Waiting"
          if path and fs.Dir(path):
              #
              # Ignore any non-existent paths, including garbage.
              # Create a new path handler if needed, or pull back
              # and existing one, and return the latest list.
              #
              with self._paths_lock:
                  if path not in self.paths:
                      self.paths[path] = Path(path, size_threshold_mb, self.N_FILES_AT_A_TIME)
                  path_handler = self.paths[path]
                  if path_handler._size_threshold_mb != size_threshold_mb:
                      path_handler.finish()
                      path_handler = self.paths[path] = Path(path, size_threshold_mb, self.N_FILES_AT_A_TIME)
                  self._paths_accessed[path] = win32timezone.utcnow()
                  files = sorted(path_handler.updated(), key=operator.attrgetter("size"), reverse=True)
                  status = path_handler.status()
  
                  #
                  # If any path hasn't been queried for at least
                  # three minutes, close the thread down and delete
                  # its entry. If it is queried again, it will just
                  # be restarted as new.
                  #
                  for path, last_accessed in self._paths_accessed.iteritems():
                      if (win32timezone.utcnow() - last_accessed).seconds > 180:
                          path_handler = self.paths.get(path)
                          if path_handler:
                              path_handler.finish()
                              del self.paths[path]
                              del self._paths_accessed[path]
  
          else:
              files = []
          return self.doc(files, status, form)
  
      def __call__(self, environ, start_response):
          """Only attempt to handle the root URI. If a refresh interval
          is requested (the default) then send a header which forces
          the refresh.
          """
          path = shift_path_info(environ).rstrip("/")
          if path == "":
              form = dict((k, v[0]) for (k, v) in cgi.parse_qs(list(environ['QUERY_STRING']).iteritems()) if v)
              if form.get("path"):
                  form['path'] = form['path'].rstrip("\\") + "\\"
              refresh_secs = int(form.get("refresh_secs", self.REFRESH_SECS) or 0)
              headers = []
              headers.append(("Content-Type", "text/html; charset=utf-8"))
              if refresh_secs:
                  headers.append(("Refresh", "%s" % refresh_secs))
              start_response("200 OK", headers)
              return (d.encode("utf8") + "\n" for d in self.handler(form))
          else:
              start_response("404 Not Found", [("Content-Type", "text/plain")])
              return []
  
      def finish(self):
          for path_handler in self.paths.itervalues():
              path_handler.finish()
  
  if __name__ == '__main__':
      misc.set_console_title("Monitor Directory")
      PORT = 8000
      HOSTNAME = socket.getfqdn()
      threading.Timer(
          2.0,
          lambda: os.startfile("http://%s:%s" % (HOSTNAME, PORT))
      ).start()
  
      app = App()
      try:
          make_server('', PORT, app).serve_forever()
      except KeyboardInterrupt:
          print("Shutting down gracefully...")
      finally:
          app.finish()
  