text: #!/usr/bin/env python
  # -*- Mode: Python; tab-width: 4; indent-tabs-mode: nil; coding: utf-8; -*-
  # vim:set ft=python ts=4 sw=4 sts=4 autoindent:
  
  # XXX: This module along with stats and annotator is pretty much pure chaos
  
  from __future__ import with_statement
  
  '''
  Document handling functionality.
  
  Author:     Pontus Stenetorp    <pontus is s u-tokyo ac jp>
              Illes Solt          <solt tmit bme hu>
  Version:    2011-04-21
  '''
  
  from os import listdir
  from os.path import abspath, dirname, isabs, isdir, normpath, getmtime
  from os.path import join as path_join
  from re import match,sub
  from errno import ENOENT, EACCES
  
  from annotation import (TextAnnotations, TEXT_FILE_SUFFIX,
          AnnotationFileNotFoundError, 
          AnnotationCollectionNotFoundError,
          JOINED_ANN_FILE_SUFF,
          open_textfile,
          BIONLP_ST_2013_COMPATIBILITY)
  from common import ProtocolError, CollectionNotAccessibleError
  from config import BASE_DIR, DATA_DIR
  from projectconfig import (ProjectConfiguration, SEPARATOR_STR, 
          SPAN_DRAWING_ATTRIBUTES, ARC_DRAWING_ATTRIBUTES,
          VISUAL_SPAN_DEFAULT, VISUAL_ARC_DEFAULT, 
          ATTR_DRAWING_ATTRIBUTES, VISUAL_ATTR_DEFAULT,
          SPECIAL_RELATION_TYPES, 
          options_get_validation, options_get_tokenization,
          options_get_ssplitter, get_annotation_config_section_labels)
  from stats import get_statistics
  from message import Messager
  from auth import allowed_to_read, AccessDeniedError
  from annlog import annotation_logging_active
  
  from itertools import chain
  
  def _fill_type_configuration(nodes, project_conf, hotkey_by_type, all_connections=None):
      # all_connections is an optimization to reduce invocations of
      # projectconfig methods such as arc_types_from_to.
      if all_connections is None:
          all_connections = project_conf.all_connections()
  
      items = []
      for node in nodes:
          if node == SEPARATOR_STR:
              items.append(None)
          else:
              item = {}
              _type = node.storage_form() 
  
              # This isn't really a great place to put this, but we need
              # to block these magic values from getting to the client.
              # TODO: resolve cleanly, preferably by not storing this with
              # other relations at all.
              if _type in SPECIAL_RELATION_TYPES:
                  continue
  
              item['name'] = project_conf.preferred_display_form(_type)
              item['type'] = _type
              item['unused'] = node.unused
              item['labels'] = project_conf.get_labels_by_type(_type)
              item['attributes'] = project_conf.attributes_for(_type)
              item['normalizations'] = node.normalizations()
  
              span_drawing_conf = project_conf.get_drawing_config_by_type(_type) 
              if span_drawing_conf is None:
                  span_drawing_conf = project_conf.get_drawing_config_by_type(VISUAL_SPAN_DEFAULT)
              if span_drawing_conf is None:
                  span_drawing_conf = {}
              for k in SPAN_DRAWING_ATTRIBUTES:
                  if k in span_drawing_conf:
                      item[k] = span_drawing_conf[k]
              
              try:
                  item['hotkey'] = hotkey_by_type[_type]
              except KeyError:
                  pass
  
              arcs = []
  
              # Note: for client, relations are represented as "arcs"
              # attached to "spans" corresponding to entity annotations.
  
              # To avoid redundant entries, fill each type at most once.
              filled_arc_type = {}
  
              for arc in chain(project_conf.relation_types_from(_type), node.arg_list):
                  if arc in filled_arc_type:
                      continue
                  filled_arc_type[arc] = True
  
                  curr_arc = {}
                  curr_arc['type'] = arc
  
                  arc_labels = project_conf.get_labels_by_type(arc)
                  curr_arc['labels'] = arc_labels if arc_labels is not None else [arc]
  
                  try:
                      curr_arc['hotkey'] = hotkey_by_type[arc]
                  except KeyError:
                      pass
                  
                  arc_drawing_conf = project_conf.get_drawing_config_by_type(arc)
                  if arc_drawing_conf is None:
                      arc_drawing_conf = project_conf.get_drawing_config_by_type(VISUAL_ARC_DEFAULT)
                  if arc_drawing_conf is None:
                      arc_drawing_conf = {}
                  for k in ARC_DRAWING_ATTRIBUTES:
                      if k in arc_drawing_conf:
                          curr_arc[k] = arc_drawing_conf[k]                    
  
                  # Client needs also possible arc 'targets',
                  # defined as the set of types (entity or event) that
                  # the arc can connect to
  
                  # This bit doesn't make sense for relations, which are
                  # already "arcs" (see comment above).
                  # TODO: determine if this should be an error: relation
                  # config should now go through _fill_relation_configuration
                  # instead.
                  if project_conf.is_relation_type(_type):
                      targets = []
                  else:
                      targets = []
  
                      if arc in all_connections[_type]:
                          targets = all_connections[_type][arc]
  
                      # TODO: this code remains here to allow for further checking of the
                      # new all_connections() functionality. Comment out to activate
                      # verification of the new implementation (above) against the old one
                      # (below, commented out).
  #                     check_targets = []
  #                     for ttype in project_conf.get_entity_types() + project_conf.get_event_types():
  #                         if arc in project_conf.arc_types_from_to(_type, ttype):
  #                             check_targets.append(ttype)
  
  #                     if targets == check_targets:
  #                         Messager.info("CHECKS OUT!")
  #                     elif sorted(targets) == sorted(check_targets):
  #                         Messager.warning("Different sort order for %s -> %s:\n%s\n%s" % (_type, arc, str(targets), str(check_targets)), 10)
  #                     else:
  #                         Messager.error("Mismatch for %s -> %s:\n%s\n%s" % (_type, arc, str(sorted(targets)), str(sorted(check_targets))), -1)
  
                  curr_arc['targets'] = targets
  
                  arcs.append(curr_arc)
                      
              # If we found any arcs, attach them
              if arcs:
                  item['arcs'] = arcs
  
              item['children'] = _fill_type_configuration(node.children,
                      project_conf, hotkey_by_type, all_connections)
              items.append(item)
      return items
  
  # TODO: duplicates part of _fill_type_configuration
  def _fill_relation_configuration(nodes, project_conf, hotkey_by_type):
      items = []
      for node in nodes:
          if node == SEPARATOR_STR:
              items.append(None)
          else:
              item = {}
              _type = node.storage_form() 
  
              if _type in SPECIAL_RELATION_TYPES:
                  continue
  
              item['name'] = project_conf.preferred_display_form(_type)
              item['type'] = _type
              item['unused'] = node.unused
              item['labels'] = project_conf.get_labels_by_type(_type)
              item['attributes'] = project_conf.attributes_for(_type)
  
              # TODO: avoid magic value
              item['properties'] = {}
              if '<REL-TYPE>' in node.special_arguments:
                  for special_argument in node.special_arguments['<REL-TYPE>']:
                      item['properties'][special_argument] = True
  
              arc_drawing_conf = project_conf.get_drawing_config_by_type(_type)
              if arc_drawing_conf is None:
                  arc_drawing_conf = project_conf.get_drawing_config_by_type(VISUAL_ARC_DEFAULT)
              if arc_drawing_conf is None:
                  arc_drawing_conf = {}
              for k in ARC_DRAWING_ATTRIBUTES:
                  if k in arc_drawing_conf:
                      item[k] = arc_drawing_conf[k]                    
              
              try:
                  item['hotkey'] = hotkey_by_type[_type]
              except KeyError:
                  pass
  
              # minimal info on argument types to allow differentiation of e.g.
              # "Equiv(Protein, Protein)" and "Equiv(Organism, Organism)"
              args = []
              for arg in node.arg_list:
                  curr_arg = {}
                  curr_arg['role'] = arg
                  # TODO: special type (e.g. "<ENTITY>") expansion via projectconf
                  curr_arg['targets'] = node.arguments[arg]
  
                  args.append(curr_arg)
  
              item['args'] = args
  
              item['children'] = _fill_relation_configuration(node.children,
                      project_conf, hotkey_by_type)
              items.append(item)
      return items
  
  
  # TODO: this may not be a good spot for this
  def _fill_attribute_configuration(nodes, project_conf):
      items = []
      for node in nodes:
          if node == SEPARATOR_STR:
              continue
          else:
              item = {}
              _type = node.storage_form() 
              item['name'] = project_conf.preferred_display_form(_type)
              item['type'] = _type
              item['unused'] = node.unused
              item['labels'] = project_conf.get_labels_by_type(_type)
  
              attr_drawing_conf = project_conf.get_drawing_config_by_type(_type)
              if attr_drawing_conf is None:
                  attr_drawing_conf = project_conf.get_drawing_config_by_type(VISUAL_ATTR_DEFAULT)
              if attr_drawing_conf is None:
                  attr_drawing_conf = {}
  
              # TODO: "special" <DEFAULT> argument
              
              # Check if the possible values for the argument are specified
              # TODO: avoid magic string
              if "Value" in node.arguments:
                  args = node.arguments["Value"]
              else:
                  # no "Value" defined; assume binary.
                  args = []
  
              if len(args) == 0:
                  # binary; use drawing config directly
                  item['values'] = { _type : {} }
                  for k in ATTR_DRAWING_ATTRIBUTES:
                      if k in attr_drawing_conf:
                          # protect against error from binary attribute
                          # having multi-valued visual config (#698)
                          if isinstance(attr_drawing_conf[k], list):
                              Messager.warning("Visual config error: expected single value for %s binary attribute '%s' config, found %d. Visuals may be wrong." % (_type, k, len(attr_drawing_conf[k])))
                              # fall back on the first just to have something.
                              item['values'][_type][k] = attr_drawing_conf[k][0]
                          else:
                              item['values'][_type][k] = attr_drawing_conf[k]
              else:
                  # has normal arguments, use these as possible values.
                  # (this is quite terrible all around, sorry.)
                  item['values'] = {}
                  for i, v in enumerate(args):
                      item['values'][v] = {}
                      # match up annotation config with drawing config by
                      # position in list of alternative values so that e.g.
                      # "Values:L1|L2|L3" can have the visual config
                      # "glyph:[1]|[2]|[3]". If only a single value is
                      # defined, apply to all.
                      for k in ATTR_DRAWING_ATTRIBUTES:
                          if k in attr_drawing_conf:
                              # (sorry about this)
                              if isinstance(attr_drawing_conf[k], list):
                                  # sufficiently many specified?
                                  if len(attr_drawing_conf[k]) > i:
                                      item['values'][v][k] = attr_drawing_conf[k][i]
                                  else:
                                      Messager.warning("Visual config error: expected %d values for %s attribute '%s' config, found only %d. Visuals may be wrong." % (len(args), v, k, len(attr_drawing_conf[k])))
                              else:
                                  # single value (presumably), apply to all
                                  item['values'][v][k] = attr_drawing_conf[k]
  
                      # if no drawing attribute was defined, fall back to
                      # using a glyph derived from the attribute value
                      if len([k for k in ATTR_DRAWING_ATTRIBUTES if
                              k in item['values'][v]]) == 0:
                          item['values'][v]['glyph'] = '['+v+']'
  
              items.append(item)
      return items
  
  def _fill_visual_configuration(types, project_conf):
      # similar to _fill_type_configuration, but for types for which
      # full annotation configuration was not found but some visual
      # configuration can be filled.
  
      # TODO: duplicates parts of _fill_type_configuration; combine?
      items = []
      for _type in types:
          item = {}
          item['name'] = project_conf.preferred_display_form(_type)
          item['type'] = _type
          item['unused'] = True
          item['labels'] = project_conf.get_labels_by_type(_type)
  
          drawing_conf = project_conf.get_drawing_config_by_type(_type) 
          # not sure if this is a good default, but let's try
          if drawing_conf is None:
              drawing_conf = project_conf.get_drawing_config_by_type(VISUAL_SPAN_DEFAULT)
          if drawing_conf is None:
              drawing_conf = {}
          # just plug in everything found, whether for a span or arc
          for k in chain(SPAN_DRAWING_ATTRIBUTES, ARC_DRAWING_ATTRIBUTES):
              if k in drawing_conf:
                  item[k] = drawing_conf[k]
  
          # TODO: anything else?
  
          items.append(item)
  
      return items
  
  # TODO: this is not a good spot for this
  def get_base_types(directory):
      project_conf = ProjectConfiguration(directory)
  
      keymap = project_conf.get_kb_shortcuts()
      hotkey_by_type = dict((v, k) for k, v in keymap.iteritems())
  
      # fill config for nodes for which annotation is configured
  
      # calculate once only (this can get heavy)
      all_connections = project_conf.all_connections()
      
      event_hierarchy = project_conf.get_event_type_hierarchy()
      event_types = _fill_type_configuration(event_hierarchy,
              project_conf, hotkey_by_type, all_connections)
  
      entity_hierarchy = project_conf.get_entity_type_hierarchy()
      entity_types = _fill_type_configuration(entity_hierarchy,
              project_conf, hotkey_by_type, all_connections)
  
      relation_hierarchy = project_conf.get_relation_type_hierarchy()
      relation_types = _fill_relation_configuration(relation_hierarchy,
              project_conf, hotkey_by_type)
  
      # make visual config available also for nodes for which there is
      # no annotation config. Note that defaults (SPAN_DEFAULT etc.)
      # are included via get_drawing_types() if defined.
      unconfigured = [l for l in (project_conf.get_labels().keys() +
                                  project_conf.get_drawing_types()) if 
                      not project_conf.is_configured_type(l)]
      unconf_types = _fill_visual_configuration(unconfigured, project_conf)
  
      return event_types, entity_types, relation_types, unconf_types
  
  def get_attribute_types(directory):
      project_conf = ProjectConfiguration(directory)
  
      entity_attribute_hierarchy = project_conf.get_entity_attribute_type_hierarchy()
      entity_attribute_types = _fill_attribute_configuration(entity_attribute_hierarchy, project_conf)
      
      relation_attribute_hierarchy = project_conf.get_relation_attribute_type_hierarchy()
      relation_attribute_types = _fill_attribute_configuration(relation_attribute_hierarchy, project_conf)
  
      event_attribute_hierarchy = project_conf.get_event_attribute_type_hierarchy()
      event_attribute_types = _fill_attribute_configuration(event_attribute_hierarchy, project_conf)
  
      return entity_attribute_types, relation_attribute_types, event_attribute_types
  
  def get_search_config(directory):
      return ProjectConfiguration(directory).get_search_config()
  
  def get_disambiguator_config(directory):
      return ProjectConfiguration(directory).get_disambiguator_config()
  
  def get_normalization_config(directory):
      return ProjectConfiguration(directory).get_normalization_config()
  
  def get_annotator_config(directory):
      # TODO: "annotator" is a very confusing term for a web service
      # that does automatic annotation in the context of a tool
      # where most annotators are expected to be human. Rethink.
      return ProjectConfiguration(directory).get_annotator_config()
  
  def assert_allowed_to_read(doc_path):
      if not allowed_to_read(doc_path):
          raise AccessDeniedError # Permission denied by access control
  
  def real_directory(directory, rel_to=DATA_DIR):
      assert isabs(directory), 'directory "%s" is not absolute' % directory
      return path_join(rel_to, directory[1:])
  
  def relative_directory(directory):
      # inverse of real_directory
      assert isabs(directory), 'directory "%s" is not absolute' % directory
      assert directory.startswith(DATA_DIR), 'directory "%s" not under DATA_DIR'
      return directory[len(DATA_DIR):]
  
  def _is_hidden(file_name):
      return file_name.startswith('hidden_') or file_name.startswith('.')
  
  def _listdir(directory):
      #return listdir(directory)
      try:
          assert_allowed_to_read(directory)
          return [f for f in listdir(directory) if not _is_hidden(f)
                  and allowed_to_read(path_join(directory, f))]
      except OSError, e:
          Messager.error("Error listing %s: %s" % (directory, e))
          raise AnnotationCollectionNotFoundError(directory)
      
  def _getmtime(file_path):
      '''
      Internal wrapper of getmtime that handles access denied and invalid paths
      according to our specification.
  
      Arguments:
  
      file_path - path to the file to get the modification time for
      '''
  
      try:
          return getmtime(file_path)
      except OSError, e:
          if e.errno in (EACCES, ENOENT):
              # The file did not exist or permission denied, we use -1 to
              #   indicate this since mtime > 0 is an actual time.
              return -1
          else:
              # We are unable to handle this exception, pass it one
              raise
  
  
  class InvalidConfiguration(ProtocolError):
      def json(self, json_dic):
          json_dic['exception'] = 'invalidConfiguration'
          return json_dic
  
  
  # TODO: Is this what we would call the configuration? It is minimal.
  def get_configuration(name):
      # TODO: Rip out this path somewhere
      config_dir = path_join(BASE_DIR, 'configurations')
      for conf_name in listdir(config_dir):
          if conf_name == name:
              config_path = path_join(config_dir, conf_name)
              break
      else:
          raise InvalidConfiguration
  
      return _inject_annotation_type_conf(config_path)
  
  def _inject_annotation_type_conf(dir_path, json_dic=None):
      if json_dic is None:
          json_dic = {}
  
      (event_types, entity_types, rel_types,
              unconf_types) = get_base_types(dir_path)
      (entity_attr_types, rel_attr_types,
              event_attr_types) = get_attribute_types(dir_path)
  
      json_dic['event_types'] = event_types
      json_dic['entity_types'] = entity_types
      json_dic['relation_types'] = rel_types
      json_dic['event_attribute_types'] = event_attr_types
      json_dic['relation_attribute_types'] = rel_attr_types
      json_dic['entity_attribute_types'] = entity_attr_types
      json_dic['unconfigured_types'] = unconf_types
  
      # inject annotation category aliases (e.g. "entities" -> "spans")
      # used in config (#903).
      section_labels = get_annotation_config_section_labels(dir_path)
      json_dic['ui_names'] = {}
      for c in ['entities', 'relations', 'events', 'attributes']:
          json_dic['ui_names'][c] = section_labels.get(c,c)
  
      return json_dic
  
  # TODO: This is not the prettiest of functions
  def get_directory_information(collection):
      directory = collection
  
      real_dir = real_directory(directory)
      
      assert_allowed_to_read(real_dir)
      
      # Get the document names
      base_names = [fn[0:-4] for fn in _listdir(real_dir)
              if fn.endswith('txt')]
  
      doclist = base_names[:]
      doclist_header = [("Document", "string")]
  
      # Then get the modification times
      doclist_with_time = []
      for file_name in doclist:
          file_path = path_join(DATA_DIR, real_dir,
              file_name + "." + JOINED_ANN_FILE_SUFF)
          doclist_with_time.append([file_name, _getmtime(file_path)])
      doclist = doclist_with_time
      doclist_header.append(("Modified", "time"))
  
      try:
          stats_types, doc_stats = get_statistics(real_dir, base_names)
      except OSError:
          # something like missing access permissions?
          raise CollectionNotAccessibleError
                  
      doclist = [doclist[i] + doc_stats[i] for i in range(len(doclist))]
      doclist_header += stats_types
  
      dirlist = [dir for dir in _listdir(real_dir)
              if isdir(path_join(real_dir, dir))]
      # just in case, and for generality
      dirlist = [[dir] for dir in dirlist]
  
      # check whether at root, ignoring e.g. possible trailing slashes
      if normpath(real_dir) != normpath(DATA_DIR):
          parent = abspath(path_join(real_dir, '..'))[len(DATA_DIR) + 1:]
          # to get consistent processing client-side, add explicitly to list
          dirlist.append([".."])
      else:
          parent = None
  
      # combine document and directory lists, adding a column
      # differentiating files from directories and an unused column (can
      # point to a specific annotation) required by the protocol.  The
      # values filled here for the first are "c" for "collection"
      # (i.e. directory) and "d" for "document".
      combolist = []
      for i in dirlist:
          combolist.append(["c", None]+i)
      for i in doclist:
          combolist.append(["d", None]+i)
  
      # plug in the search config too
      search_config = get_search_config(real_dir)
  
      # ... and the disambiguator config ... this is getting a bit much
      disambiguator_config = get_disambiguator_config(real_dir)
  
      # ... and the normalization config (TODO: rethink)
      normalization_config = get_normalization_config(real_dir)
  
      # read in README (if any) to send as a description of the
      # collection
      try:
          with open_textfile(path_join(real_dir, "README")) as txt_file:
              readme_text = txt_file.read()
      except IOError:
          readme_text = None
  
      # fill in a flag for whether annotator logging is active so that
      # the client knows whether to invoke timing actions
      ann_logging = annotation_logging_active(real_dir)
  
      # fill in NER services, if any
      ner_taggers = get_annotator_config(real_dir)
  
      return _inject_annotation_type_conf(real_dir, json_dic={
              'items': combolist,
              'header' : doclist_header,
              'parent': parent,
              'messages': [],
              'description': readme_text,
              'search_config': search_config,
              'disambiguator_config' : disambiguator_config,
              'normalization_config' : normalization_config,
              'annotation_logging': ann_logging,
              'ner_taggers': ner_taggers,
              })
  
  class UnableToReadTextFile(ProtocolError):
      def __init__(self, path):
          self.path = path
  
      def __str__(self):
          return 'Unable to read text file %s' % self.path
  
      def json(self, json_dic):
          json_dic['exception'] = 'unableToReadTextFile'
          return json_dic
  
  class IsDirectoryError(ProtocolError):
      def __init__(self, path):
          self.path = path
  
      def __str__(self):
          return ''
  
      def json(self, json_dic):
          json_dic['exception'] = 'isDirectoryError'
          return json_dic
  
  #TODO: All this enrichment isn't a good idea, at some point we need an object
  def _enrich_json_with_text(j_dic, txt_file_path, raw_text=None):
      if raw_text is not None:
          # looks like somebody read this already; nice
          text = raw_text
      else:
          # need to read raw text
          try:
              with open_textfile(txt_file_path, 'r') as txt_file:
                  text = txt_file.read()
          except IOError:
              raise UnableToReadTextFile(txt_file_path)
          except UnicodeDecodeError:
              Messager.error('Error reading text file: nonstandard encoding or binary?', -1)
              raise UnableToReadTextFile(txt_file_path)
  
      # TODO XXX huge hack, sorry, the client currently crashing on
      # chrome for two or more consecutive space, so replace every
      # second with literal non-breaking space. Note that this is just
      # for the client display -- server-side storage is not affected.
      # NOTE: it might be possible to fix this in a principled way by
      # having xml:space="preserve" on the relevant elements.
      text = text.replace("  ", ' '+unichr(0x00A0))
  
      j_dic['text'] = text
      
      from logging import info as log_info
  
      tokeniser = options_get_tokenization(dirname(txt_file_path))
  
      # First, generate tokenisation
      if tokeniser == 'mecab':
          from tokenise import jp_token_boundary_gen
          tok_offset_gen = jp_token_boundary_gen
      elif tokeniser == 'whitespace':
          from tokenise import whitespace_token_boundary_gen
          tok_offset_gen = whitespace_token_boundary_gen
      elif tokeniser == 'ptblike':
          from tokenise import gtb_token_boundary_gen
          tok_offset_gen = gtb_token_boundary_gen
      else:
          Messager.warning('Unrecognized tokenisation option '
                  ', reverting to whitespace tokenisation.')
          from tokenise import whitespace_token_boundary_gen
          tok_offset_gen = whitespace_token_boundary_gen
      j_dic['token_offsets'] = [o for o in tok_offset_gen(text)]
  
      ssplitter = options_get_ssplitter(dirname(txt_file_path))
      if ssplitter == 'newline':
          from ssplit import newline_sentence_boundary_gen
          ss_offset_gen = newline_sentence_boundary_gen
      elif ssplitter == 'regex':
          from ssplit import regex_sentence_boundary_gen
          ss_offset_gen = regex_sentence_boundary_gen
      else:
          Messager.warning('Unrecognized sentence splitting option '
                  ', reverting to newline sentence splitting.')
          from ssplit import newline_sentence_boundary_gen
          ss_offset_gen = newline_sentence_boundary_gen
      j_dic['sentence_offsets'] = [o for o in ss_offset_gen(text)]
  
      return True
  
  def _enrich_json_with_data(j_dic, ann_obj):
      # TODO: figure out if there's a reason for all the unicode()
      # invocations here; remove if not.
  
      # We collect trigger ids to be able to link the textbound later on
      trigger_ids = set()
      for event_ann in ann_obj.get_events():
          trigger_ids.add(event_ann.trigger)
          j_dic['events'].append(
                  [unicode(event_ann.id), unicode(event_ann.trigger), event_ann.args]
                  )
  
      for rel_ann in ann_obj.get_relations():
          j_dic['relations'].append(
              [unicode(rel_ann.id), unicode(rel_ann.type), 
               [(rel_ann.arg1l, rel_ann.arg1),
                (rel_ann.arg2l, rel_ann.arg2)]]
              )
  
      for tb_ann in ann_obj.get_textbounds():
          #j_tb = [unicode(tb_ann.id), tb_ann.type, tb_ann.start, tb_ann.end]
          j_tb = [unicode(tb_ann.id), tb_ann.type, tb_ann.spans]
  
          # If we spotted it in the previous pass as a trigger for an
          # event or if the type is known to be an event type, we add it
          # as a json trigger.
          # TODO: proper handling of disconnected triggers. Currently
          # these will be erroneously passed as 'entities'
          if unicode(tb_ann.id) in trigger_ids:
              j_dic['triggers'].append(j_tb)
              # special case for BioNLP ST 2013 format: send triggers
              # also as entities for those triggers that are referenced
              # from annotations other than events (#926).
              if BIONLP_ST_2013_COMPATIBILITY:
                  if tb_ann.id in ann_obj.externally_referenced_triggers:
                      try:
                          j_dic['entities'].append(j_tb)
                      except KeyError:
                          j_dic['entities'] = [j_tb, ]
          else: 
              try:
                  j_dic['entities'].append(j_tb)
              except KeyError:
                  j_dic['entities'] = [j_tb, ]
  
  
      for eq_ann in ann_obj.get_equivs():
          j_dic['equivs'].append(
                  (['*', eq_ann.type]
                      + [e for e in eq_ann.entities])
                  )
  
      for att_ann in ann_obj.get_attributes():
          j_dic['attributes'].append(
                  [unicode(att_ann.id), unicode(att_ann.type), unicode(att_ann.target), att_ann.value]
                  )
  
      for norm_ann in ann_obj.get_normalizations():
          j_dic['normalizations'].append(
                  [unicode(norm_ann.id), unicode(norm_ann.type), 
                   unicode(norm_ann.target), unicode(norm_ann.refdb), 
                   unicode(norm_ann.refid), unicode(norm_ann.reftext)]
                  )
  
      for com_ann in ann_obj.get_oneline_comments():
          comment = [unicode(com_ann.target), unicode(com_ann.type),
                  com_ann.tail.strip()]
          try:
              j_dic['comments'].append(comment)
          except KeyError:
              j_dic['comments'] = [comment, ]
  
      if ann_obj.failed_lines:
          error_msg = 'Unable to parse the following line(s):\n%s' % (
                  '\n'.join(
                  [('%s: %s' % (
                              # The line number is off by one
                              unicode(line_num + 1),
                              unicode(ann_obj[line_num])
                              )).strip()
                   for line_num in ann_obj.failed_lines])
                  )
          Messager.error(error_msg, duration=len(ann_obj.failed_lines) * 3)
  
      j_dic['mtime'] = ann_obj.ann_mtime
      j_dic['ctime'] = ann_obj.ann_ctime
  
      try:
          # XXX avoid digging the directory from the ann_obj
          import os
          docdir = os.path.dirname(ann_obj._document)
          if options_get_validation(docdir) in ('all', 'full', ):
              from verify_annotations import verify_annotation
              projectconf = ProjectConfiguration(docdir)
              issues = verify_annotation(ann_obj, projectconf)
          else:
              issues = []
      except Exception, e:
          # TODO add an issue about the failure?
          issues = []
          Messager.error('Error: verify_annotation() failed: %s' % e, -1)
  
      for i in issues:
          issue = (unicode(i.ann_id), i.type, i.description)
          try:
              j_dic['comments'].append(issue)
          except:
              j_dic['comments'] = [issue, ]
  
      # Attach the source files for the annotations and text
      from os.path import splitext
      from annotation import TEXT_FILE_SUFFIX
      ann_files = [splitext(p)[1][1:] for p in ann_obj._input_files]
      ann_files.append(TEXT_FILE_SUFFIX)
      ann_files = [p for p in set(ann_files)]
      ann_files.sort()
      j_dic['source_files'] = ann_files
  
  def _enrich_json_with_base(j_dic):
      # TODO: Make the names here and the ones in the Annotations object conform
  
      # TODO: "from offset" of what? Commented this out, remove once
      # sure that nothing is actually using this.
  #     # This is the from offset
  #     j_dic['offset'] = 0
  
      for d in (
          'entities',
          'events',
          'relations',
          'triggers',
          'modifications',
          'attributes',
          'equivs',
          'normalizations',
          'comments',
          ):
          j_dic[d] = []
  
  def _document_json_dict(document):
      #TODO: DOC!
  
      # pointing at directory instead of document?
      if isdir(document):
          raise IsDirectoryError(document)
  
      j_dic = {}
      _enrich_json_with_base(j_dic)
  
      #TODO: We don't check if the files exist, let's be more error friendly
      # Read in the textual data to make it ready to push
      _enrich_json_with_text(j_dic, document + '.' + TEXT_FILE_SUFFIX)
  
      with TextAnnotations(document) as ann_obj:
          # Note: At this stage the sentence offsets can conflict with the
          #   annotations, we thus merge any sentence offsets that lie within
          #   annotations
          # XXX: ~O(tb_ann * sentence_breaks), can be optimised
          # XXX: The merge strategy can lead to unforeseen consequences if two
          #   sentences are not adjacent (the format allows for this:
          #   S_1: [0, 10], S_2: [15, 20])
          s_breaks = j_dic['sentence_offsets']
          for tb_ann in ann_obj.get_textbounds():
              s_i = 0
              while s_i < len(s_breaks):
                  s_start, s_end = s_breaks[s_i]
                  # Does any subspan of the annotation strech over the
                  # end of the sentence?
                  found_spanning = False
                  for tb_start, tb_end in tb_ann.spans:
                      if tb_start < s_end and tb_end > s_end:
                          found_spanning = True
                          break
                  if found_spanning:
                      # Merge this sentence and the next sentence
                      s_breaks[s_i] = (s_start, s_breaks[s_i + 1][1])
                      del s_breaks[s_i + 1]
                  else:
                      s_i += 1
          
          _enrich_json_with_data(j_dic, ann_obj)
  
      return j_dic
  
  def get_document(collection, document):
      directory = collection
      real_dir = real_directory(directory)
      doc_path = path_join(real_dir, document)
      return _document_json_dict(doc_path)
  
  def get_document_timestamp(collection, document):
      directory = collection
      real_dir = real_directory(directory)
      assert_allowed_to_read(real_dir)
      doc_path = path_join(real_dir, document)
      ann_path = doc_path + '.' + JOINED_ANN_FILE_SUFF
      mtime = _getmtime(ann_path)
  
      return {
              'mtime': mtime,
              }
  