text: """Utility functions pertaining to archive metadata import."""
  
  
  from collections import defaultdict
  from datetime import date as Date, datetime as DateTime
  
  from django.db import transaction
  
  from vesper.django.app.models import (
      AnnotationConstraint, AnnotationInfo, Device, DeviceConnection,
      DeviceInput, DeviceModel, DeviceModelInput, DeviceModelOutput,
      DeviceOutput, Job, Processor, Station, StationDevice, TagInfo)
  import vesper.util.time_utils as time_utils
  import vesper.util.yaml_utils as yaml_utils
  
  
  # TODO: Allow omission of device name, in which case it is automatically
  # generated by concatenating model name and serial number.
  
  # TODO: Allow omission of connections when there is exactly one recorder,
  # only single-output microphones, and no more microphones than recorder
  # inputs, in which case microphones are connected in the order they
  # appear in the device list to recorder inputs. The period of the
  # connections is the period of association of the devices with the
  # station.
  
  # TODO: Allow specification of station devices with stations, e.g.:
  #
  #     stations:
  #     
  #         - name: Ithaca
  #           description: >
  #               Imaginary recording station in Ithaca, NY, USA.
  #               The location given for the station is within Cayuga Lake
  #               to emphasize that the station is not real!
  #           time_zone: US/Eastern
  #           latitude: 42.473168
  #           longitude: -76.516825
  #           elevation: 120
  #           devices:
  #               - name: Swift
  #                 start_time: 2018-01-01
  #                 end_time: 2019-01-01
  #
  # If we support infinite connection intervals, perhaps you could even
  # say:
  #
  #           devices: [Swift, 21c]
  
  # TODO: Allow compact specification of multiple devices in metadata YAML,
  # e.g.:
  #
  #     - name_prefix: Swift
  #       model: Swift
  #       serial_numbers: [0, 1, 2, 3]
  #
  # to specify devices "Swift 0", "Swift 1", "Swift 2", and "Swift 3".
  # 
  # Associated possible syntactic sugar for sets of numeric serial numbers
  # might include items like:
  #
  #     name_format: "Swift {n:02d}"
  #
  # where `n` is a numeric serial number (perhaps we should check the
  # format specification, constraining it to ensure safety against
  # injection attacks), and:
  #
  #     serial_number_range: [0, 3]
  #
  # to specify a range of numeric serial numbers.
  
  # TODO: Allow specification of input or output number for devices
  # that have just one input or output, e.g. "Swift Input 0", instead
  # of requiring that input or output number be omitted, e.g.
  # "Swift Input".
  
  # TODO: Allow omission of device connection start and/or end times
  # to indicate infinite time intervals.
  
  # TODO: If it isn't specified, infer time zone from location using
  # `timezonefinder` (or some such) Python module. See
  # https://stackoverflow.com/questions/16086962/
  # how-to-get-a-time-zone-from-a-location-using-latitude-and-longitude-coordinates
  
  
  class MetadataImportError(Exception):
      pass
  
  
  def import_metadata(metadata, logger=None, job_id=None):
      importer = _MetadataImporter(logger)
      with transaction.atomic():
          importer.import_metadata(metadata, job_id)
  
          
  class _MetadataImporter:
  
  
      def __init__(self, logger=None):
          self._logger = logger
          self._methods = {
              'stations': self._add_stations,
              'device_models': self._add_device_models,
              'devices': self._add_devices,
              'station_devices': self._add_station_devices,
              'detectors': self._add_detectors,
              'classifiers': self._add_classifiers,
              'annotation_constraints': self._add_annotation_constraints,
              'annotations': self._add_annotations,
              'tags': self._add_tags,
         }
  
  
      def import_metadata(self, metadata, job_id=None):
  
          self._creation_time = time_utils.get_utc_now()
          self._creating_user = None
          
          if job_id is None:
              self._creating_job = None
          else:
              self._creating_job = Job.objects.get(id=job_id)
  
          for name, data in metadata.items():
  
              method = self._methods.get(name)
  
              if method is None:
                  raise MetadataImportError(
                      f'Unrecognized metadata section name "{name}".')
  
              method(data)
  
  
      def _add_stations(self, data):
          
          for station_data in data:
          
              name = _get_required(station_data, 'name', 'station')
              
              self._log_info(f'Adding station "{name}"...')
              
              description = station_data.get('description', '')
              latitude = _get_required(station_data, 'latitude', 'station')
              longitude = _get_required(station_data, 'longitude', 'station')
              elevation = _get_required(station_data, 'elevation', 'station')
              time_zone = _get_required(station_data, 'time_zone', 'station')
              
              Station.objects.create(
                  name=name,
                  description=description,
                  latitude=latitude,
                  longitude=longitude,
                  elevation=elevation,
                  time_zone=time_zone)
  
  
      def _log_info(self, message):
          if self._logger is not None:
              self._logger.info(message)
  
  
      def _add_device_models(self, data):
          
          for model_data in data:
              model = self._add_device_model(model_data)
              self._add_ports(model, model_data, 'input', DeviceModelInput)
              self._add_ports(model, model_data, 'output', DeviceModelOutput)
              
              
      def _add_device_model(self, model_data):
          
          name = _get_required(model_data, 'name', 'device model')
          
          self._log_info(f'Adding device model "{name}"...')
  
          type_ = _get_required(model_data, 'type', 'device model')
          manufacturer = \
              _get_required(model_data, 'manufacturer', 'device model')
          model = _get_required(model_data, 'model', 'device model')
          description = model_data.get('description', '')
          
          model = DeviceModel.objects.create(
              name=name,
              type=type_,
              manufacturer=manufacturer,
              model=model,
              description=description
          )
          
          return model
              
  
      def _add_ports(self, model, model_data, port_type, port_class):
          
          port_data = self._get_port_data(model_data, port_type)
          
          for local_name, channel_num in port_data:
              
              self._log_info(
                  f'Adding device model "{model.name}" {port_type} '
                  f'"{local_name}"...')
              
              port_class.objects.create(
                  model=model,
                  local_name=local_name,
                  channel_num=channel_num)
  
  
      def _get_port_data(self, model_data, port_type):
  
          names = model_data.get(port_type + 's')
          
          if names is None:
              
              key = f'num_{port_type}s'
              num_ports = model_data.get(key, 0)
              
              if num_ports == 0:
                  names = []
                  
              elif num_ports == 1:
                  names = [port_type.capitalize()]
                  
              else:
                  names = [f'{port_type.capitalize()} {i}'
                           for i in range(num_ports)]
                  
          return [(name, i) for i, name in enumerate(names)]
                  
                  
      def _add_devices(self, data):
          
          models = _create_objects_dict(DeviceModel)
      
          for device_data in data:
              device = self._add_device(device_data, models)
              self._add_device_inputs(device)
              self._add_device_outputs(device)
              
              
      def _add_device(self, device_data, models):
          
          name = _get_required(device_data, 'name', 'device')
          
          self._log_info(f'Adding device "{name}"...')
          
          model = self._get_device_model(device_data, models)
          serial_number = _get_required(device_data, 'serial_number', 'device')
          description = device_data.get('description', '')
          
          return Device.objects.create(
              name=name,
              model=model,
              serial_number=serial_number,
              description=description)
  
  
      def _get_device_model(self, device_data, models):
  
          name = _get_required(device_data, 'model', 'device')
          
          try:
              return models[name]
          except KeyError:
              raise MetadataImportError(
                  f'Unrecognized device model name "{name}".')
  
  
      def _add_device_inputs(self, device):
          
          for model_input in device.model.inputs.all():
              
              self._log_info(
                  f'Adding device "{device.name}" input '
                  f'"{model_input.local_name}"...')
              
              DeviceInput.objects.create(
                  device=device,
                  model_input=model_input)
              
              
      def _add_device_outputs(self, device):
                  
          for model_output in device.model.outputs.all():
              
              self._log_info(
                  f'Adding device "{device.name}" output '
                  f'"{model_output.local_name}"...')
              
              DeviceOutput.objects.create(
                  device=device,
                  model_output=model_output)
  
  
      def _add_station_devices(self, data):
          
          devices = _create_objects_dict(Device)
          inputs = _create_objects_dict(DeviceInput)
          outputs = _create_objects_dict(DeviceOutput)
      
          for sd_data in data:
              
              station = self._get_station(sd_data)
              
              data_name = 'station devices array'
              
              start_time = self._get_time(
                  sd_data, 'start_time', station, data_name)
              end_time = self._get_time(
                  sd_data, 'end_time', station, data_name)
              
              device_names = _get_required(sd_data, 'devices', data_name)
              station_devices = []
              for name in device_names:
                  device = self._get_device(name, devices)
                  self._add_station_device(
                      station, device, start_time, end_time)
                  station_devices.append(device)
                  
              shorthand_inputs, shorthand_outputs = \
                  _get_shorthand_ports(station_devices)
                  
              connections = _get_required(sd_data, 'connections', data_name)
              for connection in connections:
                  output = self._get_port(
                      connection, 'output', shorthand_outputs, outputs)
                  input_ = self._get_port(
                      connection, 'input', shorthand_inputs, inputs)
                  self._add_connection(
                      station, output, input_, start_time, end_time)
                          
      
      def _get_station(self, data):
          name = _get_required(data, 'station', 'station devices item')
          try:
              return Station.objects.get(name=name)
          except Station.DoesNotExist:
              raise MetadataImportError(f'Unrecognized station "{name}".')
              
  
      def _get_time(self, data, key, station, data_name):
          dt = _get_required(data, key, data_name)
          if isinstance(dt, Date):
              dt = DateTime(dt.year, dt.month, dt.day)
          return station.local_to_utc(dt)
  
      
      def _get_device(self, name, devices):
          try:
              return devices[name]
          except KeyError:
              raise MetadataImportError(f'Unrecognized device "{name}".')
          
  
      def _add_station_device(self, station, device, start_time, end_time):
          
          self._log_info(
              f'Adding station "{station.name}" device "{device.name}" '
              f'from {str(start_time)} to {str(end_time)}"...')
      
          StationDevice.objects.create(
              station=station,
              device=device,
              start_time=start_time,
              end_time=end_time)
          
  
      def _get_port(self, connection, port_type, shorthand_ports, ports):
          
          name = _get_required(connection, port_type, 'device connection')
          
          port = shorthand_ports.get(name)
          
          if port is None:
              port = ports.get(name)
              
          if port is None:
              raise MetadataImportError(
                  f'Unrecognized device {port_type} "{name}".')
          
          else:
              return port
              
              
      def _add_connection(self, station, output, input_, start_time, end_time):
          
          self._log_info(
              f'Adding station "{station.name}" device connection '
              f'"{output.name} -> {input_.name} from {str(start_time)} '
              f'to {str(end_time)}"...')
      
          DeviceConnection.objects.create(
              output=output,
              input=input_,
              start_time=start_time,
              end_time=end_time)
  
  
      def _add_detectors(self, data):
          self._add_processors(data, 'detectors', 'detector', 'Detector')
          
          
      def _add_processors(self, data, data_key, log_type_name, db_type_name):
          
          for processor_data in data:
          
              name = _get_required(processor_data, 'name', log_type_name)
  
              self._log_info(f'Adding {log_type_name} "{name}"...')
              
              description = processor_data.get('description', '')
              
              Processor.objects.create(
                  name=name,
                  type=db_type_name,
                  description=description)
  
          
      def _add_classifiers(self, data):
          self._add_processors(data, 'classifiers', 'classifier', 'Classifier')
          
          
      def _add_annotation_constraints(self, data):
          
          for constraint_data in data:
              
              name = _get_required(
                  constraint_data, 'name', 'annotation constraint')
              
              self._log_info(f'Adding annotation constraint "{name}"...')
              
              description = constraint_data.get('description', '')
              text = yaml_utils.dump(constraint_data)
              
              AnnotationConstraint.objects.create(
                  name=name,
                  description=description,
                  text=text,
                  creation_time=self._creation_time,
                  creating_user=self._creating_user,
                  creating_job=self._creating_job)
                  
  
      def _add_annotations(self, data):
          
          for annotation_data in data:
              
              name = _get_required(annotation_data, 'name', 'annotation')
              
              self._log_info(f'Adding annotation "{name}"...')
              
              description = annotation_data.get('description', '')
              type_ = annotation_data.get('type', 'String')
              constraint = self._get_annotation_constraint(annotation_data)
              
              AnnotationInfo.objects.create(
                  name=name,
                  description=description,
                  type=type_,
                  constraint=constraint,
                  creation_time=self._creation_time,
                  creating_user=self._creating_user,
                  creating_job=self._creating_job)
      
      
      def _get_annotation_constraint(self, annotation_data):
          try:
              name = annotation_data['constraint']
          except KeyError:
              return None
          else:
              return AnnotationConstraint.objects.get(name=name)
      
          
      def _add_tags(self, bobo):
          
          for tag_data in bobo:
              
              name = _get_required(tag_data, 'name', 'tag')
              
              self._log_info(f'Adding tag "{name}"...')
              
              description = tag_data.get('description', '')
              
              TagInfo.objects.create(
                  name=name,
                  description=description,
                  creation_time=self._creation_time,
                  creating_user=self._creating_user,
                  creating_job=self._creating_job)
  
  
  def _get_required(data, key, data_name):
      try:
          return data[key]
      except KeyError:
          raise MetadataImportError(
              f'{data_name.capitalize()} missing required item "{key}".')
          
          
  def _create_objects_dict(cls):
      objects = {}
      for obj in cls.objects.all():
          objects[obj.name] = obj
          objects[obj.long_name] = obj
      return objects
  
  
  def _get_shorthand_ports(devices):
      
      # Create mapping from model names to sets of devices.
      model_devices = defaultdict(set)
      for device in devices:
          model_devices[device.model.name].add(device)
          
      # Create mappings from shorthand port names to ports. A shorthand
      # port name is like a regular port name except that it includes
      # only a model name rather than a device name. We include an item
      # in this mapping for each port of each device that is the only one
      # of its model in `devices`.
      shorthand_inputs = {}
      shorthand_outputs = {}
      for model_name, devices in model_devices.items():
          if len(devices) == 1:
              for device in devices:
                  _add_shorthand_ports(
                      shorthand_inputs, device.inputs.all(), model_name)
                  _add_shorthand_ports(
                      shorthand_outputs, device.outputs.all(), model_name)
                  
      return shorthand_inputs, shorthand_outputs
                      
                      
  def _add_shorthand_ports(shorthand_ports, ports, model_name):
      for port in ports:
          name = f'{model_name} {port.local_name}'
          shorthand_ports[name] = port
  