text: # -*- coding: utf-8 -*-
  
  """
  @author: Aaron Ponti
  """
  
  import re
  import os
  import logging
  import xml.etree.ElementTree as xml
  from datetime import datetime
  import java.io.File
  from org.apache.commons.io import FileUtils
  
  
  #
  # The Processor class performs all steps required for registering datasets
  # from the assigned dropbox folder
  #
  class Processor:
      """Registers datasets from the dropbox folder"""
  
      # A transaction object passed by openBIS
      _transaction = None
  
      # The _incoming folder to process (a java.io.File object)
      _incoming = ""
  
      # The user name
      _username = ""
  
      # The logger
      _logger = None
  
      # Constructor
      def __init__(self, transaction, logFile):
  
          self._transaction = transaction
          self._incoming = transaction.getIncoming()
          self._username = ""
  
          # Set up logging
          logging.basicConfig(filename=logFile, level=logging.DEBUG, 
                          format='%(asctime)-15s %(levelname)s: %(message)s')
          self._logger = logging.getLogger("BLSRFortessa")
  
  
      def dictToXML(self, d):
          """Converts a dictionary into an XML string."""
  
          # Create an XML node
          node = xml.Element("Parameters")
  
          # Add all attributes to the XML node
          for k, v in d.iteritems():
              node.set(k, v)
  
          # Convert to XML string
          xmlString = xml.tostring(node, encoding="UTF-8")
  
          # Return the XML string
          return xmlString
  
  
      def createExperiment(self, expId, expName):
          """Create an experiment with given Experiment ID extended with the addition
          of a string composed from current date and time.
  
          @param expID, the experiment ID
          @param expName, the experiment name
          """
  
          # Make sure to keep the code length within the limits imposed by
          # openBIS for codes
          if len(expId) > 41:
              expId = expId[0:41]
  
          # Create univocal ID
          expId = expId + "_" + self.getCustomTimeStamp()
  
          # Create the experiment
          self._logger.info("Register experiment %s" % expId)
          exp = self._transaction.createNewExperiment(expId, "LSR_FORTESSA_EXPERIMENT")
          if not exp:
              msg = "Could not create experiment " + expId + "!"
              self._logger.error(msg)
              raise Exception(msg)
          else:
              self._logger.info("Created experiment with ID " + expId + ".")
  
          # Store the name
          exp.setPropertyValue("LSR_FORTESSA_EXPERIMENT_NAME", expName)
  
          return exp
  
  
      def createSampleWithGenCode(self, spaceCode,
                                  sampleType="LSR_FORTESSA_PLATE"):
          """Create a sample with automatically generated code.
  
          @param spaceCode, the code of the space
          @param sampleType, the sample type that must already exist
          @return sample An ISample
          """
  
          # Make sure there are not slashes in the spaceCode
          spaceCode = spaceCode.replace("/", "")
  
          # Create the sample
          sample = self._transaction.createNewSampleWithGeneratedCode(spaceCode,
                                                                      sampleType)
          if not sample:
              msg = "Could not create sample with generated code"
              self._logger.error(msg)
              raise Exception(msg)
  
          return sample
  
  
      def formatExpDateForPostgreSQL(self, expDate):
          """Format the experiment date to be compatible with postgreSQL's
          'timestamp' data type.
  
          @param Date stored in the FCS file, in the form 01-JAN-2013
          @return Date in the form 2013-01-01
          """
  
          monthMapper = {'JAN': '01', 'FEB': '02', 'MAR': '03', 'APR': '04',
                         'MAY': '05', 'JUN': '06', 'JUL': '07', 'AUG': '08',
                         'SEP': '09', 'OCT': '10', 'NOV': '11', 'DEC': '12'}
  
          # Separate the date into day, month, and year
          (day, month, year) = expDate.split("-")
  
          # Try mapping the month to digits (e.g. "06"). If the mapping does
          # not work, return "NOT_FOUND"
          month = monthMapper.get(month, "NOT_FOUND")
  
          # Build the date in the correct format. If the month was not found,
          # return 01-01-1970
          if (month == "NOT_FOUND"):
              self._logger.info("Invalid experiment date %s found. " \
                           "Reverting to 1970/01/01." % expDate)
              return "1970-01-01"
          else:
              return (year + "-" + month + "-" + day)
  
  
      def getCustomTimeStamp(self):
          """Create an univocal time stamp based on the current date and time
          (works around incomplete API of Jython 2.5).
          """
  
          t = datetime.now()
          return (t.strftime("%y%d%m%H%M%S") + unicode(t)[20:])
  
  
      def getSubFolders(self):
          """Return a list of subfolders of the passed incoming directory.
  
          @return list of subfolders (String)
          """
  
          incomingStr = self._incoming.getAbsolutePath()
          return [name for name in os.listdir(incomingStr)
                  if os.path.isdir(os.path.join(incomingStr, name))]
  
  
      def processExperiment(self, experimentNode):
          """Register an IExperimentUpdatable based on the Experiment XML node.
  
          @param experimentNode An XML node corresponding to an Experiment
          @return IExperimentUpdatable experiment
          """
  
          # Get the experiment version
          expVersion = experimentNode.attrib.get("version")
          if expVersion is None:
              expVersion = "0"
  
          # Get the openBIS identifier
          openBISIdentifier = experimentNode.attrib.get("openBISIdentifier")
  
          # Get the experiment name
          expName = experimentNode.attrib.get("name")
  
          # Get the experiment date and reformat it to be compatible
          # with postgreSQL
          expDate = self.formatExpDateForPostgreSQL(experimentNode.attrib.get("date"))
  
          # Get the description
          description = experimentNode.attrib.get("description")
  
          # Get the acquisition hardware
          acqHardware = experimentNode.attrib.get("acq_hardware")
  
          # Get the acquisition software
          acqSoftware = experimentNode.attrib.get("acq_software")
  
          # Get the owner name
          owner = experimentNode.attrib.get("owner_name")
  
          # Get attachments
          attachments = experimentNode.attrib.get("attachments")
  
          # Create the experiment (with corrected ID if needed: see above)
          openBISExperiment = self.createExperiment(openBISIdentifier, expName)
          if not openBISExperiment:
              msg = "Could not create experiment " + openBISIdentifier
              self._logger.error(msg)
              raise Exception(msg)
  
          # Get comma-separated tag list
          tagList = experimentNode.attrib.get("tags")
          if tagList != None and tagList != "":
  
              # Retrieve or create the tags
              openBISTags = self.retrieveOrCreateTags(tagList)
  
              # Set the metaprojects (tags)
              for openBISTag in openBISTags:
                  openBISTag.addEntity(openBISExperiment)
  
          # Set the experiment version
          openBISExperiment.setPropertyValue("LSR_FORTESSA_EXPERIMENT_VERSION",
                                             expVersion)
  
          # Set the date
          openBISExperiment.setPropertyValue("LSR_FORTESSA_EXPERIMENT_DATE",
                                             expDate)
  
          # Set the description
          openBISExperiment.setPropertyValue("LSR_FORTESSA_EXPERIMENT_DESCRIPTION",
                                             description)
  
          # Set the acquisition hardware
          openBISExperiment.setPropertyValue("LSR_FORTESSA_EXPERIMENT_ACQ_HARDWARE",
                                             acqHardware)
  
          # Set the acquisition hardware friendly name
          openBISExperiment.setPropertyValue("LSR_FORTESSA_EXPERIMENT_ACQ_HARDWARE_FRIENDLY_NAME",
                                             self._machinename)
  
          # Set the acquisition software
          openBISExperiment.setPropertyValue("LSR_FORTESSA_EXPERIMENT_ACQ_SOFTWARE",
                                             acqSoftware)
  
          # Set the experiment owner
          openBISExperiment.setPropertyValue("LSR_FORTESSA_EXPERIMENT_OWNER",
                                             owner)
  
          # Add the attachments
          if attachments is not None:
  
              # Extract all relative file names 
              attachmentFiles = attachments.split(";")
  
              for f in attachmentFiles:
  
                  # This is an additional security step
                  if f == '':
                      continue
  
                  # Inform
                  msg = "Adding file attachment " + f 
                  self._logger.info(msg)
  
                  # Build the full path
                  attachmentFilePath = os.path.join(self._incoming.getAbsolutePath(),
                                                    f)
  
                  # Extract the file name
                  attachmentFileName = os.path.basename(attachmentFilePath)
  
                  # Read the attachment into a byte array
                  javaFile = java.io.File(attachmentFilePath)
                  byteArray = FileUtils.readFileToByteArray(javaFile)
  
                  # Add attachment
                  openBISExperiment.addAttachment(attachmentFilePath,
                                                  attachmentFileName,
                                                  "", byteArray)
  
          # Return the openBIS Experiment object
          return openBISExperiment
  
  
      def processFCSFile(self, fcsFileNode, openBISTube, openBISExperiment):
          """Register the FCS File using the parsed properties file.
  
          @param fcsFileNode An XML node corresponding to an FCS file (dataset)
          @param openBISTube  An ISample object representing a Tube or Well
          @param openBISExperiment An ISample object representing an Experiment
          """
  
          # Dataset type
          datasetType = "LSR_FORTESSA_FCSFILE"
  
          # Create a new dataset
          dataset = self._transaction.createNewDataSet()
          if not dataset:
              msg = "Could not get or create dataset"
              self._logger.error(msg)
              raise Exception(msg)
  
          # Set the dataset type
          dataset.setDataSetType(datasetType)
  
          # Assign the dataset to the sample
          dataset.setSample(openBISTube)
  
          # Assign the dataset to the experiment
          dataset.setExperiment(openBISExperiment)
  
          # Set the file type
          dataset.setFileFormatType("FCS")
  
          # Get the parameter node
          for parameterNode in fcsFileNode:
  
              if parameterNode.tag != "FCSFileParamList":
                  msg = "Expected FSC File Parameter List node!"
                  self._logger.error(msg)
                  raise Exception(msg)
  
              parametersXML = self.dictToXML(parameterNode.attrib)
  
              # Store the parameters in the LSR_FORTESSA_FCSFILE_PARAMETERS property
              dataset.setPropertyValue("LSR_FORTESSA_FCSFILE_PARAMETERS", parametersXML)
              
              # Log the parameters
              self._logger.info("FCS file parameters (XML): " + str(parametersXML))
  
          # Assign the file to the dataset (we will use the absolute path)
          fileName = fcsFileNode.attrib.get("relativeFileName")
          fileName = os.path.join(self._incoming.getAbsolutePath(), fileName)
  
          # Log
          self._logger.info("Registering file: " + fileName)
  
          # Move the file
          self._transaction.moveFile(fileName, dataset)
  
  
      def processTray(self, trayNode, openBISExperiment):
          """Register a Tray (Plate) based on the Tray XML node
          and an IExperimentUpdatable object.
  
          @param trayNode An XML node corresponding to a Tray (Plate)
          @param openBISExperiment An IExperimentUpdatable object
          @param openBISSampleType sample type (default "LSR_FORTESSA_PLATE")
          @return ISample sample, or null
          """
  
          # openBIS sample type
          openBISSampleType = "LSR_FORTESSA_PLATE"
  
          # Get the identifier of the space all relevant attributes
          openBISSpaceIdentifier = \
              trayNode.attrib.get("openBISSpaceIdentifier")
  
          # Get the tray name
          name = trayNode.attrib.get("name")
  
          # Get the tray geometry
          trayGeometry = trayNode.attrib.get("trayGeometry")
  
          # Create the sample. The Plate is configured in openBIS to
          # auto-generate its own identifier.
          openBISTray = self.createSampleWithGenCode(openBISSpaceIdentifier,
                                                     openBISSampleType)
          if not openBISTray:
              msg = "Could not create plate sample."
              self._logger.error(msg)
              raise Exception(msg)
  
          # Set the experiment for the sample
          openBISTray.setExperiment(openBISExperiment)
  
          # Set the plate name
          openBISTray.setPropertyValue("LSR_FORTESSA_PLATE_NAME", name)
  
          # Set the tray geometry
          openBISTray.setPropertyValue("LSR_FORTESSA_PLATE_GEOMETRY", trayGeometry)
  
          # Return the openBIS ISample object
          return openBISTray
  
  
      def processTubeOrWell(self, tubeNode, openBISContainerSample,
                            specimenName, openBISExperiment):
          """Register a Tube or Well (as a child of a Specimen) based on the Tube or
          Well XML node and an ISample object.
  
          The associated fcs file is attached as a IDataset
  
          @param tubeNode An XML node corresponding to a Tube or Well
          @param openBISContainerSample  An ISample object that will contain
          the Tube or Well
          @param specimenName Name of the specimen associated to the Tube or Well
          @param openBISExperiment The IExperiment to which the Tube belongs
          @param openBISSpecimenType (default "LSR_FORTESSA_TUBE"), the
          sample type. One of LSR_FORTESSA_TUBE and LSR_FORTESSA_WELL.
          @return ISample sample, or null
          """
  
          # Get the name
          name = tubeNode.attrib.get("name")
  
          # openBIS type
          if tubeNode.tag == "Tube":
              openBISSpecimenType = "LSR_FORTESSA_TUBE"
          elif tubeNode.tag == "Well":
              openBISSpecimenType = "LSR_FORTESSA_WELL"
          else:
              msg = "Unknown tube type" + tubeNode.tag
              self._logger.error(msg)
              raise Exception(msg)
  
          # Build the openBIS Identifier
          openBISSpaceIdentifier = \
              tubeNode.attrib.get("openBISSpaceIdentifier")
  
          # Create the sample. The Tube/Well is configured in openBIS to
          # auto-generate its own identifier.
          openBISTube = self.createSampleWithGenCode(openBISSpaceIdentifier,
                                                     openBISSpecimenType)
          if not openBISTube:
              msg = "Could not create sample with auto-generated identifier"
              self._logger.error(msg)
              raise Exception(msg)
  
          # Set the experiment to which it belongs
          openBISTube.setExperiment(openBISExperiment)
  
          # Set the Specimen name as a property
          openBISTube.setPropertyValue("LSR_FORTESSA_SPECIMEN", specimenName)
  
          # Set the name
          if openBISSpecimenType == "LSR_FORTESSA_TUBE":
              openBISTube.setPropertyValue("LSR_FORTESSA_TUBE_NAME", name)
          elif openBISSpecimenType == "LSR_FORTESSA_WELL":
              openBISTube.setPropertyValue("LSR_FORTESSA_WELL_NAME", name)
          else:
              msg = "Unknown value for openBISSpecimenType."
              self._logger.error(msg)
              raise Exception(msg)
  
          # Set the TubeSet container
          openBISTube.setContainer(openBISContainerSample)
  
          # Return the openBIS ISample
          return openBISTube
  
  
      def processTubeSet(self, experimentNode, openBISExperiment):
          """Register a TubeSet (virtual tube container).
  
          @param experimentNode An XML node corresponding to an Experiment
          @param openBISExperiment An IExperimentUpdatable object
          @param openBISSampleType  The TubeSet sample type
          @return ISample sample, or null
          """
  
          # Sample type
          openBISSampleType = "LSR_FORTESSA_TUBESET"
  
          # Get the identifier of the space all relevant attributes
          openBISSpaceIdentifier = \
              experimentNode.attrib.get("openBISSpaceIdentifier")
  
          # Create the sample. The Tubeset is configured in openBIS to
          # auto-generate its own identifier.
          openBISTubeSet = self.createSampleWithGenCode(openBISSpaceIdentifier,
                                                        openBISSampleType)
          if not openBISTubeSet:
              msg = "Could not get or create TubeSet"
              self._logger.error(msg)
              raise Exception(msg)
  
          # Set the experiment for the sample
          openBISTubeSet.setExperiment(openBISExperiment)
  
          self._logger.info("Created new TubeSet " \
                       "with identifier %s, sample type %s" \
                       % (openBISTubeSet.getSampleIdentifier(),
                          openBISSampleType))
  
          # Return the openBIS ISample object
          return openBISTubeSet
  
  
      def register(self, tree):
          """Register the Experiment using the parsed properties file.
  
          @param tree ElementTree parsed from the properties XML file
          """
  
          # Get the root node (obitXML)
          root = tree.getroot()
  
          # Store the username
          self._username = root.attrib.get("userName")
  
          # Store the machine name
          machinename = root.attrib.get("machineName")
          if machinename is None:
              machinename = ""
          self._machinename = machinename
  
          # Create a virtual TubeSet
          openBISTubeSet = None
  
          # Iterate over the children (Experiments)
          for experimentNode in root:
  
              # The tag of the immediate children of the root experimentNode
              # must be Experiment
              if experimentNode.tag != "Experiment":
                  msg = "Expected Experiment node, found " + experimentNode.tag
                  self._logger.error(msg)
                  raise Exception(msg)
  
              # Process an Experiment XML node and get/create an IExperimentUpdatable
              openBISExperiment = self.processExperiment(experimentNode)
  
              # Process children of the Experiment
              for childNode in experimentNode:
  
                  # The child of an Experiment can be a Tray or a Specimen
                  nodeType = childNode.tag
  
                  if nodeType == "Specimen":
  
                      # A specimen is a direct child of an experiment if there
                      # is no plate, and the FCS files are therefore associated
                      # to tubes. In this case, we create a virtual TubeSet
                      # sample container (one for all Tubes in the experiment).
                      if openBISTubeSet is None:
                          openBISTubeSet = self.processTubeSet(experimentNode,
                                                               openBISExperiment)
  
                      # The only information we need from the Specimen is its
                      # name to associate to the Tubes as property
                      specimenNameProperty = childNode.attrib.get("name")
  
                      # Now iterate over the children of the Specimen
                      for tubeNode in childNode:
  
                          # The child of a Specimen is a Tube
                          if tubeNode.tag != "Tube":
                              msg = "Expected Tube node!"
                              self._logger.error(msg)
                              raise Exception(msg)
  
                          # Process the tube node and get the openBIS object
                          openBISTube = self.processTubeOrWell(tubeNode,
                                                         openBISTubeSet,
                                                         specimenNameProperty,
                                                         openBISExperiment)
  
                          # Now process the FCS file
                          for fcsNode in tubeNode:
  
                              # The child of a Tube is an FCSFile
                              if fcsNode.tag != "FCSFile":
                                  msg = "Expected FSC File node!"
                                  self._logger.error(msg)
                                  raise Exception(msg)
  
                              # Process the FCS file node
                              self.processFCSFile(fcsNode, openBISTube,
                                                  openBISExperiment)
  
                  elif nodeType == "Tray":
  
                      # Process the tray node and get the openBIS object
                      openBISTray = self.processTray(childNode,
                                                     openBISExperiment)
  
                      # Now iterate over the children of the Tray
                      for specimenNode in childNode:
  
                          # The child of a Tray is a Specimen
                          if specimenNode.tag != "Specimen":
                              msg = "Expected Specimen node!"
                              self._logger.error(msg)
                              raise Exception(msg)
  
                          # The only information we need from the Specimen is its
                          # name to associate to the Wells as property
                          specimenNameProperty = specimenNode.attrib.get("name")
  
                          for wellNode in specimenNode:
  
                              # The child of a Specimen is a Tube
                              if wellNode.tag != "Well":
                                  msg = "Expected Well node!"
                                  self._logger.error(msg)
                                  raise Exception(msg)
  
                              # Process the tube node and get the openBIS object
                              openBISWell = self.processTubeOrWell(wellNode,
                                                             openBISTray,
                                                             specimenNameProperty,
                                                             openBISExperiment)
  
                              # Now process the FCS file
                              for fcsNode in wellNode:
  
                                  # The child of a Tube is an FCSFile
                                  if fcsNode.tag != "FCSFile":
                                      msg = "Expected FSC File node!"
                                      self._logger.error(msg)
                                      raise Exception(msg)
  
                                  # Process the FCS file node
                                  self.processFCSFile(fcsNode, openBISWell,
                                                      openBISExperiment)
  
  
                  else:
  
                      msg = "The Node must be either a Specimen or a Tray"
                      self._logger.error(msg)
                      raise Exception(msg)
  
          # Log that we are finished with the registration
          self._logger.info("Registration completed")
  
  
      def retrieveOrCreateTags(self, tagList):
          """Retrieve or create the tags (metaprojects) with specified names."""
  
          # Initialize openBISTags list
          openBISTags = []
  
          # Make sure tagList is not None
          if tagList is None:
              return []
  
          # Get the individual tag names (with no blank spaces)
          tags = ["".join(t.strip()) for t in tagList.split(",")]
  
          # Process all tags (metaprojects)
          for tag in tags:
              if len(tag) == 0:
                  continue
  
              # Retrieve the tag (metaproject)
              metaproject = self._transaction.getMetaproject(tag, self._username)
              if metaproject is None:
  
                  # Create the tag (metaproject)
                  logging("Creating metaproject " + tag)
  
                  metaproject = self._transaction.createNewMetaproject(tag,
                                                                       "",
                                                                       self._username)
  
                  # Check that creation was succcessful
                  if metaproject is None:
                      msg = "Could not create metaproject " + tag + \
                      "for user " + self._username
                      self._logger.error(msg)
                      raise Exception(msg)
  
              # Add the created metaproject to the list
              openBISTags.append(metaproject)
  
          return openBISTags
  
  
      def run(self):
          """Run the registration."""
  
          # Make sure that incoming is a folder
          if not self._incoming.isDirectory():
              msg = "Incoming MUST be a folder!"
              self._logger.error(msg)
              raise Exception(msg)
  
          # Log
          self._logger.info("Incoming folder: " + self._incoming.getAbsolutePath())
  
          # There must be just one subfolder: the user subfolder
          subFolders = self.getSubFolders()
          if len(subFolders) != 1:
              msg = "Expected user subfolder!"
              self._logger.error(msg)
              raise Exception(msg)
  
          # Set the user folder
          userFolder = os.path.join(self._incoming.getAbsolutePath(),
                                    subFolders[0])
  
          # In the user subfolder we must find the data_structure.ois file
          dataFileName = os.path.join(userFolder, "data_structure.ois")
          if not os.path.exists(dataFileName):
              msg = "File data_structure.ois not found!"
              self._logger.error(msg)
              raise Exception(msg)
  
          # Now read the data structure file and store all the pointers to
          # the properties files. The paths are stored relative to self._incoming,
          # so we can easily build the full file paths.
          propertiesFileList = []
          f = open(dataFileName)
          try:
              for line in f:
                  line = re.sub('[\r\n]', '', line)
                  propertiesFile = os.path.join(self._incoming.getAbsolutePath(),
                                              line)
                  propertiesFileList.append(propertiesFile)
          finally:
              f.close()
  
          # Process (and ultimately register) all experiments
          for propertiesFile in propertiesFileList:
  
              # Log
              self._logger.info("* * * Processing: " + propertiesFile + " * * *")
  
              # Read the properties file into an ElementTree
              tree = xml.parse(propertiesFile)
  
              # Now register the experiment
              self.register(tree)
  
  
  def process(transaction):
      """Dropbox entry point.
  
      @param transaction, the transaction object
      """
  
      # Get path to containing folder
      # __file__ does not work (reliably) in Jython
      dbPath = "../core-plugins/flow/1/dss/drop-boxes/BDLSRFortessaDropbox"
  
      # Path to the logs subfolder
      logPath = os.path.join(dbPath, "logs")
  
      # Make sure the logs subforder exist
      if not os.path.exists(logPath):
          os.makedirs(logPath)
  
      # Path for the log file
      logFile = os.path.join(logPath, "log.txt")
  
      # Create a Processor
      processor = Processor(transaction, logFile)
  
      # Run
      processor.run()
  