text: // -*- C++ -*-
  //
  // Package:    LumiProducer
  // Class:      LumiCorrectionSource
  //
  /**\class LumiCorrectionSource LumiCorrectionSource.cc RecoLuminosity/LumiProducer/src/LumiCorrectionSource.cc
  Description: A essource/esproducer for lumi correction factor and run parameters needed to deduce the corrections
        Author: Zhen Xie
  */
  
  //#include <memory>
  #include "FWCore/Framework/interface/SourceFactory.h"
  #include "FWCore/Framework/interface/ESProducer.h"
  #include "FWCore/Framework/interface/ESHandle.h"
  #include "FWCore/Framework/interface/EventSetup.h"
  
  #include "FWCore/ParameterSet/interface/ParameterSet.h"
  #include "FWCore/Framework/interface/Event.h"
  #include "FWCore/Framework/interface/LuminosityBlock.h"
  #include "FWCore/Framework/interface/Run.h"
  #include "DataFormats/Provenance/interface/LuminosityBlockID.h"
  #include "FWCore/ServiceRegistry/interface/Service.h"
  #include "FWCore/Framework/interface/IOVSyncValue.h"
  #include "CoralBase/Exception.h"
  #include "CoralBase/AttributeList.h"
  #include "CoralBase/Attribute.h"
  #include "CoralBase/AttributeSpecification.h"
  #include "CoralBase/Exception.h"
  #include "CoralKernel/Context.h"
  #include "RelationalAccess/IAuthenticationService.h"
  #include "RelationalAccess/ConnectionService.h"
  #include "CoralBase/Exception.h"
  #include "RelationalAccess/ISessionProxy.h"
  #include "RelationalAccess/ITransaction.h"
  #include "RelationalAccess/AccessMode.h"
  #include "RelationalAccess/ITypeConverter.h"
  #include "RelationalAccess/IQuery.h"
  #include "RelationalAccess/ICursor.h"
  #include "RelationalAccess/ISchema.h"
  #include "RelationalAccess/ITable.h"
  #include "CoralKernel/IPropertyManager.h"
  #include "RelationalAccess/AuthenticationServiceException.h"
  #include "RecoLuminosity/LumiProducer/interface/Exception.h"
  #include "RecoLuminosity/LumiProducer/interface/ConstantDef.h"
  #include "RecoLuminosity/LumiProducer/interface/LumiCorrectionParam.h"
  #include "RecoLuminosity/LumiProducer/interface/LumiCorrectionParamRcd.h"
  #include "RecoLuminosity/LumiProducer/interface/RevisionDML.h"
  #include "RecoLuminosity/LumiProducer/interface/NormDML.h"
  #include "RecoLuminosity/LumiProducer/interface/LumiNames.h"
  #include "LumiCorrectionSource.h"
  #include <iostream>
  #include <sstream>
  #include <string>
  #include <memory>
  #include <algorithm>
  #include <vector>
  #include <cstring>
  #include <iterator>
  #include <filesystem>
  
  #include <boost/tokenizer.hpp>
  #include <xercesc/dom/DOM.hpp>
  #include <xercesc/parsers/XercesDOMParser.hpp>
  #include "Utilities/Xerces/interface/Xerces.h"
  #include <xercesc/util/XMLString.hpp>
  
  std::string LumiCorrectionSource::x2s(const XMLCh* toTranscode) const {
    std::string tmp(xercesc::XMLString::transcode(toTranscode));
    return tmp;
  }
  
  XMLCh* LumiCorrectionSource::s2x(const std::string& temp) const {
    XMLCh* buff = xercesc::XMLString::transcode(temp.c_str());
    return buff;
  }
  
  std::string LumiCorrectionSource::toParentString(const xercesc::DOMNode& nodeToConvert) const {
    std::ostringstream oss;
    xercesc::DOMNodeList* childList = nodeToConvert.getChildNodes();
  
    unsigned int numNodes = childList->getLength();
    for (unsigned int i = 0; i < numNodes; ++i) {
      xercesc::DOMNode* childNode = childList->item(i);
      if (childNode->getNodeType() != xercesc::DOMNode::ELEMENT_NODE)
        continue;
      xercesc::DOMElement* child = static_cast<xercesc::DOMElement*>(childNode);
      xercesc::DOMNamedNodeMap* attributes = child->getAttributes();
      unsigned int numAttributes = attributes->getLength();
      for (unsigned int j = 0; j < numAttributes; ++j) {
        xercesc::DOMNode* attributeNode = attributes->item(j);
        if (attributeNode->getNodeType() != xercesc::DOMNode::ATTRIBUTE_NODE)
          continue;
        xercesc::DOMAttr* attribute = static_cast<xercesc::DOMAttr*>(attributeNode);
  
        oss << "(" << x2s(child->getTagName()) << x2s(attribute->getName()) << "=" << x2s(attribute->getValue()) << ")";
      }
    }
    return oss.str();
  }
  
  const std::string LumiCorrectionSource::servletTranslation(const std::string& servlet) const {
    std::string frontierConnect;
    std::string realconnect;
    cms::concurrency::xercesInitialize();
    std::unique_ptr<xercesc::XercesDOMParser> parser(new xercesc::XercesDOMParser);
    try {
      parser->setValidationScheme(xercesc::XercesDOMParser::Val_Auto);
      parser->setDoNamespaces(false);
      parser->parse(m_siteconfpath.c_str());
      xercesc::DOMDocument* doc = parser->getDocument();
      if (!doc) {
        return "";
      }
  
      xercesc::DOMNodeList* frontierConnectList = doc->getElementsByTagName(s2x("frontier-connect"));
      if (frontierConnectList->getLength() > 0) {
        xercesc::DOMElement* frontierConnectElement = static_cast<xercesc::DOMElement*>(frontierConnectList->item(0));
        frontierConnect = toParentString(*frontierConnectElement);
      }
      // Replace the last component of every "serverurl=" piece (up to the
      //   next close-paren) with the servlet
      std::string::size_type nextparen = 0;
      std::string::size_type serverurl, lastslash;
      std::string complexstr = "";
      while ((serverurl = frontierConnect.find("(serverurl=", nextparen)) != std::string::npos) {
        realconnect.append(frontierConnect, nextparen, serverurl - nextparen);
        nextparen = frontierConnect.find(')', serverurl);
        lastslash = frontierConnect.rfind('/', nextparen);
        realconnect.append(frontierConnect, serverurl, lastslash - serverurl + 1);
        realconnect.append(servlet);
      }
      realconnect.append(frontierConnect, nextparen, frontierConnect.length() - nextparen);
    } catch (xercesc::DOMException& e) {
    }
    return realconnect;
  }
  
  std::string LumiCorrectionSource::translateFrontierConnect(const std::string& connectStr) {
    std::string result;
    const std::string fproto("frontier://");
    std::string::size_type startservlet = fproto.length();
    std::string::size_type endservlet = connectStr.find('(', startservlet);
    if (endservlet == std::string::npos) {
      endservlet = connectStr.rfind('/', connectStr.length());
    }
    std::string servlet = connectStr.substr(startservlet, endservlet - startservlet);
    if ((!servlet.empty()) && (servlet.find_first_of(":/)[]") == std::string::npos)) {
      if (servlet == "cms_conditions_data")
        servlet = "";
      if (m_siteconfpath.length() == 0) {
        std::string url = (std::filesystem::path("SITECONF") / std::filesystem::path("local") /
                           std::filesystem::path("JobConfig") / std::filesystem::path("site-local-config.xml"))
                              .string();
        char* tmp = std::getenv("CMS_PATH");
        if (tmp) {
          m_siteconfpath = (std::filesystem::path(tmp) / std::filesystem::path(url)).string();
        }
      } else {
        if (!std::filesystem::exists(std::filesystem::path(m_siteconfpath))) {
          throw cms::Exception("Non existing path ") << m_siteconfpath;
        }
        m_siteconfpath =
            (std::filesystem::path(m_siteconfpath) / std::filesystem::path("site-local-config.xml")).string();
      }
      result = fproto + servletTranslation(servlet) + connectStr.substr(endservlet);
    }
    return result;
  }
  
  LumiCorrectionSource::LumiCorrectionSource(const edm::ParameterSet& iConfig)
      : m_connectStr(""),
        m_authfilename(""),
        m_datatag(""),
        m_globaltag(""),
        m_normtag(""),
        m_paramcachedrun(0),
        m_cachesize(0) {
    setWhatProduced(this, &LumiCorrectionSource::produceLumiCorrectionParam);
    findingRecord<LumiCorrectionParamRcd>();
    std::string connectStr = iConfig.getParameter<std::string>("connect");
    std::string globaltag;
    if (iConfig.exists("globaltag")) {
      m_globaltag = iConfig.getUntrackedParameter<std::string>("globaltag", "");
    } else {
      m_normtag = iConfig.getUntrackedParameter<std::string>("normtag", "");
    }
    m_datatag = iConfig.getUntrackedParameter<std::string>("datatag", "");
    m_cachesize = iConfig.getUntrackedParameter<unsigned int>("ncacheEntries", 3);
    m_siteconfpath = iConfig.getUntrackedParameter<std::string>("siteconfpath", "");
    const std::string fproto("frontier://");
    if (connectStr.substr(0, fproto.length()) == fproto) {
      m_connectStr = translateFrontierConnect(connectStr);
    } else if (connectStr.substr(0, 11) == "sqlite_file") {
      m_connectStr = connectStr;
    } else {
      m_connectStr = connectStr;
      std::string authpath = iConfig.getUntrackedParameter<std::string>("authpath", "");
      std::filesystem::path boostAuthPath(authpath);
      if (std::filesystem::is_directory(boostAuthPath)) {
        boostAuthPath /= std::filesystem::path("authentication.xml");
      }
      m_authfilename = boostAuthPath.string();
    }
  }
  
  LumiCorrectionSource::ReturnParamType LumiCorrectionSource::produceLumiCorrectionParam(const LumiCorrectionParamRcd&) {
    unsigned int currentrun = m_pcurrentTime->eventID().run();
    if (currentrun == 0 || currentrun == 4294967295) {
      return std::make_shared<const LumiCorrectionParam>();
    }
    if (m_paramcachedrun != currentrun) {                         //i'm in a new run
      fillparamcache(currentrun);                                 //fill cache
    } else {                                                      //i'm in an old run
      if (m_paramcache.find(currentrun) == m_paramcache.end()) {  //i'm not cached
        fillparamcache(currentrun);                               //
      }
    }
    if (m_paramcache.empty()) {
      return std::make_shared<const LumiCorrectionParam>();
    }
    m_paramresult = m_paramcache[currentrun];
    if (m_paramresult.get() == nullptr) {
      return std::make_shared<const LumiCorrectionParam>();
    }
    return m_paramresult;
  }
  
  void LumiCorrectionSource::setIntervalFor(const edm::eventsetup::EventSetupRecordKey& iKey,
                                            const edm::IOVSyncValue& iTime,
                                            edm::ValidityInterval& oValidity) {
    m_pcurrentTime = &iTime;
    oValidity.setFirst(iTime);
    oValidity.setLast(iTime);
  }
  
  void LumiCorrectionSource::reloadAuth() {
    //std::cout<<"old authfile "<<coral::Context::instance().PropertyManager().property("AuthenticationFile")->get()<<std::endl;
    coral::Context::instance().PropertyManager().property("AuthenticationFile")->set(m_authfilename);
    coral::Context::instance().loadComponent("CORAL/Services/XMLAuthenticationService");
  }
  
  void LumiCorrectionSource::fillparamcache(unsigned int runnumber) {
    m_paramcache.clear();
    m_paramcachedrun = runnumber;
    if (!m_authfilename.empty()) {
      coral::IHandle<coral::IAuthenticationService> authSvc =
          coral::Context::instance().query<coral::IAuthenticationService>();
      if (authSvc.isValid()) {
        try {
          authSvc->credentials(m_connectStr);
        } catch (const coral::UnknownConnectionException& er) {
          reloadAuth();
        }
      } else {
        reloadAuth();
      }
    }
    coral::ConnectionService* mydbservice = new coral::ConnectionService;
    if (!m_globaltag.empty()) {
      coral::ISessionProxy* gsession = mydbservice->connect(m_connectStr, coral::ReadOnly);
      gsession->transaction().start(true);
      parseGlobaltagForLumi(gsession->nominalSchema(), m_globaltag);
      gsession->transaction().commit();
      delete gsession;
    }
    coral::ISessionProxy* session = mydbservice->connect(m_connectStr, coral::ReadOnly);
    coral::ITypeConverter& tconverter = session->typeConverter();
    tconverter.setCppTypeForSqlType(std::string("float"), std::string("FLOAT(63)"));
    tconverter.setCppTypeForSqlType(std::string("unsigned int"), std::string("NUMBER(10)"));
    tconverter.setCppTypeForSqlType(std::string("unsigned short"), std::string("NUMBER(1)"));
    auto result = std::make_unique<LumiCorrectionParam>(LumiCorrectionParam::HF);
    try {
      session->transaction().start(true);
      coral::ISchema& schema = session->nominalSchema();
      lumi::RevisionDML dml;
      unsigned long long tagid = 0;
      if (m_datatag.empty()) {
        tagid = dml.currentHFDataTagId(schema);  //get datatag id
      } else {
        tagid = dml.HFDataTagIdByName(schema, m_datatag);
      }
      lumi::RevisionDML::DataID dataid = dml.dataIDForRun(schema, runnumber, tagid);  //get data id
      unsigned int lumiid = dataid.lumi_id;
      if (lumiid == 0) {
        result->setNBX(0);
        std::shared_ptr<const LumiCorrectionParam> const_result = std::move(result);
        m_paramcache.insert(std::make_pair(runnumber, const_result));
        session->transaction().commit();
        delete session;
        delete mydbservice;
        return;
      }
  
      coral::AttributeList lumidataBindVariables;
      lumidataBindVariables.extend("dataid", typeid(unsigned long long));
      lumidataBindVariables["dataid"].data<unsigned long long>() = lumiid;
      std::string conditionStr("DATA_ID=:dataid");
      coral::AttributeList lumiparamOutput;
      lumiparamOutput.extend("NCOLLIDINGBUNCHES", typeid(unsigned int));
      coral::IQuery* lumiparamQuery = schema.newQuery();
      lumiparamQuery->addToTableList(std::string("LUMIDATA"));
      lumiparamQuery->setCondition(conditionStr, lumidataBindVariables);
      lumiparamQuery->addToOutputList("NCOLLIDINGBUNCHES");
      lumiparamQuery->defineOutput(lumiparamOutput);
      coral::ICursor& lumiparamcursor = lumiparamQuery->execute();
      unsigned int ncollidingbx = 0;
      while (lumiparamcursor.next()) {
        const coral::AttributeList& row = lumiparamcursor.currentRow();
        if (!row["NCOLLIDINGBUNCHES"].isNull()) {
          ncollidingbx = row["NCOLLIDINGBUNCHES"].data<unsigned int>();
        }
        result->setNBX(ncollidingbx);
      }
      delete lumiparamQuery;
      lumi::NormDML normdml;
      unsigned long long normid = 0;
      std::map<std::string, unsigned long long> normidmap;
      if (m_normtag.empty()) {
        normdml.normIdByType(schema, normidmap, lumi::NormDML::HF, true);
        m_normtag = normidmap.begin()->first;
        normid = normidmap.begin()->second;
      } else {
        normid = normdml.normIdByName(schema, m_normtag);
      }
  
      std::map<unsigned int, lumi::NormDML::normData> normDataMap;
      normdml.normById(schema, normid, normDataMap);
  
      std::map<unsigned int, lumi::NormDML::normData>::iterator normIt = --normDataMap.end();
      if (runnumber < normIt->first) {
        normIt = normDataMap.upper_bound(runnumber);
        --normIt;
      }
      result->setNormtag(m_normtag);
      result->setcorrFunc(normIt->second.corrfunc);
      result->setnonlinearCoeff(normIt->second.coefficientmap);
      result->setafterglows(normIt->second.afterglows);
      result->setdescription(normIt->second.amodetag, normIt->second.beamegev);
      if (normIt->second.coefficientmap["DRIFT"] != 0.) {
        float intglumi = fetchIntglumi(schema, runnumber);
        result->setintglumi(intglumi);
      }
      m_paramcache.insert(std::make_pair(runnumber, std::shared_ptr<LumiCorrectionParam>(std::move(result))));
      session->transaction().commit();
    } catch (const coral::Exception& er) {
      session->transaction().rollback();
      delete session;
      delete mydbservice;
      throw cms::Exception("DatabaseError ") << er.what();
    }
    delete session;
    delete mydbservice;
  }
  void LumiCorrectionSource::parseGlobaltagForLumi(coral::ISchema& schema, const std::string& globaltag) {
    /**select i.pfn,i.tagname from TAGINVENTORY_TABLE i,TAGTREE_TABLE_GLOBALTAG v from i.tagid=v.tagid and i.recordname='LumiCorrectionParamRcd' **/
    std::string tagtreetabname("TAGTREE_TABLE_");
    tagtreetabname += std::string(globaltag);
    coral::IQuery* qHandle = schema.newQuery();
    qHandle->addToTableList("TAGINVENTORY_TABLE", "i");
    qHandle->addToTableList(tagtreetabname, "v");
    coral::AttributeList qResult;
    qResult.extend("pfn", typeid(std::string));
    qResult.extend("tagname", typeid(std::string));
    std::string conditionStr("v.tagid=i.tagid and i.recordname=:recordname");
    coral::AttributeList qCondition;
    qCondition.extend("recordname", typeid(std::string));
    qCondition["recordname"].data<std::string>() = std::string("LumiCorrectionParamRcd");
    qHandle->setCondition(conditionStr, qCondition);
    qHandle->addToOutputList("i.pfn");
    qHandle->addToOutputList("i.tagname");
    qHandle->defineOutput(qResult);
    coral::ICursor& iCursor = qHandle->execute();
    while (iCursor.next()) {
      const coral::AttributeList& row = iCursor.currentRow();
      std::string connectStr = row["pfn"].data<std::string>();
      const std::string fproto("frontier://");
      if (connectStr.substr(0, fproto.length()) == fproto) {
        m_connectStr = translateFrontierConnect(connectStr);
      } else {
        m_connectStr = connectStr;
      }
      m_normtag = row["tagname"].data<std::string>();
    }
    delete qHandle;
  }
  float LumiCorrectionSource::fetchIntglumi(coral::ISchema& schema, unsigned int runnumber) {
    float result = 0.;
    coral::IQuery* qHandle = schema.newQuery();
    qHandle->addToTableList(lumi::LumiNames::intglumiv2TableName());
    coral::AttributeList qResult;
    qResult.extend("INTGLUMI", typeid(float));
    std::string conditionStr("RUNNUM=:runnumber");
    coral::AttributeList qCondition;
    qCondition.extend("runnumber", typeid(unsigned int));
    qCondition["runnumber"].data<unsigned int>() = runnumber;
    qHandle->setCondition(conditionStr, qCondition);
    qHandle->addToOutputList("INTGLUMI");
    qHandle->defineOutput(qResult);
    coral::ICursor& intglumiCursor = qHandle->execute();
    while (intglumiCursor.next()) {
      const coral::AttributeList& row = intglumiCursor.currentRow();
      if (!row["INTGLUMI"].isNull()) {
        result = row["INTGLUMI"].data<float>();
      }
    }
    delete qHandle;
    return result;
  }
  
  LumiCorrectionSource::~LumiCorrectionSource() {}
  //define this as a plug-in
  DEFINE_FWK_EVENTSETUP_SOURCE(LumiCorrectionSource);
  