DHFPROD-2534: attempting to handle the difference between ml 9.09 and 9.10 and 10.1 in tde generation for entity services

(:
  Copyright 2012-2019 MarkLogic Corporation

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
:)
xquery version "1.0-ml";

module namespace hent = "http://marklogic.com/data-hub/hub-entities";

import module namespace es = "http://marklogic.com/entity-services"
at "/MarkLogic/entity-services/entity-services.xqy";
import module namespace tde = "http://marklogic.com/xdmp/tde"
at "/MarkLogic/tde.xqy";

declare namespace search = "http://marklogic.com/appservices/search";

declare variable $ENTITY-MODEL-COLLECTION := "http://marklogic.com/entity-services/models";

declare option xdmp:mapping "false";

declare function hent:get-model($entity-name as xs:string)
{
  hent:get-model($entity-name, ())
};

declare function hent:get-model($entity-name as xs:string, $used-models as xs:string*)
{
  let $model :=
    let $_ := fn:collection($ENTITY-MODEL-COLLECTION)[lower-case(info/title) = lower-case($entity-name)]
    return
      if (fn:count($_) > 1) then
        fn:head(fn:collection($ENTITY-MODEL-COLLECTION)[info/title = $entity-name])
      else
        $_
  where fn:exists($model)
  return
    let $model-map as map:map? := $model
    let $refs := $model//*[fn:local-name(.) = '$ref'][fn:starts-with(., "#/definitions")] ! fn:replace(., "#/definitions/", "")
    let $definitions := map:get($model-map, "definitions")
    let $_ :=
      for $ref in $refs[fn:not(. = $used-models)]
      let $m :=
        if (fn:empty(map:get($definitions, $ref))) then
          let $other-model as map:map? := hent:get-model($ref, ($used-models, $entity-name))
          let $other-defs := map:get($other-model, "definitions")
          for $key in map:keys($other-defs)
          return
            map:put($definitions, $key, map:get($other-defs, $key))
        else ()
      return ()
    return $model-map
};

declare function hent:uber-model() as map:map
{
  hent:uber-model(fn:collection($ENTITY-MODEL-COLLECTION)/object-node())
};

declare function hent:uber-model($models as object-node()*) as map:map
{
  let $uber-model := map:map()
  let $definitions :=
    let $m := map:map()
    let $_ :=
      for $model as map:map in $models
      let $defs := map:get($model, "definitions")
      for $key in map:keys($defs)
      return
        map:put($m, $key, map:get($defs, $key))
    return
      map:put($uber-model, "definitions", $m)
  return
    $uber-model
};

declare function hent:wrap-duplicates(
  $duplicate-map as map:map,
  $property-name as xs:string,
  $item as element()
) as item()
{
  if (map:contains($duplicate-map, $property-name))
  then
    comment { "This item is a duplicate and is commented out so as to create a valid artifact.
",
    xdmp:quote($item),
    "
"
    }
  else (
    map:put($duplicate-map, $property-name, true()),
    $item)
};

(:
  this method doctors the output from ES
  because of https://github.com/marklogic/entity-services/issues/359
:)
declare %private function hent:fix-options($nodes as node()*)
{
  for $n in $nodes
  return
    typeswitch($n)
      case element(search:options) return
        element { fn:node-name($n) } {
          $n/namespace::node(),
          $n/@*,
          <search:constraint name="Collection">
            <search:collection/>
          </search:constraint>,
          <search:constraint name="createdByJob">
            <search:value>
              <search:field name="datahubCreatedByJob"/>
            </search:value>
          </search:constraint>,
          <search:constraint name="createdByStep">
            <search:value>
              <search:field name="datahubCreatedByStep"/>
            </search:value>
          </search:constraint>,
          hent:fix-options($n/node())
        }
      case element(search:additional-query) return ()
      case element(search:return-facets) return <search:return-facets>true</search:return-facets>
      case element() return
        element { fn:node-name($n) } {
          $n/namespace::node(),
          $n/@*,
          hent:fix-options($n/node()),

          let $is-range-constraint := $n[self::search:range] and $n/..[self::search:constraint]
          where $is-range-constraint and fn:not($n/search:facet-option[starts-with(., "limit=")])
          return <search:facet-option>limit=25</search:facet-option>
        }
      case text() return
        fn:replace($n, "es:", "*:")
      default return $n
};

declare function hent:dump-search-options($entities as json:array)
{
  let $uber-model := hent:uber-model(json:array-values($entities) ! xdmp:to-json(.)/object-node())
  return
  (: call fix-options because of https://github.com/marklogic/entity-services/issues/359 :)
    hent:fix-options(es:search-options-generate($uber-model))
};

declare function hent:dump-pii($entities as json:array)
{
  let $uber-model := hent:uber-model(json:array-values($entities) ! xdmp:to-json(.)/object-node())
  return es:pii-generate($uber-model)
};

declare function hent:dump-indexes($entities as json:array)
{
  let $uber-model := hent:uber-model(json:array-values($entities) ! xdmp:to-json(.)/object-node())

  let $database-config := xdmp:from-json(es:database-properties-generate($uber-model))

  let $_ :=
    for $x in ("database-name", "schema-database", "triple-index", "collection-lexicon")
    return
      map:delete($database-config, $x)

  let $_ :=
    for $idx in map:get($database-config, "range-path-index") ! json:array-values(.)
    return
      map:put($idx, "path-expression", fn:replace(map:get($idx, "path-expression"), "es:", "*:"))

  let $_ := remove-duplicate-range-indexes($database-config)

  return
    xdmp:to-json($database-config)
};

(:
es:database-properties-generate will generate duplicate range indexes when e.g. two entities have properties with the
same name and namespace and are both configured to have range indexes. This function removes duplicates, where
duplicates are considered to have the same local name, namespace URI, and collation.
:)
declare function hent:remove-duplicate-range-indexes($database-config as item())
{
  let $indexes := map:get($database-config, "range-element-index")
  return
    if (fn:exists($indexes)) then
      let $index-map := map:map()
      let $_ :=
        for $index in json:array-values($indexes)
        let $key := fn:string-join(
          (
            "localname", map:get($index, "localname"),
            "namespace", map:get($index, "namespace-uri"),
            "collation", map:get($index, "collation")
          ), "-"
        )
        where fn:not(map:contains($index-map, $key))
        return map:put($index-map, $key, $index)

      let $deduplicated-indexes := json:array()
      let $_ := map:keys($index-map) ! json:array-push($deduplicated-indexes, map:get($index-map, .))
      let $_ := map:put($database-config, "range-element-index", $deduplicated-indexes)
      return ()
    else
      let $_ := map:put($database-config, "range-element-index", json:array())
      return ()
};

declare variable $generated-primary-key-column as xs:string := "DataHubGeneratedPrimaryKey";
declare variable $generated-primary-key-expression as xs:string := "xdmp:node-uri(.) || '#' || fn:position()";

declare function hent:dump-tde($entities as json:array)
{
  let $entity-values as map:map* := json:array-values($entities)
  let $uber-model := hent:uber-model($entity-values ! xdmp:to-json(.)/object-node())
  let $_set-info := $uber-model => map:put("info", fn:head($entity-values) => map:get("info"))
  let $uber-definitions := $uber-model => map:get("definitions")
  (: Primary keys are required for each definition in an entity. If the primary key is missing, we'll help out by using the doc URI and position as the primary key. :)
  let $_set-primary-keys-for-TDE :=
    for $definition-type in map:keys($uber-definitions)
    let $definition := $uber-definitions => map:get($definition-type)
    where fn:empty($definition => map:get("primaryKey"))
    return (
      $definition => map:put("primaryKey", $generated-primary-key-column),
      $definition => map:get("properties") => map:put($generated-primary-key-column, map:entry("datatype", "string"))
    )
  let $entity-model-contexts := map:keys($uber-definitions) ! ("./" || .)
  let $entity-name := map:get(map:get($uber-model, "info"), "title")
  return hent:fix-tde(es:extraction-template-generate($uber-model), $entity-model-contexts, $uber-definitions, $entity-name)
};

declare variable $default-nullable as element(tde:nullable) := element tde:nullable {fn:true()};
declare variable $default-invalid-values as element(tde:invalid-values) := element tde:invalid-values {"ignore"};
(:
  this method doctors the TDE output from ES
:)

declare function hent:fix-tde($nodes as node()*, $entity-model-contexts as xs:string*, $uber-definitions as map:map)
{
  hent:fix-tde($nodes, $entity-model-contexts, $uber-definitions, ())
};

declare function hent:fix-tde($nodes as node()*, $entity-model-contexts as xs:string*, $uber-definitions as map:map, $entity-name as xs:string?)
{
  for $n in $nodes
  return
    typeswitch($n)
      case document-node() return
        document {
          hent:fix-tde($n/node(), $entity-model-contexts, $uber-definitions, $entity-name)
        }
      case element(tde:nullable) return
        $default-nullable
      case element(tde:invalid-values) return
        $default-invalid-values
      case element(tde:val) return
        element { fn:node-name($n) } {
          $n/namespace::node(),
          let $col-name := fn:string($n/../tde:name)
          return
            if (fn:ends-with($col-name, $generated-primary-key-column)) then
              $generated-primary-key-expression
            else if (fn:starts-with($n, $col-name)) then
              let $parts := fn:tokenize($n, "/")
              let $entity-definition := $uber-definitions => map:get(fn:string($parts[2]))
              return
                if (fn:exists($entity-definition)) then
                  let $primary-key := $entity-definition => map:get("primaryKey")
                  return
                    if ($primary-key = $generated-primary-key-column) then
                      $generated-primary-key-expression
                    else
                      fn:string($n) || "/" || $primary-key
                else
                  hent:fix-tde($n/node(), $entity-model-contexts, $uber-definitions)
            else
              hent:fix-tde($n/node(), $entity-model-contexts, $uber-definitions)
        }
      case element(tde:context) return
        element { fn:node-name($n) } {
          $n/namespace::node(),
          if ($n = $entity-model-contexts) then
            fn:replace(fn:replace(fn:string($n),"^\./", ".//"), "(.)$", "$1[node()]")
          else
          if(fn:count($n) = 1) then
            let $outer-context := fn:replace(fn:string($n),"//\*:instance", "/*:envelope/*:instance")
            return if(fn:not(fn:empty($entity-name))) then
              fn:concat($outer-context, "[*:", $entity-name, "]")
            else
              $outer-context
          else
          $n/node()
        }
      case element(tde:column) return
        element { fn:node-name($n) } {
          $n/namespace::node(),
          $n/@*,
          hent:fix-tde($n/* except $n/(tde:nullable|tde:invalid-values), $entity-model-contexts, $uber-definitions),
          $default-nullable,
          $default-invalid-values
        }
      case element(tde:subject)|element(tde:predicate)|element(tde:object) return
        element { fn:node-name($n) } {
          $n/namespace::node(),
          hent:fix-tde($n/* except $n/tde:invalid-values, $entity-model-contexts, $uber-definitions),
          $default-invalid-values
        }
      case element(tde:template) return
        element { fn:node-name($n) } {
          $n/namespace::node(),
          $n/@*,
          let $context-item :=  fn:replace(fn:string($n/tde:context), "\./", "")
          let $parent-context-item := fn:replace(fn:string($n/../../tde:context), "\./", "")
          let $is-join-template := $n/tde:rows/tde:row/tde:view-name = $parent-context-item  || "_" || $context-item
          let $rows := $n/tde:rows/tde:row
          return
            if ($is-join-template) then (
              hent:fix-tde($n/tde:context, $entity-model-contexts, $uber-definitions),
              element tde:rows {
                element tde:row {
                  $rows/(tde:schema-name|tde:view-name|tde:view-layout),
                  element tde:columns {
                    let $join-prefix := $context-item || "_"
                    for $column in $rows/tde:columns/tde:column
                    return
                      element tde:column {
                        $column/@*,
                        if (fn:starts-with($column/tde:name, $join-prefix)) then (
                          hent:fix-tde($column/(tde:name|tde:scalar-type), $entity-model-contexts, $uber-definitions),
                          let $tde-val := fn:string($column/tde:val)
                          let $primary-key := $uber-definitions => map:get($tde-val) => map:get("primaryKey")
                          return
                            element tde:val {
                              if ($primary-key = $generated-primary-key-column) then
                                $generated-primary-key-expression
                              else
                                $tde-val || "/" || $primary-key
                            },
                          $default-nullable,
                          $default-invalid-values,
                          hent:fix-tde($column/(tde:default|tde:reindexing|tde:collation), $entity-model-contexts, $uber-definitions)
                        ) else
                          hent:fix-tde($column/node(), $entity-model-contexts, $uber-definitions)
                      }
                  }
                }
              }
            ) else
              hent:fix-tde($n/node(), $entity-model-contexts, $uber-definitions, $entity-name)
        }
      case element() return
        element { fn:node-name($n) } {
          $n/namespace::node(),
          hent:fix-tde(($n/@*, $n/node()), $entity-model-contexts, $uber-definitions)
        }
      case text() return
        fn:replace(
          fn:replace($n, "^\.\./(.+)$", "(../$1|parent::array-node()/../$1)"),
          "\./" || $generated-primary-key-column,
          $generated-primary-key-expression
        )
      default return $n
};

declare variable $number-types as xs:string+ := ("byte","decimal","double","float","int","integer","long","negativeInteger","nonNegativeInteger","nonPositiveInteger","positiveInteger","short","unsignedLong","unsignedInt","unsignedShort","unsignedByte");
declare variable $string-types as xs:string+ := "dateTime";

declare function hent:json-schema-generate($entity-title as xs:string, $uber-model as map:map)
{
  let $uber-model := map:new((
  (: Ensure we're not change a map for anyone else :)
  map:map(document{$uber-model}/*),
  map:entry("language", "zxx"),
  map:entry("$schema", "http://json-schema.org/draft-07/schema#")
  ))
  let $definitions := $uber-model => map:get("definitions")
  (: JSON Schema needs an extra level of wrapping to account for Entity Model label wrapping it. :)
  let $_nest-refs :=
    for $definition-type in map:keys($definitions)
    let $definition-properties := $definitions => map:get($definition-type) => map:get("properties")
    for $property-name in map:keys($definition-properties)
    let $property := $definition-properties => map:get($property-name)
    let $property-items := $property => map:get("items")
    let $datatype := $property => map:get("datatype")
    let $_set-types := (
      $property => map:put("type", if ($datatype = $number-types) then "number" else if ($datatype = $string-types) then "string" else $datatype),
      $property => map:delete("datatype"),
      if ($property-items instance of map:map) then (
        let $items-datatype := $property => map:get("datatype")
        return (
          $property-items => map:put("type", if ($items-datatype = $number-types) then "number" else if ($items-datatype = $string-types) then "string" else $datatype),
          $property-items => map:delete("datatype"))
      ) else ()
    )
    return
    (: references can be in the property or in items for arrays :)
      if ($property => map:contains("$ref")) then
        map:put($definition-properties, $property-name,
          map:new((
            map:entry("type", "object"),
            map:entry("properties",
              map:entry(fn:tokenize(map:get($property,"$ref"),"/")[fn:last()], $property)
            )
          ))
        )
      else if ($property-items instance of map:map and $property-items => map:contains("$ref")) then
        map:put($property, "items",
          map:new((
            map:entry("type", "object"),
            map:entry("properties",
              map:entry(fn:tokenize(map:get($property-items,"$ref"),"/")[fn:last()], $property-items)
            )
          ))
        )
      else ()
  let $_set-info := (
    $uber-model => map:put("properties", map:entry($entity-title, map:entry("$ref", "#/definitions/"||$entity-title)))
  )
  return xdmp:to-json($uber-model)
};

(:
  Copyright 2012-2019 MarkLogic Corporation

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
:)
xquery version "1.0-ml";

module namespace hent = "http://marklogic.com/data-hub/hub-entities";

import module namespace es = "http://marklogic.com/entity-services"
at "/MarkLogic/entity-services/entity-services.xqy";
import module namespace tde = "http://marklogic.com/xdmp/tde"
at "/MarkLogic/tde.xqy";

declare namespace search = "http://marklogic.com/appservices/search";

declare variable $ENTITY-MODEL-COLLECTION := "http://marklogic.com/entity-services/models";

declare option xdmp:mapping "false";

declare function hent:get-model($entity-name as xs:string)
{
  hent:get-model($entity-name, ())
};

declare function hent:get-model($entity-name as xs:string, $used-models as xs:string*)
{
  let $model :=
    let $_ := fn:collection($ENTITY-MODEL-COLLECTION)[lower-case(info/title) = lower-case($entity-name)]
    return
      if (fn:count($_) > 1) then
        fn:head(fn:collection($ENTITY-MODEL-COLLECTION)[info/title = $entity-name])
      else
        $_
  where fn:exists($model)
  return
    let $model-map as map:map? := $model
    let $refs := $model//*[fn:local-name(.) = '$ref'][fn:starts-with(., "#/definitions")] ! fn:replace(., "#/definitions/", "")
    let $definitions := map:get($model-map, "definitions")
    let $_ :=
      for $ref in $refs[fn:not(. = $used-models)]
      let $m :=
        if (fn:empty(map:get($definitions, $ref))) then
          let $other-model as map:map? := hent:get-model($ref, ($used-models, $entity-name))
          let $other-defs := map:get($other-model, "definitions")
          for $key in map:keys($other-defs)
          return
            map:put($definitions, $key, map:get($other-defs, $key))
        else ()
      return ()
    return $model-map
};

declare function hent:uber-model() as map:map
{
  hent:uber-model(fn:collection($ENTITY-MODEL-COLLECTION)/object-node())
};

declare function hent:uber-model($models as object-node()*) as map:map
{
  let $uber-model := map:map()
  let $definitions :=
    let $m := map:map()
    let $_ :=
      for $model as map:map in $models
      let $defs := map:get($model, "definitions")
      for $key in map:keys($defs)
      return
        map:put($m, $key, map:get($defs, $key))
    return
      map:put($uber-model, "definitions", $m)
  return
    $uber-model
};

declare function hent:wrap-duplicates(
  $duplicate-map as map:map,
  $property-name as xs:string,
  $item as element()
) as item()
{
  if (map:contains($duplicate-map, $property-name))
  then
    comment { "This item is a duplicate and is commented out so as to create a valid artifact.
",
    xdmp:quote($item),
    "
"
    }
  else (
    map:put($duplicate-map, $property-name, true()),
    $item)
};

(:
  this method doctors the output from ES
  because of https://github.com/marklogic/entity-services/issues/359
:)
declare %private function hent:fix-options($nodes as node()*)
{
  for $n in $nodes
  return
    typeswitch($n)
      case element(search:options) return
        element { fn:node-name($n) } {
          $n/namespace::node(),
          $n/@*,
          <search:constraint name="Collection">
            <search:collection/>
          </search:constraint>,
          <search:constraint name="createdByJob">
            <search:value>
              <search:field name="datahubCreatedByJob"/>
            </search:value>
          </search:constraint>,
          <search:constraint name="createdByStep">
            <search:value>
              <search:field name="datahubCreatedByStep"/>
            </search:value>
          </search:constraint>,
          hent:fix-options($n/node())
        }
      case element(search:additional-query) return ()
      case element(search:return-facets) return <search:return-facets>true</search:return-facets>
      case element() return
        element { fn:node-name($n) } {
          $n/namespace::node(),
          $n/@*,
          hent:fix-options($n/node()),

          let $is-range-constraint := $n[self::search:range] and $n/..[self::search:constraint]
          where $is-range-constraint and fn:not($n/search:facet-option[starts-with(., "limit=")])
          return <search:facet-option>limit=25</search:facet-option>
        }
      case text() return
        fn:replace($n, "es:", "*:")
      default return $n
};

declare function hent:dump-search-options($entities as json:array)
{
  let $uber-model := hent:uber-model(json:array-values($entities) ! xdmp:to-json(.)/object-node())
  return
  (: call fix-options because of https://github.com/marklogic/entity-services/issues/359 :)
    hent:fix-options(es:search-options-generate($uber-model))
};

declare function hent:dump-pii($entities as json:array)
{
  let $uber-model := hent:uber-model(json:array-values($entities) ! xdmp:to-json(.)/object-node())
  return es:pii-generate($uber-model)
};

declare function hent:dump-indexes($entities as json:array)
{
  let $uber-model := hent:uber-model(json:array-values($entities) ! xdmp:to-json(.)/object-node())

  let $database-config := xdmp:from-json(es:database-properties-generate($uber-model))

  let $_ :=
    for $x in ("database-name", "schema-database", "triple-index", "collection-lexicon")
    return
      map:delete($database-config, $x)

  let $_ :=
    for $idx in map:get($database-config, "range-path-index") ! json:array-values(.)
    return
      map:put($idx, "path-expression", fn:replace(map:get($idx, "path-expression"), "es:", "*:"))

  let $_ := remove-duplicate-range-indexes($database-config)

  return
    xdmp:to-json($database-config)
};

(:
es:database-properties-generate will generate duplicate range indexes when e.g. two entities have properties with the
same name and namespace and are both configured to have range indexes. This function removes duplicates, where
duplicates are considered to have the same local name, namespace URI, and collation.
:)
declare function hent:remove-duplicate-range-indexes($database-config as item())
{
  let $indexes := map:get($database-config, "range-element-index")
  return
    if (fn:exists($indexes)) then
      let $index-map := map:map()
      let $_ :=
        for $index in json:array-values($indexes)
        let $key := fn:string-join(
          (
            "localname", map:get($index, "localname"),
            "namespace", map:get($index, "namespace-uri"),
            "collation", map:get($index, "collation")
          ), "-"
        )
        where fn:not(map:contains($index-map, $key))
        return map:put($index-map, $key, $index)

      let $deduplicated-indexes := json:array()
      let $_ := map:keys($index-map) ! json:array-push($deduplicated-indexes, map:get($index-map, .))
      let $_ := map:put($database-config, "range-element-index", $deduplicated-indexes)
      return ()
    else
      let $_ := map:put($database-config, "range-element-index", json:array())
      return ()
};

declare variable $generated-primary-key-column as xs:string := "DataHubGeneratedPrimaryKey";
declare variable $generated-primary-key-expression as xs:string := "xdmp:node-uri(.) || '#' || fn:position()";

declare function hent:dump-tde($entities as json:array)
{
  let $entity-values as map:map* := json:array-values($entities)
  let $uber-model := hent:uber-model($entity-values ! xdmp:to-json(.)/object-node())
  let $_set-info := $uber-model => map:put("info", fn:head($entity-values) => map:get("info"))
  let $uber-definitions := $uber-model => map:get("definitions")
  (: Primary keys are required for each definition in an entity. If the primary key is missing, we'll help out by using the doc URI and position as the primary key. :)
  let $_set-primary-keys-for-TDE :=
    for $definition-type in map:keys($uber-definitions)
    let $definition := $uber-definitions => map:get($definition-type)
    where fn:empty($definition => map:get("primaryKey"))
    return (
      $definition => map:put("primaryKey", $generated-primary-key-column),
      $definition => map:get("properties") => map:put($generated-primary-key-column, map:entry("datatype", "string"))
    )
  let $entity-model-contexts := map:keys($uber-definitions) ! ("./" || .)
  let $entity-name := map:get(map:get($uber-model, "info"), "title")
  return hent:fix-tde(es:extraction-template-generate($uber-model), $entity-model-contexts, $uber-definitions, $entity-name)
};

declare variable $default-nullable as element(tde:nullable) := element tde:nullable {fn:true()};
declare variable $default-invalid-values as element(tde:invalid-values) := element tde:invalid-values {"ignore"};
(:
  this method doctors the TDE output from ES
:)

declare function hent:fix-tde($nodes as node()*, $entity-model-contexts as xs:string*, $uber-definitions as map:map)
{
  hent:fix-tde($nodes, $entity-model-contexts, $uber-definitions, ())
};

declare function hent:fix-tde($nodes as node()*, $entity-model-contexts as xs:string*, $uber-definitions as map:map, $entity-name as xs:string?)
{
  for $n in $nodes
  return
    typeswitch($n)
      case document-node() return
        document {
          hent:fix-tde($n/node(), $entity-model-contexts, $uber-definitions, $entity-name)
        }
      case element(tde:nullable) return
        $default-nullable
      case element(tde:invalid-values) return
        $default-invalid-values
      case element(tde:val) return
        element { fn:node-name($n) } {
          $n/namespace::node(),
          let $col-name := fn:string($n/../tde:name)
          return
            if (fn:ends-with($col-name, $generated-primary-key-column)) then
              $generated-primary-key-expression
            else if (fn:starts-with($n, $col-name)) then
              let $parts := fn:tokenize($n, "/")
              let $entity-definition := $uber-definitions => map:get(fn:string($parts[2]))
              return
                if (fn:exists($entity-definition)) then
                  let $primary-key := $entity-definition => map:get("primaryKey")
                  return
                    if ($primary-key = $generated-primary-key-column) then
                      $generated-primary-key-expression
                    else
                      fn:string($n) || "/" || $primary-key
                else
                  hent:fix-tde($n/node(), $entity-model-contexts, $uber-definitions)
            else
              hent:fix-tde($n/node(), $entity-model-contexts, $uber-definitions)
        }
      case element(tde:context) return
        element { fn:node-name($n) } {
          $n/namespace::node(),
          if ($n = $entity-model-contexts) then
            fn:replace(fn:replace(fn:string($n),"^\.//", "./"), "(.)$", "$1[node()]")
          else
            let $outer-context := fn:replace(fn:string($n),"//\*:instance", "/*:envelope/*:instance")
            return if(fn:not(fn:empty($entity-name))) then
              fn:concat($outer-context, "[*:", $entity-name, "]")
            else
              $outer-context
        }
      case element(tde:column) return
        element { fn:node-name($n) } {
          $n/namespace::node(),
          $n/@*,
          hent:fix-tde($n/* except $n/(tde:nullable|tde:invalid-values), $entity-model-contexts, $uber-definitions),
          $default-nullable,
          $default-invalid-values
        }
      case element(tde:subject)|element(tde:predicate)|element(tde:object) return
        element { fn:node-name($n) } {
          $n/namespace::node(),
          hent:fix-tde($n/* except $n/tde:invalid-values, $entity-model-contexts, $uber-definitions),
          $default-invalid-values
        }
      case element(tde:template) return
        element { fn:node-name($n) } {
          $n/namespace::node(),
          $n/@*,
          let $context-item :=  fn:replace(fn:string($n/tde:context), "\./", "")
          let $parent-context-item := fn:replace(fn:string($n/../../tde:context), "\./", "")
          let $is-join-template := $n/tde:rows/tde:row/tde:view-name = $parent-context-item  || "_" || $context-item
          let $rows := $n/tde:rows/tde:row
          return
            if ($is-join-template) then (
              hent:fix-tde($n/tde:context, $entity-model-contexts, $uber-definitions),
              element tde:rows {
                element tde:row {
                  $rows/(tde:schema-name|tde:view-name|tde:view-layout),
                  element tde:columns {
                    let $join-prefix := $context-item || "_"
                    for $column in $rows/tde:columns/tde:column
                    return
                      element tde:column {
                        $column/@*,
                        if (fn:starts-with($column/tde:name, $join-prefix)) then (
                          hent:fix-tde($column/(tde:name|tde:scalar-type), $entity-model-contexts, $uber-definitions),
                          let $tde-val := fn:string($column/tde:val)
                          let $primary-key := $uber-definitions => map:get($tde-val) => map:get("primaryKey")
                          return
                            element tde:val {
                              if ($primary-key = $generated-primary-key-column) then
                                $generated-primary-key-expression
                              else
                                $tde-val || "/" || $primary-key
                            },
                          $default-nullable,
                          $default-invalid-values,
                          hent:fix-tde($column/(tde:default|tde:reindexing|tde:collation), $entity-model-contexts, $uber-definitions)
                        ) else
                          hent:fix-tde($column/node(), $entity-model-contexts, $uber-definitions)
                      }
                  }
                }
              }
            ) else
              hent:fix-tde($n/node(), $entity-model-contexts, $uber-definitions, $entity-name)
        }
      case element() return
        element { fn:node-name($n) } {
          $n/namespace::node(),
          hent:fix-tde(($n/@*, $n/node()), $entity-model-contexts, $uber-definitions)
        }
      case text() return
        fn:replace(
          fn:replace($n, "^\.\./(.+)$", "(../$1|parent::array-node()/../$1)"),
          "\./" || $generated-primary-key-column,
          $generated-primary-key-expression
        )
      default return $n
};

declare variable $number-types as xs:string+ := ("byte","decimal","double","float","int","integer","long","negativeInteger","nonNegativeInteger","nonPositiveInteger","positiveInteger","short","unsignedLong","unsignedInt","unsignedShort","unsignedByte");
declare variable $string-types as xs:string+ := "dateTime";

declare function hent:json-schema-generate($entity-title as xs:string, $uber-model as map:map)
{
  let $uber-model := map:new((
  (: Ensure we're not change a map for anyone else :)
  map:map(document{$uber-model}/*),
  map:entry("language", "zxx"),
  map:entry("$schema", "http://json-schema.org/draft-07/schema#")
  ))
  let $definitions := $uber-model => map:get("definitions")
  (: JSON Schema needs an extra level of wrapping to account for Entity Model label wrapping it. :)
  let $_nest-refs :=
    for $definition-type in map:keys($definitions)
    let $definition-properties := $definitions => map:get($definition-type) => map:get("properties")
    for $property-name in map:keys($definition-properties)
    let $property := $definition-properties => map:get($property-name)
    let $property-items := $property => map:get("items")
    let $datatype := $property => map:get("datatype")
    let $_set-types := (
      $property => map:put("type", if ($datatype = $number-types) then "number" else if ($datatype = $string-types) then "string" else $datatype),
      $property => map:delete("datatype"),
      if ($property-items instance of map:map) then (
        let $items-datatype := $property => map:get("datatype")
        return (
          $property-items => map:put("type", if ($items-datatype = $number-types) then "number" else if ($items-datatype = $string-types) then "string" else $datatype),
          $property-items => map:delete("datatype"))
      ) else ()
    )
    return
    (: references can be in the property or in items for arrays :)
      if ($property => map:contains("$ref")) then
        map:put($definition-properties, $property-name,
          map:new((
            map:entry("type", "object"),
            map:entry("properties",
              map:entry(fn:tokenize(map:get($property,"$ref"),"/")[fn:last()], $property)
            )
          ))
        )
      else if ($property-items instance of map:map and $property-items => map:contains("$ref")) then
        map:put($property, "items",
          map:new((
            map:entry("type", "object"),
            map:entry("properties",
              map:entry(fn:tokenize(map:get($property-items,"$ref"),"/")[fn:last()], $property-items)
            )
          ))
        )
      else ()
  let $_set-info := (
    $uber-model => map:put("properties", map:entry($entity-title, map:entry("$ref", "#/definitions/"||$entity-title)))
  )
  return xdmp:to-json($uber-model)
};


[bugfix] Fix the incorrect path to the test document in positional-nested.xql

xquery version "3.1";

module namespace npt="http://exist-db.org/test/nested-positional-predicate";

declare namespace test="http://exist-db.org/xquery/xqsuite";

declare variable $npt:TEST_COLLECTION_NAME := "test-positional-nested";
declare variable $npt:TEST_COLLECTION_URI := "/db/" || $npt:TEST_COLLECTION_NAME;
declare variable $npt:TEST_DOC_NAME := "test.xml";
declare variable $npt:TEST_DOC_URI := $npt:TEST_COLLECTION_URI || "/" || $npt:TEST_DOC_NAME;

declare variable $npt:DATA :=
    document {
        <xml>

                B1

                B2
                <c>correct</c>

                B3
                <c>wrong</c>

        </xml>
    };

declare
    %test:setUp
function npt:setup() {
    xmldb:create-collection("/db", $npt:TEST_COLLECTION_NAME),
    xmldb:store($npt:TEST_COLLECTION_URI, $npt:TEST_DOC_NAME, $npt:DATA)
};

declare
    %test:tearDown
function npt:cleanup() {
    xmldb:remove($npt:TEST_COLLECTION_URI)
};

declare
    %test:assertEquals("<c>correct</c><c>wrong</c>")
function npt:in-memory() {
    $npt:DATA//c[../preceding-sibling::a]
};

declare
    %test:assertEquals("<c>correct</c><c>wrong</c>")
function npt:in-database() {
    doc($npt:TEST_DOC_URI)//c[../preceding-sibling::a]
};

declare
    %test:assertEquals("<c>correct</c><c>wrong</c>")
function npt:in-memory-predicate() {
    $npt:DATA//c[../preceding-sibling::a[1]]
};

declare
    %test:assertEquals("<c>correct</c><c>wrong</c>")
function npt:in-database-predicate() {
    doc($npt:TEST_DOC_URI)//c[../preceding-sibling::a[1]]
};

declare
    %test:assertEquals("<c>correct</c><c>wrong</c>")
function npt:in-memory-position() {
    $npt:DATA//c[../preceding-sibling::a[position() eq 1]]
};

declare
    %test:assertEquals("<c>correct</c><c>wrong</c>")
function npt:in-database-position() {
    doc($npt:TEST_DOC_URI)//c[../preceding-sibling::a[position() eq 1]]
};

declare
    %test:assertEquals("<c>correct</c>")
function npt:in-memory-predicate-and-path() {
    $npt:DATA//c[../preceding-sibling::a[1]/b = 'B1']
};

declare
    %test:assertEquals("<c>correct</c>")
function npt:in-database-predicate-and-path() {
    doc($npt:TEST_DOC_URI)//c[../preceding-sibling::a[1]/b = 'B1']
};

declare
    %test:assertEquals("<c>correct</c>")
function npt:in-memory-position-and-path() {
    $npt:DATA//c[../preceding-sibling::a[position() eq 1]/b = 'B1']
};

declare
    %test:assertEquals("<c>correct</c>")
function npt:in-database-position-and-path() {
    doc($npt:TEST_DOC_URI)//c[../preceding-sibling::a[position() eq 1]/b = 'B1']
};

xquery version "3.1";

module namespace npt="http://exist-db.org/test/nested-positional-predicate";

declare namespace test="http://exist-db.org/xquery/xqsuite";

declare variable $npt:DATA :=
    document {
        <xml>

                B1

                B2
                <c>correct</c>

                B3
                <c>wrong</c>

        </xml>
    };

declare
    %test:setUp
function npt:setup() {
    xmldb:create-collection("/db", "test"),
    xmldb:store("/db/test", "test.xml", $npt:DATA)
};

declare
    %test:tearDown
function npt:cleanup() {
    xmldb:remove("/db/test")
};

declare
    %test:assertEquals("<c>correct</c><c>wrong</c>")
function npt:in-memory() {
    $npt:DATA//c[../preceding-sibling::a]
};

declare
    %test:assertEquals("<c>correct</c><c>wrong</c>")
function npt:in-database() {
    doc("/db/test.xml")//c[../preceding-sibling::a]
};

declare
    %test:assertEquals("<c>correct</c><c>wrong</c>")
function npt:in-memory-predicate() {
    $npt:DATA//c[../preceding-sibling::a[1]]
};

declare
    %test:assertEquals("<c>correct</c><c>wrong</c>")
function npt:in-database-predicate() {
    doc("/db/test.xml")//c[../preceding-sibling::a[1]]
};

declare
    %test:assertEquals("<c>correct</c><c>wrong</c>")
function npt:in-memory-position() {
    $npt:DATA//c[../preceding-sibling::a[position() eq 1]]
};

declare
    %test:assertEquals("<c>correct</c><c>wrong</c>")
function npt:in-database-position() {
    doc("/db/test.xml")//c[../preceding-sibling::a[position() eq 1]]
};

declare
    %test:assertEquals("<c>correct</c>")
function npt:in-memory-predicate-and-path() {
    $npt:DATA//c[../preceding-sibling::a[1]/b = 'B1']
};

declare
    %test:assertEquals("<c>correct</c>")
function npt:in-database-predicate-and-path() {
    doc("/db/test.xml")//c[../preceding-sibling::a[1]/b = 'B1']
};

declare
    %test:assertEquals("<c>correct</c>")
function npt:in-memory-position-and-path() {
    $npt:DATA//c[../preceding-sibling::a[position() eq 1]/b = 'B1']
};

declare
    %test:assertEquals("<c>correct</c>")
function npt:in-database-position-and-path() {
    doc("/db/test.xml")//c[../preceding-sibling::a[position() eq 1]/b = 'B1']
};


fixing $version and $title (#702)

fixing $version and $title (#702)

(:
  Copyright 2012-2016 MarkLogic Corporation

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
:)
xquery version "1.0-ml";

module namespace flow = "http://marklogic.com/data-hub/flow-lib";

import module namespace consts = "http://marklogic.com/data-hub/consts"
  at "/com.marklogic.hub/lib/consts.xqy";

import module namespace debug = "http://marklogic.com/data-hub/debug"
  at "/com.marklogic.hub/lib/debug-lib.xqy";

import module namespace hul = "http://marklogic.com/data-hub/hub-utils-lib"
  at "/com.marklogic.hub/lib/hub-utils-lib.xqy";

import module namespace json="http://marklogic.com/xdmp/json"
  at "/MarkLogic/json/json.xqy";

import module namespace functx = "http://www.functx.com"
  at "/MarkLogic/functx/functx-1.0-nodoc-2007-01.xqy";

import module namespace rfc = "http://marklogic.com/data-hub/run-flow-context"
  at "/com.marklogic.hub/lib/run-flow-context.xqy";

import module namespace trace = "http://marklogic.com/data-hub/trace"
  at "/com.marklogic.hub/lib/trace-lib.xqy";

declare namespace hub = "http://marklogic.com/data-hub";

declare option xdmp:mapping "false";

(: the directory where entities live :)
declare variable $ENTITIES-DIR := "/entities/";

declare variable $PLUGIN-NS := "http://marklogic.com/data-hub/plugins";

declare function flow:get-module-ns(
  $type as xs:string) as xs:string?
{
  if ($type eq $consts:JAVASCRIPT) then ()
  else
    $PLUGIN-NS
};

(:~
 : Returns a flow by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity that owns the flow
 : @param $flow-name - name of the flow to retrieve
 : @return - xml describing the flow
 :)
declare function flow:get-flow(
  $entity-name as xs:string,
  $flow-name as xs:string,
  $flow-type as xs:string?) as element(hub:flow)?
{
  hul:run-in-modules(function() {
    /hub:flow[
      hub:entity = $entity-name and
      hub:name = $flow-name]
      [
        if (fn:exists($flow-type)) then
          hub:type = $flow-type
        else
          fn:true()
      ]
  })
};

(:~
 : Returns the flows that belong to the given entity in the database
 :
 : @param $entity-name - the name of the entity containing the flows
 : @return - xml describing the flows
 :)
declare function flow:get-flows(
  $entity-name as xs:string) as element(hub:flows)
{
  element hub:flows {
    hul:run-in-modules(function() {
      /hub:flow[hub:entity = $entity-name]
    })
  }
};

(:~
 : Returns an entity by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity to retrieve
 : @return - xml describing the entity
 :)
declare function flow:get-entity(
  $entity-name as xs:string)
{
  let $uris :=
    hul:run-in-modules(function() {
      cts:uri-match($ENTITIES-DIR || $entity-name || "/*")
    })
  return
    flow:get-entity($entity-name, $uris)
};

(:~
 : Returns a entity by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity to retrieve
 : @param $uris - uris used to build the entity xml
 : @return - xml describing the entity
 :)
declare %private function flow:get-entity(
  $entity-name as xs:string,
  $uris as xs:string*)
  as element(hub:entity)
{
  <entity xmlns="http://marklogic.com/data-hub">
    <name>{$entity-name}</name>
    {
      flow:get-flows($entity-name)
    }
  </entity>
};

(:~
 : Returns the entities in the database
 :
 : @return - xml describing the entities
 :)
declare function flow:get-entities() as element(hub:entities)
{
  let $uris := hul:run-in-modules(function() {
    cts:uri-match($ENTITIES-DIR || "*")
  })
  let $entity-names :=
    fn:distinct-values(
      let $regex := $ENTITIES-DIR || "([^/]+)/.*$"
      for $flow in $uris[fn:matches(., $regex)]
      let $name := fn:replace($flow, $regex, "$1")
      return
        $name)
  let $entities :=
    for $entity-name in $entity-names
    return
      flow:get-entity($entity-name, $uris[fn:matches(., $ENTITIES-DIR || $entity-name || "/.+")])
  return
    <entities xmlns="http://marklogic.com/data-hub">
    {
      $entities
    }
    </entities>
};

(:~
 : Runs a collector
 :
 : @param $module-uri - the uri of the collector module
 : @param $options - a map of options passed in by the client
 : @return - a sequence of strings
 :)
declare function flow:run-collector(
  $flow as element(hub:flow),
  $job-id as xs:string,
  $options as map:map) as item()*
{
    (: sanity check on required info :)
  if (fn:empty($flow/hub:collector/@module) or fn:empty($flow/hub:collector/@code-format)) then
    fn:error(xs:QName("INVALID-PLUGIN"), "The plugin definition is invalid.")
  else (),

  (: assert that we are in query mode :)
  let $_ts as xs:unsignedLong := xdmp:request-timestamp()

  let $_ := trace:set-plugin-label("collector")

  (: configure the run context :)
  let $_ := (
    rfc:with-job-id($job-id),
    rfc:with-flow($flow),
    rfc:with-module-uri($flow/hub:collector/@module),
    rfc:with-options($options)
  )

  let $func := flow:make-function(
    $flow/hub:collector/@code-format,
    "collect",
    $flow/hub:collector/@module
  )
  let $before := xdmp:elapsed-time()
  let $resp :=
    try {
      $func($options)
    }
    catch($ex) {
      debug:log(xdmp:describe($ex, (), ())),
      trace:error-trace($ex, xdmp:elapsed-time() - $before),
      xdmp:rethrow()
    }
  (: log and write the trace :)
  let $_ := (
    trace:plugin-trace($resp, xdmp:elapsed-time() - $before),
    trace:write-trace()
  )
  return
    $resp
};

(:~
 : Runs a given flow
 :
 : @param $flow - xml describing the flow
 : @param $identifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $options - a map of options passed in by the client
 : @return - nothing
 :)
declare function flow:run-flow(
  $job-id as xs:string,
  $flow as element(hub:flow),
  $identifier as xs:string,
  $options as map:map)
{
  flow:run-flow($job-id, $flow, $identifier, (), $options)
};

(:~
 : Runs a given flow
 :
 : @param $flow - xml describing the flow
 : @param $identifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $content - the content being loaded
 : @param $options - a map of options passed in by the client
 : @return - nothing
 :)
declare function flow:run-flow(
  $job-id as xs:string,
  $flow as element(hub:flow),
  $identifier as xs:string,
  $content as item()?,
  $options as map:map)
{
  (: assert that we are in query mode :)
  let $_must_run_in_query_mode as xs:unsignedLong := xdmp:request-timestamp()

  (: configure the run context :)
  let $_ := (
    rfc:with-job-id($job-id),
    rfc:with-flow($flow),
    rfc:with-id($identifier),
    rfc:with-content(
      if ($content instance of document-node()) then
        $content/node()
      else $content
    ),
    rfc:with-options($options),
    map:get($options, "target-database") ! rfc:with-target-database(.)
  )

  (: run the users main.(sjs|xqy) :)
  return
    flow:run-main($flow/hub:main)
};

declare function flow:clean-data($resp, $destination, $data-format)
{
  let $resp :=
    typeswitch($resp)
      case document-node() return
        if (fn:count($resp/node()) > 1) then
          fn:error(xs:QName("TOO_MANY_NODES"), "Too Many Nodes!. Return just 1 node")
        else
          $resp/node()
      default return
        $resp

  (: clean up output :)
  return
    typeswitch($resp)
      case object-node() | json:object return
        (: object with $type key is ES response type :)
        if ($resp instance of map:map and map:keys($resp) = "$type") then
          $resp
        else if ($data-format = $consts:XML) then
          json:transform-from-json($resp, json:config("custom"))
        else
          $resp
      case json:array return
        if ($data-format = $consts:XML) then
          json:array-values($resp)
        else
          $resp
      case empty-sequence() return
        if ($destination = "headers" and $data-format = $consts:JSON) then
          json:object()
        else if ($destination = "triples" and $data-format = $consts:JSON) then
          json:array()
        else
          $resp
      default return
        if ($data-format = $consts:JSON and
            rfc:get-code-format() = $consts:XQUERY and
            $destination = "triples") then
          json:to-array($resp)
        else
          $resp
};

(:~
 : Construct an envelope
 :
 : @param $map - a map with all the stuff in it
 : @return - the newly constructed envelope
 :)
declare function flow:make-envelope($content, $headers, $triples, $data-format)
  as document-node()
{
  let $content := flow:clean-data($content, "content", $data-format)
  let $headers := flow:clean-data($headers, "headers", $data-format)
  let $triples := flow:clean-data($triples, "triples", $data-format)
  return
    if ($data-format = $consts:JSON) then
      let $envelope :=
        let $o := json:object()
        let $_ := (
          map:put($o, "headers", $headers),
          map:put($o, "triples", $triples),
          map:put($o, "instance",
            if ($content instance of map:map and map:keys($content) = "$type") then
              let $json := flow:instance-to-canonical-json($content)
              let $info :=
                let $o :=json:object()
                let $_ := (
                  map:put($o, "title", map:get($content, "$type")),
                  map:put($o, "version",  map:get($content, "$version"))
                )
                return $o
              let $_ := map:put($json, "info", $info)
              return $json
            else
              $content
          ),
          map:put($o, "attachments",
            if ($content instance of map:map and map:keys($content) = "$attachments") then
              map:get($content, "$attachments")
            else
              ()
          )
        )
        return
          $o
      let $wrapper := json:object()
      let $_ := map:put($wrapper, "envelope", $envelope)
      return
        xdmp:to-json($wrapper)
    else if ($data-format = $consts:XML) then
      document {
        <envelope xmlns="http://marklogic.com/entity-services">
          <headers>{$headers}</headers>
          <triples>{$triples}</triples>
          <instance>
          {
            if ($content instance of map:map and map:keys($content) = "$type") then (
              <info>
                <title>{map:get($content, "$type")}</title>
                <version>{map:get($content, "$version")}</version>
              </info>,
              flow:instance-to-canonical-xml($content)
            )
            else
              $content
          }
          </instance>
          <attachments>
          {
            if ($content instance of map:map and map:keys($content) = "$attachments") then
              map:get($content, "$attachments")
            else
              ()
          }
          </attachments>
        </envelope>
      }
    else
      fn:error(xs:QName("INVALID-DATA-FORMAT"), "Invalid data format: " || $data-format)
};

declare function flow:make-legacy-envelope($content, $headers, $triples, $data-format)
  as document-node()
{
  let $content := flow:clean-data($content, "content", $data-format)
  let $headers := flow:clean-data($headers, "headers", $data-format)
  let $triples := flow:clean-data($triples, "triples", $data-format)
  return
    if ($data-format = $consts:JSON) then
      let $envelope :=
        let $o := json:object()
        let $_ := (
          map:put($o, "headers", $headers),
          map:put($o, "triples", $triples),
          map:put($o, "content", $content)
        )
        return
          $o
      let $wrapper := json:object()
      let $_ := map:put($wrapper, "envelope", $envelope)
      return
        xdmp:to-json($wrapper)
    else if ($data-format = $consts:XML) then
      document {
        <envelope xmlns="http://marklogic.com/data-hub/envelope">
          <headers>{$headers}</headers>
          <triples>{$triples}</triples>
          <content>{$content}</content>
        </envelope>
      }
    else
      fn:error(xs:QName("INVALID-DATA-FORMAT"), "Invalid data format: " || $data-format)
};

declare function flow:instance-to-canonical-json(
  $entity-instance as map:map) as json:object
{
  let $root-object := json:object()
  let $o :=
    if ( map:contains($entity-instance, "$ref") ) then
      map:get($entity-instance, "$ref")
    else
      let $o := json:object()
      let $_ := (
        for $key in map:keys($entity-instance)
        let $instance-property := map:get($entity-instance, $key)
        where ($key castable as xs:NCName and $key ne "$type")
        return
          typeswitch ($instance-property)
          (: This branch handles embedded objects.  You can choose to prune
             an entity's representation of extend it with lookups here. :)
          case json:object+ return
            for $prop in $instance-property
            return
              map:put($o, $key, flow:instance-to-canonical-json($prop))
          (: An array can also treated as multiple elements :)
          case json:array return
            let $a := json:array()
            let $_ :=
              for $val in json:array-values($instance-property)
              return
                if ($val instance of json:object) then
                  json:array-push($a, flow:instance-to-canonical-json($val))
                else
                  json:array-push($a, $val)
            return
              map:put($o, $key, $a)
          (: A sequence of values should be simply treated as multiple elements :)
          case item()+ return
            for $val in $instance-property
            return
              map:put($o, $key, $val)
          default return
            map:put($o, $key, $instance-property)
      )
      return
        $o
  let $_ := map:put($root-object, map:get($entity-instance, "$type"), $o)
  return
    $root-object
};

declare function flow:instance-to-canonical-xml(
  $entity-instance as map:map) as element()
{
  (: Construct an element that is named the same as the Entity Type :)
  element { map:get($entity-instance, "$type") }  {
    if ( map:contains($entity-instance, "$ref") ) then
      map:get($entity-instance, "$ref")
    else
      for $key in map:keys($entity-instance)
      let $instance-property := map:get($entity-instance, $key)
      where ($key castable as xs:NCName and $key ne "$type")
      return
        typeswitch ($instance-property)
        (: This branch handles embedded objects.  You can choose to prune
           an entity's representation of extend it with lookups here. :)
        case json:object+ return
          for $prop in $instance-property
          return
            element { $key } {
              flow:instance-to-canonical-xml($prop)
            }
        (: An array can also treated as multiple elements :)
        case json:array return
          for $val in json:array-values($instance-property)
          return
            if ($val instance of json:object) then
              element { $key } {
                attribute datatype { "array" },
                flow:instance-to-canonical-xml($val)
              }
            else
              element { $key } {
                attribute datatype { "array" },
                $val
              }
        (: A sequence of values should be simply treated as multiple elements :)
        case item()+ return
          for $val in $instance-property
          return
            element { $key } { $val }
        default return
          element { $key } { $instance-property }
  }
};

(: Formats the input for a trace based on the plugin type :)
declare function flow:get-trace-input(
  $plugin as element(hub:plugin),
  $data-format as xs:string,
  $content as item()?,
  $headers as item()*,
  $flow-type as xs:string)
{
  let $destination as xs:string := $plugin/@dest
  let $content :=
    if ($content instance of document-node()) then
      $content/node()
    else
      $content
  let $o := json:object()
  let $_ :=
    switch($destination)
      case "content" return
        if ($flow-type = $consts:INPUT_FLOW) then
          if ($content instance of binary()) then
            map:put($o, "rawContent", "binary document")
          else
            map:put($o, "rawContent", $content)
        else
          ()
      case "headers" return
        map:put($o, "content", $content)
      case "triples" return
      (
        map:put($o, "content", $content),
        map:put($o, "headers", ($headers, null-node{})[1])
      )
      default return ()
  return
    if ($data-format = $consts:XML) then
      for $key in map:keys($o)
      return
        element { $key } {
          let $value := map:get($o, $key)
          return
            if ($value instance of null-node()) then ()
            else $value
        }
    else
      $o
};

declare function flow:set-default-options(
  $options as map:map,
  $flow as element(hub:flow))
{
  map:put($options, "entity", fn:string($flow/hub:entity)),
  map:put($options, "flow", fn:string($flow/hub:name)),
  map:put($options, "flowType", fn:string($flow/hub:type)),
  map:put($options, "dataFormat", fn:string($flow/hub:data-format))
};

declare function flow:run-main(
  $main as element(hub:main))
{
  (: sanity check on required info :)
  if (fn:empty($main/@module) or fn:empty($main/@code-format)) then
    fn:error(xs:QName("INVALID-PLUGIN"), "The plugin definition is invalid.")
  else (),

  let $module-uri as xs:string? := $main/@module
  let $_ := rfc:with-module-uri($module-uri)
  let $func := flow:make-function($main/@code-format, "main", $module-uri)
  let $before := xdmp:elapsed-time()
  let $resp := try {
    let $options := rfc:get-options()
    let $_ := map:set-javascript-by-ref($options, fn:true())
    let $resp :=
      if (rfc:get-flow-type() eq $consts:HARMONIZE_FLOW) then
        $func(rfc:get-id(), $options)
      else
        $func(rfc:get-id(), rfc:get-content(), $options)
    (: write the trace for the current identifier :)
    let $_ := trace:write-trace()
    return
      $resp
  }
  catch($ex) {
    if ($ex/error:name eq "PLUGIN-ERROR") then
      (: plugin errors are already handled :)
      ()
    else (
      (: this is an error in main.(sjs|xqy) :)
      debug:log(xdmp:describe($ex, (), ())),

      (: log the trace event for main :)
      trace:set-plugin-label("main"),
      trace:error-trace($ex, xdmp:elapsed-time() - $before)
    ),
    xdmp:rethrow()
  }
  return
    $resp
};

(:~
 : Run a given writer
 :
 : @param $writer - xml describing the writer to run
 : @param $identifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $envelope - the envelope
 : @param $options - a map of options passed in by the client
 : @return - the output of the writer. It varies.
 :)
declare function flow:run-writer(
  $writer-function,
  $identifier as xs:string,
  $envelope as item(),
  $options as map:map)
{
  let $before := xdmp:elapsed-time()
  let $resp :=
    try {
      xdmp:eval('
        declare variable $func external;
        declare variable $identifier external;
        declare variable $envelope external;
        declare variable $options external;

        $func($identifier, $envelope, $options)
      ',
      map:new((
        map:entry("func", $writer-function),
        map:entry("identifier", $identifier),
        map:entry("envelope", $envelope),
        map:entry("options", $options)
      )),
      map:new((
        map:entry("isolation", "different-transaction"),
        map:entry("database", rfc:get-target-database()),
        map:entry("transactionMode", "update-auto-commit")
      )))
    }
    catch($ex) {
      debug:log(xdmp:describe($ex, (), ())),
      trace:error-trace($ex, xdmp:elapsed-time() - $before),
      fn:error(xs:QName("PLUGIN-ERROR"), "error in writer", $ex)
    }
  let $duration := xdmp:elapsed-time() - $before
  let $_ := trace:plugin-trace((), xdmp:elapsed-time() - $before)
  return
    $resp
};

declare function flow:make-error-json(
  $errors as json:object,
  $entity as xs:string,
  $flow  as xs:string,
  $plugin as xs:string,
  $ex
) {
  let $eo :=
    if (map:contains($errors, $entity)) then
      map:get($errors, $entity)
    else
      let $e := map:map()
      let $_ := map:put($errors, $entity, $e)
      return
        $e
  let $fo :=
    if (map:contains($eo, $flow)) then
      map:get($eo, $flow)
    else
      let $f := map:map()
      let $_ := map:put($eo, $flow, $f)
      return
        $f

  let $f := $ex/error:stack/error:frame[1]
  return
    map:put($fo, $plugin, map:new((
      map:entry("uri", $f/error:uri/fn:data()),
      map:entry("line", $f/error:line/fn:data()),
      map:entry("column", $f/error:column/fn:data()),
      map:entry("msg", $ex/error:format-string/fn:data())
    )))
};

declare function flow:validate-entities()
{
  let $errors := json:object()
  let $options := map:map()
  let $_ :=
    for $entity in flow:get-entities()/hub:entity
    for $flow in $entity/hub:flows/hub:flow
    let $data-format := $flow/hub:data-format
    (: validate collector :)
    let $_ :=
      try {
        let $collector := $flow/hub:collector
        return
          if ($collector) then
            let $module-uri := $collector/@module
            let $ns := flow:get-module-ns($collector/@code-format)
            return
              if ($collector/@code-format eq $consts:XQUERY) then
                xdmp:eval(
                  'import module namespace x = "' || $ns || '" at "' || $module-uri || '"; ' ||
                  '()',
                  map:new(map:entry("staticCheck", fn:true()))
                )
              else
                xdmp:javascript-eval(
                  'var x = require("' || $module-uri || '");',
                  map:new(map:entry("staticCheck", fn:true()))
                )
          else ()
      }
      catch($ex) {
        flow:make-error-json(
          $errors,
          $entity/hub:name,
          $flow/hub:name,
          "collector",
          $ex)
      }
    (: validate plugins :)
    let $_ :=
      for $main in $flow/hub:main
      let $module-uri := $main/@module
      let $ns := flow:get-module-ns($main/@code-format)
      return
        (:
         : Note that we are static checking the files.
         : This is because there is no reasonable way to actually
         : call the plugins and pass in data that will work for all plugins.
         :
         : The disadvantage to static checking is that we will not catch typos
         : like ctsdoc <- (missing period) because Javascript is dynamically
         : typed. Static checking will only catch syntax errors in sjs.
         :)
        try {
          if ($main/@code-format eq $consts:XQUERY) then
            xdmp:eval(
              'import module namespace x = "' || $ns || '" at "' || $module-uri || '"; ' ||
              '()',
              map:new(map:entry("staticCheck", fn:true()))
            )
          else
            xdmp:javascript-eval(
              'var x = require("' || $module-uri || '");',
              map:new(map:entry("staticCheck", fn:true()))
            )
        }
        catch($ex) {
          let $plugin :=
            let $uri := ($ex/error:stack/error:frame)[1]/error:uri
            let $name := hul:get-file-name(hul:get-file-from-uri($uri))
            return
              if ($name eq "[anonymous]") then
                fn:replace($ex/error:expr, ".+/([^.]+)\.(sjs|xqy).*", "$1")
              else
                $name
          return
            flow:make-error-json(
              $errors,
              $entity/hub:name,
              $flow/hub:name,
              $plugin,
              $ex)
        }
    return
      ()
  return
    map:new(
      map:entry("errors", $errors)
    )
};

declare function flow:safe-run($func)
{
  let $before := xdmp:elapsed-time()
  return
    try {
      let $resp := $func()
      let $duration := xdmp:elapsed-time() - $before
      let $_ := trace:plugin-trace($resp, $duration)
      return
        $resp
    }
    catch($ex) {
      debug:log(xdmp:describe($ex, (), ())),
      trace:error-trace($ex, xdmp:elapsed-time() - $before),
      fn:error(xs:QName("PLUGIN-ERROR"), "error in a plugin", $ex)
    }
};

declare %private function flow:make-function(
  $code-format as xs:string?,
  $func-name as xs:string,
  $module-uri as xs:string)
{
  let $ns := flow:get-module-ns($code-format)
  return
    xdmp:function(fn:QName($ns, $func-name), $module-uri)
};

(:
  Copyright 2012-2016 MarkLogic Corporation

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
:)
xquery version "1.0-ml";

module namespace flow = "http://marklogic.com/data-hub/flow-lib";

import module namespace consts = "http://marklogic.com/data-hub/consts"
  at "/com.marklogic.hub/lib/consts.xqy";

import module namespace debug = "http://marklogic.com/data-hub/debug"
  at "/com.marklogic.hub/lib/debug-lib.xqy";

import module namespace hul = "http://marklogic.com/data-hub/hub-utils-lib"
  at "/com.marklogic.hub/lib/hub-utils-lib.xqy";

import module namespace json="http://marklogic.com/xdmp/json"
  at "/MarkLogic/json/json.xqy";

import module namespace functx = "http://www.functx.com"
  at "/MarkLogic/functx/functx-1.0-nodoc-2007-01.xqy";

import module namespace rfc = "http://marklogic.com/data-hub/run-flow-context"
  at "/com.marklogic.hub/lib/run-flow-context.xqy";

import module namespace trace = "http://marklogic.com/data-hub/trace"
  at "/com.marklogic.hub/lib/trace-lib.xqy";

declare namespace hub = "http://marklogic.com/data-hub";

declare option xdmp:mapping "false";

(: the directory where entities live :)
declare variable $ENTITIES-DIR := "/entities/";

declare variable $PLUGIN-NS := "http://marklogic.com/data-hub/plugins";

declare function flow:get-module-ns(
  $type as xs:string) as xs:string?
{
  if ($type eq $consts:JAVASCRIPT) then ()
  else
    $PLUGIN-NS
};

(:~
 : Returns a flow by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity that owns the flow
 : @param $flow-name - name of the flow to retrieve
 : @return - xml describing the flow
 :)
declare function flow:get-flow(
  $entity-name as xs:string,
  $flow-name as xs:string,
  $flow-type as xs:string?) as element(hub:flow)?
{
  hul:run-in-modules(function() {
    /hub:flow[
      hub:entity = $entity-name and
      hub:name = $flow-name]
      [
        if (fn:exists($flow-type)) then
          hub:type = $flow-type
        else
          fn:true()
      ]
  })
};

(:~
 : Returns the flows that belong to the given entity in the database
 :
 : @param $entity-name - the name of the entity containing the flows
 : @return - xml describing the flows
 :)
declare function flow:get-flows(
  $entity-name as xs:string) as element(hub:flows)
{
  element hub:flows {
    hul:run-in-modules(function() {
      /hub:flow[hub:entity = $entity-name]
    })
  }
};

(:~
 : Returns an entity by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity to retrieve
 : @return - xml describing the entity
 :)
declare function flow:get-entity(
  $entity-name as xs:string)
{
  let $uris :=
    hul:run-in-modules(function() {
      cts:uri-match($ENTITIES-DIR || $entity-name || "/*")
    })
  return
    flow:get-entity($entity-name, $uris)
};

(:~
 : Returns a entity by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity to retrieve
 : @param $uris - uris used to build the entity xml
 : @return - xml describing the entity
 :)
declare %private function flow:get-entity(
  $entity-name as xs:string,
  $uris as xs:string*)
  as element(hub:entity)
{
  <entity xmlns="http://marklogic.com/data-hub">
    <name>{$entity-name}</name>
    {
      flow:get-flows($entity-name)
    }
  </entity>
};

(:~
 : Returns the entities in the database
 :
 : @return - xml describing the entities
 :)
declare function flow:get-entities() as element(hub:entities)
{
  let $uris := hul:run-in-modules(function() {
    cts:uri-match($ENTITIES-DIR || "*")
  })
  let $entity-names :=
    fn:distinct-values(
      let $regex := $ENTITIES-DIR || "([^/]+)/.*$"
      for $flow in $uris[fn:matches(., $regex)]
      let $name := fn:replace($flow, $regex, "$1")
      return
        $name)
  let $entities :=
    for $entity-name in $entity-names
    return
      flow:get-entity($entity-name, $uris[fn:matches(., $ENTITIES-DIR || $entity-name || "/.+")])
  return
    <entities xmlns="http://marklogic.com/data-hub">
    {
      $entities
    }
    </entities>
};

(:~
 : Runs a collector
 :
 : @param $module-uri - the uri of the collector module
 : @param $options - a map of options passed in by the client
 : @return - a sequence of strings
 :)
declare function flow:run-collector(
  $flow as element(hub:flow),
  $job-id as xs:string,
  $options as map:map) as item()*
{
    (: sanity check on required info :)
  if (fn:empty($flow/hub:collector/@module) or fn:empty($flow/hub:collector/@code-format)) then
    fn:error(xs:QName("INVALID-PLUGIN"), "The plugin definition is invalid.")
  else (),

  (: assert that we are in query mode :)
  let $_ts as xs:unsignedLong := xdmp:request-timestamp()

  let $_ := trace:set-plugin-label("collector")

  (: configure the run context :)
  let $_ := (
    rfc:with-job-id($job-id),
    rfc:with-flow($flow),
    rfc:with-module-uri($flow/hub:collector/@module),
    rfc:with-options($options)
  )

  let $func := flow:make-function(
    $flow/hub:collector/@code-format,
    "collect",
    $flow/hub:collector/@module
  )
  let $before := xdmp:elapsed-time()
  let $resp :=
    try {
      $func($options)
    }
    catch($ex) {
      debug:log(xdmp:describe($ex, (), ())),
      trace:error-trace($ex, xdmp:elapsed-time() - $before),
      xdmp:rethrow()
    }
  (: log and write the trace :)
  let $_ := (
    trace:plugin-trace($resp, xdmp:elapsed-time() - $before),
    trace:write-trace()
  )
  return
    $resp
};

(:~
 : Runs a given flow
 :
 : @param $flow - xml describing the flow
 : @param $identifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $options - a map of options passed in by the client
 : @return - nothing
 :)
declare function flow:run-flow(
  $job-id as xs:string,
  $flow as element(hub:flow),
  $identifier as xs:string,
  $options as map:map)
{
  flow:run-flow($job-id, $flow, $identifier, (), $options)
};

(:~
 : Runs a given flow
 :
 : @param $flow - xml describing the flow
 : @param $identifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $content - the content being loaded
 : @param $options - a map of options passed in by the client
 : @return - nothing
 :)
declare function flow:run-flow(
  $job-id as xs:string,
  $flow as element(hub:flow),
  $identifier as xs:string,
  $content as item()?,
  $options as map:map)
{
  (: assert that we are in query mode :)
  let $_must_run_in_query_mode as xs:unsignedLong := xdmp:request-timestamp()

  (: configure the run context :)
  let $_ := (
    rfc:with-job-id($job-id),
    rfc:with-flow($flow),
    rfc:with-id($identifier),
    rfc:with-content(
      if ($content instance of document-node()) then
        $content/node()
      else $content
    ),
    rfc:with-options($options),
    map:get($options, "target-database") ! rfc:with-target-database(.)
  )

  (: run the users main.(sjs|xqy) :)
  return
    flow:run-main($flow/hub:main)
};

declare function flow:clean-data($resp, $destination, $data-format)
{
  let $resp :=
    typeswitch($resp)
      case document-node() return
        if (fn:count($resp/node()) > 1) then
          fn:error(xs:QName("TOO_MANY_NODES"), "Too Many Nodes!. Return just 1 node")
        else
          $resp/node()
      default return
        $resp

  (: clean up output :)
  return
    typeswitch($resp)
      case object-node() | json:object return
        (: object with $type key is ES response type :)
        if ($resp instance of map:map and map:keys($resp) = "$type") then
          $resp
        else if ($data-format = $consts:XML) then
          json:transform-from-json($resp, json:config("custom"))
        else
          $resp
      case json:array return
        if ($data-format = $consts:XML) then
          json:array-values($resp)
        else
          $resp
      case empty-sequence() return
        if ($destination = "headers" and $data-format = $consts:JSON) then
          json:object()
        else if ($destination = "triples" and $data-format = $consts:JSON) then
          json:array()
        else
          $resp
      default return
        if ($data-format = $consts:JSON and
            rfc:get-code-format() = $consts:XQUERY and
            $destination = "triples") then
          json:to-array($resp)
        else
          $resp
};

(:~
 : Construct an envelope
 :
 : @param $map - a map with all the stuff in it
 : @return - the newly constructed envelope
 :)
declare function flow:make-envelope($content, $headers, $triples, $data-format)
  as document-node()
{
  let $content := flow:clean-data($content, "content", $data-format)
  let $headers := flow:clean-data($headers, "headers", $data-format)
  let $triples := flow:clean-data($triples, "triples", $data-format)
  return
    if ($data-format = $consts:JSON) then
      let $envelope :=
        let $o := json:object()
        let $_ := (
          map:put($o, "headers", $headers),
          map:put($o, "triples", $triples),
          map:put($o, "instance",
            if ($content instance of map:map and map:keys($content) = "$type") then
              let $json := flow:instance-to-canonical-json($content)
              let $info :=
                let $o :=json:object()
                let $_ := (
                  map:put($o, "$title", map:get($content, "$type")),
                  map:put($o, "$version",  map:get($content, "$version"))
                )
                return $o
              let $_ := map:put($json, "info", $info)
              return $json
            else
              $content
          ),
          map:put($o, "attachments",
            if ($content instance of map:map and map:keys($content) = "$attachments") then
              map:get($content, "$attachments")
            else
              ()
          )
        )
        return
          $o
      let $wrapper := json:object()
      let $_ := map:put($wrapper, "envelope", $envelope)
      return
        xdmp:to-json($wrapper)
    else if ($data-format = $consts:XML) then
      document {
        <envelope xmlns="http://marklogic.com/entity-services">
          <headers>{$headers}</headers>
          <triples>{$triples}</triples>
          <instance>
          {
            if ($content instance of map:map and map:keys($content) = "$type") then (
              <info>
                <title>{map:get($content, "$type")}</title>
                <version>{map:get($content, "$version")}</version>
              </info>,
              flow:instance-to-canonical-xml($content)
            )
            else
              $content
          }
          </instance>
          <attachments>
          {
            if ($content instance of map:map and map:keys($content) = "$attachments") then
              map:get($content, "$attachments")
            else
              ()
          }
          </attachments>
        </envelope>
      }
    else
      fn:error(xs:QName("INVALID-DATA-FORMAT"), "Invalid data format: " || $data-format)
};

declare function flow:make-legacy-envelope($content, $headers, $triples, $data-format)
  as document-node()
{
  let $content := flow:clean-data($content, "content", $data-format)
  let $headers := flow:clean-data($headers, "headers", $data-format)
  let $triples := flow:clean-data($triples, "triples", $data-format)
  return
    if ($data-format = $consts:JSON) then
      let $envelope :=
        let $o := json:object()
        let $_ := (
          map:put($o, "headers", $headers),
          map:put($o, "triples", $triples),
          map:put($o, "content", $content)
        )
        return
          $o
      let $wrapper := json:object()
      let $_ := map:put($wrapper, "envelope", $envelope)
      return
        xdmp:to-json($wrapper)
    else if ($data-format = $consts:XML) then
      document {
        <envelope xmlns="http://marklogic.com/data-hub/envelope">
          <headers>{$headers}</headers>
          <triples>{$triples}</triples>
          <content>{$content}</content>
        </envelope>
      }
    else
      fn:error(xs:QName("INVALID-DATA-FORMAT"), "Invalid data format: " || $data-format)
};

declare function flow:instance-to-canonical-json(
  $entity-instance as map:map) as json:object
{
  let $root-object := json:object()
  let $o :=
    if ( map:contains($entity-instance, "$ref") ) then
      map:get($entity-instance, "$ref")
    else
      let $o := json:object()
      let $_ := (
        for $key in map:keys($entity-instance)
        let $instance-property := map:get($entity-instance, $key)
        where ($key castable as xs:NCName and $key ne "$type")
        return
          typeswitch ($instance-property)
          (: This branch handles embedded objects.  You can choose to prune
             an entity's representation of extend it with lookups here. :)
          case json:object+ return
            for $prop in $instance-property
            return
              map:put($o, $key, flow:instance-to-canonical-json($prop))
          (: An array can also treated as multiple elements :)
          case json:array return
            let $a := json:array()
            let $_ :=
              for $val in json:array-values($instance-property)
              return
                if ($val instance of json:object) then
                  json:array-push($a, flow:instance-to-canonical-json($val))
                else
                  json:array-push($a, $val)
            return
              map:put($o, $key, $a)
          (: A sequence of values should be simply treated as multiple elements :)
          case item()+ return
            for $val in $instance-property
            return
              map:put($o, $key, $val)
          default return
            map:put($o, $key, $instance-property)
      )
      return
        $o
  let $_ := map:put($root-object, map:get($entity-instance, "$type"), $o)
  return
    $root-object
};

declare function flow:instance-to-canonical-xml(
  $entity-instance as map:map) as element()
{
  (: Construct an element that is named the same as the Entity Type :)
  element { map:get($entity-instance, "$type") }  {
    if ( map:contains($entity-instance, "$ref") ) then
      map:get($entity-instance, "$ref")
    else
      for $key in map:keys($entity-instance)
      let $instance-property := map:get($entity-instance, $key)
      where ($key castable as xs:NCName and $key ne "$type")
      return
        typeswitch ($instance-property)
        (: This branch handles embedded objects.  You can choose to prune
           an entity's representation of extend it with lookups here. :)
        case json:object+ return
          for $prop in $instance-property
          return
            element { $key } {
              flow:instance-to-canonical-xml($prop)
            }
        (: An array can also treated as multiple elements :)
        case json:array return
          for $val in json:array-values($instance-property)
          return
            if ($val instance of json:object) then
              element { $key } {
                attribute datatype { "array" },
                flow:instance-to-canonical-xml($val)
              }
            else
              element { $key } {
                attribute datatype { "array" },
                $val
              }
        (: A sequence of values should be simply treated as multiple elements :)
        case item()+ return
          for $val in $instance-property
          return
            element { $key } { $val }
        default return
          element { $key } { $instance-property }
  }
};

(: Formats the input for a trace based on the plugin type :)
declare function flow:get-trace-input(
  $plugin as element(hub:plugin),
  $data-format as xs:string,
  $content as item()?,
  $headers as item()*,
  $flow-type as xs:string)
{
  let $destination as xs:string := $plugin/@dest
  let $content :=
    if ($content instance of document-node()) then
      $content/node()
    else
      $content
  let $o := json:object()
  let $_ :=
    switch($destination)
      case "content" return
        if ($flow-type = $consts:INPUT_FLOW) then
          if ($content instance of binary()) then
            map:put($o, "rawContent", "binary document")
          else
            map:put($o, "rawContent", $content)
        else
          ()
      case "headers" return
        map:put($o, "content", $content)
      case "triples" return
      (
        map:put($o, "content", $content),
        map:put($o, "headers", ($headers, null-node{})[1])
      )
      default return ()
  return
    if ($data-format = $consts:XML) then
      for $key in map:keys($o)
      return
        element { $key } {
          let $value := map:get($o, $key)
          return
            if ($value instance of null-node()) then ()
            else $value
        }
    else
      $o
};

declare function flow:set-default-options(
  $options as map:map,
  $flow as element(hub:flow))
{
  map:put($options, "entity", fn:string($flow/hub:entity)),
  map:put($options, "flow", fn:string($flow/hub:name)),
  map:put($options, "flowType", fn:string($flow/hub:type)),
  map:put($options, "dataFormat", fn:string($flow/hub:data-format))
};

declare function flow:run-main(
  $main as element(hub:main))
{
  (: sanity check on required info :)
  if (fn:empty($main/@module) or fn:empty($main/@code-format)) then
    fn:error(xs:QName("INVALID-PLUGIN"), "The plugin definition is invalid.")
  else (),

  let $module-uri as xs:string? := $main/@module
  let $_ := rfc:with-module-uri($module-uri)
  let $func := flow:make-function($main/@code-format, "main", $module-uri)
  let $before := xdmp:elapsed-time()
  let $resp := try {
    let $options := rfc:get-options()
    let $_ := map:set-javascript-by-ref($options, fn:true())
    let $resp :=
      if (rfc:get-flow-type() eq $consts:HARMONIZE_FLOW) then
        $func(rfc:get-id(), $options)
      else
        $func(rfc:get-id(), rfc:get-content(), $options)
    (: write the trace for the current identifier :)
    let $_ := trace:write-trace()
    return
      $resp
  }
  catch($ex) {
    if ($ex/error:name eq "PLUGIN-ERROR") then
      (: plugin errors are already handled :)
      ()
    else (
      (: this is an error in main.(sjs|xqy) :)
      debug:log(xdmp:describe($ex, (), ())),

      (: log the trace event for main :)
      trace:set-plugin-label("main"),
      trace:error-trace($ex, xdmp:elapsed-time() - $before)
    ),
    xdmp:rethrow()
  }
  return
    $resp
};

(:~
 : Run a given writer
 :
 : @param $writer - xml describing the writer to run
 : @param $identifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $envelope - the envelope
 : @param $options - a map of options passed in by the client
 : @return - the output of the writer. It varies.
 :)
declare function flow:run-writer(
  $writer-function,
  $identifier as xs:string,
  $envelope as item(),
  $options as map:map)
{
  let $before := xdmp:elapsed-time()
  let $resp :=
    try {
      xdmp:eval('
        declare variable $func external;
        declare variable $identifier external;
        declare variable $envelope external;
        declare variable $options external;

        $func($identifier, $envelope, $options)
      ',
      map:new((
        map:entry("func", $writer-function),
        map:entry("identifier", $identifier),
        map:entry("envelope", $envelope),
        map:entry("options", $options)
      )),
      map:new((
        map:entry("isolation", "different-transaction"),
        map:entry("database", rfc:get-target-database()),
        map:entry("transactionMode", "update-auto-commit")
      )))
    }
    catch($ex) {
      debug:log(xdmp:describe($ex, (), ())),
      trace:error-trace($ex, xdmp:elapsed-time() - $before),
      fn:error(xs:QName("PLUGIN-ERROR"), "error in writer", $ex)
    }
  let $duration := xdmp:elapsed-time() - $before
  let $_ := trace:plugin-trace((), xdmp:elapsed-time() - $before)
  return
    $resp
};

declare function flow:make-error-json(
  $errors as json:object,
  $entity as xs:string,
  $flow  as xs:string,
  $plugin as xs:string,
  $ex
) {
  let $eo :=
    if (map:contains($errors, $entity)) then
      map:get($errors, $entity)
    else
      let $e := map:map()
      let $_ := map:put($errors, $entity, $e)
      return
        $e
  let $fo :=
    if (map:contains($eo, $flow)) then
      map:get($eo, $flow)
    else
      let $f := map:map()
      let $_ := map:put($eo, $flow, $f)
      return
        $f

  let $f := $ex/error:stack/error:frame[1]
  return
    map:put($fo, $plugin, map:new((
      map:entry("uri", $f/error:uri/fn:data()),
      map:entry("line", $f/error:line/fn:data()),
      map:entry("column", $f/error:column/fn:data()),
      map:entry("msg", $ex/error:format-string/fn:data())
    )))
};

declare function flow:validate-entities()
{
  let $errors := json:object()
  let $options := map:map()
  let $_ :=
    for $entity in flow:get-entities()/hub:entity
    for $flow in $entity/hub:flows/hub:flow
    let $data-format := $flow/hub:data-format
    (: validate collector :)
    let $_ :=
      try {
        let $collector := $flow/hub:collector
        return
          if ($collector) then
            let $module-uri := $collector/@module
            let $ns := flow:get-module-ns($collector/@code-format)
            return
              if ($collector/@code-format eq $consts:XQUERY) then
                xdmp:eval(
                  'import module namespace x = "' || $ns || '" at "' || $module-uri || '"; ' ||
                  '()',
                  map:new(map:entry("staticCheck", fn:true()))
                )
              else
                xdmp:javascript-eval(
                  'var x = require("' || $module-uri || '");',
                  map:new(map:entry("staticCheck", fn:true()))
                )
          else ()
      }
      catch($ex) {
        flow:make-error-json(
          $errors,
          $entity/hub:name,
          $flow/hub:name,
          "collector",
          $ex)
      }
    (: validate plugins :)
    let $_ :=
      for $main in $flow/hub:main
      let $module-uri := $main/@module
      let $ns := flow:get-module-ns($main/@code-format)
      return
        (:
         : Note that we are static checking the files.
         : This is because there is no reasonable way to actually
         : call the plugins and pass in data that will work for all plugins.
         :
         : The disadvantage to static checking is that we will not catch typos
         : like ctsdoc <- (missing period) because Javascript is dynamically
         : typed. Static checking will only catch syntax errors in sjs.
         :)
        try {
          if ($main/@code-format eq $consts:XQUERY) then
            xdmp:eval(
              'import module namespace x = "' || $ns || '" at "' || $module-uri || '"; ' ||
              '()',
              map:new(map:entry("staticCheck", fn:true()))
            )
          else
            xdmp:javascript-eval(
              'var x = require("' || $module-uri || '");',
              map:new(map:entry("staticCheck", fn:true()))
            )
        }
        catch($ex) {
          let $plugin :=
            let $uri := ($ex/error:stack/error:frame)[1]/error:uri
            let $name := hul:get-file-name(hul:get-file-from-uri($uri))
            return
              if ($name eq "[anonymous]") then
                fn:replace($ex/error:expr, ".+/([^.]+)\.(sjs|xqy).*", "$1")
              else
                $name
          return
            flow:make-error-json(
              $errors,
              $entity/hub:name,
              $flow/hub:name,
              $plugin,
              $ex)
        }
    return
      ()
  return
    map:new(
      map:entry("errors", $errors)
    )
};

declare function flow:safe-run($func)
{
  let $before := xdmp:elapsed-time()
  return
    try {
      let $resp := $func()
      let $duration := xdmp:elapsed-time() - $before
      let $_ := trace:plugin-trace($resp, $duration)
      return
        $resp
    }
    catch($ex) {
      debug:log(xdmp:describe($ex, (), ())),
      trace:error-trace($ex, xdmp:elapsed-time() - $before),
      fn:error(xs:QName("PLUGIN-ERROR"), "error in a plugin", $ex)
    }
};

declare %private function flow:make-function(
  $code-format as xs:string?,
  $func-name as xs:string,
  $module-uri as xs:string)
{
  let $ns := flow:get-module-ns($code-format)
  return
    xdmp:function(fn:QName($ns, $func-name), $module-uri)
};