refactor(xquery): typeswitch in test:atom-type() (#1004)

module namespace test = "http://www.jenitennison.com/xslt/unit-test";

(::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::)
(:  File:       generate-query-utils.xql                                    :)
(:  Author:     Jeni Tennison                                               :)
(:  URL:        http://github.com/xspec/xspec                               :)
(:  Tags:                                                                   :)
(:    Copyright (c) 2008, 2010 Jeni Tennison (see end of file.)             :)
(: ------------------------------------------------------------------------ :)

import module namespace x = "http://www.jenitennison.com/xslt/xspec"
  at "../common/xspec-utils.xquery";

declare function test:deep-equal(
    $seq1 as item()*,
    $seq2 as item()*,
    $flags as xs:string
  ) as xs:boolean
{
  if (contains($flags, '1')) then
    test:deep-equal-v1($seq1, $seq2, $flags)

  else if (empty($seq1) or empty($seq2)) then
    empty($seq1) and empty($seq2)

  else if (count($seq1) = count($seq2)) then
    every $i in (1 to count($seq1))
    satisfies test:item-deep-equal($seq1[$i], $seq2[$i], $flags)

  else if ( $seq1 instance of text() and $seq2 instance of text()+ ) then
    test:deep-equal($seq1, text { string-join($seq2, '') }, $flags)

  else
    false()
};

declare function test:deep-equal-v1(
  $seq1 as item()*,
  $seq2 as item()*,
  $flags as xs:string
) as xs:boolean
{
  let $seq2-adapted as xs:anyAtomicType? := (
    if ($seq2 instance of text()+) then
      let $seq2-string as xs:string := string-join($seq2, '')
      return
        typeswitch ($seq1)
          case xs:string  return $seq2-string
          case xs:double  return $seq2-string[. castable as xs:double]  => xs:double()
          case xs:decimal return $seq2-string[. castable as xs:decimal] => xs:decimal()
          case xs:integer return $seq2-string[. castable as xs:integer] => xs:integer()
          default         return ()
    else
      ()
  )
  return
    test:deep-equal(
      $seq1,
      ($seq2-adapted, $seq2)[1],
      translate($flags, '1', '')
    )
};

declare function test:item-deep-equal(
    $item1 as item(),
    $item2 as item(),
    $flags as xs:string
  ) as xs:boolean
{
  if ( $item1 instance of node() and $item2 instance of node() ) then
    test:node-deep-equal($item1, $item2, $flags)
  else if (not($item1 instance of node()) and not($item2 instance of node())) then
    deep-equal($item1, $item2)
  else
    false()
};

declare function test:node-deep-equal(
    $node1 as node(),
    $node2 as node(),
    $flags as xs:string
  ) as xs:boolean
{
  if ( $node1 instance of document-node() and $node2 instance of document-node() ) then
    test:deep-equal(test:sorted-children($node1, $flags), test:sorted-children($node2, $flags), $flags)

  else if ( $node1 instance of element() and $node2 instance of element() ) then
    test:element-deep-equal($node1, $node2, $flags)

  else if ( $node1 instance of text() and $node1 = '...' ) then
    true()

  else if ( $node1 instance of text() and $node2 instance of text() ) then
    string($node1) eq string($node2)

  else if ( ( $node1 instance of attribute() and $node2 instance of attribute() )
            or ( $node1 instance of processing-instruction()
                 and $node2 instance of processing-instruction())
            or ( $node1 instance of namespace-node()
                 and $node2 instance of namespace-node() ) ) then
    deep-equal(node-name($node1), node-name($node2))
      and (string($node1) eq string($node2) or string($node1) = '...')

  else if ( $node1 instance of comment() and $node2 instance of comment() ) then
    string($node1) eq string($node2) or string($node1) = '...'

  else
    false()
};

declare function test:element-deep-equal(
  $elem1 as element(),
  $elem2 as element(),
  $flags as xs:string
) as xs:boolean
{
  let $node-name-equal as xs:boolean := (node-name($elem1) eq node-name($elem2))

  let $attrs-equal as xs:boolean :=
    test:deep-equal(
      test:sort-named-nodes($elem1/attribute()),
      test:sort-named-nodes($elem2/attribute()),
      $flags
    )

  let $children-equal as xs:boolean := (
    $elem1[count(node()) eq 1][text() = '...']
    or
    test:deep-equal(
      test:sorted-children($elem1, $flags),
      test:sorted-children($elem2, $flags),
      $flags
    )
  )

  return
    ($node-name-equal and $attrs-equal and $children-equal)
};

declare function test:sorted-children(
    $node as node(),
    $flags as xs:string
  ) as node()*
{
  $node/child::node()
  except ($node/text()[not(normalize-space())][contains($flags, 'w')][not($node/self::test:ws)],
          $node/test:message)
};

declare function test:sort-named-nodes(
  $nodes as node()*
) as node()*
{
  if (empty($nodes)) then
    ()
  else
    let $idx := test:named-nodes-minimum($nodes)
      return (
        $nodes[$idx],
        test:sort-named-nodes(remove($nodes, $idx))
      )
};

(: Return the "minimum" of $nodes, using the order defined by
 : test:sort-named-nodes().
 :)
declare function test:named-nodes-minimum($nodes as node()+) as xs:integer
{
  (: if there is only one node, this is the minimum :)
  if (empty($nodes[2])) then
    1
  (: if not, init the temp minimum on the first one, then walk through the sequence :)
  else
    test:named-nodes-minimum($nodes, node-name($nodes[1]), 1, 2)
};

declare function test:named-nodes-minimum(
    $nodes as node()+,
    $min   as xs:QName,
    $idx   as xs:integer,
    $curr  as xs:integer
  ) as xs:integer
{
  if ($curr gt count($nodes)) then
    $idx
  else if (test:qname-lt(node-name($nodes[$curr]), $min)) then
    test:named-nodes-minimum($nodes, node-name($nodes[$curr]), $curr, $curr + 1)
  else
    test:named-nodes-minimum($nodes, $min, $idx, $curr + 1)
};

declare function test:qname-lt($n1 as xs:QName, $n2 as xs:QName) as xs:boolean
{
  if (namespace-uri-from-QName($n1) eq namespace-uri-from-QName($n2)) then
    local-name-from-QName($n1) lt local-name-from-QName($n2)
  else
    namespace-uri-from-QName($n1) lt namespace-uri-from-QName($n2)
};

declare function test:report-sequence(
    $sequence as item()*,
    $wrapper-name as xs:string
  ) as element()
{
  test:report-sequence($sequence, $wrapper-name, ())
};

declare function test:report-sequence(
    $sequence as item()*,
    $wrapper-name as xs:string,
    $test as attribute(test)?
  ) as element()
{
  let $wrapper-ns as xs:string := 'http://www.jenitennison.com/xslt/xspec'

  let $attribute-nodes as attribute()* := $sequence[. instance of attribute()]
  let $document-nodes as document-node()* := $sequence[. instance of document-node()]
  let $namespace-nodes as namespace-node()* := $sequence[. instance of namespace-node()]
  let $text-nodes as text()* := $sequence[. instance of text()]

  let $report-element as element() :=
    element
      { QName($wrapper-ns, $wrapper-name) }
      {
        $test,

        (
          (: Empty :)
          if (empty($sequence))
          then attribute select { "()" }

          (: One or more atomic values :)
          else if ($sequence instance of xs:anyAtomicType+)
          then (
            let $atomic-value-reports as xs:string+ :=
              (for $value in $sequence return test:report-atomic-value($value))
            return attribute select { string-join($atomic-value-reports, ', ') }
          )

          (: One or more nodes of the same type which can be a child of document node :)
          else if (
            ($sequence instance of comment()+)
            or ($sequence instance of element()+)
            or ($sequence instance of processing-instruction()+)
            or ($sequence instance of text()+)
          )
          then (
            attribute select { concat('/', x:node-type($sequence[1]), '()') },
            for $node in $sequence return test:report-node($node)
          )

          (: Single document node :)
          else if ($sequence instance of document-node())
          then (
            (: People do not always notice '/' in the report HTML. So express it more verbosely.
              Also the expression must match the one in ../reporter/format-xspec-report.xsl. :)
            attribute select { "/self::document-node()" },
            test:report-node($sequence)
          )

          (: One or more nodes which can be stored in an element safely and without losing each position.
            Those nodes include document nodes and text nodes. By storing them in an element, they will
            be unwrapped and/or merged with adjacent nodes. When it happens, the report does not
            represent the sequence precisely. That's ok, because
              * Otherwise the report will be cluttered with pseudo elements.
              * XSpec in general including its test:deep-equal() inclines to merge them. :)
          else if (($sequence instance of node()+) and not($attribute-nodes or $namespace-nodes))
          then (
            attribute select { "/node()" },
            for $node in $sequence return test:report-node($node)
          )

          (: Otherwise each item needs to be represented as a pseudo element :)
          else (
            attribute select {
              concat(
                (: Select the pseudo elements :)
                '/*',

                (
                  (: If all items are instance of node, they can be expressed in @select.
                    (Document nodes are unwrapped, though.) :)
                  if ($sequence instance of node()+)
                  then (
                    let $expressions as xs:string+ := (
                      '@*'[$attribute-nodes],
                      'namespace::*'[$namespace-nodes],
                      'node()'[$sequence except ($attribute-nodes | $namespace-nodes)]
                    )
                    let $multi-expr as xs:boolean := (count($expressions) ge 2)
                    return
                      concat(
                        '/',
                        '('[$multi-expr],
                        string-join($expressions, ' | '),
                        ')'[$multi-expr]
                      )
                  )
                  else (
                    (: Not all items can be expressed in @select. Just leave the pseudo elements selected. :)
                  )
                )
              )
            },

            for $item in $sequence
            return test:report-pseudo-item($item, $wrapper-ns)
          )
        )
      }

  (: Output the report element :)
  return (
    (: TODO: If too many nodes, save the report element as an external doc :)
    $report-element
  )
};

declare function test:report-pseudo-item(
    $item as item(),
    $wrapper-ns as xs:string
  ) as element()
{
  let $local-name-prefix as xs:string := 'pseudo-'
  return (
    if ($item instance of xs:anyAtomicType)
    then
      element
        { QName($wrapper-ns, concat($local-name-prefix, 'atomic-value')) }
        { test:report-atomic-value($item) }

    else if ($item instance of node())
    then
      element
        { QName($wrapper-ns, concat($local-name-prefix, x:node-type($item))) }
        { test:report-node($item) }

    (: TODO: function(*) including array(*) and map(*) :)

    else
      element
        { QName($wrapper-ns, concat($local-name-prefix, 'other')) }
        {}
  )
};

(:
  Copies the nodes while wrapping whitespace-only text nodes in <test:ws>
:)
declare function test:report-node(
    $node as node()
    ) as node()
{
  if (($node instance of text()) and not(normalize-space($node))) then
    element test:ws { $node }
  else if ( $node instance of document-node() ) then
    document {
      for $child in $node/child::node() return test:report-node($child)
    }
  else if ( $node instance of element() ) then
    element { node-name($node) } {
      (
        for $prefix in in-scope-prefixes($node)
          return namespace { $prefix } { namespace-uri-for-prefix($prefix, $node) }
      ),
      $node/attribute(),
      (for $child in $node/child::node() return test:report-node($child))
    }
  else $node
};

declare function test:report-atomic-value(
  $value as xs:anyAtomicType
) as xs:string
{
  typeswitch ($value)
    (: Derived types must be handled before their base types :)

    (: String types :)
    (: xs:normalizedString: Requires schema-aware processor :)
    case xs:string return x:quote-with-apos($value)

    (: Derived numeric types: Requires schema-aware processor :)

    (: Numeric types which can be expressed as numeric literals:
      http://www.w3.org/TR/xpath20/#id-literals :)
    case xs:integer return string($value)
    case xs:decimal return x:decimal-string($value)
    case xs:double
      return
        (: Do not report xs:double as a numeric literal. Report as xs:double() constructor instead.
          Justifications below.
          * Expression of xs:double as a numeric literal is a bit complicated:
            http://www.w3.org/TR/xpath-functions/#casting-to-string
          * xs:double is not used as frequently as xs:integer
          * xs:double() constructor is valid expression. It's just some more verbose than a numeric literal. :)
        test:report-atomic-value-as-constructor($value)

    case xs:QName return x:QName-expression($value)

    default return test:report-atomic-value-as-constructor($value)
};

declare function test:report-atomic-value-as-constructor($value as xs:anyAtomicType) as xs:string
{
  (: Constructor usually has the same name as type :)
  let $constructor-name as xs:string := test:atom-type($value)

  (: Cast as either xs:integer or xs:string :)
  let $casted-value as xs:anyAtomicType := (
    if ( $value instance of xs:integer )
    then
      (: Force casting down to integer, by first converting to string :)
      (string($value) cast as xs:integer)
    else
      string($value)
  )

  (: Constructor parameter:
    Either numeric literal of integer or string literal :)
  let $costructor-param as xs:string :=
    test:report-atomic-value($casted-value)

  return concat($constructor-name, '(', $costructor-param, ')')
};

declare function test:atom-type(
  $value as xs:anyAtomicType
) as xs:string
{
  let $local-name as xs:string := (
    typeswitch ($value)
      (: Grouped as the spec does: http://www.w3.org/TR/xslt20/#built-in-types
        Groups are in the reversed order so that the derived types are before the primitive types,
        otherwise xs:integer is recognised as xs:decimal, xs:yearMonthDuration as xs:duration, and so on. :)

      (: A schema-aware XSLT processor additionally supports: :)

      (:    * All other built-in types defined in [XML Schema Part 2] :)
      (: Requires schema-aware processor :)

      (: Every XSLT 2.0 processor includes the following named type definitions in the in-scope schema components: :)

      (:    * The following types defined in [XPath 2.0] :)
      case xs:yearMonthDuration return 'yearMonthDuration'
      case xs:dayTimeDuration   return 'dayTimeDuration'
      (: xs:anyAtomicType: Abstract :)
      (: xs:untyped: Not atomic :)
      case xs:untypedAtomic     return 'untypedAtomic'

      (:    * The types xs:anyType and xs:anySimpleType. :)
      (: Not atomic :)

      (:    * The derived atomic type xs:integer defined in [XML Schema Part 2]. :)
      case xs:integer return 'integer'

      (:    * All the primitive atomic types defined in [XML Schema Part 2], with the exception of xs:NOTATION. :)
      case xs:string       return 'string'
      case xs:boolean      return 'boolean'
      case xs:decimal      return 'decimal'
      case xs:double       return 'double'
      case xs:float        return 'float'
      case xs:date         return 'date'
      case xs:time         return 'time'
      case xs:dateTime     return 'dateTime'
      case xs:duration     return 'duration'
      case xs:QName        return 'QName'
      case xs:anyURI       return 'anyURI'
      case xs:gDay         return 'gDay'
      case xs:gMonthDay    return 'gMonthDay'
      case xs:gMonth       return 'gMonth'
      case xs:gYearMonth   return 'gYearMonth'
      case xs:gYear        return 'gYear'
      case xs:base64Binary return 'base64Binary'
      case xs:hexBinary    return 'hexBinary'
      default              return 'anyAtomicType'
  )
  return
    ('Q{http://www.w3.org/2001/XMLSchema}' || $local-name)
};

(: ------------------------------------------------------------------------ :)
(:  DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS COMMENT.               :)
(:                                                                          :)
(:  Copyright (c) 2008, 2010 Jeni Tennison                                  :)
(:                                                                          :)
(:  The contents of this file are subject to the MIT License (see the URI   :)
(:  http://www.opensource.org/licenses/mit-license.php for details).        :)
(:                                                                          :)
(:  Permission is hereby granted, free of charge, to any person obtaining   :)
(:  a copy of this software and associated documentation files (the         :)
(:  "Software"), to deal in the Software without restriction, including     :)
(:  without limitation the rights to use, copy, modify, merge, publish,     :)
(:  distribute, sublicense, and/or sell copies of the Software, and to      :)
(:  permit persons to whom the Software is furnished to do so, subject to   :)
(:  the following conditions:                                               :)
(:                                                                          :)
(:  The above copyright notice and this permission notice shall be          :)
(:  included in all copies or substantial portions of the Software.         :)
(:                                                                          :)
(:  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,         :)
(:  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF      :)
(:  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  :)
(:  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY    :)
(:  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,    :)
(:  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE       :)
(:  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                  :)
(: ------------------------------------------------------------------------ :)

module namespace test = "http://www.jenitennison.com/xslt/unit-test";

(::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::)
(:  File:       generate-query-utils.xql                                    :)
(:  Author:     Jeni Tennison                                               :)
(:  URL:        http://github.com/xspec/xspec                               :)
(:  Tags:                                                                   :)
(:    Copyright (c) 2008, 2010 Jeni Tennison (see end of file.)             :)
(: ------------------------------------------------------------------------ :)

import module namespace x = "http://www.jenitennison.com/xslt/xspec"
  at "../common/xspec-utils.xquery";

declare function test:deep-equal(
    $seq1 as item()*,
    $seq2 as item()*,
    $flags as xs:string
  ) as xs:boolean
{
  if (contains($flags, '1')) then
    test:deep-equal-v1($seq1, $seq2, $flags)

  else if (empty($seq1) or empty($seq2)) then
    empty($seq1) and empty($seq2)

  else if (count($seq1) = count($seq2)) then
    every $i in (1 to count($seq1))
    satisfies test:item-deep-equal($seq1[$i], $seq2[$i], $flags)

  else if ( $seq1 instance of text() and $seq2 instance of text()+ ) then
    test:deep-equal($seq1, text { string-join($seq2, '') }, $flags)

  else
    false()
};

declare function test:deep-equal-v1(
  $seq1 as item()*,
  $seq2 as item()*,
  $flags as xs:string
) as xs:boolean
{
  let $seq2-adapted as xs:anyAtomicType? := (
    if ($seq2 instance of text()+) then
      let $seq2-string as xs:string := string-join($seq2, '')
      return
        typeswitch ($seq1)
          case xs:string  return $seq2-string
          case xs:double  return $seq2-string[. castable as xs:double]  => xs:double()
          case xs:decimal return $seq2-string[. castable as xs:decimal] => xs:decimal()
          case xs:integer return $seq2-string[. castable as xs:integer] => xs:integer()
          default         return ()
    else
      ()
  )
  return
    test:deep-equal(
      $seq1,
      ($seq2-adapted, $seq2)[1],
      translate($flags, '1', '')
    )
};

declare function test:item-deep-equal(
    $item1 as item(),
    $item2 as item(),
    $flags as xs:string
  ) as xs:boolean
{
  if ( $item1 instance of node() and $item2 instance of node() ) then
    test:node-deep-equal($item1, $item2, $flags)
  else if (not($item1 instance of node()) and not($item2 instance of node())) then
    deep-equal($item1, $item2)
  else
    false()
};

declare function test:node-deep-equal(
    $node1 as node(),
    $node2 as node(),
    $flags as xs:string
  ) as xs:boolean
{
  if ( $node1 instance of document-node() and $node2 instance of document-node() ) then
    test:deep-equal(test:sorted-children($node1, $flags), test:sorted-children($node2, $flags), $flags)

  else if ( $node1 instance of element() and $node2 instance of element() ) then
    test:element-deep-equal($node1, $node2, $flags)

  else if ( $node1 instance of text() and $node1 = '...' ) then
    true()

  else if ( $node1 instance of text() and $node2 instance of text() ) then
    string($node1) eq string($node2)

  else if ( ( $node1 instance of attribute() and $node2 instance of attribute() )
            or ( $node1 instance of processing-instruction()
                 and $node2 instance of processing-instruction())
            or ( $node1 instance of namespace-node()
                 and $node2 instance of namespace-node() ) ) then
    deep-equal(node-name($node1), node-name($node2))
      and (string($node1) eq string($node2) or string($node1) = '...')

  else if ( $node1 instance of comment() and $node2 instance of comment() ) then
    string($node1) eq string($node2) or string($node1) = '...'

  else
    false()
};

declare function test:element-deep-equal(
  $elem1 as element(),
  $elem2 as element(),
  $flags as xs:string
) as xs:boolean
{
  let $node-name-equal as xs:boolean := (node-name($elem1) eq node-name($elem2))

  let $attrs-equal as xs:boolean :=
    test:deep-equal(
      test:sort-named-nodes($elem1/attribute()),
      test:sort-named-nodes($elem2/attribute()),
      $flags
    )

  let $children-equal as xs:boolean := (
    $elem1[count(node()) eq 1][text() = '...']
    or
    test:deep-equal(
      test:sorted-children($elem1, $flags),
      test:sorted-children($elem2, $flags),
      $flags
    )
  )

  return
    ($node-name-equal and $attrs-equal and $children-equal)
};

declare function test:sorted-children(
    $node as node(),
    $flags as xs:string
  ) as node()*
{
  $node/child::node()
  except ($node/text()[not(normalize-space())][contains($flags, 'w')][not($node/self::test:ws)],
          $node/test:message)
};

declare function test:sort-named-nodes(
  $nodes as node()*
) as node()*
{
  if (empty($nodes)) then
    ()
  else
    let $idx := test:named-nodes-minimum($nodes)
      return (
        $nodes[$idx],
        test:sort-named-nodes(remove($nodes, $idx))
      )
};

(: Return the "minimum" of $nodes, using the order defined by
 : test:sort-named-nodes().
 :)
declare function test:named-nodes-minimum($nodes as node()+) as xs:integer
{
  (: if there is only one node, this is the minimum :)
  if (empty($nodes[2])) then
    1
  (: if not, init the temp minimum on the first one, then walk through the sequence :)
  else
    test:named-nodes-minimum($nodes, node-name($nodes[1]), 1, 2)
};

declare function test:named-nodes-minimum(
    $nodes as node()+,
    $min   as xs:QName,
    $idx   as xs:integer,
    $curr  as xs:integer
  ) as xs:integer
{
  if ($curr gt count($nodes)) then
    $idx
  else if (test:qname-lt(node-name($nodes[$curr]), $min)) then
    test:named-nodes-minimum($nodes, node-name($nodes[$curr]), $curr, $curr + 1)
  else
    test:named-nodes-minimum($nodes, $min, $idx, $curr + 1)
};

declare function test:qname-lt($n1 as xs:QName, $n2 as xs:QName) as xs:boolean
{
  if (namespace-uri-from-QName($n1) eq namespace-uri-from-QName($n2)) then
    local-name-from-QName($n1) lt local-name-from-QName($n2)
  else
    namespace-uri-from-QName($n1) lt namespace-uri-from-QName($n2)
};

declare function test:report-sequence(
    $sequence as item()*,
    $wrapper-name as xs:string
  ) as element()
{
  test:report-sequence($sequence, $wrapper-name, ())
};

declare function test:report-sequence(
    $sequence as item()*,
    $wrapper-name as xs:string,
    $test as attribute(test)?
  ) as element()
{
  let $wrapper-ns as xs:string := 'http://www.jenitennison.com/xslt/xspec'

  let $attribute-nodes as attribute()* := $sequence[. instance of attribute()]
  let $document-nodes as document-node()* := $sequence[. instance of document-node()]
  let $namespace-nodes as namespace-node()* := $sequence[. instance of namespace-node()]
  let $text-nodes as text()* := $sequence[. instance of text()]

  let $report-element as element() :=
    element
      { QName($wrapper-ns, $wrapper-name) }
      {
        $test,

        (
          (: Empty :)
          if (empty($sequence))
          then attribute select { "()" }

          (: One or more atomic values :)
          else if ($sequence instance of xs:anyAtomicType+)
          then (
            let $atomic-value-reports as xs:string+ :=
              (for $value in $sequence return test:report-atomic-value($value))
            return attribute select { string-join($atomic-value-reports, ', ') }
          )

          (: One or more nodes of the same type which can be a child of document node :)
          else if (
            ($sequence instance of comment()+)
            or ($sequence instance of element()+)
            or ($sequence instance of processing-instruction()+)
            or ($sequence instance of text()+)
          )
          then (
            attribute select { concat('/', x:node-type($sequence[1]), '()') },
            for $node in $sequence return test:report-node($node)
          )

          (: Single document node :)
          else if ($sequence instance of document-node())
          then (
            (: People do not always notice '/' in the report HTML. So express it more verbosely.
              Also the expression must match the one in ../reporter/format-xspec-report.xsl. :)
            attribute select { "/self::document-node()" },
            test:report-node($sequence)
          )

          (: One or more nodes which can be stored in an element safely and without losing each position.
            Those nodes include document nodes and text nodes. By storing them in an element, they will
            be unwrapped and/or merged with adjacent nodes. When it happens, the report does not
            represent the sequence precisely. That's ok, because
              * Otherwise the report will be cluttered with pseudo elements.
              * XSpec in general including its test:deep-equal() inclines to merge them. :)
          else if (($sequence instance of node()+) and not($attribute-nodes or $namespace-nodes))
          then (
            attribute select { "/node()" },
            for $node in $sequence return test:report-node($node)
          )

          (: Otherwise each item needs to be represented as a pseudo element :)
          else (
            attribute select {
              concat(
                (: Select the pseudo elements :)
                '/*',

                (
                  (: If all items are instance of node, they can be expressed in @select.
                    (Document nodes are unwrapped, though.) :)
                  if ($sequence instance of node()+)
                  then (
                    let $expressions as xs:string+ := (
                      '@*'[$attribute-nodes],
                      'namespace::*'[$namespace-nodes],
                      'node()'[$sequence except ($attribute-nodes | $namespace-nodes)]
                    )
                    let $multi-expr as xs:boolean := (count($expressions) ge 2)
                    return
                      concat(
                        '/',
                        '('[$multi-expr],
                        string-join($expressions, ' | '),
                        ')'[$multi-expr]
                      )
                  )
                  else (
                    (: Not all items can be expressed in @select. Just leave the pseudo elements selected. :)
                  )
                )
              )
            },

            for $item in $sequence
            return test:report-pseudo-item($item, $wrapper-ns)
          )
        )
      }

  (: Output the report element :)
  return (
    (: TODO: If too many nodes, save the report element as an external doc :)
    $report-element
  )
};

declare function test:report-pseudo-item(
    $item as item(),
    $wrapper-ns as xs:string
  ) as element()
{
  let $local-name-prefix as xs:string := 'pseudo-'
  return (
    if ($item instance of xs:anyAtomicType)
    then
      element
        { QName($wrapper-ns, concat($local-name-prefix, 'atomic-value')) }
        { test:report-atomic-value($item) }

    else if ($item instance of node())
    then
      element
        { QName($wrapper-ns, concat($local-name-prefix, x:node-type($item))) }
        { test:report-node($item) }

    (: TODO: function(*) including array(*) and map(*) :)

    else
      element
        { QName($wrapper-ns, concat($local-name-prefix, 'other')) }
        {}
  )
};

(:
  Copies the nodes while wrapping whitespace-only text nodes in <test:ws>
:)
declare function test:report-node(
    $node as node()
    ) as node()
{
  if (($node instance of text()) and not(normalize-space($node))) then
    element test:ws { $node }
  else if ( $node instance of document-node() ) then
    document {
      for $child in $node/child::node() return test:report-node($child)
    }
  else if ( $node instance of element() ) then
    element { node-name($node) } {
      (
        for $prefix in in-scope-prefixes($node)
          return namespace { $prefix } { namespace-uri-for-prefix($prefix, $node) }
      ),
      $node/attribute(),
      (for $child in $node/child::node() return test:report-node($child))
    }
  else $node
};

declare function test:report-atomic-value(
  $value as xs:anyAtomicType
) as xs:string
{
  typeswitch ($value)
    (: Derived types must be handled before their base types :)

    (: String types :)
    (: xs:normalizedString: Requires schema-aware processor :)
    case xs:string return x:quote-with-apos($value)

    (: Derived numeric types: Requires schema-aware processor :)

    (: Numeric types which can be expressed as numeric literals:
      http://www.w3.org/TR/xpath20/#id-literals :)
    case xs:integer return string($value)
    case xs:decimal return x:decimal-string($value)
    case xs:double
      return
        (: Do not report xs:double as a numeric literal. Report as xs:double() constructor instead.
          Justifications below.
          * Expression of xs:double as a numeric literal is a bit complicated:
            http://www.w3.org/TR/xpath-functions/#casting-to-string
          * xs:double is not used as frequently as xs:integer
          * xs:double() constructor is valid expression. It's just some more verbose than a numeric literal. :)
        test:report-atomic-value-as-constructor($value)

    case xs:QName return x:QName-expression($value)

    default return test:report-atomic-value-as-constructor($value)
};

declare function test:report-atomic-value-as-constructor($value as xs:anyAtomicType) as xs:string
{
  (: Constructor usually has the same name as type :)
  let $constructor-name as xs:string := test:atom-type($value)

  (: Cast as either xs:integer or xs:string :)
  let $casted-value as xs:anyAtomicType := (
    if ( $value instance of xs:integer )
    then
      (: Force casting down to integer, by first converting to string :)
      (string($value) cast as xs:integer)
    else
      string($value)
  )

  (: Constructor parameter:
    Either numeric literal of integer or string literal :)
  let $costructor-param as xs:string :=
    test:report-atomic-value($casted-value)

  return concat($constructor-name, '(', $costructor-param, ')')
};

declare function test:atom-type($value as xs:anyAtomicType) as xs:string
{
  let $local-name as xs:string := (
    (: Grouped as the spec does: http://www.w3.org/TR/xslt20/#built-in-types
      Groups are in the reversed order so that the derived types are before the primitive types,
      otherwise xs:integer is recognised as xs:decimal, xs:yearMonthDuration as xs:duration, and so on. :)

    (: A schema-aware XSLT processor additionally supports: :)

    (:    * All other built-in types defined in [XML Schema Part 2] :)
    (: Requires schema-aware processor :)

    (: Every XSLT 2.0 processor includes the following named type definitions in the in-scope schema components: :)

    (:    * The following types defined in [XPath 2.0] :)
    if ( $value instance of xs:yearMonthDuration ) then
      'yearMonthDuration'
    else if ($value instance of xs:dayTimeDuration ) then
      'dayTimeDuration'
    (: xs:anyAtomicType: Abstract :)
    (: xs:untyped: Not atomic :)
    else if ( $value instance of xs:untypedAtomic ) then
      'untypedAtomic'

    (:    * The types xs:anyType and xs:anySimpleType. :)
    (: Not atomic :)

    (:    * The derived atomic type xs:integer defined in [XML Schema Part 2]. :)
    else if ( $value instance of xs:integer ) then
      'integer'

    (:    * All the primitive atomic types defined in [XML Schema Part 2], with the exception of xs:NOTATION. :)
    else if ( $value instance of xs:string ) then
      'string'
    else if ( $value instance of xs:boolean ) then
      'boolean'
    else if ( $value instance of xs:decimal ) then
      'decimal'
    else if ( $value instance of xs:double ) then
      'double'
    else if ( $value instance of xs:float ) then
      'float'
    else if ( $value instance of xs:date ) then
      'date'
    else if ( $value instance of xs:time ) then
      'time'
    else if ( $value instance of xs:dateTime ) then
      'dateTime'
    else if ( $value instance of xs:duration ) then
      'duration'
    else if ( $value instance of xs:QName ) then
      'QName'
    else if ( $value instance of xs:anyURI ) then
      'anyURI'
    else if ( $value instance of xs:gDay ) then
      'gDay'
    else if ( $value instance of xs:gMonthDay ) then
      'gMonthDay'
    else if ( $value instance of xs:gMonth ) then
      'gMonth'
    else if ( $value instance of xs:gYearMonth ) then
      'gYearMonth'
    else if ( $value instance of xs:gYear ) then
      'gYear'
    else if ( $value instance of xs:base64Binary ) then
      'base64Binary'
    else if ( $value instance of xs:hexBinary ) then
      'hexBinary'
    else
      'anyAtomicType'
  )
  return
    ('Q{http://www.w3.org/2001/XMLSchema}' || $local-name)
};

(: ------------------------------------------------------------------------ :)
(:  DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS COMMENT.               :)
(:                                                                          :)
(:  Copyright (c) 2008, 2010 Jeni Tennison                                  :)
(:                                                                          :)
(:  The contents of this file are subject to the MIT License (see the URI   :)
(:  http://www.opensource.org/licenses/mit-license.php for details).        :)
(:                                                                          :)
(:  Permission is hereby granted, free of charge, to any person obtaining   :)
(:  a copy of this software and associated documentation files (the         :)
(:  "Software"), to deal in the Software without restriction, including     :)
(:  without limitation the rights to use, copy, modify, merge, publish,     :)
(:  distribute, sublicense, and/or sell copies of the Software, and to      :)
(:  permit persons to whom the Software is furnished to do so, subject to   :)
(:  the following conditions:                                               :)
(:                                                                          :)
(:  The above copyright notice and this permission notice shall be          :)
(:  included in all copies or substantial portions of the Software.         :)
(:                                                                          :)
(:  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,         :)
(:  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF      :)
(:  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  :)
(:  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY    :)
(:  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,    :)
(:  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE       :)
(:  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                  :)
(: ------------------------------------------------------------------------ :)


Added instructions on how to install the script

xquery version "3.1";

(:~
: Simple script to start the replication listener.
:
: To have this script executed when eXist-db is started:
:
: - enable in conf.xml the XQueryStartupTrigger
: - carefully read the instructions about permissons
: - copy this script into /db/system/autostart
: - restart eXist-db, check logs for messages from the XQueryStartupTrigger
:)

import module namespace replication="http://exist-db.org/xquery/replication" at "java:org.exist.jms.xquery.ReplicationModule";

let $jmsConfiguration := map {
        "java.naming.factory.initial" := "org.apache.activemq.jndi.ActiveMQInitialContextFactory",
        "java.naming.provider.url" := "tcp://localhost:61616",
        "connection-factory" := "ConnectionFactory",
        "destination" := "dynamicTopics/eXistdb-replication-example",
        "subscriber.name" := "SubscriptionId",
        "connection.client-id" := "ClientId"
    }

return
    replication:register($jmsConfiguration)

xquery version "3.1";

(:~
: Simple script to start the replication listener
:)

import module namespace replication="http://exist-db.org/xquery/replication" at "java:org.exist.jms.xquery.ReplicationModule";

let $jmsConfiguration := map {
        "java.naming.factory.initial" := "org.apache.activemq.jndi.ActiveMQInitialContextFactory",
        "java.naming.provider.url" := "tcp://localhost:61616",
        "connection-factory" := "ConnectionFactory",
        "destination" := "dynamicTopics/eXistdb-replication-example",
        "subscriber.name" := "SubscriptionId",
        "connection.client-id" := "ClientId"
    }

return
    replication:register($jmsConfiguration)


DHFPROD-401 compare code, not name

DHFPROD-401 compare code, not name

(:
  Copyright 2012-2018 MarkLogic Corporation

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
:)
xquery version "1.0-ml";

module namespace flow = "http://marklogic.com/data-hub/flow-lib";

import module namespace consts = "http://marklogic.com/data-hub/consts"
  at "/MarkLogic/data-hub-framework/impl/consts.xqy";

import module namespace debug = "http://marklogic.com/data-hub/debug"
  at "/MarkLogic/data-hub-framework/impl/debug-lib.xqy";

import module namespace hul = "http://marklogic.com/data-hub/hub-utils-lib"
  at "/MarkLogic/data-hub-framework/impl/hub-utils-lib.xqy";

import module namespace json="http://marklogic.com/xdmp/json"
  at "/MarkLogic/json/json.xqy";

import module namespace functx = "http://www.functx.com"
  at "/MarkLogic/functx/functx-1.0-nodoc-2007-01.xqy";

import module namespace rfc = "http://marklogic.com/data-hub/run-flow-context"
  at "/MarkLogic/data-hub-framework/impl/run-flow-context.xqy";

import module namespace trace = "http://marklogic.com/data-hub/trace"
  at "/MarkLogic/data-hub-framework/impl/trace-lib.xqy";

declare namespace hub = "http://marklogic.com/data-hub";

declare option xdmp:mapping "false";

(: the directory where entities live :)
declare variable $ENTITIES-DIR := "/entities/";

declare variable $PLUGIN-NS := "http://marklogic.com/data-hub/plugins";

declare variable $FLOW-CACHE-KEY-PREFIX := "flow-cache-";
declare variable $MAIN-CACHE-KEY-PREFIX := "main-cache-";

declare variable $context-queue := map:map();
declare variable $writer-queue := map:map();

declare function flow:get-module-ns(
  $type as xs:string) as xs:string?
{
  if ($type eq $consts:JAVASCRIPT) then ()
  else
    $PLUGIN-NS
};

(:~
 : Returns a flow by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity that owns the flow
 : @param $flow-name - name of the flow to retrieve
 : @return - xml describing the flow
 :)
declare function flow:get-flow(
  $entity-name as xs:string,
  $flow-name as xs:string,
  $flow-type as xs:string?) as element(hub:flow)?
{
  let $duration := xs:dayTimeDuration("PT10S")
  let $key := $FLOW-CACHE-KEY-PREFIX||$entity-name||$flow-name||$flow-type
  let $flow :=  hul:from-field-cache-or-empty($key, $duration)
  return
    if ($flow) then
      $flow
    else
      hul:set-field-cache(
        $key,
        flow:get-flow-nocache($entity-name, $flow-name, $flow-type),
        $duration
      )
};

declare function flow:get-flow-nocache(
  $entity-name as xs:string,
  $flow-name as xs:string,
  $flow-type as xs:string?) as element(hub:flow)?
{

  hul:run-in-modules(function() {
    /hub:flow[
      hub:entity = $entity-name and
      hub:name = $flow-name]
      [
        if (fn:exists($flow-type)) then
          hub:type = $flow-type
        else
          fn:true()
      ]
  }) ! hul:deep-copy(.)
};

(:~
 : Returns the flows that belong to the given entity in the database
 :
 : @param $entity-name - the name of the entity containing the flows
 : @return - xml describing the flows
 :)
declare function flow:get-flows(
  $entity-name as xs:string) as element(hub:flows)
{
  element hub:flows {
    hul:run-in-modules(function() {
      /hub:flow[hub:entity = $entity-name]
    })
  }
};

(:~
 : Returns an entity by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity to retrieve
 : @return - xml describing the entity
 :)
declare function flow:get-entity(
  $entity-name as xs:string)
{
  let $uris :=
    hul:run-in-modules(function() {
      cts:uri-match($ENTITIES-DIR || $entity-name || "/*")
    })
  return
    flow:get-entity($entity-name, $uris)
};

(:~
 : Returns a entity by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity to retrieve
 : @param $uris - uris used to build the entity xml
 : @return - xml describing the entity
 :)
declare %private function flow:get-entity(
  $entity-name as xs:string,
  $uris as xs:string*)
  as element(hub:entity)
{
  <entity xmlns="http://marklogic.com/data-hub">
    <name>{$entity-name}</name>
    {
      flow:get-flows($entity-name)
    }
  </entity>
};

(:~
 : Returns the entities in the database
 :
 : @return - xml describing the entities
 :)
declare function flow:get-entities() as element(hub:entities)
{
  let $uris := hul:run-in-modules(function() {
    cts:uri-match($ENTITIES-DIR || "*")
  })
  let $entity-names :=
    fn:distinct-values(
      let $regex := $ENTITIES-DIR || "([^/]+)/.*$"
      for $flow in $uris[fn:matches(., $regex)]
      let $name := fn:replace($flow, $regex, "$1")
      return
        $name)
  let $entities :=
    for $entity-name in $entity-names
    return
      flow:get-entity($entity-name, $uris[fn:matches(., $ENTITIES-DIR || $entity-name || "/.+")])
  return
    <entities xmlns="http://marklogic.com/data-hub">
    {
      $entities
    }
    </entities>
};

(:~
 : Runs a collector
 :
 : @param $module-uri - the uri of the collector module
 : @param $options - a map of options passed in by the client
 : @return - a sequence of strings
 :)
declare function flow:run-collector(
  $flow as element(hub:flow),
  $job-id as xs:string,
  $options as map:map) as item()*
{
    (: sanity check on required info :)
  if (fn:empty($flow/hub:collector/@module) or fn:empty($flow/hub:collector/@code-format)) then
    fn:error((), "DATAHUB-INVALID-PLUGIN", "The plugin definition is invalid.")
  else (),

  (: assert that we are in query mode :)
  let $_ts as xs:unsignedLong := xdmp:request-timestamp()

  let $item-context := rfc:new-item-context()
    => rfc:with-options($options)
    => rfc:with-trace(trace:new-trace())

  let $_ := trace:set-plugin-label("collector")

  (: configure the global run context :)
  let $_ := (
    rfc:with-job-id($job-id),
    rfc:with-flow($flow),
    rfc:with-module-uri($flow/hub:collector/@module),
    rfc:with-code-format($flow/hub:collector/@code-format)
  )

  let $func := flow:make-function(
    $flow/hub:collector/@code-format,
    "collect",
    $flow/hub:collector/@module
  )
  let $before := xdmp:elapsed-time()
  let $resp :=
    try {
      $func($options)
    }
    catch($ex) {
      debug:log(xdmp:describe($ex, (), ())),
      trace:error-trace($item-context, $ex, xdmp:elapsed-time() - $before),
      xdmp:rethrow()
    }
  (: log and write the trace :)
  let $_ := (
    trace:plugin-trace($item-context, xdmp:describe($resp, 1000000, 1000000), xdmp:elapsed-time() - $before),
    trace:write-trace($item-context)
  )
  return
    $resp
};

(:~
 : Runs a given flow
 :
 : @param $flow - xml describing the flow
 : @param $identifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $options - a map of options passed in by the client
 : @return - nothing
 :)
declare function flow:run-flow(
  $job-id as xs:string,
  $flow as element(hub:flow),
  $identifier as xs:string,
  $options as map:map,
  $mainFunc)
{
  flow:run-flow($job-id, $flow, $identifier, (), $options, $mainFunc)
};

(:~
 : Runs a given flow
 :
 : @param $flow - xml describing the flow
 : @param $identifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $content - the content being loaded
 : @param $options - a map of options passed in by the client
 : @return - nothing
 :)
declare function flow:run-flow(
  $job-id as xs:string,
  $flow as element(hub:flow),
  $identifier as xs:string,
  $content as item()?,
  $options as map:map,
  $mainFunc)
{
  (: assert that we are in query mode :)
  let $_must_run_in_query_mode as xs:unsignedLong := xdmp:request-timestamp()

  (: configure the global context :)
  let $_ := (
    rfc:with-job-id($job-id),
    rfc:with-flow($flow),
    map:get($options, "target-database") ! rfc:with-target-database(.)
  )

  (: configure the item context :)
  let $item-context := rfc:new-item-context()
    => rfc:with-id($identifier)
    => rfc:with-content(
        if ($content instance of document-node()) then
          $content/node()
        else $content
      )
    => rfc:with-options($options)
    => rfc:with-trace(trace:new-trace())

  (: run the users main.(sjs|xqy) :)
  return
    flow:run-main($item-context, $mainFunc)
};

declare function flow:clean-data($resp, $destination, $data-format)
{
  let $resp :=
    typeswitch($resp)
      case document-node() return
        if (fn:count($resp/node()) > 1) then
          fn:error((), "DATAHUB-TOO-MANY-NODES", "Too Many Nodes!. Return just 1 node")
        else
          $resp/node()
      default return
        $resp

  (: clean up output :)
  return
    typeswitch($resp)
      case binary() return xs:hexBinary($resp)
      case object-node() | json:object return
        (: object with $type key is ES response type :)
        if ($resp instance of map:map and map:keys($resp) = "$type") then
          $resp
        else if ($data-format = $consts:XML) then
          json:transform-from-json($resp, json:config("custom"))
        else
          $resp
      case json:array return
        if ($data-format = $consts:XML) then
          json:array-values($resp)
        else
          $resp
      case empty-sequence() return
        if ($destination = "headers" and $data-format = $consts:JSON) then
          json:object()
        else if ($destination = "triples" and $data-format = $consts:JSON) then
          json:array()
        else
          $resp
      default return
        if ($data-format = $consts:JSON and
            rfc:get-code-format() = $consts:XQUERY and
            $destination = "triples") then
          json:to-array($resp)
        else
          $resp
};

(:~
 : Construct an envelope
 :
 : @param $map - a map with all the stuff in it
 : @return - the newly constructed envelope
 :)
declare function flow:make-envelope($content, $headers, $triples, $data-format)
  as document-node()
{
  let $content := flow:clean-data($content, "content", $data-format)
  let $headers := flow:clean-data($headers, "headers", $data-format)
  let $triples := flow:clean-data($triples, "triples", $data-format)
  return
    if ($data-format = $consts:JSON) then
      let $envelope :=
        let $o := json:object()
        let $_ := (
          map:put($o, "headers", $headers),
          map:put($o, "triples", $triples),
          map:put($o, "instance",
            if ($content instance of map:map and map:keys($content) = "$type") then
              let $json := flow:instance-to-canonical-json($content)
              let $info :=
                let $o :=json:object()
                let $_ := (
                  map:put($o, "title", map:get($content, "$type")),
                  map:put($o, "version",  map:get($content, "$version"))
                )
                return $o
              let $_ := map:put($json, "info", $info)
              return $json
            else
              $content
          ),
          map:put($o, "attachments",
            if ($content instance of map:map and map:keys($content) = "$attachments") then
              map:get($content, "$attachments")
            else
              ()
          )
        )
        return
          $o
      let $wrapper := json:object()
      let $_ := map:put($wrapper, "envelope", $envelope)
      return
        xdmp:to-json($wrapper)
    else if ($data-format = $consts:XML) then
      document {
        <envelope xmlns="http://marklogic.com/entity-services">
          <headers>{$headers}</headers>
          <triples>{$triples}</triples>
          <instance>
          {
            if ($content instance of map:map and map:keys($content) = "$type") then (
              <info>
                <title>{map:get($content, "$type")}</title>
                <version>{map:get($content, "$version")}</version>
              </info>,
              flow:instance-to-canonical-xml($content)
            )
            else
              $content
          }
          </instance>
          <attachments>
          {
            if ($content instance of map:map and map:keys($content) = "$attachments") then
              map:get($content, "$attachments")
            else
              ()
          }
          </attachments>
        </envelope>
      }
    else
      fn:error((), "RESTAPI-INVALIDCONTENT", "Invalid data format: " || $data-format)
};

declare function flow:make-legacy-envelope($content, $headers, $triples, $data-format)
  as document-node()
{
  let $content := flow:clean-data($content, "content", $data-format)
  let $headers := flow:clean-data($headers, "headers", $data-format)
  let $triples := flow:clean-data($triples, "triples", $data-format)
  return
    if ($data-format = $consts:JSON) then
      let $envelope :=
        let $o := json:object()
        let $_ := (
          map:put($o, "headers", $headers),
          map:put($o, "triples", $triples),
          map:put($o, "content", $content)
        )
        return
          $o
      let $wrapper := json:object()
      let $_ := map:put($wrapper, "envelope", $envelope)
      return
        xdmp:to-json($wrapper)
    else if ($data-format = $consts:XML) then
      document {
        <envelope xmlns="http://marklogic.com/data-hub/envelope">
          <headers>{$headers}</headers>
          <triples>{$triples}</triples>
          <content>{$content}</content>
        </envelope>
      }
    else
      fn:error((), "RESTAPI-INVALIDCONTENT", "Invalid data format: " || $data-format)
};

declare function flow:instance-to-canonical-json(
  $entity-instance as map:map) as json:object
{
  let $root-object := json:object()
  let $o :=
    if ( map:contains($entity-instance, "$ref") ) then
      map:get($entity-instance, "$ref")
    else
      let $o := json:object()
      let $_ := (
        for $key in map:keys($entity-instance)
        let $instance-property := map:get($entity-instance, $key)
        where ($key castable as xs:NCName and $key ne "$type")
        return
          typeswitch ($instance-property)
          (: This branch handles embedded objects.  You can choose to prune
             an entity's representation of extend it with lookups here. :)
          case json:object+ return
            for $prop in $instance-property
            return
              map:put($o, $key, flow:instance-to-canonical-json($prop))
          (: An array can also treated as multiple elements :)
          case json:array return
            let $a := json:array()
            let $_ :=
              for $val in json:array-values($instance-property)
              return
                if ($val instance of json:object) then
                  json:array-push($a, flow:instance-to-canonical-json($val))
                else
                  json:array-push($a, $val)
            return
              map:put($o, $key, $a)
          (: A sequence of values should be simply treated as multiple elements :)
          case item()+ return
            for $val in $instance-property
            return
              map:put($o, $key, $val)
          default return
            map:put($o, $key, $instance-property)
      )
      return
        $o
  let $_ := map:put($root-object, map:get($entity-instance, "$type"), $o)
  return
    $root-object
};

declare function flow:instance-to-canonical-xml(
  $entity-instance as map:map) as element()
{
  (: Construct an element that is named the same as the Entity Type :)
  element { map:get($entity-instance, "$type") }  {
    if ( map:contains($entity-instance, "$ref") ) then
      map:get($entity-instance, "$ref")
    else
      for $key in map:keys($entity-instance)
      let $instance-property := map:get($entity-instance, $key)
      where ($key castable as xs:NCName and $key ne "$type")
      return
        typeswitch ($instance-property)
        (: This branch handles embedded objects.  You can choose to prune
           an entity's representation of extend it with lookups here. :)
        case json:object+ return
          for $prop in $instance-property
          return
            element { $key } {
              flow:instance-to-canonical-xml($prop)
            }
        (: An array can also treated as multiple elements :)
        case json:array return
          for $val in json:array-values($instance-property)
          return
            if ($val instance of json:object) then
              element { $key } {
                attribute datatype { "array" },
                flow:instance-to-canonical-xml($val)
              }
            else
              element { $key } {
                attribute datatype { "array" },
                $val
              }
        (: A sequence of values should be simply treated as multiple elements :)
        case item()+ return
          for $val in $instance-property
          return
            element { $key } { $val }
        default return
          element { $key } { $instance-property }
  }
};

(: Formats the input for a trace based on the plugin type :)
declare function flow:get-trace-input(
  $plugin as element(hub:plugin),
  $data-format as xs:string,
  $content as item()?,
  $headers as item()*,
  $flow-type as xs:string)
{
  let $destination as xs:string := $plugin/@dest
  let $content :=
    if ($content instance of document-node()) then
      $content/node()
    else
      $content
  let $o := json:object()
  let $_ :=
    switch($destination)
      case "content" return
        if ($flow-type = $consts:INPUT_FLOW) then
          if ($content instance of binary()) then
            map:put($o, "rawContent", xs:hexBinary($content))
          else
            map:put($o, "rawContent", $content)
        else
          ()
      case "headers" return
        map:put($o, "content", $content)
      case "triples" return
      (
        map:put($o, "content", $content),
        map:put($o, "headers", ($headers, null-node{})[1])
      )
      default return ()
  return
    if ($data-format = $consts:XML) then
      for $key in map:keys($o)
      return
        element { $key } {
          let $value := map:get($o, $key)
          return
            if ($value instance of null-node()) then ()
            else $value
        }
    else
      $o
};

declare function flow:set-default-options(
  $options as map:map,
  $flow as element(hub:flow))
{
  map:put($options, "entity", fn:string($flow/hub:entity)),
  map:put($options, "flow", fn:string($flow/hub:name)),
  map:put($options, "flowType", fn:string($flow/hub:type)),
  map:put($options, "dataFormat", fn:string($flow/hub:data-format))
};

declare function flow:get-main(
  $main as element(hub:main))
{
  let $module-uri as xs:string? := $main/@module
  let $_ :=
    (: sanity check on required info :)
    if (fn:empty($module-uri) or fn:empty($main/@code-format)) then
      fn:error((), "DATAHUB-INVALID-PLUGIN", "The plugin definition is invalid.")
    else ()

  let $_ := rfc:with-module-uri($module-uri)
  let $_ := rfc:with-code-format($main/@code-format)
  let $duration := xs:dayTimeDuration("PT10S")
  let $key := $MAIN-CACHE-KEY-PREFIX||$module-uri
  let $main-func :=  hul:from-field-cache-or-empty($key, $duration)
  return
    if (fn:exists($main-func)) then $main-func
    else
      let $main-func := flow:make-function($main/@code-format, "main", $module-uri)
      let $_ := hul:set-field-cache($key, $main-func, $duration)
      return
        $main-func
};

declare function flow:run-main(
  $item-context as map:map,
  $func)
{
  let $before := xdmp:elapsed-time()
  let $_ := map:put($context-queue, rfc:get-id($item-context), $item-context)
  let $resp := try {
    let $options := rfc:get-options($item-context)
    let $_ := map:set-javascript-by-ref($options, fn:true())
    let $resp :=
      if (rfc:get-flow-type() eq $consts:HARMONIZE_FLOW) then
        $func(rfc:get-id($item-context), $options)
      else
        $func(rfc:get-id($item-context), rfc:get-content($item-context), $options)
    return
      $resp
  }
  catch($ex) {
    if ($ex/error:code eq "DATAHUB-PLUGIN-ERROR") then
      (: plugin errors are already handled :)
      ()
    else (
      (: this is an error in main.(sjs|xqy) :)
      debug:log(xdmp:describe($ex, (), ())),

      (: log the trace event for main :)
      trace:set-plugin-label(rfc:get-trace($item-context), "main"),
      trace:error-trace($item-context, $ex, xdmp:elapsed-time() - $before)
    ),
    xdmp:rethrow()
  }
  return
    $resp
};

declare function flow:queue-writer(
  $writer-function,
  $identifier as xs:string,
  $envelope as item(),
  $options as map:map)
{
  $writer-queue =>
    map:with($identifier,
      map:map()
        => map:with("writer-function", $writer-function)
        => map:with("envelope", $envelope)
        => map:with("options", $options)
    )
};

declare function flow:run-writers(
  $identifiers as xs:string*)
{
  let $updated-settings := xdmp:eval('
    import module namespace flow = "http://marklogic.com/data-hub/flow-lib"
      at "/MarkLogic/data-hub-framework/impl/flow-lib.xqy";

    import module namespace rfc = "http://marklogic.com/data-hub/run-flow-context"
      at "/MarkLogic/data-hub-framework/impl/run-flow-context.xqy";

    import module namespace trace = "http://marklogic.com/data-hub/trace"
      at "/MarkLogic/data-hub-framework/impl/trace-lib.xqy";

    declare variable $identifiers external;
    declare variable $context-queue external;
    declare variable $writer-queue external;
    declare variable $rfc-context external;
    declare variable $current-trace-settings external;

    declare option xdmp:mapping "false";

    let $_ := xdmp:set($rfc:context, $rfc-context)
    let $_ := xdmp:set($trace:current-trace-settings, $current-trace-settings)
    let $_ :=
      for $identifier in $identifiers
      let $item-context := map:get($context-queue, $identifier)
      let $writer-info := map:get($writer-queue, $identifier)
      return (
        if (fn:exists($writer-info)) then
          flow:run-writer(
            map:get($writer-info, "writer-function"),
            $item-context,
            $identifier,
            map:get($writer-info, "envelope"),
            map:get($writer-info, "options")
          )
        else ()
      )
    return
      $trace:current-trace-settings
  ',
  map:new((
    map:entry("identifiers", $identifiers),
    map:entry("context-queue", $context-queue),
    map:entry("writer-queue", $writer-queue),
    map:entry("rfc-context", $rfc:context),
    map:entry("current-trace-settings", $trace:current-trace-settings)
  )),
  map:new((
    map:entry("ignoreAmps", fn:true()),
    map:entry("isolation", "different-transaction"),
    map:entry("database", rfc:get-target-database()),
    map:entry("commit", "auto"),
    map:entry("update", "true")
  )))
  return
    xdmp:set($trace:current-trace-settings, $updated-settings)
};

(:~
 : Run a given writer
 :
 : @param $writer - xml describing the writer to run
 : @param $identifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $envelope - the envelope
 : @param $options - a map of options passed in by the client
 : @return - the output of the writer. It varies.
 :)
declare function flow:run-writer(
  $writer-function,
  $item-context as map:map,
  $identifier as xs:string,
  $envelope as item(),
  $options as map:map)
{
  let $before := xdmp:elapsed-time()
  let $trace := rfc:get-trace($item-context)
  let $current-trace := rfc:get-trace($item-context)
  let $_ := trace:set-plugin-label($current-trace, "writer")
  let $_ := trace:reset-plugin-input($current-trace)
  let $_ := trace:set-plugin-input($current-trace, "envelope", $envelope)
  let $resp :=
    try {
        $writer-function($identifier, $envelope, $options),

        trace:plugin-trace($item-context, (), xdmp:elapsed-time() - $before),

        (: write the trace for the current identifier :)
        trace:write-trace($item-context)
    }
    catch($ex) {
      debug:log(xdmp:describe($ex, (), ())),
      trace:error-trace($item-context, $ex, xdmp:elapsed-time() - $before)
    }
  let $duration := xdmp:elapsed-time() - $before
  return
    $resp
};

declare function flow:make-error-json(
  $errors as json:object,
  $entity as xs:string,
  $flow  as xs:string,
  $plugin as xs:string,
  $ex
) {
  let $eo :=
    if (map:contains($errors, $entity)) then
      map:get($errors, $entity)
    else
      let $e := map:map()
      let $_ := map:put($errors, $entity, $e)
      return
        $e
  let $fo :=
    if (map:contains($eo, $flow)) then
      map:get($eo, $flow)
    else
      let $f := map:map()
      let $_ := map:put($eo, $flow, $f)
      return
        $f

  let $f := $ex/error:stack/error:frame[1]
  return
    map:put($fo, $plugin, map:new((
      map:entry("uri", $f/error:uri/fn:data()),
      map:entry("line", $f/error:line/fn:data()),
      map:entry("column", $f/error:column/fn:data()),
      map:entry("msg", $ex/error:format-string/fn:data())
    )))
};

declare function flow:validate-entities()
{
  let $errors := json:object()
  let $options := map:map()
  let $_ :=
    for $entity in flow:get-entities()/hub:entity
    for $flow in $entity/hub:flows/hub:flow
    let $data-format := $flow/hub:data-format
    (: validate collector :)
    let $_ :=
      try {
        let $collector := $flow/hub:collector
        return
          if ($collector) then
            let $module-uri := $collector/@module
            let $ns := flow:get-module-ns($collector/@code-format)
            return
              if ($collector/@code-format eq $consts:XQUERY) then
                xdmp:eval(
                  'import module namespace x = "' || $ns || '" at "' || $module-uri || '"; ' ||
                  '()',
                  map:new((map:entry("staticCheck", fn:true())))
                )
              else
                xdmp:javascript-eval(
                  'var x = require("' || $module-uri || '");',
                  map:new((map:entry("staticCheck", fn:true())))
                )
          else ()
      }
      catch($ex) {
        flow:make-error-json(
          $errors,
          $entity/hub:name,
          $flow/hub:name,
          "collector",
          $ex)
      }
    (: validate plugins :)
    let $_ :=
      for $main in $flow/hub:main
      let $module-uri := $main/@module
      let $ns := flow:get-module-ns($main/@code-format)
      return
        (:
         : Note that we are static checking the files.
         : This is because there is no reasonable way to actually
         : call the plugins and pass in data that will work for all plugins.
         :
         : The disadvantage to static checking is that we will not catch typos
         : like ctsdoc <- (missing period) because Javascript is dynamically
         : typed. Static checking will only catch syntax errors in sjs.
         :)
        try {
          if ($main/@code-format eq $consts:XQUERY) then
            xdmp:eval(
              'import module namespace x = "' || $ns || '" at "' || $module-uri || '"; ' ||
              '()',
           map:new((map:entry("staticCheck", fn:true())))
            )
          else
            xdmp:javascript-eval(
              'var x = require("' || $module-uri || '");',
              map:new((map:entry("staticCheck", fn:true())))
            )
        }
        catch($ex) {
          let $plugin :=
            let $uri := ($ex/error:stack/error:frame)[1]/error:uri
            let $name := hul:get-file-name(hul:get-file-from-uri($uri))
            return
              if ($name eq "[anonymous]") then
                fn:replace($ex/error:expr, ".+/([^.]+)\.(sjs|xqy).*", "$1")
              else
                $name
          return
            flow:make-error-json(
              $errors,
              $entity/hub:name,
              $flow/hub:name,
              $plugin,
              $ex)
        }
    return
      ()
  return
    map:new(
      map:entry("errors", $errors)
    )
};

declare function flow:safe-run($func)
{
  let $before := xdmp:elapsed-time()
  return
    try {
      let $resp := $func()
      let $duration := xdmp:elapsed-time() - $before
      let $_ := trace:plugin-trace($rfc:item-context, $resp, $duration)
      return
        $resp
    }
    catch($ex) {
      debug:log(xdmp:describe($ex, (), ())),
      trace:error-trace($rfc:item-context, $ex, xdmp:elapsed-time() - $before),
      fn:error((), "DATAHUB-PLUGIN-ERROR", ("error in a plugin", $ex))
    }
};

declare %private function flow:make-function(
  $code-format as xs:string?,
  $func-name as xs:string,
  $module-uri as xs:string)
{
  let $ns := flow:get-module-ns($code-format)
  return
    xdmp:function(fn:QName($ns, $func-name), $module-uri)
};

(:
  Copyright 2012-2018 MarkLogic Corporation

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
:)
xquery version "1.0-ml";

module namespace flow = "http://marklogic.com/data-hub/flow-lib";

import module namespace consts = "http://marklogic.com/data-hub/consts"
  at "/MarkLogic/data-hub-framework/impl/consts.xqy";

import module namespace debug = "http://marklogic.com/data-hub/debug"
  at "/MarkLogic/data-hub-framework/impl/debug-lib.xqy";

import module namespace hul = "http://marklogic.com/data-hub/hub-utils-lib"
  at "/MarkLogic/data-hub-framework/impl/hub-utils-lib.xqy";

import module namespace json="http://marklogic.com/xdmp/json"
  at "/MarkLogic/json/json.xqy";

import module namespace functx = "http://www.functx.com"
  at "/MarkLogic/functx/functx-1.0-nodoc-2007-01.xqy";

import module namespace rfc = "http://marklogic.com/data-hub/run-flow-context"
  at "/MarkLogic/data-hub-framework/impl/run-flow-context.xqy";

import module namespace trace = "http://marklogic.com/data-hub/trace"
  at "/MarkLogic/data-hub-framework/impl/trace-lib.xqy";

declare namespace hub = "http://marklogic.com/data-hub";

declare option xdmp:mapping "false";

(: the directory where entities live :)
declare variable $ENTITIES-DIR := "/entities/";

declare variable $PLUGIN-NS := "http://marklogic.com/data-hub/plugins";

declare variable $FLOW-CACHE-KEY-PREFIX := "flow-cache-";
declare variable $MAIN-CACHE-KEY-PREFIX := "main-cache-";

declare variable $context-queue := map:map();
declare variable $writer-queue := map:map();

declare function flow:get-module-ns(
  $type as xs:string) as xs:string?
{
  if ($type eq $consts:JAVASCRIPT) then ()
  else
    $PLUGIN-NS
};

(:~
 : Returns a flow by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity that owns the flow
 : @param $flow-name - name of the flow to retrieve
 : @return - xml describing the flow
 :)
declare function flow:get-flow(
  $entity-name as xs:string,
  $flow-name as xs:string,
  $flow-type as xs:string?) as element(hub:flow)?
{
  let $duration := xs:dayTimeDuration("PT10S")
  let $key := $FLOW-CACHE-KEY-PREFIX||$entity-name||$flow-name||$flow-type
  let $flow :=  hul:from-field-cache-or-empty($key, $duration)
  return
    if ($flow) then
      $flow
    else
      hul:set-field-cache(
        $key,
        flow:get-flow-nocache($entity-name, $flow-name, $flow-type),
        $duration
      )
};

declare function flow:get-flow-nocache(
  $entity-name as xs:string,
  $flow-name as xs:string,
  $flow-type as xs:string?) as element(hub:flow)?
{

  hul:run-in-modules(function() {
    /hub:flow[
      hub:entity = $entity-name and
      hub:name = $flow-name]
      [
        if (fn:exists($flow-type)) then
          hub:type = $flow-type
        else
          fn:true()
      ]
  }) ! hul:deep-copy(.)
};

(:~
 : Returns the flows that belong to the given entity in the database
 :
 : @param $entity-name - the name of the entity containing the flows
 : @return - xml describing the flows
 :)
declare function flow:get-flows(
  $entity-name as xs:string) as element(hub:flows)
{
  element hub:flows {
    hul:run-in-modules(function() {
      /hub:flow[hub:entity = $entity-name]
    })
  }
};

(:~
 : Returns an entity by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity to retrieve
 : @return - xml describing the entity
 :)
declare function flow:get-entity(
  $entity-name as xs:string)
{
  let $uris :=
    hul:run-in-modules(function() {
      cts:uri-match($ENTITIES-DIR || $entity-name || "/*")
    })
  return
    flow:get-entity($entity-name, $uris)
};

(:~
 : Returns a entity by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity to retrieve
 : @param $uris - uris used to build the entity xml
 : @return - xml describing the entity
 :)
declare %private function flow:get-entity(
  $entity-name as xs:string,
  $uris as xs:string*)
  as element(hub:entity)
{
  <entity xmlns="http://marklogic.com/data-hub">
    <name>{$entity-name}</name>
    {
      flow:get-flows($entity-name)
    }
  </entity>
};

(:~
 : Returns the entities in the database
 :
 : @return - xml describing the entities
 :)
declare function flow:get-entities() as element(hub:entities)
{
  let $uris := hul:run-in-modules(function() {
    cts:uri-match($ENTITIES-DIR || "*")
  })
  let $entity-names :=
    fn:distinct-values(
      let $regex := $ENTITIES-DIR || "([^/]+)/.*$"
      for $flow in $uris[fn:matches(., $regex)]
      let $name := fn:replace($flow, $regex, "$1")
      return
        $name)
  let $entities :=
    for $entity-name in $entity-names
    return
      flow:get-entity($entity-name, $uris[fn:matches(., $ENTITIES-DIR || $entity-name || "/.+")])
  return
    <entities xmlns="http://marklogic.com/data-hub">
    {
      $entities
    }
    </entities>
};

(:~
 : Runs a collector
 :
 : @param $module-uri - the uri of the collector module
 : @param $options - a map of options passed in by the client
 : @return - a sequence of strings
 :)
declare function flow:run-collector(
  $flow as element(hub:flow),
  $job-id as xs:string,
  $options as map:map) as item()*
{
    (: sanity check on required info :)
  if (fn:empty($flow/hub:collector/@module) or fn:empty($flow/hub:collector/@code-format)) then
    fn:error((), "DATAHUB-INVALID-PLUGIN", "The plugin definition is invalid.")
  else (),

  (: assert that we are in query mode :)
  let $_ts as xs:unsignedLong := xdmp:request-timestamp()

  let $item-context := rfc:new-item-context()
    => rfc:with-options($options)
    => rfc:with-trace(trace:new-trace())

  let $_ := trace:set-plugin-label("collector")

  (: configure the global run context :)
  let $_ := (
    rfc:with-job-id($job-id),
    rfc:with-flow($flow),
    rfc:with-module-uri($flow/hub:collector/@module),
    rfc:with-code-format($flow/hub:collector/@code-format)
  )

  let $func := flow:make-function(
    $flow/hub:collector/@code-format,
    "collect",
    $flow/hub:collector/@module
  )
  let $before := xdmp:elapsed-time()
  let $resp :=
    try {
      $func($options)
    }
    catch($ex) {
      debug:log(xdmp:describe($ex, (), ())),
      trace:error-trace($item-context, $ex, xdmp:elapsed-time() - $before),
      xdmp:rethrow()
    }
  (: log and write the trace :)
  let $_ := (
    trace:plugin-trace($item-context, xdmp:describe($resp, 1000000, 1000000), xdmp:elapsed-time() - $before),
    trace:write-trace($item-context)
  )
  return
    $resp
};

(:~
 : Runs a given flow
 :
 : @param $flow - xml describing the flow
 : @param $identifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $options - a map of options passed in by the client
 : @return - nothing
 :)
declare function flow:run-flow(
  $job-id as xs:string,
  $flow as element(hub:flow),
  $identifier as xs:string,
  $options as map:map,
  $mainFunc)
{
  flow:run-flow($job-id, $flow, $identifier, (), $options, $mainFunc)
};

(:~
 : Runs a given flow
 :
 : @param $flow - xml describing the flow
 : @param $identifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $content - the content being loaded
 : @param $options - a map of options passed in by the client
 : @return - nothing
 :)
declare function flow:run-flow(
  $job-id as xs:string,
  $flow as element(hub:flow),
  $identifier as xs:string,
  $content as item()?,
  $options as map:map,
  $mainFunc)
{
  (: assert that we are in query mode :)
  let $_must_run_in_query_mode as xs:unsignedLong := xdmp:request-timestamp()

  (: configure the global context :)
  let $_ := (
    rfc:with-job-id($job-id),
    rfc:with-flow($flow),
    map:get($options, "target-database") ! rfc:with-target-database(.)
  )

  (: configure the item context :)
  let $item-context := rfc:new-item-context()
    => rfc:with-id($identifier)
    => rfc:with-content(
        if ($content instance of document-node()) then
          $content/node()
        else $content
      )
    => rfc:with-options($options)
    => rfc:with-trace(trace:new-trace())

  (: run the users main.(sjs|xqy) :)
  return
    flow:run-main($item-context, $mainFunc)
};

declare function flow:clean-data($resp, $destination, $data-format)
{
  let $resp :=
    typeswitch($resp)
      case document-node() return
        if (fn:count($resp/node()) > 1) then
          fn:error((), "DATAHUB-TOO-MANY-NODES", "Too Many Nodes!. Return just 1 node")
        else
          $resp/node()
      default return
        $resp

  (: clean up output :)
  return
    typeswitch($resp)
      case binary() return xs:hexBinary($resp)
      case object-node() | json:object return
        (: object with $type key is ES response type :)
        if ($resp instance of map:map and map:keys($resp) = "$type") then
          $resp
        else if ($data-format = $consts:XML) then
          json:transform-from-json($resp, json:config("custom"))
        else
          $resp
      case json:array return
        if ($data-format = $consts:XML) then
          json:array-values($resp)
        else
          $resp
      case empty-sequence() return
        if ($destination = "headers" and $data-format = $consts:JSON) then
          json:object()
        else if ($destination = "triples" and $data-format = $consts:JSON) then
          json:array()
        else
          $resp
      default return
        if ($data-format = $consts:JSON and
            rfc:get-code-format() = $consts:XQUERY and
            $destination = "triples") then
          json:to-array($resp)
        else
          $resp
};

(:~
 : Construct an envelope
 :
 : @param $map - a map with all the stuff in it
 : @return - the newly constructed envelope
 :)
declare function flow:make-envelope($content, $headers, $triples, $data-format)
  as document-node()
{
  let $content := flow:clean-data($content, "content", $data-format)
  let $headers := flow:clean-data($headers, "headers", $data-format)
  let $triples := flow:clean-data($triples, "triples", $data-format)
  return
    if ($data-format = $consts:JSON) then
      let $envelope :=
        let $o := json:object()
        let $_ := (
          map:put($o, "headers", $headers),
          map:put($o, "triples", $triples),
          map:put($o, "instance",
            if ($content instance of map:map and map:keys($content) = "$type") then
              let $json := flow:instance-to-canonical-json($content)
              let $info :=
                let $o :=json:object()
                let $_ := (
                  map:put($o, "title", map:get($content, "$type")),
                  map:put($o, "version",  map:get($content, "$version"))
                )
                return $o
              let $_ := map:put($json, "info", $info)
              return $json
            else
              $content
          ),
          map:put($o, "attachments",
            if ($content instance of map:map and map:keys($content) = "$attachments") then
              map:get($content, "$attachments")
            else
              ()
          )
        )
        return
          $o
      let $wrapper := json:object()
      let $_ := map:put($wrapper, "envelope", $envelope)
      return
        xdmp:to-json($wrapper)
    else if ($data-format = $consts:XML) then
      document {
        <envelope xmlns="http://marklogic.com/entity-services">
          <headers>{$headers}</headers>
          <triples>{$triples}</triples>
          <instance>
          {
            if ($content instance of map:map and map:keys($content) = "$type") then (
              <info>
                <title>{map:get($content, "$type")}</title>
                <version>{map:get($content, "$version")}</version>
              </info>,
              flow:instance-to-canonical-xml($content)
            )
            else
              $content
          }
          </instance>
          <attachments>
          {
            if ($content instance of map:map and map:keys($content) = "$attachments") then
              map:get($content, "$attachments")
            else
              ()
          }
          </attachments>
        </envelope>
      }
    else
      fn:error((), "RESTAPI-INVALIDCONTENT", "Invalid data format: " || $data-format)
};

declare function flow:make-legacy-envelope($content, $headers, $triples, $data-format)
  as document-node()
{
  let $content := flow:clean-data($content, "content", $data-format)
  let $headers := flow:clean-data($headers, "headers", $data-format)
  let $triples := flow:clean-data($triples, "triples", $data-format)
  return
    if ($data-format = $consts:JSON) then
      let $envelope :=
        let $o := json:object()
        let $_ := (
          map:put($o, "headers", $headers),
          map:put($o, "triples", $triples),
          map:put($o, "content", $content)
        )
        return
          $o
      let $wrapper := json:object()
      let $_ := map:put($wrapper, "envelope", $envelope)
      return
        xdmp:to-json($wrapper)
    else if ($data-format = $consts:XML) then
      document {
        <envelope xmlns="http://marklogic.com/data-hub/envelope">
          <headers>{$headers}</headers>
          <triples>{$triples}</triples>
          <content>{$content}</content>
        </envelope>
      }
    else
      fn:error((), "RESTAPI-INVALIDCONTENT", "Invalid data format: " || $data-format)
};

declare function flow:instance-to-canonical-json(
  $entity-instance as map:map) as json:object
{
  let $root-object := json:object()
  let $o :=
    if ( map:contains($entity-instance, "$ref") ) then
      map:get($entity-instance, "$ref")
    else
      let $o := json:object()
      let $_ := (
        for $key in map:keys($entity-instance)
        let $instance-property := map:get($entity-instance, $key)
        where ($key castable as xs:NCName and $key ne "$type")
        return
          typeswitch ($instance-property)
          (: This branch handles embedded objects.  You can choose to prune
             an entity's representation of extend it with lookups here. :)
          case json:object+ return
            for $prop in $instance-property
            return
              map:put($o, $key, flow:instance-to-canonical-json($prop))
          (: An array can also treated as multiple elements :)
          case json:array return
            let $a := json:array()
            let $_ :=
              for $val in json:array-values($instance-property)
              return
                if ($val instance of json:object) then
                  json:array-push($a, flow:instance-to-canonical-json($val))
                else
                  json:array-push($a, $val)
            return
              map:put($o, $key, $a)
          (: A sequence of values should be simply treated as multiple elements :)
          case item()+ return
            for $val in $instance-property
            return
              map:put($o, $key, $val)
          default return
            map:put($o, $key, $instance-property)
      )
      return
        $o
  let $_ := map:put($root-object, map:get($entity-instance, "$type"), $o)
  return
    $root-object
};

declare function flow:instance-to-canonical-xml(
  $entity-instance as map:map) as element()
{
  (: Construct an element that is named the same as the Entity Type :)
  element { map:get($entity-instance, "$type") }  {
    if ( map:contains($entity-instance, "$ref") ) then
      map:get($entity-instance, "$ref")
    else
      for $key in map:keys($entity-instance)
      let $instance-property := map:get($entity-instance, $key)
      where ($key castable as xs:NCName and $key ne "$type")
      return
        typeswitch ($instance-property)
        (: This branch handles embedded objects.  You can choose to prune
           an entity's representation of extend it with lookups here. :)
        case json:object+ return
          for $prop in $instance-property
          return
            element { $key } {
              flow:instance-to-canonical-xml($prop)
            }
        (: An array can also treated as multiple elements :)
        case json:array return
          for $val in json:array-values($instance-property)
          return
            if ($val instance of json:object) then
              element { $key } {
                attribute datatype { "array" },
                flow:instance-to-canonical-xml($val)
              }
            else
              element { $key } {
                attribute datatype { "array" },
                $val
              }
        (: A sequence of values should be simply treated as multiple elements :)
        case item()+ return
          for $val in $instance-property
          return
            element { $key } { $val }
        default return
          element { $key } { $instance-property }
  }
};

(: Formats the input for a trace based on the plugin type :)
declare function flow:get-trace-input(
  $plugin as element(hub:plugin),
  $data-format as xs:string,
  $content as item()?,
  $headers as item()*,
  $flow-type as xs:string)
{
  let $destination as xs:string := $plugin/@dest
  let $content :=
    if ($content instance of document-node()) then
      $content/node()
    else
      $content
  let $o := json:object()
  let $_ :=
    switch($destination)
      case "content" return
        if ($flow-type = $consts:INPUT_FLOW) then
          if ($content instance of binary()) then
            map:put($o, "rawContent", xs:hexBinary($content))
          else
            map:put($o, "rawContent", $content)
        else
          ()
      case "headers" return
        map:put($o, "content", $content)
      case "triples" return
      (
        map:put($o, "content", $content),
        map:put($o, "headers", ($headers, null-node{})[1])
      )
      default return ()
  return
    if ($data-format = $consts:XML) then
      for $key in map:keys($o)
      return
        element { $key } {
          let $value := map:get($o, $key)
          return
            if ($value instance of null-node()) then ()
            else $value
        }
    else
      $o
};

declare function flow:set-default-options(
  $options as map:map,
  $flow as element(hub:flow))
{
  map:put($options, "entity", fn:string($flow/hub:entity)),
  map:put($options, "flow", fn:string($flow/hub:name)),
  map:put($options, "flowType", fn:string($flow/hub:type)),
  map:put($options, "dataFormat", fn:string($flow/hub:data-format))
};

declare function flow:get-main(
  $main as element(hub:main))
{
  let $module-uri as xs:string? := $main/@module
  let $_ :=
    (: sanity check on required info :)
    if (fn:empty($module-uri) or fn:empty($main/@code-format)) then
      fn:error((), "DATAHUB-INVALID-PLUGIN", "The plugin definition is invalid.")
    else ()

  let $_ := rfc:with-module-uri($module-uri)
  let $_ := rfc:with-code-format($main/@code-format)
  let $duration := xs:dayTimeDuration("PT10S")
  let $key := $MAIN-CACHE-KEY-PREFIX||$module-uri
  let $main-func :=  hul:from-field-cache-or-empty($key, $duration)
  return
    if (fn:exists($main-func)) then $main-func
    else
      let $main-func := flow:make-function($main/@code-format, "main", $module-uri)
      let $_ := hul:set-field-cache($key, $main-func, $duration)
      return
        $main-func
};

declare function flow:run-main(
  $item-context as map:map,
  $func)
{
  let $before := xdmp:elapsed-time()
  let $_ := map:put($context-queue, rfc:get-id($item-context), $item-context)
  let $resp := try {
    let $options := rfc:get-options($item-context)
    let $_ := map:set-javascript-by-ref($options, fn:true())
    let $resp :=
      if (rfc:get-flow-type() eq $consts:HARMONIZE_FLOW) then
        $func(rfc:get-id($item-context), $options)
      else
        $func(rfc:get-id($item-context), rfc:get-content($item-context), $options)
    return
      $resp
  }
  catch($ex) {
    if ($ex/error:name eq "DATAHUB-PLUGIN-ERROR") then
      (: plugin errors are already handled :)
      ()
    else (
      (: this is an error in main.(sjs|xqy) :)
      debug:log(xdmp:describe($ex, (), ())),

      (: log the trace event for main :)
      trace:set-plugin-label(rfc:get-trace($item-context), "main"),
      trace:error-trace($item-context, $ex, xdmp:elapsed-time() - $before)
    ),
    xdmp:rethrow()
  }
  return
    $resp
};

declare function flow:queue-writer(
  $writer-function,
  $identifier as xs:string,
  $envelope as item(),
  $options as map:map)
{
  $writer-queue =>
    map:with($identifier,
      map:map()
        => map:with("writer-function", $writer-function)
        => map:with("envelope", $envelope)
        => map:with("options", $options)
    )
};

declare function flow:run-writers(
  $identifiers as xs:string*)
{
  let $updated-settings := xdmp:eval('
    import module namespace flow = "http://marklogic.com/data-hub/flow-lib"
      at "/MarkLogic/data-hub-framework/impl/flow-lib.xqy";

    import module namespace rfc = "http://marklogic.com/data-hub/run-flow-context"
      at "/MarkLogic/data-hub-framework/impl/run-flow-context.xqy";

    import module namespace trace = "http://marklogic.com/data-hub/trace"
      at "/MarkLogic/data-hub-framework/impl/trace-lib.xqy";

    declare variable $identifiers external;
    declare variable $context-queue external;
    declare variable $writer-queue external;
    declare variable $rfc-context external;
    declare variable $current-trace-settings external;

    declare option xdmp:mapping "false";

    let $_ := xdmp:set($rfc:context, $rfc-context)
    let $_ := xdmp:set($trace:current-trace-settings, $current-trace-settings)
    let $_ :=
      for $identifier in $identifiers
      let $item-context := map:get($context-queue, $identifier)
      let $writer-info := map:get($writer-queue, $identifier)
      return (
        if (fn:exists($writer-info)) then
          flow:run-writer(
            map:get($writer-info, "writer-function"),
            $item-context,
            $identifier,
            map:get($writer-info, "envelope"),
            map:get($writer-info, "options")
          )
        else ()
      )
    return
      $trace:current-trace-settings
  ',
  map:new((
    map:entry("identifiers", $identifiers),
    map:entry("context-queue", $context-queue),
    map:entry("writer-queue", $writer-queue),
    map:entry("rfc-context", $rfc:context),
    map:entry("current-trace-settings", $trace:current-trace-settings)
  )),
  map:new((
    map:entry("ignoreAmps", fn:true()),
    map:entry("isolation", "different-transaction"),
    map:entry("database", rfc:get-target-database()),
    map:entry("commit", "auto"),
    map:entry("update", "true")
  )))
  return
    xdmp:set($trace:current-trace-settings, $updated-settings)
};

(:~
 : Run a given writer
 :
 : @param $writer - xml describing the writer to run
 : @param $identifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $envelope - the envelope
 : @param $options - a map of options passed in by the client
 : @return - the output of the writer. It varies.
 :)
declare function flow:run-writer(
  $writer-function,
  $item-context as map:map,
  $identifier as xs:string,
  $envelope as item(),
  $options as map:map)
{
  let $before := xdmp:elapsed-time()
  let $trace := rfc:get-trace($item-context)
  let $current-trace := rfc:get-trace($item-context)
  let $_ := trace:set-plugin-label($current-trace, "writer")
  let $_ := trace:reset-plugin-input($current-trace)
  let $_ := trace:set-plugin-input($current-trace, "envelope", $envelope)
  let $resp :=
    try {
        $writer-function($identifier, $envelope, $options),

        trace:plugin-trace($item-context, (), xdmp:elapsed-time() - $before),

        (: write the trace for the current identifier :)
        trace:write-trace($item-context)
    }
    catch($ex) {
      debug:log(xdmp:describe($ex, (), ())),
      trace:error-trace($item-context, $ex, xdmp:elapsed-time() - $before)
    }
  let $duration := xdmp:elapsed-time() - $before
  return
    $resp
};

declare function flow:make-error-json(
  $errors as json:object,
  $entity as xs:string,
  $flow  as xs:string,
  $plugin as xs:string,
  $ex
) {
  let $eo :=
    if (map:contains($errors, $entity)) then
      map:get($errors, $entity)
    else
      let $e := map:map()
      let $_ := map:put($errors, $entity, $e)
      return
        $e
  let $fo :=
    if (map:contains($eo, $flow)) then
      map:get($eo, $flow)
    else
      let $f := map:map()
      let $_ := map:put($eo, $flow, $f)
      return
        $f

  let $f := $ex/error:stack/error:frame[1]
  return
    map:put($fo, $plugin, map:new((
      map:entry("uri", $f/error:uri/fn:data()),
      map:entry("line", $f/error:line/fn:data()),
      map:entry("column", $f/error:column/fn:data()),
      map:entry("msg", $ex/error:format-string/fn:data())
    )))
};

declare function flow:validate-entities()
{
  let $errors := json:object()
  let $options := map:map()
  let $_ :=
    for $entity in flow:get-entities()/hub:entity
    for $flow in $entity/hub:flows/hub:flow
    let $data-format := $flow/hub:data-format
    (: validate collector :)
    let $_ :=
      try {
        let $collector := $flow/hub:collector
        return
          if ($collector) then
            let $module-uri := $collector/@module
            let $ns := flow:get-module-ns($collector/@code-format)
            return
              if ($collector/@code-format eq $consts:XQUERY) then
                xdmp:eval(
                  'import module namespace x = "' || $ns || '" at "' || $module-uri || '"; ' ||
                  '()',
                  map:new((map:entry("staticCheck", fn:true())))
                )
              else
                xdmp:javascript-eval(
                  'var x = require("' || $module-uri || '");',
                  map:new((map:entry("staticCheck", fn:true())))
                )
          else ()
      }
      catch($ex) {
        flow:make-error-json(
          $errors,
          $entity/hub:name,
          $flow/hub:name,
          "collector",
          $ex)
      }
    (: validate plugins :)
    let $_ :=
      for $main in $flow/hub:main
      let $module-uri := $main/@module
      let $ns := flow:get-module-ns($main/@code-format)
      return
        (:
         : Note that we are static checking the files.
         : This is because there is no reasonable way to actually
         : call the plugins and pass in data that will work for all plugins.
         :
         : The disadvantage to static checking is that we will not catch typos
         : like ctsdoc <- (missing period) because Javascript is dynamically
         : typed. Static checking will only catch syntax errors in sjs.
         :)
        try {
          if ($main/@code-format eq $consts:XQUERY) then
            xdmp:eval(
              'import module namespace x = "' || $ns || '" at "' || $module-uri || '"; ' ||
              '()',
           map:new((map:entry("staticCheck", fn:true())))
            )
          else
            xdmp:javascript-eval(
              'var x = require("' || $module-uri || '");',
              map:new((map:entry("staticCheck", fn:true())))
            )
        }
        catch($ex) {
          let $plugin :=
            let $uri := ($ex/error:stack/error:frame)[1]/error:uri
            let $name := hul:get-file-name(hul:get-file-from-uri($uri))
            return
              if ($name eq "[anonymous]") then
                fn:replace($ex/error:expr, ".+/([^.]+)\.(sjs|xqy).*", "$1")
              else
                $name
          return
            flow:make-error-json(
              $errors,
              $entity/hub:name,
              $flow/hub:name,
              $plugin,
              $ex)
        }
    return
      ()
  return
    map:new(
      map:entry("errors", $errors)
    )
};

declare function flow:safe-run($func)
{
  let $before := xdmp:elapsed-time()
  return
    try {
      let $resp := $func()
      let $duration := xdmp:elapsed-time() - $before
      let $_ := trace:plugin-trace($rfc:item-context, $resp, $duration)
      return
        $resp
    }
    catch($ex) {
      debug:log(xdmp:describe($ex, (), ())),
      trace:error-trace($rfc:item-context, $ex, xdmp:elapsed-time() - $before),
      fn:error((), "DATAHUB-PLUGIN-ERROR", ("error in a plugin", $ex))
    }
};

declare %private function flow:make-function(
  $code-format as xs:string?,
  $func-name as xs:string,
  $module-uri as xs:string)
{
  let $ns := flow:get-module-ns($code-format)
  return
    xdmp:function(fn:QName($ns, $func-name), $module-uri)
};