Fixing fix to #3452: syntax is hard.

Fixing fix to #3452: syntax is hard.

xquery version "1.0-ml";

(:
 : This is an implementation library, not an interface to the Smart Mastering functionality.
 :
 : The process of matching starts with one document, which is not required to
 : be in the database. The match options specify what properties are to be used
 : to find matches. See match options documentation for details. The options
 : may specify multiple thresholds, each of which corresponds to an action.
 :
 : Implementation notes: the configured properties are used to generate a boost
 : query. The match part of the query identifies a set of subqueries that a
 : document must match in order to get a score above the lowest threshold.
 : Match queries all have their scores set to zero. The boost part of the query
 : is used to provide the score.

 : @see https://marklogic-community.github.io/smart-mastering-core/docs/matching-options/
 :)

module namespace match-impl = "http://marklogic.com/smart-mastering/matcher-impl";

import module namespace algorithms = "http://marklogic.com/smart-mastering/algorithms"
  at  "/com.marklogic.smart-mastering/algorithms/base.xqy",
      "/com.marklogic.smart-mastering/algorithms/standard-reduction.xqy";
import module namespace blocks-impl = "http://marklogic.com/smart-mastering/blocks-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/blocks-impl.xqy";
import module namespace const = "http://marklogic.com/smart-mastering/constants"
  at "/com.marklogic.smart-mastering/constants.xqy";
import module namespace es-helper = "http://marklogic.com/smart-mastering/entity-services" at "/com.marklogic.smart-mastering/sm-entity-services.xqy";
import module namespace helper-impl = "http://marklogic.com/smart-mastering/helper-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/helper-impl.xqy";
import module namespace json="http://marklogic.com/xdmp/json"
  at "/MarkLogic/json/json.xqy";
import module namespace opt-impl = "http://marklogic.com/smart-mastering/options-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/options-impl.xqy";
import module namespace tel = "http://marklogic.com/smart-mastering/telemetry"
  at "/com.marklogic.smart-mastering/telemetry.xqy";
import module namespace coll = "http://marklogic.com/smart-mastering/collections"
  at "/com.marklogic.smart-mastering/impl/collections.xqy";

declare namespace matcher = "http://marklogic.com/smart-mastering/matcher";
declare namespace sm = "http://marklogic.com/smart-mastering";
declare namespace es = "http://marklogic.com/entity-services";

declare option xdmp:mapping "false";

declare variable $QUERIES_WITH_WEIGHT := (
    xs:QName("cts:element-attribute-pair-geospatial-query"),xs:QName("cts:element-attribute-range-query"),
    xs:QName("cts:element-attribute-value-query"),xs:QName("cts:element-attribute-word-query"),xs:QName("cts:element-child-geospatial-query"),
    xs:QName("cts:element-geospatial-query"),xs:QName("cts:element-pair-geospatial-query"),xs:QName("cts:element-range-query"),
    xs:QName("cts:element-value-query"),xs:QName("cts:element-word-query"),xs:QName("cts:field-range-query"),
    xs:QName("cts:field-value-query"),xs:QName("cts:field-word-query"),xs:QName("cts:geospatial-region-query"),
    xs:QName("cts:json-property-child-geospatial-query"),xs:QName("cts:json-property-geospatial-query"),
    xs:QName("cts:json-property-pair-geospatial-query"),xs:QName("cts:json-property-range-query"),xs:QName("cts:json-property-value-query"),
    xs:QName("cts:json-property-word-query"),xs:QName("cts:lsqt-query"),xs:QName("cts:near-query"),xs:QName("cts:not-query"),
    xs:QName("cts:path-geospatial-query"),xs:QName("cts:path-range-query"),xs:QName("cts:range-query"),xs:QName("cts:registered-query"),
    xs:QName("cts:reverse-query"),xs:QName("cts:similar-query"),xs:QName("cts:triple-range-query"),xs:QName("cts:word-query"));

declare variable $_cached-compiled-match-options as map:map := map:map();

(:
 : Calculate queries once per unique match options in request to reduce repeat logic
 : @param $match-options  Options specifying how documents will be matched
 : @param $document-is-json  Is a JSON document we are matching?
 : @param $minimum-threshold  Minimum threshold for search results to meet
 : @return map:map with compiled information about match options
 :)
declare function match-impl:compile-match-options(
  $match-options as node(),
  $document-is-json as xs:boolean,
  $minimum-threshold as xs:double
) {
  let $cache-id :=
      fn:generate-id($match-options) || "|doc-is-json:"|| $document-is-json || "|min-threshold:" || $minimum-threshold
  return
  if (map:contains($_cached-compiled-match-options, $cache-id)) then
    map:get($_cached-compiled-match-options, $cache-id)
  else
    let $options :=
      if ($match-options instance of object-node()) then
        opt-impl:options-from-json($match-options)
      else
        $match-options
    let $data-format := $options/matcher:data-format
    let $options :=
          if (fn:empty($data-format)) then
            element matcher:options {
              element matcher:data-format {
                if ($document-is-json) then $const:FORMAT-JSON else $const:FORMAT-XML
              },
              $options/*
            }
          else
            $options
    let $scoring := $options/matcher:scoring
    let $max-property-score := fn:max(($scoring/@weight ! fn:number(.)))
    let $score-ratio :=
        if ($max-property-score le 64) then
          1
        else
          (64 idiv $max-property-score)
    let $algorithms := algorithms:build-algorithms-map($options/matcher:algorithms)
    let $target-entity := $options/matcher:target-entity ! fn:string(.)
    let $target-entity-def := es-helper:get-entity-def($target-entity)
    let $queries := (
        for $score in $scoring/(matcher:add|matcher:expand)
        let $type := fn:local-name($score)
        let $weight := fn:number($score/@weight)
        let $full-property-name := $score/@property-name
        let $helper-query := helper-impl:property-name-to-query($options, $full-property-name)
        where fn:exists($helper-query)
        order by $weight descending
        return
          let $qname := helper-impl:property-name-to-qname($options, $full-property-name)
          let $base-values-query :=
              if ($type eq "add") then
                $helper-query(?, $weight)
              else if ($type eq "expand") then
                let $algorithm := map:get($algorithms, $score/@algorithm-ref)
                where fn:exists($algorithm)
                return algorithms:execute-algorithm($algorithm, ?, $score, $options)
              else ()
          return map:new((
            map:entry("queryID", sem:uuid-string()),
            map:entry("propertyName",$full-property-name),
            map:entry("type",$type),
            map:entry("weight",$weight),
            map:entry("qname", $qname),
            map:entry(
              "valuesToQueryFunction",
              $base-values-query
            )
          )),
        for $score in $scoring/matcher:reduce
        let $type := fn:local-name($score)
        let $weight := fn:number($score/@weight)
        let $algorithm-ref := $score/@algorithm-ref
        return
          map:new((
            map:entry("queryID", sem:uuid-string()),
            map:entry("type",$type),
            map:entry("weight",$weight),
            map:entry("algorithm", fn:head(($algorithm-ref, "standard"))),
            map:entry(
              "valuesToQueryFunction",
              if ($type eq "reduce") then
                let $algorithm := $algorithm-ref ! map:get($algorithms, .)
                return
                  if (fn:exists($algorithm)) then
                    algorithms:execute-algorithm($algorithm, ?, $score, $options)
                  else
                    algorithms:standard-reduction-query(?, $score, $options)
              else ()
            )
          ))
      )
    let $minimum-threshold-combinations :=
        match-impl:minimum-threshold-combinations($queries[fn:not(. => map:get("type") = "reduce")], $minimum-threshold)
    let $compiled-match-options := map:new((
        map:entry("options", $options),
        map:entry("targetEntity", $target-entity),
        map:entry("scoreRatio", $score-ratio),
        map:entry("algorithms", $algorithms),
        map:entry("queries", $queries),
        map:entry("orderedThresholds",
          for $threshold in $options/matcher:thresholds/matcher:threshold
          order by $threshold/@above cast as xs:decimal descending
          return $threshold
        ),
        map:entry("minimumThresholdCombinations", $minimum-threshold-combinations),
        map:entry("baseContentQuery",
          if (fn:exists($target-entity)) then
            if ($document-is-json) then
              cts:json-property-scope-query(
                "info",
                cts:json-property-value-query("title", fn:string($target-entity-def/entityTitle), (), 0)
              )
            else
              cts:element-query(
                xs:QName("es:info"),
                cts:element-value-query(xs:QName("es:title"), fn:string($target-entity-def/entityTitle), (), 0)
              )
          else
            match-impl:build-collection-query(coll:content-collections($options))
        )
      ))
    return (
      map:put(
        $_cached-compiled-match-options,
        $cache-id,
        $compiled-match-options
      ),
      $compiled-match-options
    )
};

(:
 : Find documents that are potential matches for the provided document.
 : @param $document  a source document to draw values from
 : @param $options  XML or JSON representation of match options
 : @param $start  paging: 1-based index
 : @param $page-length  paging: number of results to return
 : @param $minimum-threshold  the required score for the lowest-scoring
 :                            threshold (see match options)
 : @param $include-matches  if true, the response will include, for each result,
 :                          the properties that earned points for the match
 :                          (similar) to snippets
 : @param $filter-query  a cts:query that reduces the scope of documents that
 :                       will be searched for matches
 : @return results specify document URIs that matches for provided document
 :)
declare function match-impl:find-document-matches-by-options(
  $document as node()?,
  $options as item(),
  $start as xs:integer,
  $page-length as xs:integer,
  $minimum-threshold as xs:double,
  $include-matches as xs:boolean,
  $filter-query as cts:query
) as element(results)
{
  (: increment usage count :)
  tel:increment(),
  match-impl:find-document-matches-by-options(
    $document,
    $options,
    $start,
    $page-length,
    $minimum-threshold,
    $include-matches,
    $filter-query,
    fn:true()
  )
};

declare function match-impl:find-document-matches-by-options(
  $document as node()?,
  $options as item(),
  $start as xs:integer,
  $page-length as xs:integer,
  $minimum-threshold as xs:double,
  $include-matches as xs:boolean,
  $filter-query as cts:query,
  $include-results as xs:boolean
) as element(results)
{
  match-impl:find-document-matches-by-options(
    $document,
    $options,
    $start,
    $page-length,
    $minimum-threshold,
    $include-matches,
    $filter-query,
    $include-results,
    (: by default don't short-circuit redundant queries :)
    fn:false()
  )
};

(: A map to track match queries previously searched on. Useful for process-impl:build-match-summary :)
declare variable $map-of-queries-previously-run := map:map();

declare function match-impl:find-document-matches-by-options(
  $document as node()?,
  $options as item(),
  $start as xs:integer,
  $page-length as xs:integer,
  $minimum-threshold as xs:double,
  $include-matches as xs:boolean,
  $filter-query as cts:query,
  $include-results as xs:boolean,
  $short-circuit-redundant-queries as xs:boolean
) as element(results)
{
  if (fn:exists($document)) then (
    let $start-elapsed := xdmp:elapsed-time()
    let $is-json := (xdmp:node-kind($document) = "object" or fn:exists($document/(object-node()|array-node())))
    let $compiled-options := match-impl:compile-match-options($options, $is-json, $minimum-threshold)
    let $values-by-qname := match-impl:values-by-qname($document, $compiled-options)
    let $query-prov as map:map? :=
        if ($include-results and $include-matches) then
          map:map()
        else ()
    let $cached-queries := map:map()
    let $minimum-threshold-combinations-queries :=
      for $query-set in $compiled-options => map:get("minimumThresholdCombinations")
      let $query-maps := $query-set => map:get("queries")
      let $queries :=
        for $query-map in $query-maps
        (: if there are no values from the document for a query, then match-impl:query-map-to-query returns an empty sequence  :)
        return match-impl:query-map-to-query($query-map, $values-by-qname, $cached-queries, $query-prov)
      (: We want to be certain that we have values for each of the queries in a min threshold combo :)
      where fn:exists($queries) and fn:count($queries) eq fn:count($query-maps)
      return
        let $queries := if (fn:count($queries) gt 1) then
          helper-impl:group-queries-by-scope($queries, cts:and-query#1)
        else
          $queries
        return
          if (fn:count($queries) gt 1) then
            cts:and-query($queries)
          else
            $queries
    (: We want to ignore redundant queries. This only applies to expanded searches to find additional merges that may be added via process-impl:build-match-summary
     : If the queries generated by a document are the same as document previously searched on, it won't result in additional URIs.
     :)
    let $has-redundant-match-combo := $short-circuit-redundant-queries and (
      every $min-query in $minimum-threshold-combinations-queries satisfies
        map:contains($map-of-queries-previously-run, xdmp:md5(document{$min-query}))
    )
    let $_cache-minimum-threshold := if ($short-circuit-redundant-queries and fn:not($has-redundant-match-combo)) then
      for $min-query in $minimum-threshold-combinations-queries
      return map:put($map-of-queries-previously-run, xdmp:md5(document{$min-query}), fn:true())
    else ()
    (: If there are no match queries, just return a cts:false-query  :)
    let $minimum-threshold-combinations-query :=
      if (fn:exists($minimum-threshold-combinations-queries)) then
        helper-impl:group-queries-by-scope($minimum-threshold-combinations-queries, cts:or-query#1)
      else
        cts:false-query()
    (: Exclude the current document and any blocked matches :)
    let $excluded-uris := (
      xdmp:node-uri($document),
      blocks-impl:get-blocks(fn:base-uri($document))/node()
    )
    let $match-base-query := cts:and-query((
          $compiled-options => map:get("baseContentQuery"),
          $minimum-threshold-combinations-query
        ))
    let $match-query :=
      if (fn:exists($excluded-uris)) then
        cts:and-not-query(
          $match-base-query,
          cts:document-query($excluded-uris)
        )
      else
        $match-base-query
    let $match-query :=
      match-impl:instance-query-wrapper(
        if (fn:not($filter-query instance of cts:true-query)) then
          cts:and-query((
            $filter-query,
            $match-query
          ))
        else
          $match-query,
        $is-json
      )
    let $serialized-match-query :=
      element match-query {
        $match-query
      }
    return
      (: If minimum threshold can't be met or we're short-circuiting a query already run, don't bother with estimate and search :)
      if ($minimum-threshold-combinations-query instance of cts:false-query or $has-redundant-match-combo) then
        match-impl:build-empty-results($start, $page-length, $serialized-match-query)
      else
        let $estimate := xdmp:estimate(cts:search(fn:collection(), match-impl:instance-query-wrapper($match-query, $is-json), "unfiltered"))
        return (
          element results {
            attribute total { $estimate },
            attribute page-length { $page-length },
            attribute start { $start },
            $serialized-match-query,
            if ($include-results and $estimate gt 0) then
              let $boost-query := match-impl:build-boost-query($document, $values-by-qname, $compiled-options, $cached-queries, $query-prov)
              return
                  match-impl:search(
                    $match-query,
                    $boost-query,
                    $minimum-threshold,
                    $start,
                    $page-length,
                    $compiled-options,
                    $include-matches,
                    $is-json,
                    $query-prov
                  )
            else (),
            if (xdmp:trace-enabled($const:TRACE-PERFORMANCE)) then
              xdmp:trace($const:TRACE-PERFORMANCE, "match-impl:find-document-matches-by-options: " || (xdmp:elapsed-time() - $start-elapsed))
            else ()
          }
        )
  ) else
    match-impl:build-empty-results($start,$page-length, ())
};

declare function match-impl:build-empty-results(
  $start as xs:integer,
  $page-length as xs:integer,
  $serialize-match-query as node()?
) {
  element results {
    attribute total { 0 },
    attribute page-length { $page-length },
    attribute start { $start },
    $serialize-match-query
  }
};

(:
 :
 :)
declare function match-impl:build-collection-query($collections as xs:string*)
{
  if (fn:empty($collections)) then
    ()
  else if (fn:count($collections) > 1) then
    cts:and-query($collections ! cts:collection-query(.))
  else
    cts:collection-query($collections)
};

(:
 : Does each item in $s1 appear in $s2?
 :)
declare function match-impl:seq-contains($s1, $s2)
{
  every $s in $s1 satisfies $s = $s2
};

(:
 : Build the boost query as specified by the match options.
 : @param $document  the source document from which property values are drawn
 : @param $values-by-qname  map:map that organizes document values by QName
 : @param $compiled-options map:map with compiled details about match options
 : @param $cached-queries map:map containing previously created queries
 : @param $query-prov map:map to optionally track the provenance of queries contributing to match
 : @return a cts:or-query that will be used as a boost query
 :)
declare function match-impl:build-boost-query(
  $document as node(),
  $values-by-qname as map:map,
  $compiled-options as map:map,
  $cached-queries as map:map,
  $query-prov as map:map?
)
{
  cts:or-query((
    for $query-map in map:get($compiled-options, "queries")
    return
      if ($query-map => map:get("type") = "reduce") then
        ($query-map => map:get("valuesToQueryFunction"))($document)
      else
        match-impl:query-map-to-query($query-map, $values-by-qname, $cached-queries, $query-prov)
  ))
};

(:
 : Convert map:map describing query to cts:query
 : @param $query-map  map:map describing a query to generate
 : @param $values-by-qname  map:map that organizes document values by QName
 : @param $cached-queries  map:map caching previously generated queries
 : @param $query-prov map:map to optionally track the provenance of queries contributing to match
 : @return cts:query?
 :)
declare function match-impl:query-map-to-query(
  $query-map as map:map,
  $values-by-qname as map:map,
  $cached-queries as map:map,
  $query-prov as map:map?
)
{
  if (map:contains($query-map, "queryID") and map:contains($cached-queries, $query-map => map:get("queryID"))) then
    map:get($cached-queries, $query-map => map:get("queryID"))
  else
    let $query :=
        let $qname := $query-map => map:get("qname")
        let $values := $values-by-qname => map:get(xdmp:key-from-QName($qname))
        where fn:exists($values)
        return
          ($query-map => map:get("valuesToQueryFunction"))($values)
    return (
      map:put($cached-queries, $query-map => map:get("queryID"), $query),
      if (fn:exists($query-prov)) then
        let $query-hashes := fn:distinct-values(cts:element-walk(document{$query}, $QUERIES_WITH_WEIGHT, element query-hash{xdmp:md5(document {$cts:node})})//query-hash !
          fn:string(.))
        for $query-hash in $query-hashes
        return map:put($query-prov, $query-hash, $query-map)
      else (),
      $query
    )
};

(:
 : Organize values by QName
 : @param $document  document with property values
 : @param $compiled-options  map:map with compiled details about match options
 : @return map:map of values organized by QName
 :)
declare function match-impl:values-by-qname(
  $document as node()?,
  $compiled-options as map:map
)
{
  map:new(
    for $qname in fn:distinct-values((map:get($compiled-options, "queries") ! map:get(.,"qname")))
    let $values := fn:distinct-values($document/(self::*:envelope|*:envelope)/*:instance//*[fn:node-name(.) eq $qname][fn:normalize-space(fn:string(.))]/fn:data(.))
    where fn:exists($values)
    return
      map:entry(xdmp:key-from-QName($qname), $values)
  )
};

(:
 : Execute the generated search and construct the response.
 : @param $match-query  a query built such that any matches will score at least
 :                      high enough to reach the lowest threshold
 : @param $boosting-query  a query that is used to score matches
 : @param $filter-query  a query to reduce the universe of match candidates
 : @param $min-threshold  lowest score required to hit a threshold
 : @param $start  paging: 1-based index
 : @param $page-length  paging
 : @param $algorithms  map derived from match options
 : @param $options  full match options; included to pass to reduce algorithm
 : @param $include-matches
 : @param $is-json
 : @param $query-prov an optional map for tracking how options/algorithms contributed to matches
 : @return results specify document URIs that matches for provided document
 :)
declare function match-impl:search(
  $match-query,
  $boosting-query,
  $min-threshold as xs:double,
  $start as xs:int,
  $page-length as xs:int,
  $compiled-options as map:map,
  $include-matches as xs:boolean,
  $is-json as xs:boolean,
  $query-prov as map:map?
) {
  let $range := $start to ($start + $page-length - 1)
  let $cts-walk-query :=
    if ($include-matches) then
      match-impl:instance-query-wrapper(
        $boosting-query,
        $is-json
      )
    else ()
  let $thresholds := $compiled-options => map:get("orderedThresholds")
  let $individual-matching-queries := cts:or-query-queries($boosting-query)
  for $result at $pos in cts:search(
    fn:collection(),
    $match-query,
    ("unfiltered", "score-simple"),
    0
  )[fn:position() = $range]
  let $uri := xdmp:node-uri($result)
  let $matching-queries := $individual-matching-queries[cts:contains($result, .)]
  let $matching-weights-map := map:entry("values", ())
  let $_accumulate-scores :=
    cts:element-walk(document{$matching-queries}, $QUERIES_WITH_WEIGHT,
      let $weight := fn:head(($cts:node/@weight, 1))
      return
      (
        $matching-weights-map => map:put("values",
          ($matching-weights-map => map:get("values"), $weight)),
        $matching-weights-map => map:put(xdmp:md5(document {$cts:node}), $weight)
      )
    )
  let $score :=
      fn:sum(
        $matching-weights-map => map:get("values")
      )
  where $score ge $min-threshold
  return
    element result {
      attribute uri {$uri},
      attribute index {$range[fn:position() = $pos]},
      attribute score {$score},
      let $selected-threshold := fn:head($thresholds[$score ge @above])
      return (
        attribute threshold { fn:string($selected-threshold/@label) },
        attribute action { fn:string($selected-threshold/(matcher:action|@action)) }
      ),
      if ($include-matches) then
        element matches {
          (: rather than store the entire node and risk mixing
             content type (json != xml) we store the path to the
             node instead :)
          cts:walk(
            $result/*:envelope/*:instance,
            cts:or-query($matching-queries),
            (let $query-hashes := fn:distinct-values($cts:queries !
                cts:element-walk(document{.}, $QUERIES_WITH_WEIGHT, element query-hash{xdmp:md5(document {$cts:node})})//query-hash !
                fn:string(.))
            let $weight := fn:sum($query-hashes ! ($matching-weights-map => map:get(.)))
            let $node-name := fn:string(fn:head((fn:node-name($cts:node),fn:node-name($cts:node/..))))
            return <match weight="{$weight}">
                      {
                        for $query-hash in $query-hashes
                        let $query-map := map:get($query-prov, $query-hash)
                        return
                          <contributions>
                            <algorithm>{fn:string(fn:head(($query-map ! map:get(., "algorithm") ! fn:string(.),"standard")))}</algorithm>
                            <weight>{fn:string($matching-weights-map => map:get($query-hash))}</weight>
                          </contributions>
                      }
                      <nodeName>{$node-name}</nodeName>
                      <value>{fn:data($cts:node)}</value>
                      <path>{xdmp:path($cts:node, fn:true())}</path>
                   </match>)
          )
        }
      else ()
    }
};

(: Configuration used to convert XML match results to JSON. :)
declare variable $results-json-config := match-impl:_results-json-config();

declare function match-impl:_results-json-config()
{
  let $config := json:config("custom")
  return (
    map:put($config, "array-element-names", ("result","matches","algorithms",xs:QName("cts:option"),xs:QName("cts:text"),xs:QName("cts:element"))),
    map:put($config, "full-element-names",
      (xs:QName("cts:query"),
      xs:QName("cts:and-query"),
      xs:QName("cts:near-query"),
      xs:QName("cts:or-query"))
    ),
    map:put($config, "json-children", "queries"),
    map:put($config, "attribute-names",
      ("name","localname", "namespace", "function", "weight",
      "at", "property-name", "weight", "above", "label","algorithm-ref")
    ),
    map:put($config, "camel-case", fn:true()),
    $config
  )
};

(:
 : Identify a sequence of queries whose scores add up to the $threshold. A document must match at least one of these
 : queries in order to be returned as a potential match.
 :
 : @param $query-results  a sequence of queries with weights
 : @param $threshold  minimum weighted-score for a match to be relevant
 : @return a sequence of queries; a document that matches any of these will have at least $threshold as a score
 :)
declare function match-impl:minimum-threshold-combinations($query-results, $threshold as xs:double)
  as map:map*
{
  (: Each of $queries-ge-threshold has a weight high enough to hit the $threshold :)
  let $queries-ge-threshold := $query-results[(. => map:get("weight")) ge $threshold]
  let $queries-lt-threshold := $query-results[(. => map:get("weight")) lt $threshold]
  return (
    $queries-ge-threshold ! map:entry("queries", .),
    match-impl:filter-for-required-queries($queries-lt-threshold, 0, $threshold, ())
  )
};

(:
 : Find combinations of queries whose weights are individually below the threshold, but combined are above it.
 :
 : @param $remaining-queries  sequence of queries ordered by their weights, descending
 : @param $combined-weight
 : @param $threshold  the target value
 : @param $accumulated-queries  accumlated sequence, building up to see whether it can hit the $threshold.
 : @return a sequence of cts:and-queries, one for each required filter
 : note: return type left off to allow for tail recursion optimization.
 :)
declare function match-impl:filter-for-required-queries(
  $remaining-queries as map:map*,
  $combined-weight,
  $threshold,
  $accumulated-queries as map:map*
)
{
  if ($threshold eq 0 or $combined-weight ge $threshold) then (
    if (fn:exists($accumulated-queries)) then
      map:entry(
        "queries",
        $accumulated-queries
      )
    else
      ()
  )
  else
    for $query at $pos in $remaining-queries
    let $query-weight := fn:head(($query => map:get("weight"), 1))
    let $new-combined-weight := $combined-weight + $query-weight
    return (
      match-impl:filter-for-required-queries(
        fn:subsequence($remaining-queries, $pos + 1),
        $new-combined-weight,
        $threshold,
        ($accumulated-queries, $query)
      )
    )
};

(:
 : Convert XML match results to JSON.
 :)
declare function match-impl:results-to-json($results-xml)
  as object-node()?
{
  if (fn:exists($results-xml)) then
    xdmp:to-json(
      json:transform-to-json-object($results-xml, $results-json-config)
    )/node()
  else ()
};

declare function match-impl:instance-query-wrapper(
  $query as cts:query,
  $is-json as xs:boolean
) {
  if ($is-json) then
    if (fn:exists($const:JSON-INSTANCE)) then
      cts:json-property-scope-query($const:JSON-INSTANCE, $query)
    else ()
  else
    if (fn:exists($const:XML-INSTANCE)) then
      cts:element-query($const:XML-INSTANCE, $query)
    else ()
};

xquery version "1.0-ml";

(:
 : This is an implementation library, not an interface to the Smart Mastering functionality.
 :
 : The process of matching starts with one document, which is not required to
 : be in the database. The match options specify what properties are to be used
 : to find matches. See match options documentation for details. The options
 : may specify multiple thresholds, each of which corresponds to an action.
 :
 : Implementation notes: the configured properties are used to generate a boost
 : query. The match part of the query identifies a set of subqueries that a
 : document must match in order to get a score above the lowest threshold.
 : Match queries all have their scores set to zero. The boost part of the query
 : is used to provide the score.

 : @see https://marklogic-community.github.io/smart-mastering-core/docs/matching-options/
 :)

module namespace match-impl = "http://marklogic.com/smart-mastering/matcher-impl";

import module namespace algorithms = "http://marklogic.com/smart-mastering/algorithms"
  at  "/com.marklogic.smart-mastering/algorithms/base.xqy",
      "/com.marklogic.smart-mastering/algorithms/standard-reduction.xqy";
import module namespace blocks-impl = "http://marklogic.com/smart-mastering/blocks-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/blocks-impl.xqy";
import module namespace const = "http://marklogic.com/smart-mastering/constants"
  at "/com.marklogic.smart-mastering/constants.xqy";
import module namespace es-helper = "http://marklogic.com/smart-mastering/entity-services" at "/com.marklogic.smart-mastering/sm-entity-services.xqy";
import module namespace helper-impl = "http://marklogic.com/smart-mastering/helper-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/helper-impl.xqy";
import module namespace json="http://marklogic.com/xdmp/json"
  at "/MarkLogic/json/json.xqy";
import module namespace opt-impl = "http://marklogic.com/smart-mastering/options-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/options-impl.xqy";
import module namespace tel = "http://marklogic.com/smart-mastering/telemetry"
  at "/com.marklogic.smart-mastering/telemetry.xqy";
import module namespace coll = "http://marklogic.com/smart-mastering/collections"
  at "/com.marklogic.smart-mastering/impl/collections.xqy";

declare namespace matcher = "http://marklogic.com/smart-mastering/matcher";
declare namespace sm = "http://marklogic.com/smart-mastering";
declare namespace es = "http://marklogic.com/entity-services";

declare option xdmp:mapping "false";

declare variable $QUERIES_WITH_WEIGHT := (
    xs:QName("cts:element-attribute-pair-geospatial-query"),xs:QName("cts:element-attribute-range-query"),
    xs:QName("cts:element-attribute-value-query"),xs:QName("cts:element-attribute-word-query"),xs:QName("cts:element-child-geospatial-query"),
    xs:QName("cts:element-geospatial-query"),xs:QName("cts:element-pair-geospatial-query"),xs:QName("cts:element-range-query"),
    xs:QName("cts:element-value-query"),xs:QName("cts:element-word-query"),xs:QName("cts:field-range-query"),
    xs:QName("cts:field-value-query"),xs:QName("cts:field-word-query"),xs:QName("cts:geospatial-region-query"),
    xs:QName("cts:json-property-child-geospatial-query"),xs:QName("cts:json-property-geospatial-query"),
    xs:QName("cts:json-property-pair-geospatial-query"),xs:QName("cts:json-property-range-query"),xs:QName("cts:json-property-value-query"),
    xs:QName("cts:json-property-word-query"),xs:QName("cts:lsqt-query"),xs:QName("cts:near-query"),xs:QName("cts:not-query"),
    xs:QName("cts:path-geospatial-query"),xs:QName("cts:path-range-query"),xs:QName("cts:range-query"),xs:QName("cts:registered-query"),
    xs:QName("cts:reverse-query"),xs:QName("cts:similar-query"),xs:QName("cts:triple-range-query"),xs:QName("cts:word-query"));

declare variable $_cached-compiled-match-options as map:map := map:map();

(:
 : Calculate queries once per unique match options in request to reduce repeat logic
 : @param $match-options  Options specifying how documents will be matched
 : @param $document-is-json  Is a JSON document we are matching?
 : @param $minimum-threshold  Minimum threshold for search results to meet
 : @return map:map with compiled information about match options
 :)
declare function match-impl:compile-match-options(
  $match-options as node(),
  $document-is-json as xs:boolean,
  $minimum-threshold as xs:double
) {
  let $cache-id :=
      fn:generate-id($match-options) || "|doc-is-json:"|| $document-is-json || "|min-threshold:" || $minimum-threshold
  return
  if (map:contains($_cached-compiled-match-options, $cache-id)) then
    map:get($_cached-compiled-match-options, $cache-id)
  else
    let $options :=
      if ($match-options instance of object-node()) then
        opt-impl:options-from-json($match-options)
      else
        $match-options
    let $data-format := $options/matcher:data-format
    let $options :=
          if (fn:empty($data-format)) then
            element matcher:options {
              element matcher:data-format {
                if ($document-is-json) then $const:FORMAT-JSON else $const:FORMAT-XML
              },
              $options/*
            }
          else
            $options
    let $scoring := $options/matcher:scoring
    let $max-property-score := fn:max(($scoring/@weight ! fn:number(.)))
    let $score-ratio :=
        if ($max-property-score le 64) then
          1
        else
          (64 idiv $max-property-score)
    let $algorithms := algorithms:build-algorithms-map($options/matcher:algorithms)
    let $target-entity := $options/matcher:target-entity ! fn:string(.)
    let $target-entity-def := es-helper:get-entity-def($target-entity)
    let $queries := (
        for $score in $scoring/(matcher:add|matcher:expand)
        let $type := fn:local-name($score)
        let $weight := fn:number($score/@weight)
        let $full-property-name := $score/@property-name
        let $helper-query := helper-impl:property-name-to-query($options, $full-property-name)
        where fn:exists($helper-query)
        order by $weight descending
        return
          let $qname := helper-impl:property-name-to-qname($options, $full-property-name)
          let $base-values-query :=
              if ($type eq "add") then
                $helper-query(?, $weight)
              else if ($type eq "expand") then
                let $algorithm := map:get($algorithms, $score/@algorithm-ref)
                where fn:exists($algorithm)
                return algorithms:execute-algorithm($algorithm, ?, $score, $options)
              else ()
          return map:new((
            map:entry("queryID", sem:uuid-string()),
            map:entry("propertyName",$full-property-name),
            map:entry("type",$type),
            map:entry("weight",$weight),
            map:entry("qname", $qname),
            map:entry(
              "valuesToQueryFunction",
              $base-values-query
            )
          )),
        for $score in $scoring/matcher:reduce
        let $type := fn:local-name($score)
        let $weight := fn:number($score/@weight)
        let $algorithm-ref := $score/@algorithm-ref
        return
          map:new((
            map:entry("queryID", sem:uuid-string()),
            map:entry("type",$type),
            map:entry("weight",$weight),
            map:entry("algorithm", fn:head(($algorithm-ref, "standard"))),
            map:entry(
              "valuesToQueryFunction",
              if ($type eq "reduce") then
                let $algorithm := $algorithm-ref ! map:get($algorithms, .)
                return
                  if (fn:exists($algorithm)) then
                    algorithms:execute-algorithm($algorithm, ?, $score, $options)
                  else
                    algorithms:standard-reduction-query(?, $score, $options)
              else ()
            )
          ))
      )
    let $minimum-threshold-combinations :=
        match-impl:minimum-threshold-combinations($queries[fn:not(. => map:get("type") = "reduce")], $minimum-threshold)
    let $compiled-match-options := map:new((
        map:entry("options", $options),
        map:entry("targetEntity", $target-entity),
        map:entry("scoreRatio", $score-ratio),
        map:entry("algorithms", $algorithms),
        map:entry("queries", $queries),
        map:entry("orderedThresholds",
          for $threshold in $options/matcher:thresholds/matcher:threshold
          order by $threshold/@above cast as xs:decimal descending
          return $threshold
        ),
        map:entry("minimumThresholdCombinations", $minimum-threshold-combinations),
        map:entry("baseContentQuery",
          if (fn:exists($target-entity)) then
            if ($document-is-json) then
              cts:json-property-scope-query(
                "info",
                cts:json-property-value-query("title", fn:string($target-entity-def/entityTitle), (), 0)
              )
            else
              cts:element-query(
                xs:QName("es:info"),
                cts:element-value-query(xs:QName("es:title"), fn:string($target-entity-def/entityTitle), (), 0)
              )
            ,
            $document-is-json
          else
            match-impl:build-collection-query(coll:content-collections($options))
        )
      ))
    return (
      map:put(
        $_cached-compiled-match-options,
        $cache-id,
        $compiled-match-options
      ),
      $compiled-match-options
    )
};

(:
 : Find documents that are potential matches for the provided document.
 : @param $document  a source document to draw values from
 : @param $options  XML or JSON representation of match options
 : @param $start  paging: 1-based index
 : @param $page-length  paging: number of results to return
 : @param $minimum-threshold  the required score for the lowest-scoring
 :                            threshold (see match options)
 : @param $include-matches  if true, the response will include, for each result,
 :                          the properties that earned points for the match
 :                          (similar) to snippets
 : @param $filter-query  a cts:query that reduces the scope of documents that
 :                       will be searched for matches
 : @return results specify document URIs that matches for provided document
 :)
declare function match-impl:find-document-matches-by-options(
  $document as node()?,
  $options as item(),
  $start as xs:integer,
  $page-length as xs:integer,
  $minimum-threshold as xs:double,
  $include-matches as xs:boolean,
  $filter-query as cts:query
) as element(results)
{
  (: increment usage count :)
  tel:increment(),
  match-impl:find-document-matches-by-options(
    $document,
    $options,
    $start,
    $page-length,
    $minimum-threshold,
    $include-matches,
    $filter-query,
    fn:true()
  )
};

declare function match-impl:find-document-matches-by-options(
  $document as node()?,
  $options as item(),
  $start as xs:integer,
  $page-length as xs:integer,
  $minimum-threshold as xs:double,
  $include-matches as xs:boolean,
  $filter-query as cts:query,
  $include-results as xs:boolean
) as element(results)
{
  match-impl:find-document-matches-by-options(
    $document,
    $options,
    $start,
    $page-length,
    $minimum-threshold,
    $include-matches,
    $filter-query,
    $include-results,
    (: by default don't short-circuit redundant queries :)
    fn:false()
  )
};

(: A map to track match queries previously searched on. Useful for process-impl:build-match-summary :)
declare variable $map-of-queries-previously-run := map:map();

declare function match-impl:find-document-matches-by-options(
  $document as node()?,
  $options as item(),
  $start as xs:integer,
  $page-length as xs:integer,
  $minimum-threshold as xs:double,
  $include-matches as xs:boolean,
  $filter-query as cts:query,
  $include-results as xs:boolean,
  $short-circuit-redundant-queries as xs:boolean
) as element(results)
{
  if (fn:exists($document)) then (
    let $start-elapsed := xdmp:elapsed-time()
    let $is-json := (xdmp:node-kind($document) = "object" or fn:exists($document/(object-node()|array-node())))
    let $compiled-options := match-impl:compile-match-options($options, $is-json, $minimum-threshold)
    let $values-by-qname := match-impl:values-by-qname($document, $compiled-options)
    let $query-prov as map:map? :=
        if ($include-results and $include-matches) then
          map:map()
        else ()
    let $cached-queries := map:map()
    let $minimum-threshold-combinations-queries :=
      for $query-set in $compiled-options => map:get("minimumThresholdCombinations")
      let $query-maps := $query-set => map:get("queries")
      let $queries :=
        for $query-map in $query-maps
        (: if there are no values from the document for a query, then match-impl:query-map-to-query returns an empty sequence  :)
        return match-impl:query-map-to-query($query-map, $values-by-qname, $cached-queries, $query-prov)
      (: We want to be certain that we have values for each of the queries in a min threshold combo :)
      where fn:exists($queries) and fn:count($queries) eq fn:count($query-maps)
      return
        let $queries := if (fn:count($queries) gt 1) then
          helper-impl:group-queries-by-scope($queries, cts:and-query#1)
        else
          $queries
        return
          if (fn:count($queries) gt 1) then
            cts:and-query($queries)
          else
            $queries
    (: We want to ignore redundant queries. This only applies to expanded searches to find additional merges that may be added via process-impl:build-match-summary
     : If the queries generated by a document are the same as document previously searched on, it won't result in additional URIs.
     :)
    let $has-redundant-match-combo := $short-circuit-redundant-queries and (
      every $min-query in $minimum-threshold-combinations-queries satisfies
        map:contains($map-of-queries-previously-run, xdmp:md5(document{$min-query}))
    )
    let $_cache-minimum-threshold := if ($short-circuit-redundant-queries and fn:not($has-redundant-match-combo)) then
      for $min-query in $minimum-threshold-combinations-queries
      return map:put($map-of-queries-previously-run, xdmp:md5(document{$min-query}), fn:true())
    else ()
    (: If there are no match queries, just return a cts:false-query  :)
    let $minimum-threshold-combinations-query :=
      if (fn:exists($minimum-threshold-combinations-queries)) then
        helper-impl:group-queries-by-scope($minimum-threshold-combinations-queries, cts:or-query#1)
      else
        cts:false-query()
    (: Exclude the current document and any blocked matches :)
    let $excluded-uris := (
      xdmp:node-uri($document),
      blocks-impl:get-blocks(fn:base-uri($document))/node()
    )
    let $match-base-query := cts:and-query((
          $compiled-options => map:get("baseContentQuery"),
          $minimum-threshold-combinations-query
        ))
    let $match-query :=
      if (fn:exists($excluded-uris)) then
        cts:and-not-query(
          $match-base-query,
          cts:document-query($excluded-uris)
        )
      else
        $match-base-query
    let $match-query :=
      match-impl:instance-query-wrapper(
        if (fn:not($filter-query instance of cts:true-query)) then
          cts:and-query((
            $filter-query,
            $match-query
          ))
        else
          $match-query,
        $is-json
      )
    let $serialized-match-query :=
      element match-query {
        $match-query
      }
    return
      (: If minimum threshold can't be met or we're short-circuiting a query already run, don't bother with estimate and search :)
      if ($minimum-threshold-combinations-query instance of cts:false-query or $has-redundant-match-combo) then
        match-impl:build-empty-results($start, $page-length, $serialized-match-query)
      else
        let $estimate := xdmp:estimate(cts:search(fn:collection(), match-impl:instance-query-wrapper($match-query, $is-json), "unfiltered"))
        return (
          element results {
            attribute total { $estimate },
            attribute page-length { $page-length },
            attribute start { $start },
            $serialized-match-query,
            if ($include-results and $estimate gt 0) then
              let $boost-query := match-impl:build-boost-query($document, $values-by-qname, $compiled-options, $cached-queries, $query-prov)
              return
                  match-impl:search(
                    $match-query,
                    $boost-query,
                    $minimum-threshold,
                    $start,
                    $page-length,
                    $compiled-options,
                    $include-matches,
                    $is-json,
                    $query-prov
                  )
            else (),
            if (xdmp:trace-enabled($const:TRACE-PERFORMANCE)) then
              xdmp:trace($const:TRACE-PERFORMANCE, "match-impl:find-document-matches-by-options: " || (xdmp:elapsed-time() - $start-elapsed))
            else ()
          }
        )
  ) else
    match-impl:build-empty-results($start,$page-length, ())
};

declare function match-impl:build-empty-results(
  $start as xs:integer,
  $page-length as xs:integer,
  $serialize-match-query as node()?
) {
  element results {
    attribute total { 0 },
    attribute page-length { $page-length },
    attribute start { $start },
    $serialize-match-query
  }
};

(:
 :
 :)
declare function match-impl:build-collection-query($collections as xs:string*)
{
  if (fn:empty($collections)) then
    ()
  else if (fn:count($collections) > 1) then
    cts:and-query($collections ! cts:collection-query(.))
  else
    cts:collection-query($collections)
};

(:
 : Does each item in $s1 appear in $s2?
 :)
declare function match-impl:seq-contains($s1, $s2)
{
  every $s in $s1 satisfies $s = $s2
};

(:
 : Build the boost query as specified by the match options.
 : @param $document  the source document from which property values are drawn
 : @param $values-by-qname  map:map that organizes document values by QName
 : @param $compiled-options map:map with compiled details about match options
 : @param $cached-queries map:map containing previously created queries
 : @param $query-prov map:map to optionally track the provenance of queries contributing to match
 : @return a cts:or-query that will be used as a boost query
 :)
declare function match-impl:build-boost-query(
  $document as node(),
  $values-by-qname as map:map,
  $compiled-options as map:map,
  $cached-queries as map:map,
  $query-prov as map:map?
)
{
  cts:or-query((
    for $query-map in map:get($compiled-options, "queries")
    return
      if ($query-map => map:get("type") = "reduce") then
        ($query-map => map:get("valuesToQueryFunction"))($document)
      else
        match-impl:query-map-to-query($query-map, $values-by-qname, $cached-queries, $query-prov)
  ))
};

(:
 : Convert map:map describing query to cts:query
 : @param $query-map  map:map describing a query to generate
 : @param $values-by-qname  map:map that organizes document values by QName
 : @param $cached-queries  map:map caching previously generated queries
 : @param $query-prov map:map to optionally track the provenance of queries contributing to match
 : @return cts:query?
 :)
declare function match-impl:query-map-to-query(
  $query-map as map:map,
  $values-by-qname as map:map,
  $cached-queries as map:map,
  $query-prov as map:map?
)
{
  if (map:contains($query-map, "queryID") and map:contains($cached-queries, $query-map => map:get("queryID"))) then
    map:get($cached-queries, $query-map => map:get("queryID"))
  else
    let $query :=
        let $qname := $query-map => map:get("qname")
        let $values := $values-by-qname => map:get(xdmp:key-from-QName($qname))
        where fn:exists($values)
        return
          ($query-map => map:get("valuesToQueryFunction"))($values)
    return (
      map:put($cached-queries, $query-map => map:get("queryID"), $query),
      if (fn:exists($query-prov)) then
        let $query-hashes := fn:distinct-values(cts:element-walk(document{$query}, $QUERIES_WITH_WEIGHT, element query-hash{xdmp:md5(document {$cts:node})})//query-hash !
          fn:string(.))
        for $query-hash in $query-hashes
        return map:put($query-prov, $query-hash, $query-map)
      else (),
      $query
    )
};

(:
 : Organize values by QName
 : @param $document  document with property values
 : @param $compiled-options  map:map with compiled details about match options
 : @return map:map of values organized by QName
 :)
declare function match-impl:values-by-qname(
  $document as node()?,
  $compiled-options as map:map
)
{
  map:new(
    for $qname in fn:distinct-values((map:get($compiled-options, "queries") ! map:get(.,"qname")))
    let $values := fn:distinct-values($document/(self::*:envelope|*:envelope)/*:instance//*[fn:node-name(.) eq $qname][fn:normalize-space(fn:string(.))]/fn:data(.))
    where fn:exists($values)
    return
      map:entry(xdmp:key-from-QName($qname), $values)
  )
};

(:
 : Execute the generated search and construct the response.
 : @param $match-query  a query built such that any matches will score at least
 :                      high enough to reach the lowest threshold
 : @param $boosting-query  a query that is used to score matches
 : @param $filter-query  a query to reduce the universe of match candidates
 : @param $min-threshold  lowest score required to hit a threshold
 : @param $start  paging: 1-based index
 : @param $page-length  paging
 : @param $algorithms  map derived from match options
 : @param $options  full match options; included to pass to reduce algorithm
 : @param $include-matches
 : @param $is-json
 : @param $query-prov an optional map for tracking how options/algorithms contributed to matches
 : @return results specify document URIs that matches for provided document
 :)
declare function match-impl:search(
  $match-query,
  $boosting-query,
  $min-threshold as xs:double,
  $start as xs:int,
  $page-length as xs:int,
  $compiled-options as map:map,
  $include-matches as xs:boolean,
  $is-json as xs:boolean,
  $query-prov as map:map?
) {
  let $range := $start to ($start + $page-length - 1)
  let $cts-walk-query :=
    if ($include-matches) then
      match-impl:instance-query-wrapper(
        $boosting-query,
        $is-json
      )
    else ()
  let $thresholds := $compiled-options => map:get("orderedThresholds")
  let $individual-matching-queries := cts:or-query-queries($boosting-query)
  for $result at $pos in cts:search(
    fn:collection(),
    $match-query,
    ("unfiltered", "score-simple"),
    0
  )[fn:position() = $range]
  let $uri := xdmp:node-uri($result)
  let $matching-queries := $individual-matching-queries[cts:contains($result, .)]
  let $matching-weights-map := map:entry("values", ())
  let $_accumulate-scores :=
    cts:element-walk(document{$matching-queries}, $QUERIES_WITH_WEIGHT,
      let $weight := fn:head(($cts:node/@weight, 1))
      return
      (
        $matching-weights-map => map:put("values",
          ($matching-weights-map => map:get("values"), $weight)),
        $matching-weights-map => map:put(xdmp:md5(document {$cts:node}), $weight)
      )
    )
  let $score :=
      fn:sum(
        $matching-weights-map => map:get("values")
      )
  where $score ge $min-threshold
  return
    element result {
      attribute uri {$uri},
      attribute index {$range[fn:position() = $pos]},
      attribute score {$score},
      let $selected-threshold := fn:head($thresholds[$score ge @above])
      return (
        attribute threshold { fn:string($selected-threshold/@label) },
        attribute action { fn:string($selected-threshold/(matcher:action|@action)) }
      ),
      if ($include-matches) then
        element matches {
          (: rather than store the entire node and risk mixing
             content type (json != xml) we store the path to the
             node instead :)
          cts:walk(
            $result/*:envelope/*:instance,
            cts:or-query($matching-queries),
            (let $query-hashes := fn:distinct-values($cts:queries !
                cts:element-walk(document{.}, $QUERIES_WITH_WEIGHT, element query-hash{xdmp:md5(document {$cts:node})})//query-hash !
                fn:string(.))
            let $weight := fn:sum($query-hashes ! ($matching-weights-map => map:get(.)))
            let $node-name := fn:string(fn:head((fn:node-name($cts:node),fn:node-name($cts:node/..))))
            return <match weight="{$weight}">
                      {
                        for $query-hash in $query-hashes
                        let $query-map := map:get($query-prov, $query-hash)
                        return
                          <contributions>
                            <algorithm>{fn:string(fn:head(($query-map ! map:get(., "algorithm") ! fn:string(.),"standard")))}</algorithm>
                            <weight>{fn:string($matching-weights-map => map:get($query-hash))}</weight>
                          </contributions>
                      }
                      <nodeName>{$node-name}</nodeName>
                      <value>{fn:data($cts:node)}</value>
                      <path>{xdmp:path($cts:node, fn:true())}</path>
                   </match>)
          )
        }
      else ()
    }
};

(: Configuration used to convert XML match results to JSON. :)
declare variable $results-json-config := match-impl:_results-json-config();

declare function match-impl:_results-json-config()
{
  let $config := json:config("custom")
  return (
    map:put($config, "array-element-names", ("result","matches","algorithms",xs:QName("cts:option"),xs:QName("cts:text"),xs:QName("cts:element"))),
    map:put($config, "full-element-names",
      (xs:QName("cts:query"),
      xs:QName("cts:and-query"),
      xs:QName("cts:near-query"),
      xs:QName("cts:or-query"))
    ),
    map:put($config, "json-children", "queries"),
    map:put($config, "attribute-names",
      ("name","localname", "namespace", "function", "weight",
      "at", "property-name", "weight", "above", "label","algorithm-ref")
    ),
    map:put($config, "camel-case", fn:true()),
    $config
  )
};

(:
 : Identify a sequence of queries whose scores add up to the $threshold. A document must match at least one of these
 : queries in order to be returned as a potential match.
 :
 : @param $query-results  a sequence of queries with weights
 : @param $threshold  minimum weighted-score for a match to be relevant
 : @return a sequence of queries; a document that matches any of these will have at least $threshold as a score
 :)
declare function match-impl:minimum-threshold-combinations($query-results, $threshold as xs:double)
  as map:map*
{
  (: Each of $queries-ge-threshold has a weight high enough to hit the $threshold :)
  let $queries-ge-threshold := $query-results[(. => map:get("weight")) ge $threshold]
  let $queries-lt-threshold := $query-results[(. => map:get("weight")) lt $threshold]
  return (
    $queries-ge-threshold ! map:entry("queries", .),
    match-impl:filter-for-required-queries($queries-lt-threshold, 0, $threshold, ())
  )
};

(:
 : Find combinations of queries whose weights are individually below the threshold, but combined are above it.
 :
 : @param $remaining-queries  sequence of queries ordered by their weights, descending
 : @param $combined-weight
 : @param $threshold  the target value
 : @param $accumulated-queries  accumlated sequence, building up to see whether it can hit the $threshold.
 : @return a sequence of cts:and-queries, one for each required filter
 : note: return type left off to allow for tail recursion optimization.
 :)
declare function match-impl:filter-for-required-queries(
  $remaining-queries as map:map*,
  $combined-weight,
  $threshold,
  $accumulated-queries as map:map*
)
{
  if ($threshold eq 0 or $combined-weight ge $threshold) then (
    if (fn:exists($accumulated-queries)) then
      map:entry(
        "queries",
        $accumulated-queries
      )
    else
      ()
  )
  else
    for $query at $pos in $remaining-queries
    let $query-weight := fn:head(($query => map:get("weight"), 1))
    let $new-combined-weight := $combined-weight + $query-weight
    return (
      match-impl:filter-for-required-queries(
        fn:subsequence($remaining-queries, $pos + 1),
        $new-combined-weight,
        $threshold,
        ($accumulated-queries, $query)
      )
    )
};

(:
 : Convert XML match results to JSON.
 :)
declare function match-impl:results-to-json($results-xml)
  as object-node()?
{
  if (fn:exists($results-xml)) then
    xdmp:to-json(
      json:transform-to-json-object($results-xml, $results-json-config)
    )/node()
  else ()
};

declare function match-impl:instance-query-wrapper(
  $query as cts:query,
  $is-json as xs:boolean
) {
  if ($is-json) then
    if (fn:exists($const:JSON-INSTANCE)) then
      cts:json-property-scope-query($const:JSON-INSTANCE, $query)
    else ()
  else
    if (fn:exists($const:XML-INSTANCE)) then
      cts:element-query($const:XML-INSTANCE, $query)
    else ()
};


CSV module: refactored internal function to avoid item sequence materialization.

(:
 : Copyright 2006-2009 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
:)

(:~
 : Function library providing convertors from CSV/TXT to XML and back.
 : The functions are optimized to work with large amounts of data, in a streaming way.
 :
 : @author Daniel Turcanu
 :)
module namespace zorba-csv = "http://www.zorba-xquery.com/modules/csv";

(:~
 : Import module for checking if csv options element is validated.
 :)
import module namespace zorba-schema = "http://www.zorba-xquery.com/modules/schema";

(:~
 : Contains the definitions of the csv options element.
  :)
import schema namespace csv-options = "http://www.zorba-xquery.com/modules/csv-options";

(:~
 : Errors namespace URI.
:)
declare variable $zorba-csv:csvNS as xs:string := "http://www.zorba-xquery.com/modules/csv";

(:~
 : Error code for wrong parameter situations.

 : Possible error messages:

 : * "Options field must be of element options instead of ..."

:)
declare variable $zorba-csv:errWrongParam as xs:QName := fn:QName($zorba-csv:csvNS, "zorba-csv:WrongParam");

(:~
 : Parse a CSV or fixed size text and convert to XML.

 : By default each line is converted to a <row> element, and each field to a <column> element inside <row>.

 : @param $csv the string containing the csv or fixed size text.
 : @param $options having format

 :    <csv-options:options>

 :        <csv  [separator="default comma ,"] ?

 :          [quote-char="default double quotes &quote;"]?

 :          [quote-escape="default double double quotes &quote;&quote;"]? />

 :

 :        or

 :        <column-widths>

 :          <column-width>[column fixed width, unsigned int]<column-width>*

 :		  	</column-widths>

 :

 :        or

 :        <column-positions>

 :          <column-position>[column position on line, unsigned int]<column-position>*

 :		  	</column-positions>

 :

 :        <first-row-is-header [line="first_line[-last_line]?"]?/>?

 :        <start-from-row line="first_line[-last_line]?"/>?

 :        <add-last-void-columns/>?

 :        <xml-nodes>

 :          [<row-name>

 :            [<column-name/>]?

 :          </row-name>]?

 :        </xml-nodes>?

 :    </csv-options:options>

 :

 :    All the parameters are optional and can appear in any order.

 :    All the parameters are case sensitive. The namespace used is "http://www.zorba-xquery.com/modules/csv-options".

 :    All strings must have UTF-8 encoding.

 :    Parameters csv, column-widths, column-positions are mutualy exclusive. If none is specified,
 :    the input string is assumed to be csv.

 :    Description of parameters:
 :

 :
csv

 :
 Specifies the parameters for parsing a csv string.

 :

 :
separator

 :
The character or group of characters used to separating fields in a row.
 :            If it is not specified, it defaults to comma ','.
 :

 :
quote-char

 :
The character or group of characters used for quoting the fields that may contain special characters,
 :             like separator, new line or this quote char. The default value is double quote ".

 :

 :
quote-escape

 :
The group of characters used for escaping the quote char inside a field. The whole quote escape group
 :           is translated to a quote char during parsing. The default value is double double quotes "".

 :

 :

 :

 :

 :
column-widths

 :
Specifies the column widths for fixed size text. It contains multiple column-width child elements
 :        specifying the fixed width of each column, from left to right.

 :        If the line has more fields than specified, they are ignored.
 :

 :
column-positions

 :
This is an alternative to column-widths, and specifies instead the starting position of each column.
 :        Column positions are 1 based, and are specified in order from left to right.
 :        The last column is read until end of line. The first column position can be greater than 1, if you want
 :        to parse only a part of the input text.
 :

 :
first-row-is-header

 :
The presence of this element indicates that the first row is to be treated as the name of the columns.
 :        If it is not present, then each field is enclosed in a <column> element,
 :        or how it is specified in <xml-nodes> parameter.

 :        If the first row is the header, then each field is enclosed in an element with the corresponding name from the header.

 :        For example, the csv:

 :        ID,Name,Occupation

 :        1,John,student

 :

 :        is parsed into

 :        <row>

 :        <ID>1</ID>

 :        <Name>John</Name>

 :        <Occupation>student</Occupation>

 :        </row>

 :

 :        If the header names contain characters that cannot be used in a QName, they are replaced with underscore '_'.

 :        The namespace for the header QNames is taken from the column name specified in xml-nodes parameter, or from
 :        the row name, or if that doesn't exist either then empty namespace is used.

 :        If the header is not the first line in the input string, the starting line can be specified in the line attribute.

 :        If a column does not have a name, a new name is constructed in the form columnN where N is the position of the column,
 :        starting from 1.

 :        Subheaders

 :				If the header consists of more than one line, this can be specified in the line attribute in the form
 :        "first_line - last_line". Having more lines as the header translates into a hierarchy of elements in the xml.

 :        For example, the csv:

 :        ID,Name,,Occupation

 :        ,First Name,Last Name,

 :        1,John,Howard,student

 :

 :        is parsed into

 :        <row>

 :        <ID>1</ID>

 :        <Name>

 :          <First_Name>John</First_Name>

 :          <Last_Name>Howard</Last_Name>

 :        </Name>

 :        <Occupation>student</Occupation>

 :        </row>

 :

 :
start-from-row

 :
If the data does not start from line 1 or imediately after the header,
 :        you can specify the starting line in the line attribute.

 :        Also you can use this attribute in the form "first_line - last_line" to specify also the last line
 :        if you don't want the whole csv to be parsed.
 :

 :
add-last-void-columns

 :
In the case when using headers and some data lines are shorter than the header, by default the excess columns are ignored
 :          for those lines. You can set the add-last-void-columns parameter to make all the columns appear in xml even if they are void.
 :

 :
xml-nodes

 :
With this parameter you can specify the names for the row element and for the column element if there is no header.

 :        The first element child of this element specifies the desired QName of the row element in the output xml.
 :        The name of this element will be used as the name of the row element.

 :        The element child of this row element is the column element, and its name will be used as the name of the column elements
 :        that enclose the fields in the output xml if there is no header.

 :        If the csv has a header, only the namespace is used from the column element.

 :        For example, with parameter:

 :        <xml-nodes>

 :        <r>

 :          <c/>

 :        </r>

 :        </xml-nodes>

 :

 :        the output for each line will look like

 :        <r>

 :          <c>field1</c>

 :          <c>field2</c>

 :          .......

 :        </r>

 :

 :        If this parameter is not specified, the row name is by default "row" and the column name is by default "column".
 :

 :

 : @return a sequence of row elements, one for each line in csv
 : @error zorba-csv:WrongParam if the options parameter doesn't have the name "options".
 : @example csv_parse1.xq
 : @example csv_parse2.xq
 : @example csv_parse3.xq
 : @example csv_parse6.xq
 : @example csv_parse11.xq
 : @example csv_parse_utf8_11.xq
 : @example txt_parse5.xq
 : @example txt_parse8.xq
:)
declare function zorba-csv:parse($csv as xs:string,
                                 $options as element(csv-options:options)?) as element()*
{
  let $validated-options :=
  if(empty($options)) then
    $options
  else
  if(zorba-schema:is-validated($options)) then
    $options
  else
    validate{$options}
  return
    zorba-csv:parse-internal($csv, $validated-options)
};

declare %private function zorba-csv:parse-internal($csv as xs:string,
                                 $options as element(csv-options:options, csv-options:optionsType)?) as element()* external;

(:~
 : Convert XML into CSV or fixed size text.

 : Note: if you want to serialize out the result, make sure that the serializer method is set to Text.
 :  For example, in zorba app, you have to set the param --serialize-text.
 :  When using file:write function, you have to set the serialization option <method>text</method>.

 :
 : @param $xml a sequence of elements, each element representing a row. The name of each row element is ignored.
 :     The childs of each row are the column fields.
 : @param $options having format

 :    <csv-options:options>

 :        <csv  [separator="default comma ,"] ?

 :          [quote-char="default double quotes &quote;"]?

 :          [quote-escape="default double double quotes &quote;&quote;"]? />

 :

 :        or

 :        <column-widths [align="left|right"]?>

 :          <column-width [align="left|right"]?>[column fixed width, unsigned int]<column-width>*

 :		  	</column-widths>

 :

 :        or

 :        <column-positions [align="left|right"]?>

 :          <column-position [align="left|right"]?>[column position on line, unsigned int]<column-position>*

 :		  	</column-positions>

 :

 :        <first-row-is-header/>?

 :    </csv-options:options>
 :

 :    All the parameters are optional and can appear in any order.

 :    All the parameters are case sensitive. The namespace used is "http://www.zorba-xquery.com/modules/csv-options".

 :    All strings must have UTF-8 encoding.

 :    Parameters csv, column-widths, column-positions are mutualy exclusive. If none is specified,
 :    the xml is converted to csv.

 :    Description of parameters:
 :

 :
csv

 :
 Specifies the parameters for converting to csv.

 :

 :
separator

 :
The character or group of characters used to separating fields in a row.
 :            If it is not specified, it defaults to comma ','.
 :

 :
quote-char

 :
The character or group of characters used for quoting the fields that may contain special characters,
 :             like separator, new line or this quote char. The default value is double quote ".

 :

 :
quote-escape

 :
The group of characters used for escaping the quote char inside a field. The whole quote escape group
 :           is translated to a quote char during parsing. The default value is double double quotes "".

 :

 :

 :

 :

 :
column-widths

 :
Specifies the column widths for fixed size text. It contains multiple column-width child elements
 :        specifying the fixed width of each column, from left to right.

 :        With the attribute align you can specify how to align fields that are smaller than the column width.
 :        The default alignment is left.

 :

 :
column-positions

 :
This is an alternative to column-widths, and specifies instead the starting position of each column.
 :        Column positions are 1 based, and are specified in order from left to right.
 :        The last column has a variable length.

 :        With the attribute align you can specify how to align fields that are smaller than the column width.
 :        The default alignment is left. The last column does not need alignment.

 :

 :
first-row-is-header

 :
The presence of this element indicates that the first row will contain the header, that is, the names of
 :        the column elements. Only the column names from the first row element are taken into account.

 :        For example, the row xml:

 :        <row>

 :        <ID>1</ID>

 :        <Name>John</Name>

 :        <Occupation>student</Occupation>

 :        </row>

 :

 :        is converted to

 :        ID,Name,Occupation

 :        1,John,student

 :

 :        The header names are the localnames of the column elements, and the namespace is ignored.

 :        Subheaders

 :        If the row-column hierarchy is more complex, then subheaders are also generated on subsequent lines.
 :        The number of subheaders depends on the depth of the column hierarchy.

 :        When generating the subheaders, the non-whitespace text nodes are also taken into account,
 :        and a separate column is generated for them too.

 :        For example, the xml row element:

 :        <row>

 :        <ID>1</ID>

 :        <Name>

 :          Mr.

 :          <First_Name>John</First_Name>

 :          <Last_Name>Howard</Last_Name>

 :        </Name>

 :        <Occupation>student</Occupation>

 :        </row>

 :        is converted to

 :        ID,Name,,Occupation

 :        ,,First Name,Last Name,

 :        1,Mr.,John,Howard,student

 :

 :        If first-row-is-header is not specified and the columns have a deeper hierarchy,
 :          only the first layer of columns is processed, and the fields are the string values of each column.

 :

 :

 : @return the csv or fixed size text as string containing all the lines
 : @example csv_serialize1.xq
 : @example csv_serialize2.xq
 : @example csv_serialize3.xq
 : @example csv_serialize5.xq
 : @example csv_serialize6.xq
 : @example csv_parse_serialize6.xq
 : @example txt_serialize4.xq
 : @example txt_serialize6.xq
 : @example txt_parse_serialize6.xq
:)
declare function zorba-csv:serialize($xml as element()*,
									$options as element(csv-options:options)?) as xs:string
{
  let $validated-options :=
  if(empty($options)) then
    $options
  else
  if(zorba-schema:is-validated($options)) then
    $options
  else
    validate{$options}
  return
    zorba-csv:serialize-internal($xml, $validated-options)
};

declare %private function zorba-csv:serialize-internal($xml as element()*,
									$options as element(csv-options:options, csv-options:optionsType)?) as xs:string external;

(:
 : Copyright 2006-2009 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
:)

(:~
 : Function library providing convertors from CSV/TXT to XML and back.
 : The functions are optimized to work with large amounts of data, in a streaming way.
 :
 : @author Daniel Turcanu
 :)
module namespace zorba-csv = "http://www.zorba-xquery.com/modules/csv";

(:~
 : Import module for checking if csv options element is validated.
 :)
import module namespace zorba-schema = "http://www.zorba-xquery.com/modules/schema";

(:~
 : Contains the definitions of the csv options element.
  :)
import schema namespace csv-options = "http://www.zorba-xquery.com/modules/csv-options";

(:~
 : Errors namespace URI.
:)
declare variable $zorba-csv:csvNS as xs:string := "http://www.zorba-xquery.com/modules/csv";

(:~
 : Error code for wrong parameter situations.

 : Possible error messages:

 : * "Options field must be of element options instead of ..."

:)
declare variable $zorba-csv:errWrongParam as xs:QName := fn:QName($zorba-csv:csvNS, "zorba-csv:WrongParam");

(:~
 : Parse a CSV or fixed size text and convert to XML.

 : By default each line is converted to a <row> element, and each field to a <column> element inside <row>.

 : @param $csv the string containing the csv or fixed size text.
 : @param $options having format

 :    <csv-options:options>

 :        <csv  [separator="default comma ,"] ?

 :          [quote-char="default double quotes &quote;"]?

 :          [quote-escape="default double double quotes &quote;&quote;"]? />

 :

 :        or

 :        <column-widths>

 :          <column-width>[column fixed width, unsigned int]<column-width>*

 :		  	</column-widths>

 :

 :        or

 :        <column-positions>

 :          <column-position>[column position on line, unsigned int]<column-position>*

 :		  	</column-positions>

 :

 :        <first-row-is-header [line="first_line[-last_line]?"]?/>?

 :        <start-from-row line="first_line[-last_line]?"/>?

 :        <add-last-void-columns/>?

 :        <xml-nodes>

 :          [<row-name>

 :            [<column-name/>]?

 :          </row-name>]?

 :        </xml-nodes>?

 :    </csv-options:options>

 :

 :    All the parameters are optional and can appear in any order.

 :    All the parameters are case sensitive. The namespace used is "http://www.zorba-xquery.com/modules/csv-options".

 :    All strings must have UTF-8 encoding.

 :    Parameters csv, column-widths, column-positions are mutualy exclusive. If none is specified,
 :    the input string is assumed to be csv.

 :    Description of parameters:
 :

 :
csv

 :
 Specifies the parameters for parsing a csv string.

 :

 :
separator

 :
The character or group of characters used to separating fields in a row.
 :            If it is not specified, it defaults to comma ','.
 :

 :
quote-char

 :
The character or group of characters used for quoting the fields that may contain special characters,
 :             like separator, new line or this quote char. The default value is double quote ".

 :

 :
quote-escape

 :
The group of characters used for escaping the quote char inside a field. The whole quote escape group
 :           is translated to a quote char during parsing. The default value is double double quotes "".

 :

 :

 :

 :

 :
column-widths

 :
Specifies the column widths for fixed size text. It contains multiple column-width child elements
 :        specifying the fixed width of each column, from left to right.

 :        If the line has more fields than specified, they are ignored.
 :

 :
column-positions

 :
This is an alternative to column-widths, and specifies instead the starting position of each column.
 :        Column positions are 1 based, and are specified in order from left to right.
 :        The last column is read until end of line. The first column position can be greater than 1, if you want
 :        to parse only a part of the input text.
 :

 :
first-row-is-header

 :
The presence of this element indicates that the first row is to be treated as the name of the columns.
 :        If it is not present, then each field is enclosed in a <column> element,
 :        or how it is specified in <xml-nodes> parameter.

 :        If the first row is the header, then each field is enclosed in an element with the corresponding name from the header.

 :        For example, the csv:

 :        ID,Name,Occupation

 :        1,John,student

 :

 :        is parsed into

 :        <row>

 :        <ID>1</ID>

 :        <Name>John</Name>

 :        <Occupation>student</Occupation>

 :        </row>

 :

 :        If the header names contain characters that cannot be used in a QName, they are replaced with underscore '_'.

 :        The namespace for the header QNames is taken from the column name specified in xml-nodes parameter, or from
 :        the row name, or if that doesn't exist either then empty namespace is used.

 :        If the header is not the first line in the input string, the starting line can be specified in the line attribute.

 :        If a column does not have a name, a new name is constructed in the form columnN where N is the position of the column,
 :        starting from 1.

 :        Subheaders

 :				If the header consists of more than one line, this can be specified in the line attribute in the form
 :        "first_line - last_line". Having more lines as the header translates into a hierarchy of elements in the xml.

 :        For example, the csv:

 :        ID,Name,,Occupation

 :        ,First Name,Last Name,

 :        1,John,Howard,student

 :

 :        is parsed into

 :        <row>

 :        <ID>1</ID>

 :        <Name>

 :          <First_Name>John</First_Name>

 :          <Last_Name>Howard</Last_Name>

 :        </Name>

 :        <Occupation>student</Occupation>

 :        </row>

 :

 :
start-from-row

 :
If the data does not start from line 1 or imediately after the header,
 :        you can specify the starting line in the line attribute.

 :        Also you can use this attribute in the form "first_line - last_line" to specify also the last line
 :        if you don't want the whole csv to be parsed.
 :

 :
add-last-void-columns

 :
In the case when using headers and some data lines are shorter than the header, by default the excess columns are ignored
 :          for those lines. You can set the add-last-void-columns parameter to make all the columns appear in xml even if they are void.
 :

 :
xml-nodes

 :
With this parameter you can specify the names for the row element and for the column element if there is no header.

 :        The first element child of this element specifies the desired QName of the row element in the output xml.
 :        The name of this element will be used as the name of the row element.

 :        The element child of this row element is the column element, and its name will be used as the name of the column elements
 :        that enclose the fields in the output xml if there is no header.

 :        If the csv has a header, only the namespace is used from the column element.

 :        For example, with parameter:

 :        <xml-nodes>

 :        <r>

 :          <c/>

 :        </r>

 :        </xml-nodes>

 :

 :        the output for each line will look like

 :        <r>

 :          <c>field1</c>

 :          <c>field2</c>

 :          .......

 :        </r>

 :

 :        If this parameter is not specified, the row name is by default "row" and the column name is by default "column".
 :

 :

 : @return a sequence of row elements, one for each line in csv
 : @error zorba-csv:WrongParam if the options parameter doesn't have the name "options".
 : @example csv_parse1.xq
 : @example csv_parse2.xq
 : @example csv_parse3.xq
 : @example csv_parse6.xq
 : @example csv_parse11.xq
 : @example csv_parse_utf8_11.xq
 : @example txt_parse5.xq
 : @example txt_parse8.xq
:)
declare function zorba-csv:parse($csv as xs:string,
                                 $options as element(csv-options:options)?) as element()*
{
  if(empty($options)) then
    zorba-csv:parse-internal($csv, $options)
  else
  if(zorba-schema:is-validated($options)) then
    zorba-csv:parse-internal($csv, $options)
  else
    zorba-csv:parse-internal($csv, validate{$options})
};

declare %private function zorba-csv:parse-internal($csv as xs:string,
                                 $options as element(csv-options:options, csv-options:optionsType)?) as element()* external;

(:~
 : Convert XML into CSV or fixed size text.

 : Note: if you want to serialize out the result, make sure that the serializer method is set to Text.
 :  For example, in zorba app, you have to set the param --serialize-text.
 :  When using file:write function, you have to set the serialization option <method>text</method>.

 :
 : @param $xml a sequence of elements, each element representing a row. The name of each row element is ignored.
 :     The childs of each row are the column fields.
 : @param $options having format

 :    <csv-options:options>

 :        <csv  [separator="default comma ,"] ?

 :          [quote-char="default double quotes &quote;"]?

 :          [quote-escape="default double double quotes &quote;&quote;"]? />

 :

 :        or

 :        <column-widths [align="left|right"]?>

 :          <column-width [align="left|right"]?>[column fixed width, unsigned int]<column-width>*

 :		  	</column-widths>

 :

 :        or

 :        <column-positions [align="left|right"]?>

 :          <column-position [align="left|right"]?>[column position on line, unsigned int]<column-position>*

 :		  	</column-positions>

 :

 :        <first-row-is-header/>?

 :    </csv-options:options>
 :

 :    All the parameters are optional and can appear in any order.

 :    All the parameters are case sensitive. The namespace used is "http://www.zorba-xquery.com/modules/csv-options".

 :    All strings must have UTF-8 encoding.

 :    Parameters csv, column-widths, column-positions are mutualy exclusive. If none is specified,
 :    the xml is converted to csv.

 :    Description of parameters:
 :

 :
csv

 :
 Specifies the parameters for converting to csv.

 :

 :
separator

 :
The character or group of characters used to separating fields in a row.
 :            If it is not specified, it defaults to comma ','.
 :

 :
quote-char

 :
The character or group of characters used for quoting the fields that may contain special characters,
 :             like separator, new line or this quote char. The default value is double quote ".

 :

 :
quote-escape

 :
The group of characters used for escaping the quote char inside a field. The whole quote escape group
 :           is translated to a quote char during parsing. The default value is double double quotes "".

 :

 :

 :

 :

 :
column-widths

 :
Specifies the column widths for fixed size text. It contains multiple column-width child elements
 :        specifying the fixed width of each column, from left to right.

 :        With the attribute align you can specify how to align fields that are smaller than the column width.
 :        The default alignment is left.

 :

 :
column-positions

 :
This is an alternative to column-widths, and specifies instead the starting position of each column.
 :        Column positions are 1 based, and are specified in order from left to right.
 :        The last column has a variable length.

 :        With the attribute align you can specify how to align fields that are smaller than the column width.
 :        The default alignment is left. The last column does not need alignment.

 :

 :
first-row-is-header

 :
The presence of this element indicates that the first row will contain the header, that is, the names of
 :        the column elements. Only the column names from the first row element are taken into account.

 :        For example, the row xml:

 :        <row>

 :        <ID>1</ID>

 :        <Name>John</Name>

 :        <Occupation>student</Occupation>

 :        </row>

 :

 :        is converted to

 :        ID,Name,Occupation

 :        1,John,student

 :

 :        The header names are the localnames of the column elements, and the namespace is ignored.

 :        Subheaders

 :        If the row-column hierarchy is more complex, then subheaders are also generated on subsequent lines.
 :        The number of subheaders depends on the depth of the column hierarchy.

 :        When generating the subheaders, the non-whitespace text nodes are also taken into account,
 :        and a separate column is generated for them too.

 :        For example, the xml row element:

 :        <row>

 :        <ID>1</ID>

 :        <Name>

 :          Mr.

 :          <First_Name>John</First_Name>

 :          <Last_Name>Howard</Last_Name>

 :        </Name>

 :        <Occupation>student</Occupation>

 :        </row>

 :        is converted to

 :        ID,Name,,Occupation

 :        ,,First Name,Last Name,

 :        1,Mr.,John,Howard,student

 :

 :        If first-row-is-header is not specified and the columns have a deeper hierarchy,
 :          only the first layer of columns is processed, and the fields are the string values of each column.

 :

 :

 : @return the csv or fixed size text as string containing all the lines
 : @example csv_serialize1.xq
 : @example csv_serialize2.xq
 : @example csv_serialize3.xq
 : @example csv_serialize5.xq
 : @example csv_serialize6.xq
 : @example csv_parse_serialize6.xq
 : @example txt_serialize4.xq
 : @example txt_serialize6.xq
 : @example txt_parse_serialize6.xq
:)
declare function zorba-csv:serialize($xml as element()*,
									$options as element(csv-options:options)?) as xs:string
{
  if(empty($options)) then
    zorba-csv:serialize-internal($xml, $options)
  else
  if(zorba-schema:is-validated($options)) then
    zorba-csv:serialize-internal($xml, $options)
  else
    zorba-csv:serialize-internal($xml, validate{$options})
};

declare %private function zorba-csv:serialize-internal($xml as element()*,
									$options as element(csv-options:options, csv-options:optionsType)?) as xs:string external;