DHFPROD-3425: Don't use minimum combo query where document is missing values

xquery version "1.0-ml";

(:
 : This is an implementation library, not an interface to the Smart Mastering functionality.
 :
 : The process of matching starts with one document, which is not required to
 : be in the database. The match options specify what properties are to be used
 : to find matches. See match options documentation for details. The options
 : may specify multiple thresholds, each of which corresponds to an action.
 :
 : Implementation notes: the configured properties are used to generate a boost
 : query. The match part of the query identifies a set of subqueries that a
 : document must match in order to get a score above the lowest threshold.
 : Match queries all have their scores set to zero. The boost part of the query
 : is used to provide the score.

 : @see https://marklogic-community.github.io/smart-mastering-core/docs/matching-options/
 :)

module namespace match-impl = "http://marklogic.com/smart-mastering/matcher-impl";

import module namespace algorithms = "http://marklogic.com/smart-mastering/algorithms"
  at  "/com.marklogic.smart-mastering/algorithms/base.xqy",
      "/com.marklogic.smart-mastering/algorithms/standard-reduction.xqy";
import module namespace blocks-impl = "http://marklogic.com/smart-mastering/blocks-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/blocks-impl.xqy";
import module namespace const = "http://marklogic.com/smart-mastering/constants"
  at "/com.marklogic.smart-mastering/constants.xqy";
import module namespace es-helper = "http://marklogic.com/smart-mastering/entity-services" at "/com.marklogic.smart-mastering/sm-entity-services.xqy";
import module namespace helper-impl = "http://marklogic.com/smart-mastering/helper-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/helper-impl.xqy";
import module namespace json="http://marklogic.com/xdmp/json"
  at "/MarkLogic/json/json.xqy";
import module namespace opt-impl = "http://marklogic.com/smart-mastering/options-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/options-impl.xqy";
import module namespace tel = "http://marklogic.com/smart-mastering/telemetry"
  at "/com.marklogic.smart-mastering/telemetry.xqy";
import module namespace coll = "http://marklogic.com/smart-mastering/collections"
  at "/com.marklogic.smart-mastering/impl/collections.xqy";

declare namespace matcher = "http://marklogic.com/smart-mastering/matcher";
declare namespace sm = "http://marklogic.com/smart-mastering";
declare namespace es = "http://marklogic.com/entity-services";

declare option xdmp:mapping "false";

declare variable $QUERIES_WITH_WEIGHT := (
    xs:QName("cts:element-attribute-pair-geospatial-query"),xs:QName("cts:element-attribute-range-query"),
    xs:QName("cts:element-attribute-value-query"),xs:QName("cts:element-attribute-word-query"),xs:QName("cts:element-child-geospatial-query"),
    xs:QName("cts:element-geospatial-query"),xs:QName("cts:element-pair-geospatial-query"),xs:QName("cts:element-range-query"),
    xs:QName("cts:element-value-query"),xs:QName("cts:element-word-query"),xs:QName("cts:field-range-query"),
    xs:QName("cts:field-value-query"),xs:QName("cts:field-word-query"),xs:QName("cts:geospatial-region-query"),
    xs:QName("cts:json-property-child-geospatial-query"),xs:QName("cts:json-property-geospatial-query"),
    xs:QName("cts:json-property-pair-geospatial-query"),xs:QName("cts:json-property-range-query"),xs:QName("cts:json-property-value-query"),
    xs:QName("cts:json-property-word-query"),xs:QName("cts:lsqt-query"),xs:QName("cts:near-query"),xs:QName("cts:not-query"),
    xs:QName("cts:path-geospatial-query"),xs:QName("cts:path-range-query"),xs:QName("cts:range-query"),xs:QName("cts:registered-query"),
    xs:QName("cts:reverse-query"),xs:QName("cts:similar-query"),xs:QName("cts:triple-range-query"),xs:QName("cts:word-query"));

declare variable $_cached-compiled-match-options as map:map := map:map();

(:
 : Calculate queries once per unique match options in request to reduce repeat logic
 : @param $match-options  Options specifying how documents will be matched
 : @param $document-is-json  Is a JSON document we are matching?
 : @param $minimum-threshold  Minimum threshold for search results to meet
 : @return map:map with compiled information about match options
 :)
declare function match-impl:compile-match-options(
  $match-options as node(),
  $document-is-json as xs:boolean,
  $minimum-threshold as xs:double
) {
  let $cache-id :=
      fn:generate-id($match-options) || "|doc-is-json:"|| $document-is-json || "|min-threshold:" || $minimum-threshold
  return
  if (map:contains($_cached-compiled-match-options, $cache-id)) then
    map:get($_cached-compiled-match-options, $cache-id)
  else
    let $options :=
      if ($match-options instance of object-node()) then
        opt-impl:options-from-json($match-options)
      else
        $match-options
    let $data-format := $options/matcher:data-format
    let $options :=
          if (fn:empty($data-format)) then
            element matcher:options {
              element matcher:data-format {
                if ($document-is-json) then $const:FORMAT-JSON else $const:FORMAT-XML
              },
              $options/*
            }
          else
            $options
    let $scoring := $options/matcher:scoring
    let $max-property-score := fn:max(($scoring/@weight ! fn:number(.)))
    let $score-ratio :=
        if ($max-property-score le 64) then
          1
        else
          (64 idiv $max-property-score)
    let $algorithms := algorithms:build-algorithms-map($options/matcher:algorithms)
    let $target-entity := $options/matcher:target-entity ! fn:string(.)
    let $target-entity-def := es-helper:get-entity-def($target-entity)
    let $queries := (
        for $score in $scoring/(matcher:add|matcher:expand)
        let $type := fn:local-name($score)
        let $weight := fn:number($score/@weight)
        let $full-property-name := $score/@property-name
        let $helper-query := helper-impl:property-name-to-query($options, $full-property-name)
        where fn:exists($helper-query)
        order by $weight descending
        return
          let $qname := helper-impl:property-name-to-qname($options, $full-property-name)
          let $base-values-query :=
              if ($type eq "add") then
                $helper-query(?, $weight)
              else if ($type eq "expand") then
                let $algorithm := map:get($algorithms, $score/@algorithm-ref)
                where fn:exists($algorithm)
                return algorithms:execute-algorithm($algorithm, ?, $score, $options)
              else ()
          return map:new((
            map:entry("queryID", sem:uuid-string()),
            map:entry("propertyName",$full-property-name),
            map:entry("type",$type),
            map:entry("weight",$weight),
            map:entry("qname", $qname),
            map:entry(
              "valuesToQueryFunction",
              $base-values-query
            )
          )),
        for $score in $scoring/matcher:reduce
        let $type := fn:local-name($score)
        let $weight := fn:number($score/@weight)
        let $algorithm-ref := $score/@algorithm-ref
        return
          map:new((
            map:entry("queryID", sem:uuid-string()),
            map:entry("type",$type),
            map:entry("weight",$weight),
            map:entry("algorithm", fn:head(($algorithm-ref, "standard"))),
            map:entry(
              "valuesToQueryFunction",
              if ($type eq "reduce") then
                let $algorithm := $algorithm-ref ! map:get($algorithms, .)
                return
                  if (fn:exists($algorithm)) then
                    algorithms:execute-algorithm($algorithm, ?, $score, $options)
                  else
                    algorithms:standard-reduction-query(?, $score, $options)
              else ()
            )
          ))
      )
    let $minimum-threshold-combinations :=
        match-impl:minimum-threshold-combinations($queries[fn:not(. => map:get("type") = "reduce")], $minimum-threshold)
    let $compiled-match-options := map:new((
        map:entry("options", $options),
        map:entry("targetEntity", $target-entity),
        map:entry("scoreRatio", $score-ratio),
        map:entry("algorithms", $algorithms),
        map:entry("queries", $queries),
        map:entry("orderedThresholds",
          for $threshold in $options/matcher:thresholds/matcher:threshold
          order by $threshold/@above cast as xs:decimal descending
          return $threshold
        ),
        map:entry("minimumThresholdCombinations", $minimum-threshold-combinations),
        map:entry("baseContentQuery",
          if (fn:exists($target-entity)) then
            instance-query-wrapper(
              if ($document-is-json) then
                cts:json-property-scope-query(
                  "info",
                  cts:json-property-value-query("title", fn:string($target-entity-def/entityTitle), (), 0)
                )
              else
                cts:element-query(
                  xs:QName("es:info"),
                  cts:element-value-query(xs:QName("es:title"), fn:string($target-entity-def/entityTitle), (), 0)
                )
              ,
              $document-is-json
            )
          else
            match-impl:build-collection-query(coll:content-collections($options))
        )
      ))
    return (
      map:put(
        $_cached-compiled-match-options,
        $cache-id,
        $compiled-match-options
      ),
      $compiled-match-options
    )
};

(:
 : Find documents that are potential matches for the provided document.
 : @param $document  a source document to draw values from
 : @param $options  XML or JSON representation of match options
 : @param $start  paging: 1-based index
 : @param $page-length  paging: number of results to return
 : @param $minimum-threshold  the required score for the lowest-scoring
 :                            threshold (see match options)
 : @param $include-matches  if true, the response will include, for each result,
 :                          the properties that earned points for the match
 :                          (similar) to snippets
 : @param $filter-query  a cts:query that reduces the scope of documents that
 :                       will be searched for matches
 : @return results specify document URIs that matches for provided document
 :)
declare function match-impl:find-document-matches-by-options(
  $document as node()?,
  $options as item(),
  $start as xs:integer,
  $page-length as xs:integer,
  $minimum-threshold as xs:double,
  $include-matches as xs:boolean,
  $filter-query as cts:query
) as element(results)
{
  (: increment usage count :)
  tel:increment(),
  match-impl:find-document-matches-by-options(
    $document,
    $options,
    $start,
    $page-length,
    $minimum-threshold,
    $include-matches,
    $filter-query,
    fn:true()
  )
};

declare function match-impl:find-document-matches-by-options(
  $document as node()?,
  $options as item(),
  $start as xs:integer,
  $page-length as xs:integer,
  $minimum-threshold as xs:double,
  $include-matches as xs:boolean,
  $filter-query as cts:query,
  $include-results as xs:boolean
) as element(results)
{
  match-impl:find-document-matches-by-options(
    $document,
    $options,
    $start,
    $page-length,
    $minimum-threshold,
    $include-matches,
    $filter-query,
    $include-results,
    (: by default don't short-circuit redundant queries :)
    fn:false()
  )
};

(: A map to track match queries previously searched on. Useful for process-impl:build-match-summary :)
declare variable $map-of-queries-previously-run := map:map();

declare function match-impl:find-document-matches-by-options(
  $document as node()?,
  $options as item(),
  $start as xs:integer,
  $page-length as xs:integer,
  $minimum-threshold as xs:double,
  $include-matches as xs:boolean,
  $filter-query as cts:query,
  $include-results as xs:boolean,
  $short-circuit-redundant-queries as xs:boolean
) as element(results)
{
  if (fn:exists($document)) then (
    let $start-elapsed := xdmp:elapsed-time()
    let $is-json := (xdmp:node-kind($document) = "object" or fn:exists($document/(object-node()|array-node())))
    let $compiled-options := match-impl:compile-match-options($options, $is-json, $minimum-threshold)
    let $values-by-qname := match-impl:values-by-qname($document, $compiled-options)
    let $query-prov as map:map? :=
        if ($include-results and $include-matches) then
          map:map()
        else ()
    let $cached-queries := map:map()
    let $minimum-threshold-combinations-queries :=
      for $query-set in $compiled-options => map:get("minimumThresholdCombinations")
      let $query-maps := $query-set => map:get("queries")
      let $queries :=
        for $query-map in $query-maps
        (: if there are no values from the document for a query, then match-impl:query-map-to-query returns an empty sequence  :)
        return match-impl:query-map-to-query($query-map, $values-by-qname, $cached-queries, $query-prov)
      (: We want to be certain that we have values for each of the queries in a min threshold combo :)
      where fn:exists($queries) and fn:count($queries) eq fn:count($query-maps)
      return
        let $queries := if (fn:count($queries) gt 1) then
          helper-impl:group-queries-by-scope($queries, cts:and-query#1)
        else
          $queries
        return
          if (fn:count($queries) gt 1) then
            cts:and-query($queries)
          else
            $queries
    (: We want to ignore redundant queries. This only applies to expanded searches to find additional merges that may be added via process-impl:build-match-summary
     : If the queries generated by a document are the same as document previously searched on, it won't result in additional URIs.
     :)
    let $has-redundant-match-combo := $short-circuit-redundant-queries and (
      every $min-query in $minimum-threshold-combinations-queries satisfies
        map:contains($map-of-queries-previously-run, xdmp:md5(document{$min-query}))
    )
    let $_cache-minimum-threshold := if ($short-circuit-redundant-queries and fn:not($has-redundant-match-combo)) then
      for $min-query in $minimum-threshold-combinations-queries
      return map:put($map-of-queries-previously-run, xdmp:md5(document{$min-query}), fn:true())
    else ()
    (: If there are no match queries, just return a cts:false-query  :)
    let $minimum-threshold-combinations-query :=
      if (fn:exists($minimum-threshold-combinations-queries)) then
        helper-impl:group-queries-by-scope($minimum-threshold-combinations-queries, cts:or-query#1)
      else
        cts:false-query()
    (: Exclude the current document and any blocked matches :)
    let $excluded-uris := (
      xdmp:node-uri($document),
      blocks-impl:get-blocks(fn:base-uri($document))/node()
    )
    let $match-base-query := cts:and-query((
          $compiled-options => map:get("baseContentQuery"),
          $minimum-threshold-combinations-query
        ))
    let $match-query :=
      if (fn:exists($excluded-uris)) then
        cts:and-not-query(
          $match-base-query,
          cts:document-query($excluded-uris)
        )
      else
        $match-base-query
    let $match-query :=
      match-impl:instance-query-wrapper(
        if (fn:not($filter-query instance of cts:true-query)) then
          cts:and-query((
            $filter-query,
            $match-query
          ))
        else
          $match-query,
        $is-json
      )
    let $serialized-match-query :=
      element match-query {
        $match-query
      }
    return
      (: If minimum threshold can't be met or we're short-circuiting a query already run, don't bother with estimate and search :)
      if ($minimum-threshold-combinations-query instance of cts:false-query or $has-redundant-match-combo) then
        match-impl:build-empty-results($start, $page-length, $serialized-match-query)
      else
        let $estimate := xdmp:estimate(cts:search(fn:collection(), match-impl:instance-query-wrapper($match-query, $is-json), "unfiltered"))
        return (
          element results {
            attribute total { $estimate },
            attribute page-length { $page-length },
            attribute start { $start },
            $serialized-match-query,
            if ($include-results and $estimate gt 0) then
              let $boost-query := match-impl:build-boost-query($document, $values-by-qname, $compiled-options, $cached-queries, $query-prov)
              return
                  match-impl:search(
                    $match-query,
                    $boost-query,
                    $minimum-threshold,
                    $start,
                    $page-length,
                    $compiled-options,
                    $include-matches,
                    $is-json,
                    $query-prov
                  )
            else (),
            if (xdmp:trace-enabled($const:TRACE-PERFORMANCE)) then
              xdmp:trace($const:TRACE-PERFORMANCE, "match-impl:find-document-matches-by-options: " || (xdmp:elapsed-time() - $start-elapsed))
            else ()
          }
        )
  ) else
    match-impl:build-empty-results($start,$page-length, ())
};

declare function match-impl:build-empty-results(
  $start as xs:integer,
  $page-length as xs:integer,
  $serialize-match-query as node()?
) {
  element results {
    attribute total { 0 },
    attribute page-length { $page-length },
    attribute start { $start },
    $serialize-match-query
  }
};

(:
 :
 :)
declare function match-impl:build-collection-query($collections as xs:string*)
{
  if (fn:empty($collections)) then
    ()
  else if (fn:count($collections) > 1) then
    cts:and-query($collections ! cts:collection-query(.))
  else
    cts:collection-query($collections)
};

(:
 : Does each item in $s1 appear in $s2?
 :)
declare function match-impl:seq-contains($s1, $s2)
{
  every $s in $s1 satisfies $s = $s2
};

(:
 : Build the boost query as specified by the match options.
 : @param $document  the source document from which property values are drawn
 : @param $values-by-qname  map:map that organizes document values by QName
 : @param $compiled-options map:map with compiled details about match options
 : @param $cached-queries map:map containing previously created queries
 : @param $query-prov map:map to optionally track the provenance of queries contributing to match
 : @return a cts:or-query that will be used as a boost query
 :)
declare function match-impl:build-boost-query(
  $document as node(),
  $values-by-qname as map:map,
  $compiled-options as map:map,
  $cached-queries as map:map,
  $query-prov as map:map?
)
{
  cts:or-query((
    for $query-map in map:get($compiled-options, "queries")
    return
      if ($query-map => map:get("type") = "reduce") then
        ($query-map => map:get("valuesToQueryFunction"))($document)
      else
        match-impl:query-map-to-query($query-map, $values-by-qname, $cached-queries, $query-prov)
  ))
};

(:
 : Convert map:map describing query to cts:query
 : @param $query-map  map:map describing a query to generate
 : @param $values-by-qname  map:map that organizes document values by QName
 : @param $cached-queries  map:map caching previously generated queries
 : @param $query-prov map:map to optionally track the provenance of queries contributing to match
 : @return cts:query?
 :)
declare function match-impl:query-map-to-query(
  $query-map as map:map,
  $values-by-qname as map:map,
  $cached-queries as map:map,
  $query-prov as map:map?
)
{
  if (map:contains($query-map, "queryID") and map:contains($cached-queries, $query-map => map:get("queryID"))) then
    map:get($cached-queries, $query-map => map:get("queryID"))
  else
    let $query :=
        let $qname := $query-map => map:get("qname")
        let $values := $values-by-qname => map:get(xdmp:key-from-QName($qname))
        where fn:exists($values)
        return
          ($query-map => map:get("valuesToQueryFunction"))($values)
    return (
      map:put($cached-queries, $query-map => map:get("queryID"), $query),
      if (fn:exists($query-prov)) then
        let $query-hashes := fn:distinct-values(cts:element-walk(document{$query}, $QUERIES_WITH_WEIGHT, element query-hash{xdmp:md5(document {$cts:node})})//query-hash !
          fn:string(.))
        for $query-hash in $query-hashes
        return map:put($query-prov, $query-hash, $query-map)
      else (),
      $query
    )
};

(:
 : Organize values by QName
 : @param $document  document with property values
 : @param $compiled-options  map:map with compiled details about match options
 : @return map:map of values organized by QName
 :)
declare function match-impl:values-by-qname(
  $document as node()?,
  $compiled-options as map:map
)
{
  map:new(
    for $qname in fn:distinct-values((map:get($compiled-options, "queries") ! map:get(.,"qname")))
    let $values := fn:distinct-values($document/(self::*:envelope|*:envelope)/*:instance//*[fn:node-name(.) eq $qname][fn:normalize-space(fn:string(.))]/fn:data(.))
    where fn:exists($values)
    return
      map:entry(xdmp:key-from-QName($qname), $values)
  )
};

(:
 : Execute the generated search and construct the response.
 : @param $match-query  a query built such that any matches will score at least
 :                      high enough to reach the lowest threshold
 : @param $boosting-query  a query that is used to score matches
 : @param $filter-query  a query to reduce the universe of match candidates
 : @param $min-threshold  lowest score required to hit a threshold
 : @param $start  paging: 1-based index
 : @param $page-length  paging
 : @param $algorithms  map derived from match options
 : @param $options  full match options; included to pass to reduce algorithm
 : @param $include-matches
 : @param $is-json
 : @param $query-prov an optional map for tracking how options/algorithms contributed to matches
 : @return results specify document URIs that matches for provided document
 :)
declare function match-impl:search(
  $match-query,
  $boosting-query,
  $min-threshold as xs:double,
  $start as xs:int,
  $page-length as xs:int,
  $compiled-options as map:map,
  $include-matches as xs:boolean,
  $is-json as xs:boolean,
  $query-prov as map:map?
) {
  let $range := $start to ($start + $page-length - 1)
  let $cts-walk-query :=
    if ($include-matches) then
      match-impl:instance-query-wrapper(
        $boosting-query,
        $is-json
      )
    else ()
  let $thresholds := $compiled-options => map:get("orderedThresholds")
  let $individual-matching-queries := cts:or-query-queries($boosting-query)
  for $result at $pos in cts:search(
    fn:collection(),
    $match-query,
    ("unfiltered", "score-simple"),
    0
  )[fn:position() = $range]
  let $uri := xdmp:node-uri($result)
  let $matching-queries := $individual-matching-queries[cts:contains($result, .)]
  let $matching-weights-map := map:entry("values", ())
  let $_accumulate-scores :=
    cts:element-walk(document{$matching-queries}, $QUERIES_WITH_WEIGHT,
      let $weight := fn:head(($cts:node/@weight, 1))
      return
      (
        $matching-weights-map => map:put("values",
          ($matching-weights-map => map:get("values"), $weight)),
        $matching-weights-map => map:put(xdmp:md5(document {$cts:node}), $weight)
      )
    )
  let $score :=
      fn:sum(
        $matching-weights-map => map:get("values")
      )
  where $score ge $min-threshold
  return
    element result {
      attribute uri {$uri},
      attribute index {$range[fn:position() = $pos]},
      attribute score {$score},
      let $selected-threshold := fn:head($thresholds[$score ge @above])
      return (
        attribute threshold { fn:string($selected-threshold/@label) },
        attribute action { fn:string($selected-threshold/(matcher:action|@action)) }
      ),
      if ($include-matches) then
        element matches {
          (: rather than store the entire node and risk mixing
             content type (json != xml) we store the path to the
             node instead :)
          cts:walk(
            $result/*:envelope/*:instance,
            cts:or-query($matching-queries),
            (let $query-hashes := fn:distinct-values($cts:queries !
                cts:element-walk(document{.}, $QUERIES_WITH_WEIGHT, element query-hash{xdmp:md5(document {$cts:node})})//query-hash !
                fn:string(.))
            let $weight := fn:sum($query-hashes ! ($matching-weights-map => map:get(.)))
            let $node-name := fn:string(fn:head((fn:node-name($cts:node),fn:node-name($cts:node/..))))
            return <match weight="{$weight}">
                      {
                        for $query-hash in $query-hashes
                        let $query-map := map:get($query-prov, $query-hash)
                        return
                          <contributions>
                            <algorithm>{fn:string(fn:head(($query-map ! map:get(., "algorithm") ! fn:string(.),"standard")))}</algorithm>
                            <weight>{fn:string($matching-weights-map => map:get($query-hash))}</weight>
                          </contributions>
                      }
                      <nodeName>{$node-name}</nodeName>
                      <value>{fn:data($cts:node)}</value>
                      <path>{xdmp:path($cts:node, fn:true())}</path>
                   </match>)
          )
        }
      else ()
    }
};

(: Configuration used to convert XML match results to JSON. :)
declare variable $results-json-config := match-impl:_results-json-config();

declare function match-impl:_results-json-config()
{
  let $config := json:config("custom")
  return (
    map:put($config, "array-element-names", ("result","matches","algorithms",xs:QName("cts:option"),xs:QName("cts:text"),xs:QName("cts:element"))),
    map:put($config, "full-element-names",
      (xs:QName("cts:query"),
      xs:QName("cts:and-query"),
      xs:QName("cts:near-query"),
      xs:QName("cts:or-query"))
    ),
    map:put($config, "json-children", "queries"),
    map:put($config, "attribute-names",
      ("name","localname", "namespace", "function", "weight",
      "at", "property-name", "weight", "above", "label","algorithm-ref")
    ),
    map:put($config, "camel-case", fn:true()),
    $config
  )
};

(:
 : Identify a sequence of queries whose scores add up to the $threshold. A document must match at least one of these
 : queries in order to be returned as a potential match.
 :
 : @param $query-results  a sequence of queries with weights
 : @param $threshold  minimum weighted-score for a match to be relevant
 : @return a sequence of queries; a document that matches any of these will have at least $threshold as a score
 :)
declare function match-impl:minimum-threshold-combinations($query-results, $threshold as xs:double)
  as map:map*
{
  (: Each of $queries-ge-threshold has a weight high enough to hit the $threshold :)
  let $queries-ge-threshold := $query-results[(. => map:get("weight")) ge $threshold]
  let $queries-lt-threshold := $query-results[(. => map:get("weight")) lt $threshold]
  return (
    $queries-ge-threshold ! map:entry("queries", .),
    match-impl:filter-for-required-queries($queries-lt-threshold, 0, $threshold, ())
  )
};

(:
 : Find combinations of queries whose weights are individually below the threshold, but combined are above it.
 :
 : @param $remaining-queries  sequence of queries ordered by their weights, descending
 : @param $combined-weight
 : @param $threshold  the target value
 : @param $accumulated-queries  accumlated sequence, building up to see whether it can hit the $threshold.
 : @return a sequence of cts:and-queries, one for each required filter
 : note: return type left off to allow for tail recursion optimization.
 :)
declare function match-impl:filter-for-required-queries(
  $remaining-queries as map:map*,
  $combined-weight,
  $threshold,
  $accumulated-queries as map:map*
)
{
  if ($threshold eq 0 or $combined-weight ge $threshold) then (
    if (fn:exists($accumulated-queries)) then
      map:entry(
        "queries",
        $accumulated-queries
      )
    else
      ()
  )
  else
    for $query at $pos in $remaining-queries
    let $query-weight := fn:head(($query => map:get("weight"), 1))
    let $new-combined-weight := $combined-weight + $query-weight
    return (
      match-impl:filter-for-required-queries(
        fn:subsequence($remaining-queries, $pos + 1),
        $new-combined-weight,
        $threshold,
        ($accumulated-queries, $query)
      )
    )
};

(:
 : Convert XML match results to JSON.
 :)
declare function match-impl:results-to-json($results-xml)
  as object-node()?
{
  if (fn:exists($results-xml)) then
    xdmp:to-json(
      json:transform-to-json-object($results-xml, $results-json-config)
    )/node()
  else ()
};

declare function match-impl:instance-query-wrapper(
  $query as cts:query,
  $is-json as xs:boolean
) {
  if ($is-json) then
    if (fn:exists($const:JSON-INSTANCE)) then
      cts:json-property-scope-query($const:JSON-INSTANCE, $query)
    else ()
  else
    if (fn:exists($const:XML-INSTANCE)) then
      cts:element-query($const:XML-INSTANCE, $query)
    else ()
};

xquery version "1.0-ml";

(:
 : This is an implementation library, not an interface to the Smart Mastering functionality.
 :
 : The process of matching starts with one document, which is not required to
 : be in the database. The match options specify what properties are to be used
 : to find matches. See match options documentation for details. The options
 : may specify multiple thresholds, each of which corresponds to an action.
 :
 : Implementation notes: the configured properties are used to generate a boost
 : query. The match part of the query identifies a set of subqueries that a
 : document must match in order to get a score above the lowest threshold.
 : Match queries all have their scores set to zero. The boost part of the query
 : is used to provide the score.

 : @see https://marklogic-community.github.io/smart-mastering-core/docs/matching-options/
 :)

module namespace match-impl = "http://marklogic.com/smart-mastering/matcher-impl";

import module namespace algorithms = "http://marklogic.com/smart-mastering/algorithms"
  at  "/com.marklogic.smart-mastering/algorithms/base.xqy",
      "/com.marklogic.smart-mastering/algorithms/standard-reduction.xqy";
import module namespace blocks-impl = "http://marklogic.com/smart-mastering/blocks-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/blocks-impl.xqy";
import module namespace const = "http://marklogic.com/smart-mastering/constants"
  at "/com.marklogic.smart-mastering/constants.xqy";
import module namespace es-helper = "http://marklogic.com/smart-mastering/entity-services" at "/com.marklogic.smart-mastering/sm-entity-services.xqy";
import module namespace helper-impl = "http://marklogic.com/smart-mastering/helper-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/helper-impl.xqy";
import module namespace json="http://marklogic.com/xdmp/json"
  at "/MarkLogic/json/json.xqy";
import module namespace opt-impl = "http://marklogic.com/smart-mastering/options-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/options-impl.xqy";
import module namespace tel = "http://marklogic.com/smart-mastering/telemetry"
  at "/com.marklogic.smart-mastering/telemetry.xqy";
import module namespace coll = "http://marklogic.com/smart-mastering/collections"
  at "/com.marklogic.smart-mastering/impl/collections.xqy";

declare namespace matcher = "http://marklogic.com/smart-mastering/matcher";
declare namespace sm = "http://marklogic.com/smart-mastering";
declare namespace es = "http://marklogic.com/entity-services";

declare option xdmp:mapping "false";

declare variable $QUERIES_WITH_WEIGHT := (
    xs:QName("cts:element-attribute-pair-geospatial-query"),xs:QName("cts:element-attribute-range-query"),
    xs:QName("cts:element-attribute-value-query"),xs:QName("cts:element-attribute-word-query"),xs:QName("cts:element-child-geospatial-query"),
    xs:QName("cts:element-geospatial-query"),xs:QName("cts:element-pair-geospatial-query"),xs:QName("cts:element-range-query"),
    xs:QName("cts:element-value-query"),xs:QName("cts:element-word-query"),xs:QName("cts:field-range-query"),
    xs:QName("cts:field-value-query"),xs:QName("cts:field-word-query"),xs:QName("cts:geospatial-region-query"),
    xs:QName("cts:json-property-child-geospatial-query"),xs:QName("cts:json-property-geospatial-query"),
    xs:QName("cts:json-property-pair-geospatial-query"),xs:QName("cts:json-property-range-query"),xs:QName("cts:json-property-value-query"),
    xs:QName("cts:json-property-word-query"),xs:QName("cts:lsqt-query"),xs:QName("cts:near-query"),xs:QName("cts:not-query"),
    xs:QName("cts:path-geospatial-query"),xs:QName("cts:path-range-query"),xs:QName("cts:range-query"),xs:QName("cts:registered-query"),
    xs:QName("cts:reverse-query"),xs:QName("cts:similar-query"),xs:QName("cts:triple-range-query"),xs:QName("cts:word-query"));

declare variable $_cached-compiled-match-options as map:map := map:map();

(:
 : Calculate queries once per unique match options in request to reduce repeat logic
 : @param $match-options  Options specifying how documents will be matched
 : @param $document-is-json  Is a JSON document we are matching?
 : @param $minimum-threshold  Minimum threshold for search results to meet
 : @return map:map with compiled information about match options
 :)
declare function match-impl:compile-match-options(
  $match-options as node(),
  $document-is-json as xs:boolean,
  $minimum-threshold as xs:double
) {
  let $cache-id :=
      fn:generate-id($match-options) || "|doc-is-json:"|| $document-is-json || "|min-threshold:" || $minimum-threshold
  return
  if (map:contains($_cached-compiled-match-options, $cache-id)) then
    map:get($_cached-compiled-match-options, $cache-id)
  else
    let $options :=
      if ($match-options instance of object-node()) then
        opt-impl:options-from-json($match-options)
      else
        $match-options
    let $data-format := $options/matcher:data-format
    let $options :=
          if (fn:empty($data-format)) then
            element matcher:options {
              element matcher:data-format {
                if ($document-is-json) then $const:FORMAT-JSON else $const:FORMAT-XML
              },
              $options/*
            }
          else
            $options
    let $scoring := $options/matcher:scoring
    let $max-property-score := fn:max(($scoring/@weight ! fn:number(.)))
    let $score-ratio :=
        if ($max-property-score le 64) then
          1
        else
          (64 idiv $max-property-score)
    let $algorithms := algorithms:build-algorithms-map($options/matcher:algorithms)
    let $target-entity := $options/matcher:target-entity ! fn:string(.)
    let $target-entity-def := es-helper:get-entity-def($target-entity)
    let $queries := (
        for $score in $scoring/(matcher:add|matcher:expand)
        let $type := fn:local-name($score)
        let $weight := fn:number($score/@weight)
        let $full-property-name := $score/@property-name
        let $helper-query := helper-impl:property-name-to-query($options, $full-property-name)
        where fn:exists($helper-query)
        order by $weight descending
        return
          let $qname := helper-impl:property-name-to-qname($options, $full-property-name)
          let $base-values-query :=
              if ($type eq "add") then
                $helper-query(?, $weight)
              else if ($type eq "expand") then
                let $algorithm := map:get($algorithms, $score/@algorithm-ref)
                where fn:exists($algorithm)
                return algorithms:execute-algorithm($algorithm, ?, $score, $options)
              else ()
          return map:new((
            map:entry("queryID", sem:uuid-string()),
            map:entry("propertyName",$full-property-name),
            map:entry("type",$type),
            map:entry("weight",$weight),
            map:entry("qname", $qname),
            map:entry(
              "valuesToQueryFunction",
              $base-values-query
            )
          )),
        for $score in $scoring/matcher:reduce
        let $type := fn:local-name($score)
        let $weight := fn:number($score/@weight)
        let $algorithm-ref := $score/@algorithm-ref
        return
          map:new((
            map:entry("queryID", sem:uuid-string()),
            map:entry("type",$type),
            map:entry("weight",$weight),
            map:entry("algorithm", fn:head(($algorithm-ref, "standard"))),
            map:entry(
              "valuesToQueryFunction",
              if ($type eq "reduce") then
                let $algorithm := $algorithm-ref ! map:get($algorithms, .)
                return
                  if (fn:exists($algorithm)) then
                    algorithms:execute-algorithm($algorithm, ?, $score, $options)
                  else
                    algorithms:standard-reduction-query(?, $score, $options)
              else ()
            )
          ))
      )
    let $minimum-threshold-combinations :=
        match-impl:minimum-threshold-combinations($queries[fn:not(. => map:get("type") = "reduce")], $minimum-threshold)
    let $compiled-match-options := map:new((
        map:entry("options", $options),
        map:entry("targetEntity", $target-entity),
        map:entry("scoreRatio", $score-ratio),
        map:entry("algorithms", $algorithms),
        map:entry("queries", $queries),
        map:entry("orderedThresholds",
          for $threshold in $options/matcher:thresholds/matcher:threshold
          order by $threshold/@above cast as xs:decimal descending
          return $threshold
        ),
        map:entry("minimumThresholdCombinations", $minimum-threshold-combinations),
        map:entry("baseContentQuery",
          if (fn:exists($target-entity)) then
            instance-query-wrapper(
              if ($document-is-json) then
                cts:json-property-scope-query(
                  "info",
                  cts:json-property-value-query("title", fn:string($target-entity-def/entityTitle), (), 0)
                )
              else
                cts:element-query(
                  xs:QName("es:info"),
                  cts:element-value-query(xs:QName("es:title"), fn:string($target-entity-def/entityTitle), (), 0)
                )
              ,
              $document-is-json
            )
          else
            match-impl:build-collection-query(coll:content-collections($options))
        )
      ))
    return (
      map:put(
        $_cached-compiled-match-options,
        $cache-id,
        $compiled-match-options
      ),
      $compiled-match-options
    )
};

(:
 : Find documents that are potential matches for the provided document.
 : @param $document  a source document to draw values from
 : @param $options  XML or JSON representation of match options
 : @param $start  paging: 1-based index
 : @param $page-length  paging: number of results to return
 : @param $minimum-threshold  the required score for the lowest-scoring
 :                            threshold (see match options)
 : @param $include-matches  if true, the response will include, for each result,
 :                          the properties that earned points for the match
 :                          (similar) to snippets
 : @param $filter-query  a cts:query that reduces the scope of documents that
 :                       will be searched for matches
 : @return results specify document URIs that matches for provided document
 :)
declare function match-impl:find-document-matches-by-options(
  $document as node()?,
  $options as item(),
  $start as xs:integer,
  $page-length as xs:integer,
  $minimum-threshold as xs:double,
  $include-matches as xs:boolean,
  $filter-query as cts:query
) as element(results)
{
  (: increment usage count :)
  tel:increment(),
  match-impl:find-document-matches-by-options(
    $document,
    $options,
    $start,
    $page-length,
    $minimum-threshold,
    $include-matches,
    $filter-query,
    fn:true()
  )
};

declare function match-impl:find-document-matches-by-options(
  $document as node()?,
  $options as item(),
  $start as xs:integer,
  $page-length as xs:integer,
  $minimum-threshold as xs:double,
  $include-matches as xs:boolean,
  $filter-query as cts:query,
  $include-results as xs:boolean
) as element(results)
{
  match-impl:find-document-matches-by-options(
    $document,
    $options,
    $start,
    $page-length,
    $minimum-threshold,
    $include-matches,
    $filter-query,
    $include-results,
    (: by default don't short-circuit redundant queries :)
    fn:false()
  )
};

(: A map to track match queries previously searched on. Useful for process-impl:build-match-summary :)
declare variable $map-of-queries-previously-run := map:map();

declare function match-impl:find-document-matches-by-options(
  $document as node()?,
  $options as item(),
  $start as xs:integer,
  $page-length as xs:integer,
  $minimum-threshold as xs:double,
  $include-matches as xs:boolean,
  $filter-query as cts:query,
  $include-results as xs:boolean,
  $short-circuit-redundant-queries as xs:boolean
) as element(results)
{
  if (fn:exists($document)) then (
    let $start-elapsed := xdmp:elapsed-time()
    let $is-json := (xdmp:node-kind($document) = "object" or fn:exists($document/(object-node()|array-node())))
    let $compiled-options := match-impl:compile-match-options($options, $is-json, $minimum-threshold)
    let $values-by-qname := match-impl:values-by-qname($document, $compiled-options)
    let $query-prov as map:map? :=
        if ($include-results and $include-matches) then
          map:map()
        else ()
    let $cached-queries := map:map()
    let $minimum-threshold-combinations-queries :=
      for $query-set in $compiled-options => map:get("minimumThresholdCombinations")
      let $query-maps := $query-set => map:get("queries")
      let $queries :=
        for $query-map in $query-maps
        return match-impl:query-map-to-query($query-map, $values-by-qname, $cached-queries, $query-prov)
      where fn:exists($queries)
      return
        if (fn:count($queries) gt 1) then
          helper-impl:group-queries-by-scope($queries, cts:and-query#1)
        else
          $queries
    (: We want to ignore redundant queries. This only applies to expanded searches to find additional merges that may be added via process-impl:build-match-summary
     : If the queries generated by a document are the same as document previously searched on, it won't result in additional URIs.
     :)
    let $has-redundant-match-combo := $short-circuit-redundant-queries and (
      every $min-query in $minimum-threshold-combinations-queries satisfies (
        let $md5 := xdmp:md5(document{$min-query})
        return (map:contains($map-of-queries-previously-run, $md5), map:put($map-of-queries-previously-run, $md5, fn:true()))
      )
    )
    (: If there are no match queries, just return a cts:false-query  :)
    let $minimum-threshold-combinations-query :=
      if (fn:exists($minimum-threshold-combinations-queries)) then
        helper-impl:group-queries-by-scope($minimum-threshold-combinations-queries, cts:or-query#1)
      else
        cts:false-query()
    (: Exclude the current document and any blocked matches :)
    let $excluded-uris := (
      xdmp:node-uri($document),
      blocks-impl:get-blocks(fn:base-uri($document))/node()
    )
    let $match-base-query := cts:and-query((
          $compiled-options => map:get("baseContentQuery"),
          $minimum-threshold-combinations-query
        ))
    let $match-query :=
      if (fn:exists($excluded-uris)) then
        cts:and-not-query(
          $match-base-query,
          cts:document-query($excluded-uris)
        )
      else
        $match-base-query
    let $match-query :=
      match-impl:instance-query-wrapper(
        if (fn:not($filter-query instance of cts:true-query)) then
          cts:and-query((
            $filter-query,
            $match-query
          ))
        else
          $match-query,
        $is-json
      )
    let $serialized-match-query :=
      element match-query {
        $match-query
      }
    return
      (: If minimum threshold can't be met or we're short-circuiting a query already run, don't bother with estimate and search :)
      if ($minimum-threshold-combinations-query instance of cts:false-query or $has-redundant-match-combo) then
        match-impl:build-empty-results($start, $page-length, $serialized-match-query)
      else
        let $estimate := xdmp:estimate(cts:search(fn:collection(), match-impl:instance-query-wrapper($match-query, $is-json), "unfiltered"))
        return (
          element results {
            attribute total { $estimate },
            attribute page-length { $page-length },
            attribute start { $start },
            $serialized-match-query,
            if ($include-results and $estimate gt 0) then
              let $boost-query := match-impl:build-boost-query($document, $values-by-qname, $compiled-options, $cached-queries, $query-prov)
              return
                  match-impl:search(
                    $match-query,
                    $boost-query,
                    $minimum-threshold,
                    $start,
                    $page-length,
                    $compiled-options,
                    $include-matches,
                    $is-json,
                    $query-prov
                  )
            else (),
            if (xdmp:trace-enabled($const:TRACE-PERFORMANCE)) then
              xdmp:trace($const:TRACE-PERFORMANCE, "match-impl:find-document-matches-by-options: " || (xdmp:elapsed-time() - $start-elapsed))
            else ()
          }
        )
  ) else
    match-impl:build-empty-results($start,$page-length, ())
};

declare function match-impl:build-empty-results(
  $start as xs:integer,
  $page-length as xs:integer,
  $serialize-match-query as node()?
) {
  element results {
    attribute total { 0 },
    attribute page-length { $page-length },
    attribute start { $start },
    $serialize-match-query
  }
};

(:
 :
 :)
declare function match-impl:build-collection-query($collections as xs:string*)
{
  if (fn:empty($collections)) then
    ()
  else if (fn:count($collections) > 1) then
    cts:and-query($collections ! cts:collection-query(.))
  else
    cts:collection-query($collections)
};

(:
 : Does each item in $s1 appear in $s2?
 :)
declare function match-impl:seq-contains($s1, $s2)
{
  every $s in $s1 satisfies $s = $s2
};

(:
 : Build the boost query as specified by the match options.
 : @param $document  the source document from which property values are drawn
 : @param $values-by-qname  map:map that organizes document values by QName
 : @param $compiled-options map:map with compiled details about match options
 : @param $cached-queries map:map containing previously created queries
 : @param $query-prov map:map to optionally track the provenance of queries contributing to match
 : @return a cts:or-query that will be used as a boost query
 :)
declare function match-impl:build-boost-query(
  $document as node(),
  $values-by-qname as map:map,
  $compiled-options as map:map,
  $cached-queries as map:map,
  $query-prov as map:map?
)
{
  cts:or-query((
    for $query-map in map:get($compiled-options, "queries")
    return
      if ($query-map => map:get("type") = "reduce") then
        ($query-map => map:get("valuesToQueryFunction"))($document)
      else
        match-impl:query-map-to-query($query-map, $values-by-qname, $cached-queries, $query-prov)
  ))
};

(:
 : Convert map:map describing query to cts:query
 : @param $query-map  map:map describing a query to generate
 : @param $values-by-qname  map:map that organizes document values by QName
 : @param $cached-queries  map:map caching previously generated queries
 : @param $query-prov map:map to optionally track the provenance of queries contributing to match
 : @return cts:query?
 :)
declare function match-impl:query-map-to-query(
  $query-map as map:map,
  $values-by-qname as map:map,
  $cached-queries as map:map,
  $query-prov as map:map?
)
{
  if (map:contains($query-map, "queryID") and map:contains($cached-queries, $query-map => map:get("queryID"))) then
    map:get($cached-queries, $query-map => map:get("queryID"))
  else
    let $query :=
        let $qname := $query-map => map:get("qname")
        let $values := $values-by-qname => map:get(xdmp:key-from-QName($qname))
        where fn:exists($values)
        return
          ($query-map => map:get("valuesToQueryFunction"))($values)
    return (
      map:put($cached-queries, $query-map => map:get("queryID"), $query),
      if (fn:exists($query-prov)) then
        let $query-hashes := fn:distinct-values(cts:element-walk(document{$query}, $QUERIES_WITH_WEIGHT, element query-hash{xdmp:md5(document {$cts:node})})//query-hash !
          fn:string(.))
        for $query-hash in $query-hashes
        return map:put($query-prov, $query-hash, $query-map)
      else (),
      $query
    )
};

(:
 : Organize values by QName
 : @param $document  document with property values
 : @param $compiled-options  map:map with compiled details about match options
 : @return map:map of values organized by QName
 :)
declare function match-impl:values-by-qname(
  $document as node()?,
  $compiled-options as map:map
)
{
  map:new(
    for $qname in fn:distinct-values((map:get($compiled-options, "queries") ! map:get(.,"qname")))
    let $values := fn:distinct-values($document/(self::*:envelope|*:envelope)/*:instance//*[fn:node-name(.) eq $qname][fn:normalize-space(fn:string(.))]/fn:data(.))
    where fn:exists($values)
    return
      map:entry(xdmp:key-from-QName($qname), $values)
  )
};

(:
 : Execute the generated search and construct the response.
 : @param $match-query  a query built such that any matches will score at least
 :                      high enough to reach the lowest threshold
 : @param $boosting-query  a query that is used to score matches
 : @param $filter-query  a query to reduce the universe of match candidates
 : @param $min-threshold  lowest score required to hit a threshold
 : @param $start  paging: 1-based index
 : @param $page-length  paging
 : @param $algorithms  map derived from match options
 : @param $options  full match options; included to pass to reduce algorithm
 : @param $include-matches
 : @param $is-json
 : @param $query-prov an optional map for tracking how options/algorithms contributed to matches
 : @return results specify document URIs that matches for provided document
 :)
declare function match-impl:search(
  $match-query,
  $boosting-query,
  $min-threshold as xs:double,
  $start as xs:int,
  $page-length as xs:int,
  $compiled-options as map:map,
  $include-matches as xs:boolean,
  $is-json as xs:boolean,
  $query-prov as map:map?
) {
  let $range := $start to ($start + $page-length - 1)
  let $cts-walk-query :=
    if ($include-matches) then
      match-impl:instance-query-wrapper(
        $boosting-query,
        $is-json
      )
    else ()
  let $thresholds := $compiled-options => map:get("orderedThresholds")
  let $individual-matching-queries := cts:or-query-queries($boosting-query)
  for $result at $pos in cts:search(
    fn:collection(),
    $match-query,
    ("unfiltered", "score-simple"),
    0
  )[fn:position() = $range]
  let $uri := xdmp:node-uri($result)
  let $matching-queries := $individual-matching-queries[cts:contains($result, .)]
  let $matching-weights-map := map:entry("values", ())
  let $_accumulate-scores :=
    cts:element-walk(document{$matching-queries}, $QUERIES_WITH_WEIGHT,
      let $weight := fn:head(($cts:node/@weight, 1))
      return
      (
        $matching-weights-map => map:put("values",
          ($matching-weights-map => map:get("values"), $weight)),
        $matching-weights-map => map:put(xdmp:md5(document {$cts:node}), $weight)
      )
    )
  let $score :=
      fn:sum(
        $matching-weights-map => map:get("values")
      )
  where $score ge $min-threshold
  return
    element result {
      attribute uri {$uri},
      attribute index {$range[fn:position() = $pos]},
      attribute score {$score},
      let $selected-threshold := fn:head($thresholds[$score ge @above])
      return (
        attribute threshold { fn:string($selected-threshold/@label) },
        attribute action { fn:string($selected-threshold/(matcher:action|@action)) }
      ),
      if ($include-matches) then
        element matches {
          (: rather than store the entire node and risk mixing
             content type (json != xml) we store the path to the
             node instead :)
          cts:walk(
            $result/*:envelope/*:instance,
            cts:or-query($matching-queries),
            (let $query-hashes := fn:distinct-values($cts:queries !
                cts:element-walk(document{.}, $QUERIES_WITH_WEIGHT, element query-hash{xdmp:md5(document {$cts:node})})//query-hash !
                fn:string(.))
            let $weight := fn:sum($query-hashes ! ($matching-weights-map => map:get(.)))
            let $node-name := fn:string(fn:head((fn:node-name($cts:node),fn:node-name($cts:node/..))))
            return <match weight="{$weight}">
                      {
                        for $query-hash in $query-hashes
                        let $query-map := map:get($query-prov, $query-hash)
                        return
                          <contributions>
                            <algorithm>{fn:string(fn:head(($query-map ! map:get(., "algorithm") ! fn:string(.),"standard")))}</algorithm>
                            <weight>{fn:string($matching-weights-map => map:get($query-hash))}</weight>
                          </contributions>
                      }
                      <nodeName>{$node-name}</nodeName>
                      <value>{fn:data($cts:node)}</value>
                      <path>{xdmp:path($cts:node, fn:true())}</path>
                   </match>)
          )
        }
      else ()
    }
};

(: Configuration used to convert XML match results to JSON. :)
declare variable $results-json-config := match-impl:_results-json-config();

declare function match-impl:_results-json-config()
{
  let $config := json:config("custom")
  return (
    map:put($config, "array-element-names", ("result","matches","algorithms",xs:QName("cts:option"),xs:QName("cts:text"),xs:QName("cts:element"))),
    map:put($config, "full-element-names",
      (xs:QName("cts:query"),
      xs:QName("cts:and-query"),
      xs:QName("cts:near-query"),
      xs:QName("cts:or-query"))
    ),
    map:put($config, "json-children", "queries"),
    map:put($config, "attribute-names",
      ("name","localname", "namespace", "function", "weight",
      "at", "property-name", "weight", "above", "label","algorithm-ref")
    ),
    map:put($config, "camel-case", fn:true()),
    $config
  )
};

(:
 : Identify a sequence of queries whose scores add up to the $threshold. A document must match at least one of these
 : queries in order to be returned as a potential match.
 :
 : @param $query-results  a sequence of queries with weights
 : @param $threshold  minimum weighted-score for a match to be relevant
 : @return a sequence of queries; a document that matches any of these will have at least $threshold as a score
 :)
declare function match-impl:minimum-threshold-combinations($query-results, $threshold as xs:double)
  as map:map*
{
  (: Each of $queries-ge-threshold has a weight high enough to hit the $threshold :)
  let $queries-ge-threshold := $query-results[(. => map:get("weight")) ge $threshold]
  let $queries-lt-threshold := $query-results[(. => map:get("weight")) lt $threshold]
  return (
    $queries-ge-threshold ! map:entry("queries", .),
    match-impl:filter-for-required-queries($queries-lt-threshold, 0, $threshold, ())
  )
};

(:
 : Find combinations of queries whose weights are individually below the threshold, but combined are above it.
 :
 : @param $remaining-queries  sequence of queries ordered by their weights, descending
 : @param $combined-weight
 : @param $threshold  the target value
 : @param $accumulated-queries  accumlated sequence, building up to see whether it can hit the $threshold.
 : @return a sequence of cts:and-queries, one for each required filter
 : note: return type left off to allow for tail recursion optimization.
 :)
declare function match-impl:filter-for-required-queries(
  $remaining-queries as map:map*,
  $combined-weight,
  $threshold,
  $accumulated-queries as map:map*
)
{
  if ($threshold eq 0 or $combined-weight ge $threshold) then (
    if (fn:exists($accumulated-queries)) then
      map:entry(
        "queries",
        $accumulated-queries
      )
    else
      ()
  )
  else
    for $query at $pos in $remaining-queries
    let $query-weight := fn:head(($query => map:get("weight"), 1))
    let $new-combined-weight := $combined-weight + $query-weight
    return (
      match-impl:filter-for-required-queries(
        fn:subsequence($remaining-queries, $pos + 1),
        $new-combined-weight,
        $threshold,
        ($accumulated-queries, $query)
      )
    )
};

(:
 : Convert XML match results to JSON.
 :)
declare function match-impl:results-to-json($results-xml)
  as object-node()?
{
  if (fn:exists($results-xml)) then
    xdmp:to-json(
      json:transform-to-json-object($results-xml, $results-json-config)
    )/node()
  else ()
};

declare function match-impl:instance-query-wrapper(
  $query as cts:query,
  $is-json as xs:boolean
) {
  if ($is-json) then
    if (fn:exists($const:JSON-INSTANCE)) then
      cts:json-property-scope-query($const:JSON-INSTANCE, $query)
    else ()
  else
    if (fn:exists($const:XML-INSTANCE)) then
      cts:element-query($const:XML-INSTANCE, $query)
    else ()
};


Annotate the MarkLogic math functions with versioned type restrictions.

xquery version "1.0-ml";
(:~
 : MarkLogic math functions
 :
 : @see https://docs.marklogic.com/math
 :)
module namespace math = "http://marklogic.com/xdmp/math";

declare namespace a = "http://reecedunn.co.uk/xquery/annotations";
declare namespace o = "http://reecedunn.co.uk/xquery/options";

declare option o:requires "marklogic/5.0";

declare %a:since("marklogic", "6.0") %a:until("marklogic", "7.0", "math:pi#0") function math:PI() as xs:double external;
declare %a:since("marklogic", "5.0") function math:acos($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:asin($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:atan($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:atan2($y as xs:double, $x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:ceil($x as xs:double) as xs:double external;
declare %a:since("marklogic", "6.0") function math:correlation($arg as json:array*) as xs:double? external;
declare %a:since("marklogic", "5.0") function math:cos($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:cosh($x as xs:double) as xs:double external;
declare %a:since("marklogic", "6.0") function math:cot($x as xs:double) as xs:double external;
declare %a:since("marklogic", "6.0") function math:covariance($arg as json:array*) as xs:double? external;
declare %a:since("marklogic", "6.0") function math:covariance-p($arg as json:array*) as xs:double? external;
declare %a:since("marklogic", "6.0") function math:degrees($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:exp($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:fabs($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:floor($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:fmod($x as xs:double, $y as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:frexp($x as xs:double) as (xs:double, xs:integer) external;
declare %a:since("marklogic", "5.0") function math:ldexp($y as xs:double, $i as xs:integer) as xs:double external;
declare %a:since("marklogic", "6.0") function math:linear-model($arg as json:array*) as math:linear-model? external;
declare %a:since("marklogic", "6.0") function math:linear-model-coeff($linear-model as math:linear-model) as xs:double* external;
declare %a:since("marklogic", "6.0") function math:linear-model-intercept($linear-model as math:linear-model) as xs:double external;
declare %a:since("marklogic", "6.0") function math:linear-model-rsquared($linear-model as math:linear-model) as xs:double external;
declare %a:since("marklogic", "5.0") function math:log($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:log10($x as xs:double) as xs:double external;
declare %a:since("marklogic", "6.0") function math:median($arg as xs:double*) as xs:double? external;
declare %a:since("marklogic", "6.0") function math:mode($arg as xs:anyAtomicType*) as xs:anyAtomicType* external;
declare %a:since("marklogic", "6.0") function math:mode($arg as xs:anyAtomicType*, $options as xs:string*) as xs:anyAtomicType* external;
declare %a:restrict-until("return", "marklogic", "8.0", "(xs:double,xs:double)")
        %a:restrict-since("return", "marklogic", "8.0", "(xs:double,xs:integer)")
        %a:since("marklogic", "5.0") function math:modf($x as xs:double) as (xs:double,(xs:double|xs:integer)) external;
declare %a:since("marklogic", "6.0") function math:percent-rank($arg as xs:anyAtomicType*, $value as xs:anyAtomicType) as xs:double? external;
declare %a:since("marklogic", "6.0") function math:percent-rank($arg as xs:anyAtomicType*, $value as xs:anyAtomicType, $options as xs:string*) as xs:double? external;
declare %a:since("marklogic", "6.0") function math:percentile($arg as xs:double*, $p as xs:double*) as xs:double* external;
declare %a:since("marklogic", "7.0") function math:pi() as xs:double external;
declare %a:since("marklogic", "5.0") function math:pow($x as xs:double, $y as xs:double) as xs:double external;
declare %a:since("marklogic", "6.0") function math:radians($x as xs:double) as xs:double external;
declare %a:since("marklogic", "6.0") function math:rank($arg1 as xs:anyAtomicType*, $arg2 as xs:anyAtomicType) as xs:integer? external;
declare %a:since("marklogic", "6.0") function math:rank($arg1 as xs:anyAtomicType*, $arg2 as xs:anyAtomicType, $options as xs:string*) as xs:integer? external;
declare %a:since("marklogic", "5.0") function math:sin($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:sinh($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:sqrt($x as xs:double) as xs:double external;
declare %a:since("marklogic", "6.0") function math:stddev($arg as xs:double*) as xs:double? external;
declare %a:since("marklogic", "6.0") function math:stddev-p($arg as xs:double*) as xs:double? external;
declare %a:since("marklogic", "5.0") function math:tan($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:tanh($x as xs:double) as xs:double external;
declare %a:since("marklogic", "8.0") function math:trunc($arg as xs:numeric?) as xs:numeric? external;
declare %a:since("marklogic", "6.0") function math:trunc($arg as xs:numeric?, $n as xs:integer) as xs:numeric? external;
declare %a:since("marklogic", "6.0") function math:variance($arg as xs:double*) as xs:double? external;
declare %a:since("marklogic", "6.0") function math:variance-p($arg as xs:double*) as xs:double? external;

xquery version "1.0-ml";
(:~
 : MarkLogic math functions
 :
 : @see https://docs.marklogic.com/math
 :)
module namespace math = "http://marklogic.com/xdmp/math";

declare namespace a = "http://reecedunn.co.uk/xquery/annotations";
declare namespace o = "http://reecedunn.co.uk/xquery/options";

declare option o:requires "marklogic/5.0";

declare %a:since("marklogic", "6.0") %a:until("marklogic", "7.0", "math:pi#0") function math:PI() as xs:double external;
declare %a:since("marklogic", "5.0") function math:acos($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:asin($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:atan($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:atan2($y as xs:double, $x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:ceil($x as xs:double) as xs:double external;
declare %a:since("marklogic", "6.0") function math:correlation($arg as json:array*) as xs:double? external;
declare %a:since("marklogic", "5.0") function math:cos($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:cosh($x as xs:double) as xs:double external;
declare %a:since("marklogic", "6.0") function math:cot($x as xs:double) as xs:double external;
declare %a:since("marklogic", "6.0") function math:covariance($arg as json:array*) as xs:double? external;
declare %a:since("marklogic", "6.0") function math:covariance-p($arg as json:array*) as xs:double? external;
declare %a:since("marklogic", "6.0") function math:degrees($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:exp($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:fabs($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:floor($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:fmod($x as xs:double, $y as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:frexp($x as xs:double) as (xs:double, xs:integer) external;
declare %a:since("marklogic", "5.0") function math:ldexp($y as xs:double, $i as xs:integer) as xs:double external;
declare %a:since("marklogic", "6.0") function math:linear-model($arg as json:array*) as math:linear-model? external;
declare %a:since("marklogic", "6.0") function math:linear-model-coeff($linear-model as math:linear-model) as xs:double* external;
declare %a:since("marklogic", "6.0") function math:linear-model-intercept($linear-model as math:linear-model) as xs:double external;
declare %a:since("marklogic", "6.0") function math:linear-model-rsquared($linear-model as math:linear-model) as xs:double external;
declare %a:since("marklogic", "5.0") function math:log($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:log10($x as xs:double) as xs:double external;
declare %a:since("marklogic", "6.0") function math:median($arg as xs:double*) as xs:double? external;
declare %a:since("marklogic", "6.0") function math:mode($arg as xs:anyAtomicType*) as xs:anyAtomicType* external;
declare %a:since("marklogic", "6.0") function math:mode($arg as xs:anyAtomicType*, $options as xs:string*) as xs:anyAtomicType* external;
declare %a:since("marklogic", "5.0") function math:modf($x as xs:double) (: as [5.0](xs:double,xs:double) [8.0](xs:double,xs:integer) :) external;
declare %a:since("marklogic", "6.0") function math:percent-rank($arg as xs:anyAtomicType*, $value as xs:anyAtomicType) as xs:double? external;
declare %a:since("marklogic", "6.0") function math:percent-rank($arg as xs:anyAtomicType*, $value as xs:anyAtomicType, $options as xs:string*) as xs:double? external;
declare %a:since("marklogic", "6.0") function math:percentile($arg as xs:double*, $p as xs:double*) as xs:double* external;
declare %a:since("marklogic", "7.0") function math:pi() as xs:double external;
declare %a:since("marklogic", "5.0") function math:pow($x as xs:double, $y as xs:double) as xs:double external;
declare %a:since("marklogic", "6.0") function math:radians($x as xs:double) as xs:double external;
declare %a:since("marklogic", "6.0") function math:rank($arg1 as xs:anyAtomicType*, $arg2 as xs:anyAtomicType) as xs:integer? external;
declare %a:since("marklogic", "6.0") function math:rank($arg1 as xs:anyAtomicType*, $arg2 as xs:anyAtomicType, $options as xs:string*) as xs:integer? external;
declare %a:since("marklogic", "5.0") function math:sin($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:sinh($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:sqrt($x as xs:double) as xs:double external;
declare %a:since("marklogic", "6.0") function math:stddev($arg as xs:double*) as xs:double? external;
declare %a:since("marklogic", "6.0") function math:stddev-p($arg as xs:double*) as xs:double? external;
declare %a:since("marklogic", "5.0") function math:tan($x as xs:double) as xs:double external;
declare %a:since("marklogic", "5.0") function math:tanh($x as xs:double) as xs:double external;
declare %a:since("marklogic", "8.0") function math:trunc($arg as xs:numeric?) as xs:numeric? external;
declare %a:since("marklogic", "6.0") function math:trunc($arg as xs:numeric?, $n as xs:integer) as xs:numeric? external;
declare %a:since("marklogic", "6.0") function math:variance($arg as xs:double*) as xs:double? external;
declare %a:since("marklogic", "6.0") function math:variance-p($arg as xs:double*) as xs:double? external;


- removed comments

- removed comments

xquery version "3.0";

(:
 : Copyright 2008-2011 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
:)

(:~
 : Convert an XQDoc document into an HTML document.
 :
 : This module contains a single convert() function
 : that transform an XQDoc document into an HTML document.
 : Usage:
 :

 : let $xqdoc := xqdoc:xqdoc("http://expath.org/ns/file")
 : return html:convert($xqdoc)
 :

 :
 : @author William Candillon wcandillon at gmail dot com
 : @project Zorba/XQDoc/HTML
 :)

module namespace html = "http://www.zorba-xquery.com/modules/xqdoc/html";

declare namespace xq = "http://www.xqdoc.org/1.0";
declare namespace o = "http://www.w3.org/2010/xslt-xquery-serialization";

declare copy-namespaces no-preserve, inherit;

declare namespace ver = "http://www.zorba-xquery.com/options/versioning";
declare option ver:module-version "2.0";

declare %private variable $html:empty-tags-to-delete := ("tt");

declare function html:convert($xqdoc as element(xq:xqdoc))
as element(div)
{
let $module := $xqdoc/xq:module
let $ns := normalize-space($module/xq:uri/text())
let $comment := $module/xq:comment
let $namespaces := $module/xq:custom[@tag = "namespaces"]/xq:namespace
let $prefix := $xqdoc/xq:functions/xq:function[1]/xq:name/text()/substring-before(., ":")
let $prefix := if(exists($prefix)) then $prefix else $xqdoc/xq:variables/xq:variable[1]/xq:uri/text()/substring-before(., ":")
let $variables := for $variable in $xqdoc/xq:variables/xq:variable order by $variable/xq:name/text() ascending return $variable
let $functions := for $function in $xqdoc/xq:functions/xq:function[not(html:is-function-private(.))] order by concat($function/xq:name/text(), $function/@arity) ascending return $function
let $description := $comment/xq:description/node()
return
html:normalize(

{$ns}

Description

Before using any of the functions below please remember to import the module namespace:

import module namespace {$prefix} = "{$ns}";

    {
      if($description instance of text()) then

{$description}

      else
        $comment/xq:description/node()
    }
    {
      let $schemas := $xqdoc/xq:imports//xq:import[@type = "schema"]
      return
        if(exists($schemas)) then

Imported Schemas

            {
              for $schema in $schemas
              let $uri  := $schema/xq:uri/text()
              let $href := replace(replace($uri, "http://", ""), "/", "_") || ".xsd"
              return

                {$uri}

            }

Please note that the schemas are not automatically imported in the modules that import this module.

In order to import and use the schemas, please add:

{
            let $namespaces := $xqdoc/xq:module//xq:custom[@tag = "namespaces"]
            return
              string-join(
                for $import in $xqdoc/xq:imports//xq:import[@type = "schema"]
                return
                  concat('import schema namespace ',
                         $namespaces//xq:namespace[(@uri = $import/xq:uri/text()) and (@isSchema = "true")]/@prefix,
                         ' =  "',
                         string($import/xq:uri/text()),'";'),"
             ")
           }

        else
          ()
    }
    {
      let $sees := $comment/xq:see
      return
        if(exists($sees)) then

See also

          {
            for $see in $sees
            let $see := if(exists($see/a) and exists($see/text())) then $see/node() else if(exists($see/node())) then {$see/node()} else ""
            return
{$see}

          }

        else ()
    }
    {
      let $authors := $comment/xq:author
      return
        if(exists($authors)) then

{concat("Author", if(count($authors) gt 1) then "s" else "")}

           {
             for $author in $authors
             return
{$author/node()}

           }

        else ()
    }
    {
      let $version  := $comment/xq:custom[@tag =  "XQuery version"]/text()
      let $encoding := $comment/xq:custom[@tag = "encoding"]/text()
      return
        if(exists($version)) then

XQuery version and encoding

xquery version "{$version}"{if(exists($encoding)) then concat(' encoding "', $encoding, '"')  else ()};

        else ()
    }

  {
    let $examples := $module//xq:custom[@tag="example"]/text()
    let $module-name := tokenize($module/xq:uri/text(),"/")[last()]
    return
      if(exists($examples)) then

Examples

           {
             for $example in $examples
             let $example-name := tokenize($example,"/")[last()]
             return
{$example-name}

           }

      else ()
  }
  {
    if(exists($namespaces)) then

Namespaces

        {
          for $ns in $namespaces
          return

              {string($ns/@prefix)}
              {string($ns/@uri)}

        }

    else ()
  }
  {
    if(exists($variables)) then

Variable Summary

        {
          for $variable in $variables
          let $varname := $variable/xq:uri/text()
          return

            {$varname}

        }

    else()
  }
  {
    if(exists($functions)) then

Function Summary

        {
          for $function in $functions
          let $name  := $function/xq:name/text()/substring-after(., ":")
          let $arity := number($function/@arity)
          return

            {html:function-properties($function)}

              {if(exists($function//xq:deprecated)) then attribute class { "deprecated" } else ()}
              {$name}{substring-after($function/xq:signature/text(), $function/xq:name/text())}
              {
                let $description := $function/xq:comment/xq:description
                return
                  if(exists($description)) then
                    html:description-summary($description)
                  else ()
              }

        }

    else ()
  }
  {
    if(exists($variables)) then

Variables

        {
          for $variable at $i in $variables
          let $name := $variable/xq:uri/text()
          let $type := $variable/xq:comment/xq:custom[@tag = "type"]/text()
          return

${$name}{if(exists($type)) then concat(" as ", $type) else ()}

{$variable/xq:comment/xq:description/node()}

        }

    else ()
  }
  {
    if(exists($functions)) then

Functions

        {
          for $function at $i in $functions
          let $qname  := $function/xq:name/text()
          let $name   := $qname/substring-after(., ":")
          let $arity  := string($function/@arity)
          let $params := $function/xq:parameters/xq:parameter
          let $return := $function/xq:return
          let $errors := $function/xq:comment/xq:error
          return

{$name}#{$arity}

{concat("declare", html:serialize-annotations($function/xq:annotations), if(html:is-function-updating($function)) then " updating " else " ")}function {$qname}({html:serialize-params($function/xq:parameters)}) as {let $return := $function/xq:return/xq:type/text() return if(empty($return)) then "item()*" else $return}{if(html:is-function-external($function)) then " external" else ()}

            {if(exists($function//xq:deprecated)) then
This function is deprecated
 else ()}

{$function/xq:comment/xq:description/node()}

            {
               if(exists($params)) then

Parameters

                  {
                    for $param in $params
                    return

                    {concat("$", $param/xq:name/text())} as {concat($param/xq:type/text(), $param/xq:type/@occurence)}
                    {
                      let $variable := concat('$', $param/xq:name/text())
                      let $description := $function/xq:comment/xq:param[starts-with(./text()[1], $variable)][1]
                      let $description := if(exists($description)) then
                                            copy $d := $description
                                            modify (
                                              rename node $d as "div",
                                              let $text-node := $d/text()[starts-with(., $variable)]
                                              return
                                                replace value of node $text-node with replace($text-node, concat("\$", $param/xq:name/text()), "")
                                            )
                                            return $d
                                          else ()
                      return $description
                    }

                  }

               else ()
            }
            {
              if(exists($return)) then

Returns

                      {$return/xq:type/text()}

{$function/xq:comment/xq:return/node()}

              else ()
            }
            {
              if(exists($errors)) then

Errors

                  {
                    for $error in $errors
                    return

                    {$error/node()}

                  }

              else ()
            }
            {
              let $examples := $function//xq:custom[@tag="example"]/text()
              let $module-name := tokenize($module/xq:uri/text(),"/")[last()]
              return
                if(exists($examples)) then

Examples

                     {
                       for $example in $examples
                       let $example-name := tokenize($example,"/")[last()]
                       return
{$example-name}

                     }

                else ()
            }
            {
              let $sees := $function/xq:comment/xq:see
              return
                if(exists($sees)) then

See also

                    {
                      for $see in $sees
                      let $see := if(exists($see/a) and exists($see/text())) then $see/node() else if(exists($see/node())) then {$see/node()} else ""
                      return
{$see}

                    }

                else ()
            }

        }

    else ()
  }

)
};

declare %private function html:normalize-pre($pre as element(pre))
as element(pre)
{

  {$pre/attribute()}
  {serialize($pre/node(), <o:serialization-parameters></o:serialization-parameters>)}

};

declare %private function html:normalize($nodes as node()*)
as item()*
{
  copy $html := $nodes
  modify (
    for $node in $html//*[empty(./node()[not(. instance of attribute())]) and local-name(.) = $html:empty-tags-to-delete]
    return delete node $node,
    for $pre in $html//pre[exists(*)]
    return replace node $pre with html:normalize-pre($pre)
  )
  return $html
};

declare %private function html:text($nodes as node()*)
as xs:string?
{
  string-join(
    for $node in $nodes
    return
      if($node instance of text()) then
        $node
      else
        html:text($node/node())
  , " ")
};

declare %private function html:description-summary($description as element(xq:description))
as element(p)
{
  let $text := html:text($description)
  let $text := if(contains($text, ".")) then concat(substring-before($text, "."), ".") else $text
  return

{$text}

};

declare %private function html:serialize-params($params as element(xq:parameters)?)
as xs:string*
{
  let $params :=
    for $param in $params/xq:parameter
    return
      concat("$", $param/xq:name/text(), " as ", $param/xq:type/text(), $param/xq:type/@occurrence)
  return
    if(exists($params)) then
      concat("
    ", string-join($params, ",
    "), "
")
    else ()
};

declare %private function html:serialize-annotations($annotations as element(xq:annotations)?)
as xs:string?
{
  let $result :=
    string-join(
      let $namespaces := $annotations/ancestor::*[empty(..)]//xq:module/xq:custom[@tag = "namespaces"]/xq:namespace
      for $annotation in $annotations/xq:annotation
      let $ns := $annotation/@namespace/string(.)
      let $prefix := $annotation/@prefix/string(.)
      return
        concat("%", if(exists($prefix)) then concat($prefix, ":") else (), $annotation/@localname)
    , " ")
  return
    if($result != "") then
      concat(" ", $result)
    else
      $result
};

declare %private function html:function-properties($function as element(xq:function))
as element(span)*
{
  if(html:is-function-external($function)) then

  else(),
  if(html:is-function-sequential($function)) then

  else(),
  if(html:is-function-updating($function)) then

  else (),
  if(html:is-function-variadic($function)) then

  else (),
  if(html:is-function-streamable($function)) then

  else (),
  if(html:is-function-nondeterministic($function)) then

  else ()
};

declare %private function html:is-function-updating($function as element(xq:function))
as xs:boolean
{
  $function/xq:signature/text()/contains(substring-before(., $function/xq:name/text()), " updating ")
};

declare %private function html:is-function-private($function as element(xq:function))
as xs:boolean
{
  exists($function/xq:annotations/xq:annotation[@localname = "private" and @namespace = "http://www.w3.org/2005/xpath-functions"])
};

declare %private function html:is-function-nondeterministic($function as element(xq:function))
as xs:boolean
{
  exists($function//xq:annotation[@namespace = "http://www.zorba-xquery.com/annotations" and @localname = "nondeterministic"])
};

declare %private function html:is-function-streamable($function as element(xq:function))
as xs:boolean
{
  exists($function//xq:annotation[@namespace = "http://www.zorba-xquery.com/annotations" and @localname = "streamable"])
};

declare %private function html:is-function-variadic($function as element(xq:function))
as xs:boolean
{
  exists($function//xq:annotation[@namespace = "http://www.zorba-xquery.com/annotations" and @localname = "variadic"])
};

declare %private function html:is-function-sequential($function as element(xq:function))
as xs:boolean
{
  exists($function//xq:annotation[@namespace = "http://www.zorba-xquery.com/annotations" and @localname = "sequential"])
};

declare %private function html:is-function-external($function as element(xq:function))
as xs:boolean
{
  $function/xq:signature/text()/ends-with(., " external")
};

xquery version "3.0";

(:
 : Copyright 2008-2011 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
:)

(:~
 : Convert an XQDoc document into an HTML document.
 :
 : This module contains a single convert() function
 : that transform an XQDoc document into an HTML document.
 : Usage:
 :

 : let $xqdoc := xqdoc:xqdoc("http://expath.org/ns/file")
 : return html:convert($xqdoc)
 :

 :
 : @author William Candillon wcandillon at gmail dot com
 : @project Zorba/XQDoc/HTML
 :)

module namespace html = "http://www.zorba-xquery.com/modules/xqdoc/html";

declare namespace xq = "http://www.xqdoc.org/1.0";
declare namespace o = "http://www.w3.org/2010/xslt-xquery-serialization";

declare copy-namespaces no-preserve, inherit;

declare namespace ver = "http://www.zorba-xquery.com/options/versioning";
declare option ver:module-version "2.0";

declare %private variable $html:empty-tags-to-delete := ("tt");

declare function html:convert($xqdoc as element(xq:xqdoc))
as element(div)
{
let $module := $xqdoc/xq:module
let $ns := normalize-space($module/xq:uri/text())
let $comment := $module/xq:comment
let $namespaces := $module/xq:custom[@tag = "namespaces"]/xq:namespace
let $prefix := $xqdoc/xq:functions/xq:function[1]/xq:name/text()/substring-before(., ":")
let $prefix := if(exists($prefix)) then $prefix else $xqdoc/xq:variables/xq:variable[1]/xq:uri/text()/substring-before(., ":")
let $variables := for $variable in $xqdoc/xq:variables/xq:variable order by $variable/xq:name/text() ascending return $variable
let $functions := for $function in $xqdoc/xq:functions/xq:function[not(html:is-function-private(.))] order by concat($function/xq:name/text(), $function/@arity) ascending return $function
let $description := $comment/xq:description/node()
return
html:normalize(

{$ns}

Description

Before using any of the functions below please remember to import the module namespace:

import module namespace {$prefix} = "{$ns}";

    {
      if($description instance of text()) then

{$description}

      else
        $comment/xq:description/node()
    }
    {
      let $schemas := $xqdoc/xq:imports//xq:import[@type = "schema"]
      return
        if(exists($schemas)) then

Imported Schemas

            {
              for $schema in $schemas
              let $uri  := $schema/xq:uri/text()
              let $href := replace(replace($uri, "http://", ""), "/", "_") || ".xsd"
              return

                {$uri}

            }

Please note that the schemas are not automatically imported in the modules that import this module.

In order to import and use the schemas, please add:

{
            let $namespaces := $xqdoc/xq:module//xq:custom[@tag = "namespaces"]
            return
              string-join(
                for $import in $xqdoc/xq:imports//xq:import[@type = "schema"]
                return
                  concat('import schema namespace ',
                         $namespaces//xq:namespace[(@uri = $import/xq:uri/text()) and (@isSchema = "true")]/@prefix,
                         ' =  "',
                         string($import/xq:uri/text()),'";'),"
             ")
           }

        else
          ()
    }
    {
      let $sees := $comment/xq:see
      return
        if(exists($sees)) then

See also

          {
            for $see in $sees
            let $see := if(exists($see/a) and exists($see/text())) then $see/node() else if(exists($see/node())) then {$see/node()} else ""
            return
{$see}

          }

        else ()
    }
    {
      let $authors := $comment/xq:author
      return
        if(exists($authors)) then

{concat("Author", if(count($authors) gt 1) then "s" else "")}

           {
             for $author in $authors
             return
{$author/node()}

           }

        else ()
    }
    {
      let $version  := $comment/xq:custom[@tag =  "XQuery version"]/text()
      let $encoding := $comment/xq:custom[@tag = "encoding"]/text()
      return
        if(exists($version)) then

XQuery version and encoding

xquery version "{$version}"{if(exists($encoding)) then concat(' encoding "', $encoding, '"')  else ()};

        else ()
    }

  {
    let $examples := $module//xq:custom[@tag="example"]/text()
    let $module-name := tokenize($module/xq:uri/text(),"/")[last()]
    return
      if(exists($examples)) then

Examples

           {
             for $example in $examples
             let $example-name := tokenize($example,"/")[last()]
             return
{$example-name}

           }

      else ()
  }
  {
    if(exists($namespaces)) then

Namespaces

        {
          for $ns in $namespaces
          return

              {string($ns/@prefix)}
              {string($ns/@uri)}

        }

    else ()
  }
  {
    if(exists($variables)) then

Variable Summary

        {
          for $variable in $variables
          let $varname := $variable/xq:uri/text()
          return

            {$varname}

        }

    else()
  }
  {
    if(exists($functions)) then

Function Summary

        {
          for $function in $functions
          let $name  := $function/xq:name/text()/substring-after(., ":")
          let $arity := number($function/@arity)
          return

            {html:function-properties($function)}

              {if(exists($function//xq:deprecated)) then attribute class { "deprecated" } else ()}
              {$name}{substring-after($function/xq:signature/text(), $function/xq:name/text())}
              {
                let $description := $function/xq:comment/xq:description
                return
                  if(exists($description)) then
                    html:description-summary($description)
                  else ()
              }

        }

    else ()
  }
  {
    if(exists($variables)) then

Variables

        {
          for $variable at $i in $variables
          let $name := $variable/xq:uri/text()
          let $type := $variable/xq:comment/xq:custom[@tag = "type"]/text()
          return

${$name}{if(exists($type)) then concat(" as ", $type) else ()}

{$variable/xq:comment/xq:description/node()}

        }

    else ()
  }
  {
    if(exists($functions)) then

Functions

        {
          for $function at $i in $functions
          let $qname  := $function/xq:name/text()
          let $name   := $qname/substring-after(., ":")
          let $arity  := string($function/@arity)
          let $params := $function/xq:parameters/xq:parameter
          let $return := $function/xq:return
          let $errors := $function/xq:comment/xq:error
          return

{$name}#{$arity}

{concat("declare", html:serialize-annotations($function/xq:annotations), if(html:is-function-updating($function)) then " updating " else " ")}function {$qname}({html:serialize-params($function/xq:parameters)}) as {let $return := $function/xq:return/xq:type/text() return if(empty($return)) then "item()*" else $return}{if(html:is-function-external($function)) then " external" else ()}

            {if(exists($function//xq:deprecated)) then
This function is deprecated
 else ()}

{$function/xq:comment/xq:description/node()}

            {
               if(exists($params)) then

Parameters

                  {
                    for $param in $params
                    return

                    {concat("$", $param/xq:name/text())} as {concat($param/xq:type/text(), $param/xq:type/@occurence)}
                    {
                      let $variable := concat('$', $param/xq:name/text())
                      let $description := $function/xq:comment/xq:param[starts-with(./text()[1], $variable)][1]
                      let $description := if(exists($description)) then
                                            copy $d := $description
                                            modify (
                                              rename node $d as "div",
                                              let $text-node := $d/text()[starts-with(., $variable)]
                                              return
                                                replace value of node $text-node with replace($text-node, concat("\$", $param/xq:name/text()), "")
                                            )
                                            return $d
                                          else ()
                      return $description
                    }

                  }

               else ()
            }
            {
              if(exists($return)) then

Returns

                      {$return/xq:type/text()}

{$function/xq:comment/xq:return/node()}

              else ()
            }
            {
              if(exists($errors)) then

Errors

                  {
                    for $error in $errors
                    return

                    {$error/node()}

                  }

              else ()
            }
            {(:
              let $examples := $function//xq:custom[@tag="example"]/text()
              let $module-name := tokenize($module/xq:uri/text(),"/")[last()]
              return
                if(exists($examples)) then

Examples

                     {
                       for $example in $examples
                       let $example-name := tokenize($example,"/")[last()]
                       return
{$example-name}

                     }

                else ():)
            }
            {
              let $sees := $function/xq:comment/xq:see
              return
                if(exists($sees)) then

See also

                    {
                      for $see in $sees
                      let $see := if(exists($see/a) and exists($see/text())) then $see/node() else if(exists($see/node())) then {$see/node()} else ""
                      return
{$see}

                    }

                else ()
            }

        }

    else ()
  }

)
};

declare %private function html:normalize-pre($pre as element(pre))
as element(pre)
{

  {$pre/attribute()}
  {serialize($pre/node(), <o:serialization-parameters></o:serialization-parameters>)}

};

declare %private function html:normalize($nodes as node()*)
as item()*
{
  copy $html := $nodes
  modify (
    for $node in $html//*[empty(./node()[not(. instance of attribute())]) and local-name(.) = $html:empty-tags-to-delete]
    return delete node $node,
    for $pre in $html//pre[exists(*)]
    return replace node $pre with html:normalize-pre($pre)
  )
  return $html
};

declare %private function html:text($nodes as node()*)
as xs:string?
{
  string-join(
    for $node in $nodes
    return
      if($node instance of text()) then
        $node
      else
        html:text($node/node())
  , " ")
};

declare %private function html:description-summary($description as element(xq:description))
as element(p)
{
  let $text := html:text($description)
  let $text := if(contains($text, ".")) then concat(substring-before($text, "."), ".") else $text
  return

{$text}

};

declare %private function html:serialize-params($params as element(xq:parameters)?)
as xs:string*
{
  let $params :=
    for $param in $params/xq:parameter
    return
      concat("$", $param/xq:name/text(), " as ", $param/xq:type/text(), $param/xq:type/@occurrence)
  return
    if(exists($params)) then
      concat("
    ", string-join($params, ",
    "), "
")
    else ()
};

declare %private function html:serialize-annotations($annotations as element(xq:annotations)?)
as xs:string?
{
  let $result :=
    string-join(
      let $namespaces := $annotations/ancestor::*[empty(..)]//xq:module/xq:custom[@tag = "namespaces"]/xq:namespace
      for $annotation in $annotations/xq:annotation
      let $ns := $annotation/@namespace/string(.)
      let $prefix := $annotation/@prefix/string(.)
      return
        concat("%", if(exists($prefix)) then concat($prefix, ":") else (), $annotation/@localname)
    , " ")
  return
    if($result != "") then
      concat(" ", $result)
    else
      $result
};

declare %private function html:function-properties($function as element(xq:function))
as element(span)*
{
  if(html:is-function-external($function)) then

  else(),
  if(html:is-function-sequential($function)) then

  else(),
  if(html:is-function-updating($function)) then

  else (),
  if(html:is-function-variadic($function)) then

  else (),
  if(html:is-function-streamable($function)) then

  else (),
  if(html:is-function-nondeterministic($function)) then

  else ()
};

declare %private function html:is-function-updating($function as element(xq:function))
as xs:boolean
{
  $function/xq:signature/text()/contains(substring-before(., $function/xq:name/text()), " updating ")
};

declare %private function html:is-function-private($function as element(xq:function))
as xs:boolean
{
  exists($function/xq:annotations/xq:annotation[@localname = "private" and @namespace = "http://www.w3.org/2005/xpath-functions"])
};

declare %private function html:is-function-nondeterministic($function as element(xq:function))
as xs:boolean
{
  exists($function//xq:annotation[@namespace = "http://www.zorba-xquery.com/annotations" and @localname = "nondeterministic"])
};

declare %private function html:is-function-streamable($function as element(xq:function))
as xs:boolean
{
  exists($function//xq:annotation[@namespace = "http://www.zorba-xquery.com/annotations" and @localname = "streamable"])
};

declare %private function html:is-function-variadic($function as element(xq:function))
as xs:boolean
{
  exists($function//xq:annotation[@namespace = "http://www.zorba-xquery.com/annotations" and @localname = "variadic"])
};

declare %private function html:is-function-sequential($function as element(xq:function))
as xs:boolean
{
  exists($function//xq:annotation[@namespace = "http://www.zorba-xquery.com/annotations" and @localname = "sequential"])
};

declare %private function html:is-function-external($function as element(xq:function))
as xs:boolean
{
  $function/xq:signature/text()/ends-with(., " external")
};