remove unused function

remove unused function

svn path=/trunk/eXist/; revision=12866

xquery version "1.0";

module namespace sharing = "http://exist-db.org/mods/sharing";

import module namespace config = "http://exist-db.org/mods/config" at "config.xqm";
import module namespace security = "http://exist-db.org/mods/security" at "security.xqm";
declare namespace group = "http://exist-db.org/mods/sharing/group";

declare variable $sharing:groups-collection := fn:concat($config:mods-root, "/groups");

declare function sharing:get-groups() as element(group:group)*
{
    fn:collection($sharing:groups-collection)/group:group
};

declare function sharing:group-exists($groupId as xs:string) as xs:boolean
{
    exists(fn:collection($sharing:groups-collection)/group:group[@id eq $groupId])
};

declare function sharing:get-group-members($groupId) as xs:string*
{
        let $group := sharing:__group-id-to-system-group-name($groupId) return
            if($group)then(
                security:get-group-members($group)
            )else()
};

declare function sharing:get-group-id($collection as xs:string) as xs:string?
{
    let $security-group := security:get-group($collection) return
        fn:string(fn:collection($sharing:groups-collection)/group:group[group:system/group:group eq $security-group]/@id)
};

declare function sharing:group-readable($collection as xs:string) as xs:boolean
{
    let $group := security:get-group($collection) return
        if($group)then(
            security:group-can-read-collection($group, $collection)
        ) else (
            false()
        )
};

declare function sharing:group-readable($collection as xs:string, $groupId as xs:string) as xs:boolean
{
    let $group := sharing:__group-id-to-system-group-name($groupId) return
        if($group)then(
            security:group-can-read-collection($group, $collection)
        ) else (
            false()
        )
};

declare function sharing:group-writeable($collection as xs:string) as xs:boolean
{
    let $group := security:get-group($collection) return
        if($group)then(
            security:group-can-write-collection($group, $collection)
        ) else (
            false()
        )
};

declare function sharing:group-writeable($collection as xs:string, $groupId as xs:string) as xs:boolean
{
    let $group := sharing:__group-id-to-system-group-name($groupId) return
        if($group)then(
            security:group-can-write-collection($group, $collection)
        ) else (
            false()
        )
};

declare function sharing:other-readable($collection as xs:string) as xs:boolean
{
    security:other-can-read-collection($collection)
};

declare function sharing:other-writeable($collection as xs:string) as xs:boolean
{
    security:other-can-write-collection($collection)
};

declare function sharing:__group-id-to-system-group-name($groupId as xs:string) as xs:string?
{
    fn:collection($sharing:groups-collection)/group:group[@id eq $groupId]/group:system/group:group
};

declare function sharing:share-with-other($collection as xs:string, $read as xs:boolean, $write as xs:boolean) as xs:boolean
{
    security:set-other-can-read-collection($collection, $read)
    and
    security:set-other-can-write-collection($collection, $write)
};

declare function sharing:share-with-group($collection as xs:string, $read as xs:boolean, $write as xs:boolean) as xs:boolean
{
    security:set-group-can-read-collection($collection, $read)
    and
    security:set-group-can-write-collection($collection, $write)
};

declare function sharing:share-with-group($collection as xs:string, $groupId as xs:string, $read as xs:boolean, $write as xs:boolean) as xs:boolean
{
    let $group := sharing:__group-id-to-system-group-name($groupId) return
    if($group)then
    (
        security:set-group-can-read-collection($collection, $group, $read)
        and
        security:set-group-can-write-collection($collection, $group, $write)
    )
    else
    (
        false()
    )
};

declare function sharing:is-group-owner($group-id as xs:string, $user as xs:string) as xs:boolean
{
    exists(sharing:get-group($group-id)/group:system[group:owner eq $user])
};

declare function sharing:create-group($group-name as xs:string, $owner as xs:string, $group-member as xs:string*) as xs:string?
{
    let $new-group-id := util:uuid(),
    $system-group-name :=  fn:concat($owner, ".", fn:replace($group-name, "[^a-zA-Z0-9]", "")) return

        if(security:create-group($system-group-name, $group-member))then
        (
            let $group-doc := xmldb:store($sharing:groups-collection, (),
                <group:group id="{$new-group-id}">
                    <group:system>
                        <group:owner>{$owner}</group:owner>
                        <group:group>{$system-group-name}</group:group>
                    </group:system>
                    <group:name>{$group-name}</group:name>
                </group:group>
            ) return
                 security:set-resource-permissions($group-doc, $owner, $group-name, true(), true(), true(), false(), false(), false()),
                 $new-group-id
        )
        else()
};

declare function sharing:get-users-groups($user as xs:string) as element(group:group)*
{
    fn:collection($sharing:groups-collection)/group:group[group:system/group:group = security:get-groups($user)]
};

declare function sharing:get-group($group-id as xs:string) as element(group:group) {
    fn:collection($sharing:groups-collection)/group:group[@id eq $group-id]
};

declare function sharing:get-group-collections($group-id as xs:string, $collection as xs:string) as xs:string*
{
    let $system-group := sharing:get-group($group-id) return
        xmldb:get-child-collections($collection) (:[security:get-group(.) eq $system-group]:)
};


Annotate the BaseX fetch functions with versioned type restrictions.

xquery version "3.1";
(:~
 : BaseX Fetch Module functions
 :
 : @see http://docs.basex.org/wiki/Fetch_Module
 :)
module namespace fetch = "http://basex.org/modules/fetch";

declare namespace a = "http://reecedunn.co.uk/xquery/annotations";
declare namespace o = "http://reecedunn.co.uk/xquery/options";

declare option o:requires "basex/7.6";

declare %a:since("basex", "7.6") function fetch:binary($uri as xs:string) as xs:base64Binary external;
declare %a:since("basex", "7.6") function fetch:text($uri as xs:string) as xs:string external;
declare %a:since("basex", "7.6") function fetch:text($uri as xs:string, $encoding as xs:string) as xs:string external;
declare %a:since("basex", "8.5") function fetch:text($uri as xs:string, $encoding as xs:string, $fallback as xs:boolean) as xs:string  external;
declare %a:since("basex", "8.0") function fetch:xml($uri as xs:string) as document-node() external;
declare %a:restrict-until("$options", "basex", "8.6.7", "map(*)")
        %a:since("basex", "8.0") function fetch:xml($uri as xs:string, $options as map(*)?) as document-node() external;
declare %a:since("basex", "9.0") function fetch:xml-binary($data as xs:base64Binary) as document-node() external;
declare %a:since("basex", "9.0") function fetch:xml-binary($data as xs:base64Binary, $options as map(*)?) as document-node() external;
declare %a:since("basex", "7.6") function fetch:content-type($uri as xs:string) as xs:string external;

xquery version "3.1";
(:~
 : BaseX Fetch Module functions
 :
 : @see http://docs.basex.org/wiki/Fetch_Module
 :)
module namespace fetch = "http://basex.org/modules/fetch";

declare namespace a = "http://reecedunn.co.uk/xquery/annotations";
declare namespace o = "http://reecedunn.co.uk/xquery/options";

declare option o:requires "basex/7.6";

declare %a:since("basex", "7.6") function fetch:binary($uri as xs:string) as xs:base64Binary external;
declare %a:since("basex", "7.6") function fetch:text($uri as xs:string) as xs:string external;
declare %a:since("basex", "7.6") function fetch:text($uri as xs:string, $encoding as xs:string) as xs:string external;
declare %a:since("basex", "8.5") function fetch:text($uri as xs:string, $encoding as xs:string, $fallback as xs:boolean) as xs:string  external;
declare %a:since("basex", "8.0") function fetch:xml($uri as xs:string) as document-node() external;
declare %a:since("basex", "8.0") function fetch:xml($uri as xs:string, $options as map(*)? (: $options [8.0] as map(*) [8.6.7] as map(*)? :)) as document-node() external;
declare %a:since("basex", "9.0") function fetch:xml-binary($data as xs:base64Binary) as document-node() external;
declare %a:since("basex", "9.0") function fetch:xml-binary($data as xs:base64Binary, $options as map(*)?) as document-node() external;
declare %a:since("basex", "7.6") function fetch:content-type($uri as xs:string) as xs:string external;


Fixed test to show command(0 route for aggregation function

xquery version "3.1";

module namespace aggregate="http://exist-db.org/mongodb/test/aggregate";

import module namespace xqjson = "http://xqilla.sourceforge.net/lib/xqjson";

import module namespace test="http://exist-db.org/xquery/xqsuite"
                at "resource:org/exist/xquery/lib/xqsuite/xqsuite.xql";

import module namespace mongodb = "http://expath.org/ns/mongo"
                at "java:org.exist.mongodb.xquery.MongodbModule";

import module namespace support = "http://exist-db.org/mongrel/test/support"
                at "./support.xqm";

(:
 :  Example test taken from http://docs.mongodb.org/ecosystem/tutorial/use-aggregation-framework-with-java-driver/
 :)

(: Connect to mongodb, store token :)
declare %test:setUp function aggregate:setup()
{
    let $setup := support:setup()
    let $mongodbClientId := support:getToken()
    return
        (
            mongodb:insert($mongodbClientId, $support:database, $support:mongoCollection,
                            '{ "employee" : 1 , "department" : "Sales" , "amount" : 71 , "type" : "airfare"}'),
            mongodb:insert($mongodbClientId, $support:database, $support:mongoCollection,
                            '{ "employee" : 2 , "department" : "Engineering" , "amount" : 15 , "type" : "airfare"}'),
            mongodb:insert($mongodbClientId, $support:database, $support:mongoCollection,
                             '{ "employee" : 4 , "department" : "Human Resources" , "amount" : 5 , "type" : "airfare"}'),
            mongodb:insert($mongodbClientId, $support:database, $support:mongoCollection,
                             '{ "employee" : 42 , "department" : "Sales" , "amount" : 77 , "type" : "airfare"}')
        )

};

(: Disconnect from mongodb, cleanup token :)
declare %test:tearDown function aggregate:cleanup()
{
    support:cleanup()
};

(: ----------------------------
 : Actual tests below this line
 : ---------------------------- :)

(:
 : collection#aggregate(query)
 :
 : { "_id" : "Human Resources" , "average" : 5.0}
 : { "_id" : "Engineering" , "average" : 15.0}
 : { "_id" : "Sales" , "average" : 74.0}
 :)
declare
    %test:assertEquals("5.0", "15.0", "74.0")
function aggregate:aggregate_simple() {
    let $mongodbClientId := support:getToken()

    let $result := mongodb:aggregate($mongodbClientId, $support:database, $support:mongoCollection,
                   (
                        '{ "$match" : { "type" : "airfare"}}',
                        '{ "$project" : { "department" : 1 , "amount" : 1 , "_id" : 0}}',
                        '{ "$group" : { "_id" : "$department" , "average" : { "$avg" : "$amount"}}}',
                        '{ "$sort" : { "amount" : -1}}'   )
                    )

    let $formatted := <result>{for $one in $result
    return xqjson:parse-json($one)}</result>

    return $formatted//pair[@name eq 'average']/text()

};

declare
    %test:assertEquals(5, 15, 74)
function aggregate:aggregate_simple_xq31() {
    let $mongodbClientId := support:getToken()

    let $options := map { "liberal": true(), "duplicates": "use-last" }

    let $match := parse-json('{ "$match" : { "type" : "airfare"}}', $options)
    let $project := parse-json('{ "$project" : { "department" : 1 , "amount" : 1 , "_id" : 0}}', $options)
    let $group := parse-json('{ "$group" : { "_id" : "$department" , "average" : { "$avg" : "$amount"}}}', $options)
    let $sort := parse-json('{ "$sort" : { "amount" : -1}}', $options)

    let $result := mongodb:aggregate($mongodbClientId, $support:database, $support:mongoCollection,
                   ($match, $project, $group, $sort))

    return
        for $one in $result
        return parse-json($one, $options)?average

};

(:
 : Run same tests directly with command function
 :)
declare
    %test:assertEquals(5, 15, 74)
function aggregate:aggregate_command_xq31() {
    let $mongodbClientId := support:getToken()

    let $options := map { "liberal": true(), "duplicates": "use-last" }

    let $command := '{
  aggregate: "mongodbTest",
  pipeline: [
    {
        "$match": {
            "type": "airfare"
        }
    },
    {
        "$project": {
            "_id": 0,
            "amount": 1,
            "department": 1
        }
    },
    {
        "$group": {
            "_id": "$department",
            "average": {
                "$avg": "$amount"
            }
        }
    },
    {
        "$sort": {
            "amount": -1
        }
    }
]
}'

    let $cmdresult := mongodb:command($mongodbClientId, $support:database, $command)
    let $result := parse-json($cmdresult, $options)

    return
        map:get($result, "result")?*?average

};

xquery version "3.1";

module namespace aggregate="http://exist-db.org/mongodb/test/aggregate";

import module namespace xqjson = "http://xqilla.sourceforge.net/lib/xqjson";

import module namespace test="http://exist-db.org/xquery/xqsuite"
                at "resource:org/exist/xquery/lib/xqsuite/xqsuite.xql";

import module namespace mongodb = "http://expath.org/ns/mongo"
                at "java:org.exist.mongodb.xquery.MongodbModule";

import module namespace support = "http://exist-db.org/mongrel/test/support"
                at "./support.xqm";

(:
 :  Example test taken from http://docs.mongodb.org/ecosystem/tutorial/use-aggregation-framework-with-java-driver/
 :)

(: Connect to mongodb, store token :)
declare %test:setUp function aggregate:setup()
{
    let $setup := support:setup()
    let $mongodbClientId := support:getToken()
    return
        (
            mongodb:insert($mongodbClientId, $support:database, $support:mongoCollection,
                            '{ "employee" : 1 , "department" : "Sales" , "amount" : 71 , "type" : "airfare"}'),
            mongodb:insert($mongodbClientId, $support:database, $support:mongoCollection,
                            '{ "employee" : 2 , "department" : "Engineering" , "amount" : 15 , "type" : "airfare"}'),
            mongodb:insert($mongodbClientId, $support:database, $support:mongoCollection,
                             '{ "employee" : 4 , "department" : "Human Resources" , "amount" : 5 , "type" : "airfare"}'),
            mongodb:insert($mongodbClientId, $support:database, $support:mongoCollection,
                             '{ "employee" : 42 , "department" : "Sales" , "amount" : 77 , "type" : "airfare"}')
        )

};

(: Disconnect from mongodb, cleanup token :)
declare %test:tearDown function aggregate:cleanup()
{
    support:cleanup()
};

(: ----------------------------
 : Actual tests below this line
 : ---------------------------- :)

(:
 : collection#aggregate(query)
 :
 : { "_id" : "Human Resources" , "average" : 5.0}
 : { "_id" : "Engineering" , "average" : 15.0}
 : { "_id" : "Sales" , "average" : 74.0}
 :)
declare
    %test:assertEquals("5.0", "15.0", "74.0")
function aggregate:aggregate_simple() {
    let $mongodbClientId := support:getToken()

    let $result := mongodb:aggregate($mongodbClientId, $support:database, $support:mongoCollection,
                   (
                        '{ "$match" : { "type" : "airfare"}}',
                        '{ "$project" : { "department" : 1 , "amount" : 1 , "_id" : 0}}',
                        '{ "$group" : { "_id" : "$department" , "average" : { "$avg" : "$amount"}}}',
                        '{ "$sort" : { "amount" : -1}}'   )
                    )

    let $formatted := <result>{for $one in $result
    return xqjson:parse-json($one)}</result>

    return $formatted//pair[@name eq 'average']/text()

};

declare
    %test:assertEquals(5, 15, 74)
function aggregate:aggregate_simple_xq31() {
    let $mongodbClientId := support:getToken()

    let $options := map { "liberal": true(), "duplicates": "use-last" }

    let $match := parse-json('{ "$match" : { "type" : "airfare"}}', $options)
    let $project := parse-json('{ "$project" : { "department" : 1 , "amount" : 1 , "_id" : 0}}', $options)
    let $group := parse-json('{ "$group" : { "_id" : "$department" , "average" : { "$avg" : "$amount"}}}', $options)
    let $sort := parse-json('{ "$sort" : { "amount" : -1}}', $options)

    let $result := mongodb:aggregate($mongodbClientId, $support:database, $support:mongoCollection,
                   ($match, $project, $group, $sort))

    return
        for $one in $result
        return parse-json($one, $options)?average

};

declare
    %test:assertEquals('{ "serverUsed" : "miniserver.local:27017" , "result" : [ ] , "ok" : 1.0}')
function aggregate:aggregate_command_xq31() {
    let $mongodbClientId := support:getToken()

    let $options := map { "liberal": true(), "duplicates": "use-last" }

    let $command := '{
  aggregate: "mongoTest",
  pipeline: [
    {
        "$match": {
            "type": "airfare"
        }
    },
    {
        "$project": {
            "_id": 0,
            "amount": 1,
            "department": 1
        }
    },
    {
        "$group": {
            "_id": "$department",
            "average": {
                "$avg": "$amount"
            }
        }
    },
    {
        "$sort": {
            "amount": -1
        }
    }
]
}'

    let $json := parse-json($command, $options)

    let $result := mongodb:command($mongodbClientId, $support:database, $command)

    return
        $result

};


adjust tests to include @joewiz's remarks

xquery version "3.0";

module namespace flwor="http://exist-db.org/xquery/test/flwor";

declare namespace test="http://exist-db.org/xquery/xqsuite";

declare variable $flwor:COLLECTION_CONFIG :=
    <collection xmlns="http://exist-db.org/collection-config/1.0">
        <index xmlns:xs="http://www.w3.org/2001/XMLSchema">
            <fulltext default="none" attributes="false"/>
            <range>
                <create qname="place">
                    <field name="place-id" type="xs:string" match="@xml:id"/>
                    <field name="place-name" type="xs:string" match="placeName"/>
                </create>
            </range>
        </index>
    </collection>;

declare variable $flwor:DATA :=
    <listPlace>
         <place xml:id="warsaw">
             <placeName>Warsaw</placeName>
         </place>
         <place xml:id="berlin">
             <placeName>Berlin</placeName>
         </place>
    </listPlace>;

declare variable $flwor:COLLECTION_NAME := "flwortest";
declare variable $flwor:COLLECTION := "/db/" || $flwor:COLLECTION_NAME;

declare
    %test:setUp
function flwor:setup() {
    xmldb:create-collection("/db/system/config/db", $flwor:COLLECTION_NAME),
    xmldb:store("/db/system/config/db/" || $flwor:COLLECTION_NAME, "collection.xconf", $flwor:COLLECTION_CONFIG),
    xmldb:create-collection("/db", $flwor:COLLECTION_NAME),
    xmldb:store($flwor:COLLECTION, "test.xml", $flwor:DATA)
};

declare
    %test:tearDown
function flwor:cleanup() {
    xmldb:remove($flwor:COLLECTION),
    xmldb:remove("/db/system/config/db/" || $flwor:COLLECTION_NAME)
};

declare function flwor:test($name) {
    collection($flwor:COLLECTION_NAME)//place[placeName = $name]/string()
};

declare
    %test:assertEquals("Berlin", "Berlin")
function flwor:order-by-with-range() {
    for $i in 1 to 2
    order by $i
    return
        flwor:test("Berlin")
};

(: https://github.com/eXist-db/exist/issues/739#issuecomment-130997865 :)
declare
    %test:assertEquals(1, 2, 1, 2, 3, 1, 2, 3, 4)
function flwor:orderby-for-multi() {
    for $x in (3, 2, 4)
    order by $x
    for $y in (1 to $x)
    return $y
};

declare
    %test:assertEquals("a1", "a2", "b1", "b2")
function flwor:orderby-multi() {
  for $a in ("b", "a")
  order by $a
  for $b in (2, 1)
  order by $b
  return
    $a || $b
};

declare
    %test:assertEquals("a1", "a2", "b1", "b2")
function flwor:orderby-where-multi() {
    for $a in ("b", "a", "c")
    order by $a
    where $a = ("a", "b")
    for $b in (2, 1, 5, 6, 9)
    where $b < 4
    order by $b
    return
        $a || $b
};

declare
    %test:assertEquals(2, 4)
function flwor:where-multi() {
    for $a in 1 to 10
    where $a mod 2 = 0
    where $a < 5
    return
        $a
};

declare
    %test:assertEquals(1, 1)
function flwor:where-multi-groupby() {
    let $xml :=
        <t>
            <n class="a">1</n>
            <n class="b">2</n>
            <n class="a">1</n>
            <n class="a">3</n>
        </t>
    for $a in $xml/n
    where $a != 3
    group by $c := $a/@class
    where count($a) > 1
    return
        $a/string()
};

declare
    %test:args(0)
    %test:assertEquals("[]")
    %test:args(1)
    %test:assertEquals("[1]")
    %test:args(6)
    %test:assertEquals("[1]", "[2]", "[3]", "[4]", "[6]")
function flwor:allowing-empty($n as xs:integer) {
    for $x allowing empty in 1 to $n
    where not($x = 5)
    return concat("[", $x, "]")
};

xquery version "3.0";

module namespace flwor="http://exist-db.org/xquery/test/flwor";

declare namespace test="http://exist-db.org/xquery/xqsuite";

declare variable $flwor:COLLECTION_CONFIG :=
    <collection xmlns="http://exist-db.org/collection-config/1.0">
        <index xmlns:xs="http://www.w3.org/2001/XMLSchema">
            <fulltext default="none" attributes="false"/>
            <range>
                <create qname="place">
                    <field name="place-id" type="xs:string" match="@xml:id"/>
                    <field name="place-name" type="xs:string" match="placeName"/>
                </create>
            </range>
        </index>
    </collection>;

declare variable $flwor:DATA :=
    <listPlace>
         <place xml:id="warsaw">
             <placeName>Warsaw</placeName>
         </place>
         <place xml:id="berlin">
             <placeName>Berlin</placeName>
         </place>
    </listPlace>;

declare variable $flwor:COLLECTION_NAME := "flwortest";
declare variable $flwor:COLLECTION := "/db/" || $flwor:COLLECTION_NAME;

declare
    %test:setUp
function flwor:setup() {
    xmldb:create-collection("/db/system/config/db", $flwor:COLLECTION_NAME),
    xmldb:store("/db/system/config/db/" || $flwor:COLLECTION_NAME, "collection.xconf", $flwor:COLLECTION_CONFIG),
    xmldb:create-collection("/db", $flwor:COLLECTION_NAME),
    xmldb:store($flwor:COLLECTION, "test.xml", $flwor:DATA)
};

declare
    %test:tearDown
function flwor:cleanup() {
    xmldb:remove($flwor:COLLECTION),
    xmldb:remove("/db/system/config/db/" || $flwor:COLLECTION_NAME)
};

declare function flwor:test($a) {
    collection($flwor:COLLECTION_NAME)//place[placeName = "berlin"]/string()
};

declare
    %test:assertEquals("Berlin")
function flwor:order-by-with-range() {
    for $i in 1 to 2
    order by $i
    return
        flwor:test("Berlin")
};

(: https://github.com/eXist-db/exist/issues/739#issuecomment-130997865 :)
declare
    %test:assertEquals(1, 2, 1, 2, 3, 1, 2, 3, 4)
function flwor:orderby-for-multi() {
    for $x in (3, 2, 4)
    order by $x
    for $y in (1 to $x)
    return $y
};

declare
    %test:assertEquals("a1", "a2", "b1", "b2")
function flwor:orderby-multi() {
  for $a in ("b", "a")
  order by $a
  for $b in (2, 1)
  order by $b
  return
    $a || $b
};

declare
    %test:assertEquals("a1", "a2", "b1", "b2")
function flwor:orderby-where-multi() {
    for $a in ("b", "a", "c")
    order by $a
    where $a = ("a", "b")
    for $b in (2, 1, 5, 6, 9)
    where $b < 4
    order by $b
    return
        $a || $b
};

declare
    %test:assertEquals(2, 4)
function flwor:where-multi() {
    for $a in 1 to 10
    where $a mod 2 = 0
    where $a < 5
    return
        $a
};

declare
    %test:assertEquals(1, 1)
function flwor:where-multi-groupby() {
    let $xml :=
        <t>
            <n class="a">1</n>
            <n class="b">2</n>
            <n class="a">1</n>
            <n class="a">3</n>
        </t>
    for $a in $xml/n
    where $a != 3
    group by $c := $a/@class
    where count($a) > 1
    return
        $a/string()
};

declare
    %test:args(0)
    %test:assertEquals("[]")
    %test:args(1)
    %test:assertEquals("[1]")
    %test:args(6)
    %test:assertEquals("[1]", "[2]", "[3]", "[4]", "[6]")
function flwor:allowing-empty($n as xs:integer) {
    for $x allowing empty in 1 to $n
    where not($x = 5)
    return concat("[", $x, "]")
};


ode-tools: updated entity-graph link in entity listing

xquery version "3.0" encoding "UTF-8";

declare namespace dc = "http://purl.org/dc/elements/1.1/";
declare namespace pm = "http://www.politicalmashup.nl";
declare namespace pmx = "http://www.politicalmashup.nl/extra";
declare namespace folia = "http://ilk.uvt.nl/FoLiA";
declare namespace local = "local";

import module namespace settings="http://politicalmashup.nl/modules/settings" at '/db/apps/modules/settings.xqm';
import module namespace export="http://politicalmashup.nl/modules/export";
import module namespace functx="http://www.functx.com";

declare function local:select-collection($request) {
  if ($request/@collection) then collection(concat($settings:data-root,'/',$request/@collection))
  else ()
};

declare function local:summarise-entity-names($entity-names as xs:string*) as xs:string {
  let $collapsed := for $en in $entity-names
                    let $name := xs:string($en)
                    group by $name
                    return <en n="{$name}" c="{count($en)}"/>
(:                    return concat($name,' (',count($en),')'):)
  let $strings := for $en in $collapsed order by xs:decimal($en/@c) descending return concat($en/@n,' (',$en/@c,')')
  return string-join($strings,', ')
};

declare function local:entities($collection, $request as element(request)) {
  let $headers := (
    export:xml-util-description('List of entities'),
    export:xml-util-headers( ('entity search','entity graph','nr of documents','words','wikilink') )
    )

   let $counts :=
    for $link in $collection//dc:relation/pm:link
      let $wiki := string($link)
      group by $wiki
      (:return <e w="{$wiki}" c="{count($link)}" p="{string-join(distinct-values(for $l in $link return $l/pmx:reference/@pmx:entity-name),', ')}"/>:)
      return <e w="{$wiki}" c="{count($link)}" p="{local:summarise-entity-names(for $l in $link return $l/pmx:reference/@pmx:entity-name)}"/>

  let $items :=
    for $l in $counts
    order by xs:integer($l/@c) descending
    return
      export:xml-row(
        (
        export:xml-item(concat('entity-search.xq?collection=',$request/@collection,'&entity=',$l/@w),<options display="{util:unescape-uri($l/@w,'UTF-8')}" link="true"/>),
        export:xml-item(concat('timeline/entity-graph-monthly.xq?collection=',$request/@collection,'&entity=',$l/@w),<options display="{util:unescape-uri($l/@w,'UTF-8')}" link="true"/>),
        export:xml-item($l/@c),
        export:xml-item($l/@p),
        export:xml-item($l/@w,<options link="true"/>)
        )
      )

  return export:xml-output( ($headers, $items) )
};

(: Escape input entity query, because xml (and index!) contains escaped strings. :)
declare function local:fix-wikilink-input($request) {
  (: Cut off the escaped part after the wiki link, then unescape the entity (in case input is actually escaped already), and escape the string, then combine again. :)
  concat(
    substring-before($request/@entity,'/wiki/'), '/wiki/',
    encode-for-uri(
      util:unescape-uri(substring-after($request/@entity,'/wiki/'),'UTF-8')
    )
  )
};

let $accepted-collections := string-join(export:build-collection-tree(true()),',')

let $count-queries := 'entities,documents'

let $request := export:request-parameters( (<view default="table" accept="csv,table,xml"/>,
                                            (:<count accept="{$count-queries}"/>,:)
                                            (:<entity/>,:)
                                            <collection accept="{$accepted-collections}"/>) )

let $options := export:options( (
                                  (:<count explanation="choose example query" select="{$count-queries}"/>,:)
                                  <view explanation="view output as html table or plain text csv" select="csv,table"/>,
                                  (:<entity explanation="entity as wikilink to search for"/>,:)
                                  <collection explanation="available leaf collections" select="{$accepted-collections}"/>
                                  (:,
                                  <entity-escaped value="{local:fix-wikilink-input($request)}" explanation="escaped wiki link used for querying"/>:)
                                ),
                                $request)

let $collection := local:select-collection($request)

(:let $xml-output := if ($request/@count eq 'entities') then local:entities($collection,$request)
                   else if ($request/@count eq 'documents') then local:documents($collection,$request)
                   else ():)
let $xml-output := local:entities($collection,$request)

let $introduction-html :=

List pre-computed entities occuring in the data.

let $search-form := export:html-util-generate-search-form($options, $request)

let $output := if ($request/@view eq 'table') then ($introduction-html, $search-form, export:html-output($xml-output))
               else if ($request/@view eq 'csv') then export:csv-output($xml-output)
               else if ($request/@view eq 'xml') then $xml-output
               else ()

let $output := export:output($request/@view, $output, 'ODE-II FoLiA Entity Search')

let $serialization := export:set-serialization( if ($request/@view eq 'table') then 'html' else if ($request/@view eq 'csv') then 'text' else 'xml' )

return $output

xquery version "3.0" encoding "UTF-8";

declare namespace dc = "http://purl.org/dc/elements/1.1/";
declare namespace pm = "http://www.politicalmashup.nl";
declare namespace pmx = "http://www.politicalmashup.nl/extra";
declare namespace folia = "http://ilk.uvt.nl/FoLiA";
declare namespace local = "local";

import module namespace settings="http://politicalmashup.nl/modules/settings" at '/db/apps/modules/settings.xqm';
import module namespace export="http://politicalmashup.nl/modules/export";
import module namespace functx="http://www.functx.com";

declare function local:select-collection($request) {
  if ($request/@collection) then collection(concat($settings:data-root,'/',$request/@collection))
  else ()
};

declare function local:summarise-entity-names($entity-names as xs:string*) as xs:string {
  let $collapsed := for $en in $entity-names
                    let $name := xs:string($en)
                    group by $name
                    return <en n="{$name}" c="{count($en)}"/>
(:                    return concat($name,' (',count($en),')'):)
  let $strings := for $en in $collapsed order by xs:decimal($en/@c) descending return concat($en/@n,' (',$en/@c,')')
  return string-join($strings,', ')
};

declare function local:entities($collection, $request as element(request)) {
  let $headers := (
    export:xml-util-description('List of entities'),
    export:xml-util-headers( ('entity search','entity graph','nr of documents','words','wikilink') )
    )

   let $counts :=
    for $link in $collection//dc:relation/pm:link
      let $wiki := string($link)
      group by $wiki
      (:return <e w="{$wiki}" c="{count($link)}" p="{string-join(distinct-values(for $l in $link return $l/pmx:reference/@pmx:entity-name),', ')}"/>:)
      return <e w="{$wiki}" c="{count($link)}" p="{local:summarise-entity-names(for $l in $link return $l/pmx:reference/@pmx:entity-name)}"/>

  let $items :=
    for $l in $counts
    order by xs:integer($l/@c) descending
    return
      export:xml-row(
        (
        export:xml-item(concat('entity-search.xq?collection=',$request/@collection,'&entity=',$l/@w),<options display="{util:unescape-uri($l/@w,'UTF-8')}" link="true"/>),
        export:xml-item(concat('entity-graph.xq?collection=',$request/@collection,'&entity=',$l/@w),<options display="{util:unescape-uri($l/@w,'UTF-8')}" link="true"/>),
        export:xml-item($l/@c),
        export:xml-item($l/@p),
        export:xml-item($l/@w,<options link="true"/>)
        )
      )

  return export:xml-output( ($headers, $items) )
};

(: Escape input entity query, because xml (and index!) contains escaped strings. :)
declare function local:fix-wikilink-input($request) {
  (: Cut off the escaped part after the wiki link, then unescape the entity (in case input is actually escaped already), and escape the string, then combine again. :)
  concat(
    substring-before($request/@entity,'/wiki/'), '/wiki/',
    encode-for-uri(
      util:unescape-uri(substring-after($request/@entity,'/wiki/'),'UTF-8')
    )
  )
};

let $accepted-collections := string-join(export:build-collection-tree(true()),',')

let $count-queries := 'entities,documents'

let $request := export:request-parameters( (<view default="table" accept="csv,table,xml"/>,
                                            (:<count accept="{$count-queries}"/>,:)
                                            (:<entity/>,:)
                                            <collection accept="{$accepted-collections}"/>) )

let $options := export:options( (
                                  (:<count explanation="choose example query" select="{$count-queries}"/>,:)
                                  <view explanation="view output as html table or plain text csv" select="csv,table"/>,
                                  (:<entity explanation="entity as wikilink to search for"/>,:)
                                  <collection explanation="available leaf collections" select="{$accepted-collections}"/>
                                  (:,
                                  <entity-escaped value="{local:fix-wikilink-input($request)}" explanation="escaped wiki link used for querying"/>:)
                                ),
                                $request)

let $collection := local:select-collection($request)

(:let $xml-output := if ($request/@count eq 'entities') then local:entities($collection,$request)
                   else if ($request/@count eq 'documents') then local:documents($collection,$request)
                   else ():)
let $xml-output := local:entities($collection,$request)

let $introduction-html :=

List pre-computed entities occuring in the data.

let $search-form := export:html-util-generate-search-form($options, $request)

let $output := if ($request/@view eq 'table') then ($introduction-html, $search-form, export:html-output($xml-output))
               else if ($request/@view eq 'csv') then export:csv-output($xml-output)
               else if ($request/@view eq 'xml') then $xml-output
               else ()

let $output := export:output($request/@view, $output, 'ODE-II FoLiA Entity Search')

let $serialization := export:set-serialization( if ($request/@view eq 'table') then 'html' else if ($request/@view eq 'csv') then 'text' else 'xml' )

return $output


fixed issue in facility read service

fixed issue in facility read service

import module namespace csd_bl = "https://github.com/openhie/openinfoman/csd_bl";
declare default element  namespace   "urn:ihe:iti:csd:2013";
declare variable $careServicesRequest as item() external;

(:
   The query will be executed against the root element of the CSD document.

   The dynamic context of this query has $careServicesRequest set to contain any of the search
   and limit paramaters as sent by the Service Finder
:)

let $facs0 := if (exists($careServicesRequest/organization/@entityID)) then /CSD/providerDirectory/*  else ()
let $facs1 := if (exists($careServicesRequest/organization/service/@position)) then $facs0  else ()
let $facs2 := if (exists($careServicesRequest/id/@entityID)) then csd_bl:filter_by_primary_id($facs1,$careServicesRequest/id) else ()
let $facs3 :=
  if (count($facs2) = 1)
    then
    let $facility :=  $facs2[1]
    return
    <facility entityID="{$facility/@entityID}">
      {
	if (exists($careServicesRequest/organization/@entityID) and exists($careServicesRequest/organization/service/@position))
	  then
	  <organizations>
	    <organization entityID="{$careServicesRequest/organization/@entityID}">
	    {
	      for $srvc in $facility/organizations/organization[upper-case(@entityID) = upper-case($careServicesRequest/organization/@entityID)]/service[position() = $careServicesRequest/organization/service/@position]
	      return       <service position="{$careServicesRequest/organization/service/@position}" entityID="{$srvc/@entityID}">{$srvc/*}</service>
	  }
	    </organization>
	  </organizations>
	else
	  ()
      }
      {$facility/record}
    </facility>
  else ()

return csd_bl:wrap_facilities($facs3)

import module namespace csd_bl = "https://github.com/openhie/openinfoman/csd_bl";
declare default element  namespace   "urn:ihe:iti:csd:2013";
declare variable $careServicesRequest as item() external;

(:
   The query will be executed against the root element of the CSD document.

   The dynamic context of this query has $careServicesRequest set to contain any of the search
   and limit paramaters as sent by the Service Finder
:)
!
let $facs0 := if (exists($careServicesRequest/organization/@entityID)) then /CSD/providerDirectory/*  else ()
let $facs1 := if (exists($careServicesRequest/organization/service/@position)) then $facs0  else ()
let $facs2 := if (exists($careServicesRequest/id/@entityID)) then csd_bl:filter_by_primary_id($facs1,$careServicesRequest/id) else ()
let $facs3 :=
  if (count($facs2) = 1)
    then
    let $facility :=  $facs2[1]
    return
    <facility entityID="{$facility/@entityID}">
      {
	if (exists($careServicesRequest/organization/@entityID) and exists($careServicesRequest/organization/service/@position))
	  then
	  <organizations>
	    <organization entityID="{$careServicesRequest/organization/@entityID}">
	    {
	      for $srvc in $facility/organizations/organization[upper-case(@entityID) = upper-case($careServicesRequest/organization/@entityID)]/service[position() = $careServicesRequest/organization/service/@position]
	      return       <service position="{$careServicesRequest/organization/service/@position}" entityID="{$srvc/@entityID}">{$srvc/*}</service>
	  }
	    </organization>
	  </organizations>
	else
	  ()
      }
      {$facility/record}
    </facility>
  else ()

return csd_bl:wrap_facilities($facs3)


[ignore] rename tests to avoid duplicate

corrects a problem noted in PR #1593

xquery version "3.1";

(:~
 : Test various F+O functions
 :)
module namespace fnt="http://exist-db.org/xquery/test/fn";

import module namespace test="http://exist-db.org/xquery/xqsuite" at "resource:org/exist/xquery/lib/xqsuite/xqsuite.xql";

declare variable $fnt:composed := 'Åström';
declare variable $fnt:decomposed := 'Åström';

declare
    %test:setUp
function fnt:store() {
    let $col := xmldb:create-collection("/db", "fn-test")
    return
        (
            xmldb:store($col, "test.xml", <books><book/></books>),
            xmldb:store($col, "test.bin", "some binary", "application/octet-stream")
        )
};

declare
    %test:tearDown
function fnt:cleanup() {
    xmldb:remove("/db/fn-test")
};

declare
    %test:args("NFC")
    %test:assertEquals(6, 6)
    %test:args("NFD")
    %test:assertEquals(8, 8)
    %test:args("NFKD")
    %test:assertEquals(8, 8)
    %test:args("NFKC")
    %test:assertEquals(6, 6)
    %test:args("XYZ")
    %test:assertError
function fnt:normalize-unicode($normalization-form as xs:string) {
    count(string-to-codepoints(normalize-unicode($fnt:composed, $normalization-form))),
    count(string-to-codepoints(normalize-unicode($fnt:decomposed, $normalization-form)))
};

declare
    %test:args("test.xml")
    %test:assertEquals("true")
    %test:args("test.bin")
    %test:assertEquals("false")
    %test:args("no-such-file.xml")
    %test:assertEquals("false")
    %test:args("no-such-file.bin")
    %test:assertEquals("false")
function fnt:doc-available($filename as xs:string) {
    fn:doc-available("/db/fn-test/" || $filename)
};

declare
    %test:args("test.xml")
    %test:assertEquals("true")
function fnt:doc-returns-document-node($filename as xs:string) {
    fn:doc("/db/fn-test/" || $filename) instance of document-node()
};

declare
    %test:args("test.xml")
    %test:assertEmpty
function fnt:doc-does-not-return-element-node($filename as xs:string) {
    fn:doc("/db/fn-test/" || $filename)/book
};

declare
    %test:assertEquals(0, 0)
function fnt:tokenize-empty() {
     count(() => tokenize("\s")),
     count(tokenize((), "\s"))
};

declare
    %test:args("hello adam")
    %test:assertEquals("hello", "adam")
    %test:args("hello   adam")
    %test:assertEquals("hello", "adam")
function fnt:tokenize-onearg($str as xs:string) {
     tokenize($str)
};

xquery version "3.1";

(:~
 : Test various F+O functions
 :)
module namespace fnt="http://exist-db.org/xquery/test/fn";

import module namespace test="http://exist-db.org/xquery/xqsuite" at "resource:org/exist/xquery/lib/xqsuite/xqsuite.xql";

declare variable $fnt:composed := 'Åström';
declare variable $fnt:decomposed := 'Åström';

declare
    %test:setUp
function fnt:store() {
    let $col := xmldb:create-collection("/db", "fn-test")
    return
        (
            xmldb:store($col, "test.xml", <books><book/></books>),
            xmldb:store($col, "test.bin", "some binary", "application/octet-stream")
        )
};

declare
    %test:tearDown
function fnt:cleanup() {
    xmldb:remove("/db/fn-test")
};

declare
    %test:args("NFC")
    %test:assertEquals(6, 6)
    %test:args("NFD")
    %test:assertEquals(8, 8)
    %test:args("NFKD")
    %test:assertEquals(8, 8)
    %test:args("NFKC")
    %test:assertEquals(6, 6)
    %test:args("XYZ")
    %test:assertError
function fnt:normalize-unicode($normalization-form as xs:string) {
    count(string-to-codepoints(normalize-unicode($fnt:composed, $normalization-form))),
    count(string-to-codepoints(normalize-unicode($fnt:decomposed, $normalization-form)))
};

declare
    %test:args("test.xml")
    %test:assertEquals("true")
    %test:args("test.bin")
    %test:assertEquals("false")
    %test:args("no-such-file.xml")
    %test:assertEquals("false")
    %test:args("no-such-file.bin")
    %test:assertEquals("false")
function fnt:doc-available($filename as xs:string) {
    fn:doc-available("/db/fn-test/" || $filename)
};

declare
    %test:args("test.xml")
    %test:assertEquals("true")
function fnt:doc($filename as xs:string) {
    fn:doc("/db/fn-test/" || $filename) instance of document-node()
};

declare
    %test:args("test.xml")
    %test:assertEmpty
function fnt:doc($filename as xs:string) {
    fn:doc("/db/fn-test/" || $filename)/book
};

declare
    %test:assertEquals(0, 0)
function fnt:tokenize-empty() {
     count(() => tokenize("\s")),
     count(tokenize((), "\s"))
};

declare
    %test:args("hello adam")
    %test:assertEquals("hello", "adam")
    %test:args("hello   adam")
    %test:assertEquals("hello", "adam")
function fnt:tokenize-onearg($str as xs:string) {
     tokenize($str)
};


DHFPROD-3400: Don't check the same URIs repeatedly for additional merges

xquery version "1.0-ml";

(:
 : This is an implementation library, not an interface to the Smart Mastering functionality.
 :
 : Match-and-merge combines the two primary functions of Smart Mastering in a
 : single call. This means that both happen in the same transaction. When
 : called this way, the actions configured on thresholds in the match options
 : are taken automatically, rather than individually by the client.
 :)

module namespace proc-impl = "http://marklogic.com/smart-mastering/process-records/impl";

import module namespace blocks-impl = "http://marklogic.com/smart-mastering/blocks-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/blocks-impl.xqy";
import module namespace const = "http://marklogic.com/smart-mastering/constants"
  at "/com.marklogic.smart-mastering/constants.xqy";
import module namespace fun-ext = "http://marklogic.com/smart-mastering/function-extension"
  at "../function-extension/base.xqy";
import module namespace matcher = "http://marklogic.com/smart-mastering/matcher"
  at "/com.marklogic.smart-mastering/matcher.xqy";
import module namespace match-impl = "http://marklogic.com/smart-mastering/matcher-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/matcher-impl.xqy";
import module namespace match-opt-impl = "http://marklogic.com/smart-mastering/options-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/options-impl.xqy";
import module namespace merging = "http://marklogic.com/smart-mastering/merging"
  at "/com.marklogic.smart-mastering/merging.xqy";
import module namespace merge-impl = "http://marklogic.com/smart-mastering/survivorship/merging"
  at "/com.marklogic.smart-mastering/survivorship/merging/base.xqy",
    "/com.marklogic.smart-mastering/survivorship/merging/options.xqy";
import module namespace coll-impl = "http://marklogic.com/smart-mastering/survivorship/collections"
  at "/com.marklogic.smart-mastering/survivorship/merging/collections.xqy";
import module namespace tel = "http://marklogic.com/smart-mastering/telemetry"
  at "/com.marklogic.smart-mastering/telemetry.xqy";

declare option xdmp:mapping "false";

declare function proc-impl:process-match-and-merge($input as item()*)
  as item()*
{
  let $merging-options := merging:get-options($const:FORMAT-XML)
  return
    if (fn:exists($merging-options)) then
      for $merge-options in $merging-options
      let $match-options := matcher:get-options(fn:string($merge-options/merging:match-options), $const:FORMAT-XML)
      return
        proc-impl:process-match-and-merge-with-options-save($input, $merge-options, $match-options, cts:true-query())
    else
      fn:error($const:NO-MERGE-OPTIONS-ERROR, "No Merging Options are present. See: https://marklogic-community.github.io/smart-mastering-core/docs/merging-options/")
};

declare function proc-impl:process-match-and-merge(
  $input as item()*,
  $option-name as xs:string,
  $filter-query as cts:query)
  as item()*
{
  let $all-options := xdmp:invoke-function(function() {
      let $merge-options := merging:get-options($option-name, $const:FORMAT-XML)
      let $match-options := matcher:get-options(fn:string($merge-options/merging:match-options), $const:FORMAT-XML)
      return
        map:map()
          => map:with("merge-options", $merge-options)
          => map:with("match-options", $match-options)
    }, map:entry("update", "false"))
  return
    proc-impl:process-match-and-merge-with-options-save(
      $input,
      $all-options => map:get("merge-options"),
      $all-options => map:get("match-options"),
      $filter-query
    )
};

declare variable $STRING-TOKEN := "##";

(:
 : Given a map with keys that are URIs and values that are the result elements from running the match function against
 : that URI's document, produce a map where the key is a generated unique ID and the values are sequences of URIs to be
 : merged. We want to eliminate redundant cases, such as merge(docA, docB) and merge(docB, docA).
 : @param $matches - map:map stores search results by URI
 : @param $match-options - the match options for the query
 : @param $max-scan - max number of search results to return
 : @param $merge-threshold - max score needed for a merge
 : @param $filter-query - cts:query that restricts the search
 : @return  map(unique ID -> sequence of URIs)
 :)
declare function proc-impl:consolidate-merges(
  $matches as map:map,
  $matching-options,
  $max-scan,
  $merge-threshold,
  $filter-query
) as map:map
{
  map:new((
    let $merges :=
      fn:distinct-values(
        for $key in map:keys($matches)
        let $merge-uris as xs:string* := map:get($matches, $key)/result[@action=$const:MERGE-ACTION]/@uri
        where fn:exists($merge-uris)
        return
          let $additional-uris :=
            proc-impl:expand-uris-for-merge(
              $merge-uris,
              $merge-uris,
              $matches,
              $matching-options,
              $max-scan,
              $merge-threshold,
              $filter-query
            )
          return
            fn:string-join(
              for $uri in fn:distinct-values(($key, $merge-uris, $additional-uris))
              let $_lock-on-uri := if (fn:not(map:contains($matches,$uri))) then merge-impl:lock-for-update($uri) else ()
              order by $uri
              return $uri,
              $STRING-TOKEN
            )
      )
    for $merge in $merges
    let $uris := fn:tokenize($merge, $STRING-TOKEN)
    let $merge-id := fn:head((
      $uris[fn:starts-with(., $merge-impl:MERGED-DIR)] ! fn:replace(fn:substring-after(., $merge-impl:MERGED-DIR), "\.(json|xml)", ""),
      xdmp:md5($merge)
    ))
    return
      map:entry($merge-id, $uris)
  ))
};

(:
 : This expands out searches to find any documents that may exist
 : @param $current-uris - xs:string* URIs that we just discovered with the last search
 : @param accumlated-uris - xs:string* URIs that we've discovered so far
 : @param $matches - map:map stores search results by URI
 : @param $match-options - the match options for the query
 : @param $max-scan - max number of search results to return
 : @param $merge-threshold - max score needed for a merge
 : @param $filter-query - cts:query that restricts the search
 : @return  xs:string* URIs found for the merge
 :)
declare function proc-impl:expand-uris-for-merge(
  $current-uris as xs:string*,
  $accumlated-uris as xs:string*,
  $matches as map:map,
  $matching-options,
  $max-scan,
  $merge-threshold,
  $filter-query
) (: as xs:string* ~leaving off for tail recursion~ :)
{
  let $additional-uris :=
    fn:distinct-values(
      for $merge-uri in $current-uris
      return
        if (map:contains($matches, $merge-uri)) then
          map:get($matches, $merge-uri)/result[@action = $const:MERGE-ACTION]/@uri ! fn:string(.)
        else
          let $_lock-for-update := merge-impl:lock-for-update($merge-uri)
          let $results :=
            match-impl:find-document-matches-by-options(
              fn:doc($merge-uri),
              $matching-options,
              1,
              $max-scan,
              $merge-threshold,
              (: don't include detailed match information :)
              fn:false(),
              if ($filter-query instance of cts:true-query) then
                cts:not-query(cts:document-query($accumlated-uris))
              else
                cts:and-not-query($filter-query, cts:document-query($accumlated-uris)),
              (: return results :)
              fn:true()
            )[result/@action = $const:MERGE-ACTION]
          where fn:exists($results)
          return (
            map:put($matches, $merge-uri, $results),
            $results/result[@action = $const:MERGE-ACTION]/@uri ! fn:string(.)
          )
    )
  let $new-uris := $additional-uris[fn:not(. = $accumlated-uris)]
  return
    if (fn:empty($new-uris)) then
      $accumlated-uris
    else
      proc-impl:expand-uris-for-merge(
        $new-uris,
        fn:distinct-values(($new-uris,$accumlated-uris)),
        $matches,
        $matching-options,
        $max-scan,
        $merge-threshold,
        $filter-query
      )
};

declare function proc-impl:consolidate-notifies($all-matches as map:map, $merged-into as map:map)
  as xs:string*
{
  fn:distinct-values(
    for $key in map:keys($all-matches)
    for $updated-key in
      (if (map:contains($merged-into, $key)) then
        map:get($merged-into, $key)
      else
        $key)
    let $key-notifications := map:get($all-matches, $key)/result[@action=$const:NOTIFY-ACTION]
    let $key-thresholds := fn:distinct-values($key-notifications/@threshold)
    for $key-threshold in $key-thresholds
    let $updated-notification-uris :=
      for $key-notification in $key-notifications[@threshold = $key-threshold]
      let $key-uri as xs:string := $key-notification/@uri
      let $_lock-on-uri := if (fn:not(map:contains($all-matches, $key))) then merge-impl:lock-for-update($key-uri) else ()
      let $updated-uri :=
        if (map:contains($merged-into, $key-uri)) then
          map:get($merged-into, $key-uri)
        else
          $key-uri
      return $updated-uri
    return
      fn:string-join((
        $key-threshold,
        for $uri in fn:distinct-values(($updated-key, $updated-notification-uris))
        order by $uri
        return $uri
      ), $STRING-TOKEN)
  )
};

(: The following will store URIs of documents merged in transaction :)
declare variable $merges-in-transaction as map:map := map:map();
(: The following will store URIs of documents notified in transaction :)
declare variable $notifications-in-transaction as map:map := map:map();
(: The following will store URIs of documents notified in transaction :)
declare variable $no-matches-in-transaction as map:map := map:map();

(:
 : The workhorse function.
 :)
declare function proc-impl:process-match-and-merge-with-options(
  $input as item()*,
  $merge-options as item(),
  $match-options as item(),
  $filter-query as cts:query) as json:array
{
  (: increment usage count :)
  tel:increment(),
  let $start-elapsed := xdmp:elapsed-time()
  let $matching-options :=
      if ($match-options instance of object-node()) then
        match-opt-impl:options-from-json($match-options)
      else
        $match-options
  let $merge-options :=
      if ($merge-options instance of object-node()) then
        merge-impl:options-from-json($merge-options)
      else
        $merge-options
  let $archived-collection := coll-impl:on-archive(map:map(), $merge-options)
  let $normalized-input :=
    if ($input instance of xs:string*) then
      for $doc in cts:search(fn:doc(), cts:and-not-query(cts:document-query($input), cts:collection-query($archived-collection)), "unfiltered")
      return
        proc-impl:build-write-object-for-doc($doc)
    else if ($input instance of map:map*) then
      $input
    else ()
  let $write-objects-by-uri := fn:fold-left(
    function($map, $map-addition) { $map => map:with($map-addition => map:get("uri"), $map-addition) },
    map:map(),
    $normalized-input
  )
  let $uris := map:keys($write-objects-by-uri)
  let $_prime-blocks-cache := blocks-impl:get-blocks-of-uris($uris)
  let $_ := if (xdmp:trace-enabled($const:TRACE-MATCH-RESULTS)) then
        xdmp:trace($const:TRACE-MATCH-RESULTS, "processing: " || fn:string-join($uris, ", "))
      else ()
  let $target-entity := $matching-options/matcher:target-entity ! fn:string(.)
  let $actions := fn:distinct-values(($matching-options/matcher:actions/matcher:action/@name ! fn:string(.), $const:MERGE-ACTION, $const:NOTIFY-ACTION))
  let $thresholds := $matching-options/matcher:thresholds/matcher:threshold[(@action|matcher:action) = $actions]
  let $threshold-labels := $thresholds/(@label|matcher:label)
  let $minimum-threshold :=
    fn:min(
      $thresholds[(@label|matcher:label) = $threshold-labels]/(@above|matcher:above) ! fn:number(.)
    )
  let $merge-threshold :=
    fn:min((
      $thresholds[(@action|matcher:action) = $const:MERGE-ACTION]/(@above|matcher:above) ! fn:number(.),
      999
    ))
  let $_min-threshold-err :=
            if (fn:empty($minimum-threshold)) then
              fn:error($const:NO-THRESHOLD-ACTION-FOUND, "No threshold actions to act on.", ($matching-options/matcher:thresholds))
            else ()
  let $max-scan := fn:head((
    $matching-options//*:max-scan ! xs:integer(.),
    500
  ))
  let $all-matches :=
    let $start-elapsed := xdmp:elapsed-time()
    let $matches :=
        map:new((
          $normalized-input !
            map:entry(
              (. => map:get("uri")),
              matcher:find-document-matches-by-options(
                (. => map:get("value")),
                $matching-options,
                1,
                $max-scan,
                $minimum-threshold,
                fn:false(),
                $filter-query
              )
            )
        ))
    return (
      $matches,
      if (xdmp:trace-enabled($const:TRACE-PERFORMANCE)) then
        xdmp:trace($const:TRACE-PERFORMANCE, "proc-impl:process-match-and-merge-with-options: Matches: " || (xdmp:elapsed-time() - $start-elapsed))
      else ()
    )
  let $consolidated-merges := proc-impl:consolidate-merges($all-matches,
    $matching-options,
    $max-scan,
    $merge-threshold,
    $filter-query)
  let $merged-uris := map:keys($consolidated-merges)
  let $merged-into := map:map()
  let $on-merge-options := $merge-options/merging:algorithms/merging:collections/merging:on-merge
  let $processed-merges :=
    (: Process merges :)
    let $start-elapsed := xdmp:elapsed-time()
    let $merges :=
      for $new-uri in $merged-uris
      where fn:not(map:contains($merges-in-transaction, $new-uri))
      return (
        map:put($merges-in-transaction, $new-uri, fn:true()),
        let $distinct-uris := fn:distinct-values(map:get($consolidated-merges, $new-uri))
        let $merged-doc-def := merge-impl:build-merge-models-by-uri($distinct-uris, $merge-options, $new-uri)
        let $merged-doc := $merged-doc-def => map:get("value")
        let $merge-uri := merge-impl:build-merge-uri(
          $new-uri,
          if ($merged-doc instance of element() or
            $merged-doc instance of document-node(element())) then
            $const:FORMAT-XML
          else
            $const:FORMAT-JSON
        )
        return (
          $distinct-uris ! map:put($merged-into, ., $merge-uri),
          $merged-doc-def
            => map:get("audit-trace")
            => map:with("hidden", fn:true()),
          map:new((
            map:entry("uri", $merge-uri),
            map:entry("value",
              $merged-doc
            ),
            map:entry("context",
              map:new((
                map:entry("collections",
                  (
                    coll-impl:on-merge(
                      map:new((
                        for $uri in $distinct-uris
                        let $write-object := proc-impl:retrieve-write-object($write-objects-by-uri, $uri)
                        return
                          map:entry(
                            $uri,
                            $write-object
                              => map:get("context")
                              => map:get("collections")
                          )
                      )),
                      $on-merge-options
                    ),
                    $target-entity
                  )
                ),
                map:entry("permissions",
                  (
                    xdmp:default-permissions($merge-uri, "objects"),
                    for $uri in $distinct-uris
                    let $write-object := proc-impl:retrieve-write-object($write-objects-by-uri, $uri)
                    return
                      $write-object
                        => map:get("context")
                        => map:get("permissions")
                  )
                )
              ))
            )
          ))
        )
      )
    return (
      $merges,
      if (xdmp:trace-enabled($const:TRACE-PERFORMANCE)) then
        xdmp:trace($const:TRACE-PERFORMANCE, "proc-impl:process-match-and-merge-with-options: ["||xdmp:request()||"] Merges: " || (xdmp:elapsed-time() - $start-elapsed))
      else ()
    )
  let $uris-that-were-merged := map:keys($merged-into)
  let $_archived-updates :=
    (: Process collections on no matches :)
    let $start-elapsed := xdmp:elapsed-time()
    let $archived :=
      let $on-archive := $merge-options/merging:algorithms/merging:collections/merging:on-archive
      for $uri in $uris-that-were-merged
      let $write-object := proc-impl:retrieve-write-object($write-objects-by-uri, $uri)
      let $write-context := $write-object
          => map:get("context")
      let $current-collections := $write-context
          => map:get("collections")
      let $new-collections := coll-impl:on-archive(
        map:entry($uri, $current-collections),
        $on-archive
      )
      return $write-context
        => map:put("collections", $new-collections)
    return (
      $archived,
      if (xdmp:trace-enabled($const:TRACE-PERFORMANCE)) then
        xdmp:trace($const:TRACE-PERFORMANCE, "proc-impl:process-match-and-merge-with-options:["||xdmp:request()||"] Archived: " || (xdmp:elapsed-time() - $start-elapsed))
      else ()
    )
  let $consolidated-notifies := proc-impl:consolidate-notifies($all-matches, $merged-into)
  let $_no-matches-updates :=
    (: Process collections on no matches :)
    let $start-elapsed := xdmp:elapsed-time()
    let $no-matches :=
      let $on-no-match := $merge-options/merging:algorithms/merging:collections/merging:on-no-match
      for $uri in $uris[fn:not(. = $uris-that-were-merged)]
      let $write-object := proc-impl:retrieve-write-object($write-objects-by-uri, $uri)
      let $doc := $write-object
        => map:get("value")
      let $has-merges :=
        if (map:contains($all-matches, $uri)) then
          fn:number(map:get($all-matches, $uri)/@total) gt 0
        else
          fn:number(
            match-impl:find-document-matches-by-options(
              $doc,
              $matching-options,
              1,
              1,
              $merge-threshold,
              (: don't include detailed match information :)
              fn:false(),
              $filter-query,
              (: don't return results. we just want the estimate. :)
              fn:false()
            )/@total
          ) gt 0
      return
      (: If there are merges with a doc outside of this batch then leave it alone. :)
        if ($has-merges) then
          map:delete($write-objects-by-uri, $uri)
        (: otherwise, we want to pick up new collections :)
        else
          let $write-context := $write-object
          => map:get("context")
          let $current-collections := $write-context
          => map:get("collections")
          let $new-collections := (
            coll-impl:on-no-match(
              map:entry($uri, $current-collections),
              $on-no-match
            ),
            $target-entity
          )
          let $_update := $write-context
          => map:put("collections", $new-collections)
          return
            if (fn:not(map:contains($all-matches, $uri))) then
              xdmp:lock-for-update($uri)
            else ()
    return (
        $no-matches,
        if (xdmp:trace-enabled($const:TRACE-PERFORMANCE)) then
          xdmp:trace($const:TRACE-PERFORMANCE, "proc-impl:process-match-and-merge-with-options:["||xdmp:request()||"] No matches: " || (xdmp:elapsed-time() - $start-elapsed))
        else ()
    )
  let $processed-notifications :=
    let $start-elapsed := xdmp:elapsed-time()
    let $notifications :=
      (: Process notifications :)
      for $notification in $consolidated-notifies
      let $parts := fn:tokenize($notification, $STRING-TOKEN)
      let $threshold := fn:head($parts)
      let $uris := fn:tail($parts)
      let $match-write-object := matcher:build-match-notification($threshold, $uris, $merge-options)
      return
        $match-write-object
    return (
      $notifications,
      if (xdmp:trace-enabled($const:TRACE-PERFORMANCE)) then
        xdmp:trace($const:TRACE-PERFORMANCE, "proc-impl:process-match-and-merge-with-options:["||xdmp:request()||"] Notifications: " || (xdmp:elapsed-time() - $start-elapsed))
      else ()
    )
  (: Ensure that we are only writing once to a URI and merges take precedence over archives :)
  let $_processed-merges :=
    for $processed-merge in $processed-merges
    return
      $write-objects-by-uri => map:put(($processed-merge => map:get("uri")), $processed-merge)
  let $_processed-notifications :=
    for $processed-notification in $processed-notifications
    return
      $write-objects-by-uri => map:put(($processed-notification => map:get("uri")), $processed-notification)
  return json:to-array((
    if (xdmp:trace-enabled($const:TRACE-MATCH-RESULTS)) then (
      xdmp:trace($const:TRACE-MATCH-RESULTS, "All matches: " || xdmp:describe($all-matches, (),())),
      xdmp:trace($const:TRACE-MATCH-RESULTS, "Matching options: " || xdmp:describe($matching-options, (),())),
      xdmp:trace($const:TRACE-MATCH-RESULTS, "Merge options: " || xdmp:describe($merge-options, (),())),
      xdmp:trace($const:TRACE-MATCH-RESULTS, "Consolidated merges: " || xdmp:quote($consolidated-merges)),
      xdmp:trace($const:TRACE-MATCH-RESULTS, "Consolidated notifications: " || xdmp:quote($consolidated-notifies))
    )
    else (),
    for $uri in map:keys($write-objects-by-uri)
    return
      $write-objects-by-uri => map:get($uri),
    (: Process custom actions :)
    let $action-map :=
      map:new((
        let $custom-action-names := $matching-options/matcher:thresholds/matcher:threshold/(matcher:action|@action)[fn:not(. = $const:NOTIFY-ACTION or . = $const:MERGE-ACTION)]
        for $custom-action-name in fn:distinct-values($custom-action-names)
        let $action-xml := $matching-options/matcher:actions/matcher:action[@name = $custom-action-name]
        return
          map:entry(
            $custom-action-name,
            fun-ext:function-lookup(
              fn:string($action-xml/@function),
              fn:string($action-xml/@namespace),
              fn:string($action-xml/@at),
              ()
            )
          )
      ))
    for $uri in map:keys($all-matches)
    for $custom-action-match in map:get($all-matches, $uri)/result[fn:not(./@action = $const:NOTIFY-ACTION or ./@action = $const:MERGE-ACTION)]
    let $action-func := map:get($action-map, $custom-action-match/@action)
    return
      if (fn:exists($action-func)) then
        if (fn:ends-with(xdmp:function-module($action-func), "sjs")) then
          xdmp:apply(
            $action-func,
            $uri,
            proc-impl:matches-to-json($custom-action-match),
            merge-impl:options-to-json($merge-options)
          )
        else
          xdmp:apply($action-func, $uri, $custom-action-match, $merge-options)
      else
        fn:error(xs:QName("SM-CONFIGURATION"), "Threshold action is not configured or not found", $custom-action-match),
      if (xdmp:trace-enabled($const:TRACE-PERFORMANCE)) then
        xdmp:trace($const:TRACE-PERFORMANCE, "proc-impl:process-match-and-merge-with-options:["||xdmp:request()||"] " || (xdmp:elapsed-time() - $start-elapsed))
      else ()
  ))
};

(:
 : The workhorse function.
 :)
declare function proc-impl:process-match-and-merge-with-options-save(
  $input as item()*,
  $merge-options as item(),
  $match-options as item(),
  $filter-query as cts:query)
{
  let $actions as item()* := json:array-values(proc-impl:process-match-and-merge-with-options(
      $input,
      $merge-options,
      $match-options,
      $filter-query
    ))
  for $action in $actions
  return
    if ($action instance of map:map) then
      let $uri as xs:string := $action => map:get("uri")
      let $context as map:map? := $action => map:get("context")
      let $permissions := $context => map:get("permissions")
      return (
        if (fn:not(($action => map:get("hidden")))) then
          ($action => map:get("value"))[fn:empty(xdmp:node-uri(.))]
        else (),
        xdmp:document-insert(
          $uri,
          $action => map:get("value"),
          map:new((
            map:entry("collections", $context => map:get("collections")),
            map:entry("permissions", if (fn:exists($permissions)) then $permissions else xdmp:default-permissions($uri, "objects")),
            map:entry("metadata", $context => map:get("metadata"))
          ))
        )
      )
    else
      $action
};

(:
 : Convert the result elements into JSON objects.
 : TODO -- does not yet convert result/match elements to JSON. This is okay for now as there is no way to turn on the
 : $include-matches parameter from process-match-and-merge.
 :)
declare function proc-impl:matches-to-json($filtered-matches as element(result)*)
{
  array-node {
    for $match in $filtered-matches
    return object-node {
      "uri": $match/@uri/fn:string(),
      "score": $match/@score/fn:data(),
      "threshold": $match/@threshold/fn:string()
    }
  }
};

declare function proc-impl:build-write-object-for-doc($doc as document-node())
  as map:map
{
  map:new((
    map:entry("uri", xdmp:node-uri($doc)),
    map:entry("value", $doc),
    map:entry("context", map:new((
      map:entry("collections", xdmp:node-collections($doc)),
      map:entry("metadata", xdmp:node-metadata($doc)),
      map:entry("permissions", xdmp:node-permissions($doc, "objects"))
    )))
  ))
};

declare function proc-impl:retrieve-write-object(
  $write-objects-by-uri as map:map,
  $uri as xs:string
) as map:map?
{
  if (map:contains($write-objects-by-uri, $uri)) then
    $write-objects-by-uri
    => map:get($uri)
  else
    let $write-obj := proc-impl:build-write-object-for-doc(fn:doc($uri))
    return (
      map:put($write-objects-by-uri, $uri, $write-obj),
      $write-obj
    )
};

xquery version "1.0-ml";

(:
 : This is an implementation library, not an interface to the Smart Mastering functionality.
 :
 : Match-and-merge combines the two primary functions of Smart Mastering in a
 : single call. This means that both happen in the same transaction. When
 : called this way, the actions configured on thresholds in the match options
 : are taken automatically, rather than individually by the client.
 :)

module namespace proc-impl = "http://marklogic.com/smart-mastering/process-records/impl";

import module namespace blocks-impl = "http://marklogic.com/smart-mastering/blocks-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/blocks-impl.xqy";
import module namespace const = "http://marklogic.com/smart-mastering/constants"
  at "/com.marklogic.smart-mastering/constants.xqy";
import module namespace fun-ext = "http://marklogic.com/smart-mastering/function-extension"
  at "../function-extension/base.xqy";
import module namespace matcher = "http://marklogic.com/smart-mastering/matcher"
  at "/com.marklogic.smart-mastering/matcher.xqy";
import module namespace match-impl = "http://marklogic.com/smart-mastering/matcher-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/matcher-impl.xqy";
import module namespace match-opt-impl = "http://marklogic.com/smart-mastering/options-impl"
  at "/com.marklogic.smart-mastering/matcher-impl/options-impl.xqy";
import module namespace merging = "http://marklogic.com/smart-mastering/merging"
  at "/com.marklogic.smart-mastering/merging.xqy";
import module namespace merge-impl = "http://marklogic.com/smart-mastering/survivorship/merging"
  at "/com.marklogic.smart-mastering/survivorship/merging/base.xqy",
    "/com.marklogic.smart-mastering/survivorship/merging/options.xqy";
import module namespace coll-impl = "http://marklogic.com/smart-mastering/survivorship/collections"
  at "/com.marklogic.smart-mastering/survivorship/merging/collections.xqy";
import module namespace tel = "http://marklogic.com/smart-mastering/telemetry"
  at "/com.marklogic.smart-mastering/telemetry.xqy";

declare option xdmp:mapping "false";

declare function proc-impl:process-match-and-merge($input as item()*)
  as item()*
{
  let $merging-options := merging:get-options($const:FORMAT-XML)
  return
    if (fn:exists($merging-options)) then
      for $merge-options in $merging-options
      let $match-options := matcher:get-options(fn:string($merge-options/merging:match-options), $const:FORMAT-XML)
      return
        proc-impl:process-match-and-merge-with-options-save($input, $merge-options, $match-options, cts:true-query())
    else
      fn:error($const:NO-MERGE-OPTIONS-ERROR, "No Merging Options are present. See: https://marklogic-community.github.io/smart-mastering-core/docs/merging-options/")
};

declare function proc-impl:process-match-and-merge(
  $input as item()*,
  $option-name as xs:string,
  $filter-query as cts:query)
  as item()*
{
  let $all-options := xdmp:invoke-function(function() {
      let $merge-options := merging:get-options($option-name, $const:FORMAT-XML)
      let $match-options := matcher:get-options(fn:string($merge-options/merging:match-options), $const:FORMAT-XML)
      return
        map:map()
          => map:with("merge-options", $merge-options)
          => map:with("match-options", $match-options)
    }, map:entry("update", "false"))
  return
    proc-impl:process-match-and-merge-with-options-save(
      $input,
      $all-options => map:get("merge-options"),
      $all-options => map:get("match-options"),
      $filter-query
    )
};

declare variable $STRING-TOKEN := "##";

(:
 : Given a map with keys that are URIs and values that are the result elements from running the match function against
 : that URI's document, produce a map where the key is a generated unique ID and the values are sequences of URIs to be
 : merged. We want to eliminate redundant cases, such as merge(docA, docB) and merge(docB, docA).
 : @param $matches - map:map stores search results by URI
 : @param $match-options - the match options for the query
 : @param $max-scan - max number of search results to return
 : @param $merge-threshold - max score needed for a merge
 : @param $filter-query - cts:query that restricts the search
 : @return  map(unique ID -> sequence of URIs)
 :)
declare function proc-impl:consolidate-merges(
  $matches as map:map,
  $matching-options,
  $max-scan,
  $merge-threshold,
  $filter-query
) as map:map
{
  map:new((
    let $merges :=
      fn:distinct-values(
        for $key in map:keys($matches)
        let $merge-uris as xs:string* := map:get($matches, $key)/result[@action=$const:MERGE-ACTION]/@uri
        where fn:exists($merge-uris)
        return
          let $additional-uris :=
            proc-impl:expand-uris-for-merge(
              $merge-uris,
              $merge-uris,
              $matches,
              $matching-options,
              $max-scan,
              $merge-threshold,
              $filter-query
            )
          return
            fn:string-join(
              for $uri in fn:distinct-values(($key, $merge-uris, $additional-uris))
              let $_lock-on-uri := if (fn:not(map:contains($matches,$uri))) then merge-impl:lock-for-update($uri) else ()
              order by $uri
              return $uri,
              $STRING-TOKEN
            )
      )
    for $merge in $merges
    let $uris := fn:tokenize($merge, $STRING-TOKEN)
    let $merge-id := fn:head((
      $uris[fn:starts-with(., $merge-impl:MERGED-DIR)] ! fn:replace(fn:substring-after(., $merge-impl:MERGED-DIR), "\.(json|xml)", ""),
      xdmp:md5($merge)
    ))
    return
      map:entry($merge-id, $uris)
  ))
};

(:
 : This expands out searches to find any documents that may exist
 : @param $current-uris - xs:string* URIs that we just discovered with the last search
 : @param accumlated-uris - xs:string* URIs that we've discovered so far
 : @param $matches - map:map stores search results by URI
 : @param $match-options - the match options for the query
 : @param $max-scan - max number of search results to return
 : @param $merge-threshold - max score needed for a merge
 : @param $filter-query - cts:query that restricts the search
 : @return  xs:string* URIs found for the merge
 :)
declare function proc-impl:expand-uris-for-merge(
  $current-uris as xs:string*,
  $accumlated-uris as xs:string*,
  $matches as map:map,
  $matching-options,
  $max-scan,
  $merge-threshold,
  $filter-query
) (: as xs:string* ~leaving off for tail recursion~ :)
{
  let $additional-uris :=
    for $merge-uri in $current-uris
    return
      if (map:contains($matches,$merge-uri)) then
        map:get($matches, $merge-uri)/result[@action=$const:MERGE-ACTION]/@uri ! fn:string(.)
      else
        let $_lock-for-update := merge-impl:lock-for-update($merge-uri)
        let $results :=
          match-impl:find-document-matches-by-options(
            fn:doc($merge-uri),
            $matching-options,
            1,
            $max-scan,
            $merge-threshold,
            (: don't include detailed match information :)
            fn:false(),
            if ($filter-query instance of cts:true-query) then
              cts:not-query(cts:document-query($accumlated-uris))
            else
              cts:and-not-query($filter-query, cts:document-query($accumlated-uris)),
            (: return results :)
            fn:true()
          )[result/@action = $const:MERGE-ACTION]
        where fn:exists($results)
        return (
          map:put($matches, $merge-uri, $results),
          $results/result[@action=$const:MERGE-ACTION]/@uri ! fn:string(.)
        )
  let $new-uris := $additional-uris[fn:not(. = $accumlated-uris)]
  return
    if (fn:empty($new-uris)) then
      $accumlated-uris
    else
      proc-impl:expand-uris-for-merge(
        $new-uris,
        fn:distinct-values(($new-uris,$accumlated-uris)),
        $matches,
        $matching-options,
        $max-scan,
        $merge-threshold,
        $filter-query
      )
};

declare function proc-impl:consolidate-notifies($all-matches as map:map, $merged-into as map:map)
  as xs:string*
{
  fn:distinct-values(
    for $key in map:keys($all-matches)
    for $updated-key in
      (if (map:contains($merged-into, $key)) then
        map:get($merged-into, $key)
      else
        $key)
    let $key-notifications := map:get($all-matches, $key)/result[@action=$const:NOTIFY-ACTION]
    let $key-thresholds := fn:distinct-values($key-notifications/@threshold)
    for $key-threshold in $key-thresholds
    let $updated-notification-uris :=
      for $key-notification in $key-notifications[@threshold = $key-threshold]
      let $key-uri as xs:string := $key-notification/@uri
      let $_lock-on-uri := if (fn:not(map:contains($all-matches, $key))) then merge-impl:lock-for-update($key-uri) else ()
      let $updated-uri :=
        if (map:contains($merged-into, $key-uri)) then
          map:get($merged-into, $key-uri)
        else
          $key-uri
      return $updated-uri
    return
      fn:string-join((
        $key-threshold,
        for $uri in fn:distinct-values(($updated-key, $updated-notification-uris))
        order by $uri
        return $uri
      ), $STRING-TOKEN)
  )
};

(: The following will store URIs of documents merged in transaction :)
declare variable $merges-in-transaction as map:map := map:map();
(: The following will store URIs of documents notified in transaction :)
declare variable $notifications-in-transaction as map:map := map:map();
(: The following will store URIs of documents notified in transaction :)
declare variable $no-matches-in-transaction as map:map := map:map();

(:
 : The workhorse function.
 :)
declare function proc-impl:process-match-and-merge-with-options(
  $input as item()*,
  $merge-options as item(),
  $match-options as item(),
  $filter-query as cts:query) as json:array
{
  (: increment usage count :)
  tel:increment(),
  let $start-elapsed := xdmp:elapsed-time()
  let $matching-options :=
      if ($match-options instance of object-node()) then
        match-opt-impl:options-from-json($match-options)
      else
        $match-options
  let $merge-options :=
      if ($merge-options instance of object-node()) then
        merge-impl:options-from-json($merge-options)
      else
        $merge-options
  let $archived-collection := coll-impl:on-archive(map:map(), $merge-options)
  let $normalized-input :=
    if ($input instance of xs:string*) then
      for $doc in cts:search(fn:doc(), cts:and-not-query(cts:document-query($input), cts:collection-query($archived-collection)), "unfiltered")
      return
        proc-impl:build-write-object-for-doc($doc)
    else if ($input instance of map:map*) then
      $input
    else ()
  let $write-objects-by-uri := fn:fold-left(
    function($map, $map-addition) { $map => map:with($map-addition => map:get("uri"), $map-addition) },
    map:map(),
    $normalized-input
  )
  let $uris := map:keys($write-objects-by-uri)
  let $_prime-blocks-cache := blocks-impl:get-blocks-of-uris($uris)
  let $_ := if (xdmp:trace-enabled($const:TRACE-MATCH-RESULTS)) then
        xdmp:trace($const:TRACE-MATCH-RESULTS, "processing: " || fn:string-join($uris, ", "))
      else ()
  let $target-entity := $matching-options/matcher:target-entity ! fn:string(.)
  let $actions := fn:distinct-values(($matching-options/matcher:actions/matcher:action/@name ! fn:string(.), $const:MERGE-ACTION, $const:NOTIFY-ACTION))
  let $thresholds := $matching-options/matcher:thresholds/matcher:threshold[(@action|matcher:action) = $actions]
  let $threshold-labels := $thresholds/(@label|matcher:label)
  let $minimum-threshold :=
    fn:min(
      $thresholds[(@label|matcher:label) = $threshold-labels]/(@above|matcher:above) ! fn:number(.)
    )
  let $merge-threshold :=
    fn:min((
      $thresholds[(@action|matcher:action) = $const:MERGE-ACTION]/(@above|matcher:above) ! fn:number(.),
      999
    ))
  let $_min-threshold-err :=
            if (fn:empty($minimum-threshold)) then
              fn:error($const:NO-THRESHOLD-ACTION-FOUND, "No threshold actions to act on.", ($matching-options/matcher:thresholds))
            else ()
  let $max-scan := fn:head((
    $matching-options//*:max-scan ! xs:integer(.),
    500
  ))
  let $all-matches :=
    let $start-elapsed := xdmp:elapsed-time()
    let $matches :=
        map:new((
          $normalized-input !
            map:entry(
              (. => map:get("uri")),
              matcher:find-document-matches-by-options(
                (. => map:get("value")),
                $matching-options,
                1,
                $max-scan,
                $minimum-threshold,
                fn:false(),
                $filter-query
              )
            )
        ))
    return (
      $matches,
      if (xdmp:trace-enabled($const:TRACE-PERFORMANCE)) then
        xdmp:trace($const:TRACE-PERFORMANCE, "proc-impl:process-match-and-merge-with-options: Matches: " || (xdmp:elapsed-time() - $start-elapsed))
      else ()
    )
  let $consolidated-merges := proc-impl:consolidate-merges($all-matches,
    $matching-options,
    $max-scan,
    $merge-threshold,
    $filter-query)
  let $merged-uris := map:keys($consolidated-merges)
  let $merged-into := map:map()
  let $on-merge-options := $merge-options/merging:algorithms/merging:collections/merging:on-merge
  let $processed-merges :=
    (: Process merges :)
    let $start-elapsed := xdmp:elapsed-time()
    let $merges :=
      for $new-uri in $merged-uris
      where fn:not(map:contains($merges-in-transaction, $new-uri))
      return (
        map:put($merges-in-transaction, $new-uri, fn:true()),
        let $distinct-uris := fn:distinct-values(map:get($consolidated-merges, $new-uri))
        let $merged-doc-def := merge-impl:build-merge-models-by-uri($distinct-uris, $merge-options, $new-uri)
        let $merged-doc := $merged-doc-def => map:get("value")
        let $merge-uri := merge-impl:build-merge-uri(
          $new-uri,
          if ($merged-doc instance of element() or
            $merged-doc instance of document-node(element())) then
            $const:FORMAT-XML
          else
            $const:FORMAT-JSON
        )
        return (
          $distinct-uris ! map:put($merged-into, ., $merge-uri),
          $merged-doc-def
            => map:get("audit-trace")
            => map:with("hidden", fn:true()),
          map:new((
            map:entry("uri", $merge-uri),
            map:entry("value",
              $merged-doc
            ),
            map:entry("context",
              map:new((
                map:entry("collections",
                  (
                    coll-impl:on-merge(
                      map:new((
                        for $uri in $distinct-uris
                        let $write-object := proc-impl:retrieve-write-object($write-objects-by-uri, $uri)
                        return
                          map:entry(
                            $uri,
                            $write-object
                              => map:get("context")
                              => map:get("collections")
                          )
                      )),
                      $on-merge-options
                    ),
                    $target-entity
                  )
                ),
                map:entry("permissions",
                  (
                    xdmp:default-permissions($merge-uri, "objects"),
                    for $uri in $distinct-uris
                    let $write-object := proc-impl:retrieve-write-object($write-objects-by-uri, $uri)
                    return
                      $write-object
                        => map:get("context")
                        => map:get("permissions")
                  )
                )
              ))
            )
          ))
        )
      )
    return (
      $merges,
      if (xdmp:trace-enabled($const:TRACE-PERFORMANCE)) then
        xdmp:trace($const:TRACE-PERFORMANCE, "proc-impl:process-match-and-merge-with-options: ["||xdmp:request()||"] Merges: " || (xdmp:elapsed-time() - $start-elapsed))
      else ()
    )
  let $uris-that-were-merged := map:keys($merged-into)
  let $_archived-updates :=
    (: Process collections on no matches :)
    let $start-elapsed := xdmp:elapsed-time()
    let $archived :=
      let $on-archive := $merge-options/merging:algorithms/merging:collections/merging:on-archive
      for $uri in $uris-that-were-merged
      let $write-object := proc-impl:retrieve-write-object($write-objects-by-uri, $uri)
      let $write-context := $write-object
          => map:get("context")
      let $current-collections := $write-context
          => map:get("collections")
      let $new-collections := coll-impl:on-archive(
        map:entry($uri, $current-collections),
        $on-archive
      )
      return $write-context
        => map:put("collections", $new-collections)
    return (
      $archived,
      if (xdmp:trace-enabled($const:TRACE-PERFORMANCE)) then
        xdmp:trace($const:TRACE-PERFORMANCE, "proc-impl:process-match-and-merge-with-options:["||xdmp:request()||"] Archived: " || (xdmp:elapsed-time() - $start-elapsed))
      else ()
    )
  let $consolidated-notifies := proc-impl:consolidate-notifies($all-matches, $merged-into)
  let $_no-matches-updates :=
    (: Process collections on no matches :)
    let $start-elapsed := xdmp:elapsed-time()
    let $no-matches :=
      let $on-no-match := $merge-options/merging:algorithms/merging:collections/merging:on-no-match
      for $uri in $uris[fn:not(. = $uris-that-were-merged)]
      let $write-object := proc-impl:retrieve-write-object($write-objects-by-uri, $uri)
      let $doc := $write-object
        => map:get("value")
      let $has-merges :=
        if (map:contains($all-matches, $uri)) then
          fn:number(map:get($all-matches, $uri)/@total) gt 0
        else
          fn:number(
            match-impl:find-document-matches-by-options(
              $doc,
              $matching-options,
              1,
              1,
              $merge-threshold,
              (: don't include detailed match information :)
              fn:false(),
              $filter-query,
              (: don't return results. we just want the estimate. :)
              fn:false()
            )/@total
          ) gt 0
      return
      (: If there are merges with a doc outside of this batch then leave it alone. :)
        if ($has-merges) then
          map:delete($write-objects-by-uri, $uri)
        (: otherwise, we want to pick up new collections :)
        else
          let $write-context := $write-object
          => map:get("context")
          let $current-collections := $write-context
          => map:get("collections")
          let $new-collections := (
            coll-impl:on-no-match(
              map:entry($uri, $current-collections),
              $on-no-match
            ),
            $target-entity
          )
          let $_update := $write-context
          => map:put("collections", $new-collections)
          return
            if (fn:not(map:contains($all-matches, $uri))) then
              xdmp:lock-for-update($uri)
            else ()
    return (
        $no-matches,
        if (xdmp:trace-enabled($const:TRACE-PERFORMANCE)) then
          xdmp:trace($const:TRACE-PERFORMANCE, "proc-impl:process-match-and-merge-with-options:["||xdmp:request()||"] No matches: " || (xdmp:elapsed-time() - $start-elapsed))
        else ()
    )
  let $processed-notifications :=
    let $start-elapsed := xdmp:elapsed-time()
    let $notifications :=
      (: Process notifications :)
      for $notification in $consolidated-notifies
      let $parts := fn:tokenize($notification, $STRING-TOKEN)
      let $threshold := fn:head($parts)
      let $uris := fn:tail($parts)
      let $match-write-object := matcher:build-match-notification($threshold, $uris, $merge-options)
      return
        $match-write-object
    return (
      $notifications,
      if (xdmp:trace-enabled($const:TRACE-PERFORMANCE)) then
        xdmp:trace($const:TRACE-PERFORMANCE, "proc-impl:process-match-and-merge-with-options:["||xdmp:request()||"] Notifications: " || (xdmp:elapsed-time() - $start-elapsed))
      else ()
    )
  (: Ensure that we are only writing once to a URI and merges take precedence over archives :)
  let $_processed-merges :=
    for $processed-merge in $processed-merges
    return
      $write-objects-by-uri => map:put(($processed-merge => map:get("uri")), $processed-merge)
  let $_processed-notifications :=
    for $processed-notification in $processed-notifications
    return
      $write-objects-by-uri => map:put(($processed-notification => map:get("uri")), $processed-notification)
  return json:to-array((
    if (xdmp:trace-enabled($const:TRACE-MATCH-RESULTS)) then (
      xdmp:trace($const:TRACE-MATCH-RESULTS, "All matches: " || xdmp:describe($all-matches, (),())),
      xdmp:trace($const:TRACE-MATCH-RESULTS, "Matching options: " || xdmp:describe($matching-options, (),())),
      xdmp:trace($const:TRACE-MATCH-RESULTS, "Merge options: " || xdmp:describe($merge-options, (),())),
      xdmp:trace($const:TRACE-MATCH-RESULTS, "Consolidated merges: " || xdmp:quote($consolidated-merges)),
      xdmp:trace($const:TRACE-MATCH-RESULTS, "Consolidated notifications: " || xdmp:quote($consolidated-notifies))
    )
    else (),
    for $uri in map:keys($write-objects-by-uri)
    return
      $write-objects-by-uri => map:get($uri),
    (: Process custom actions :)
    let $action-map :=
      map:new((
        let $custom-action-names := $matching-options/matcher:thresholds/matcher:threshold/(matcher:action|@action)[fn:not(. = $const:NOTIFY-ACTION or . = $const:MERGE-ACTION)]
        for $custom-action-name in fn:distinct-values($custom-action-names)
        let $action-xml := $matching-options/matcher:actions/matcher:action[@name = $custom-action-name]
        return
          map:entry(
            $custom-action-name,
            fun-ext:function-lookup(
              fn:string($action-xml/@function),
              fn:string($action-xml/@namespace),
              fn:string($action-xml/@at),
              ()
            )
          )
      ))
    for $uri in map:keys($all-matches)
    for $custom-action-match in map:get($all-matches, $uri)/result[fn:not(./@action = $const:NOTIFY-ACTION or ./@action = $const:MERGE-ACTION)]
    let $action-func := map:get($action-map, $custom-action-match/@action)
    return
      if (fn:exists($action-func)) then
        if (fn:ends-with(xdmp:function-module($action-func), "sjs")) then
          xdmp:apply(
            $action-func,
            $uri,
            proc-impl:matches-to-json($custom-action-match),
            merge-impl:options-to-json($merge-options)
          )
        else
          xdmp:apply($action-func, $uri, $custom-action-match, $merge-options)
      else
        fn:error(xs:QName("SM-CONFIGURATION"), "Threshold action is not configured or not found", $custom-action-match),
      if (xdmp:trace-enabled($const:TRACE-PERFORMANCE)) then
        xdmp:trace($const:TRACE-PERFORMANCE, "proc-impl:process-match-and-merge-with-options:["||xdmp:request()||"] " || (xdmp:elapsed-time() - $start-elapsed))
      else ()
  ))
};

(:
 : The workhorse function.
 :)
declare function proc-impl:process-match-and-merge-with-options-save(
  $input as item()*,
  $merge-options as item(),
  $match-options as item(),
  $filter-query as cts:query)
{
  let $actions as item()* := json:array-values(proc-impl:process-match-and-merge-with-options(
      $input,
      $merge-options,
      $match-options,
      $filter-query
    ))
  for $action in $actions
  return
    if ($action instance of map:map) then
      let $uri as xs:string := $action => map:get("uri")
      let $context as map:map? := $action => map:get("context")
      let $permissions := $context => map:get("permissions")
      return (
        if (fn:not(($action => map:get("hidden")))) then
          ($action => map:get("value"))[fn:empty(xdmp:node-uri(.))]
        else (),
        xdmp:document-insert(
          $uri,
          $action => map:get("value"),
          map:new((
            map:entry("collections", $context => map:get("collections")),
            map:entry("permissions", if (fn:exists($permissions)) then $permissions else xdmp:default-permissions($uri, "objects")),
            map:entry("metadata", $context => map:get("metadata"))
          ))
        )
      )
    else
      $action
};

(:
 : Convert the result elements into JSON objects.
 : TODO -- does not yet convert result/match elements to JSON. This is okay for now as there is no way to turn on the
 : $include-matches parameter from process-match-and-merge.
 :)
declare function proc-impl:matches-to-json($filtered-matches as element(result)*)
{
  array-node {
    for $match in $filtered-matches
    return object-node {
      "uri": $match/@uri/fn:string(),
      "score": $match/@score/fn:data(),
      "threshold": $match/@threshold/fn:string()
    }
  }
};

declare function proc-impl:build-write-object-for-doc($doc as document-node())
  as map:map
{
  map:new((
    map:entry("uri", xdmp:node-uri($doc)),
    map:entry("value", $doc),
    map:entry("context", map:new((
      map:entry("collections", xdmp:node-collections($doc)),
      map:entry("metadata", xdmp:node-metadata($doc)),
      map:entry("permissions", xdmp:node-permissions($doc, "objects"))
    )))
  ))
};

declare function proc-impl:retrieve-write-object(
  $write-objects-by-uri as map:map,
  $uri as xs:string
) as map:map?
{
  if (map:contains($write-objects-by-uri, $uri)) then
    $write-objects-by-uri
    => map:get($uri)
  else
    let $write-obj := proc-impl:build-write-object-for-doc(fn:doc($uri))
    return (
      map:put($write-objects-by-uri, $uri, $write-obj),
      $write-obj
    )
};