XQTS: made output more clear on exist version, build date etc.

svn path=/trunk/eXist-1.0/; revision=3652

declare option exist:serialize "method=xml indent=yes";
<test-suite-result xmlns="http://www.w3.org/2005/02/query-test-XQTSResult"
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">

        <implementation name="{util:system-property("product-version")}" version="{util:system-property("product-build")}" anonymous-result-column="false">

                <organization name="eXist Open Source native XML database"  website="http://www.exist-db.org" anonymous="false"/>

                <submittor name="eXist XQTS testsuite runner" email="me@not.com"/>

                <description>

eXist is an Open Source native XML database featuring efficient,
                        index-based XQuery processing, automatic indexing, extensions for full-text
                        search,  XUpdate  support and tight integration with existing XML development tools.

                </description>

                <implementation-defined-items>

                </implementation-defined-items>

                <features>
                        <feature name="Minimum Conformance" supported="true"/>
                </features>

                <context-properties>

                </context-properties>

        </implementation>

        <syntax>XQuery</syntax>

        <test-run dateRun="{fn:current-dateTime()}">
                <test-suite version="{doc('/db/XQTS/XQTSCatalog.xml')/*:test-suite/@version}"/>
                <transformation>

No unusual transformations.

                </transformation>
                <comparison>

No unusual comparisons

                </comparison>
                <otherComments>

The test run took n seconds.

                </otherComments>
        </test-run>

{
    for $case in fn:collection(  '/db/XQTS' )//test-case
    return <test-case name="{$case/@name}" result="{$case/@result}"  />
 }
</test-suite-result>

<test-suite-result xmlns="http://www.w3.org/2005/02/query-test-XQTSResult"
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">

        <implementation name="eXist" version="1.0" anonymous-result-column="false">

                <organization name="eXist Open Source native XML database"
                        website="http://www.exist-db.org" anonymous="false"/>

                <submittor name="eXist XQTS testsuite runner" email="me@not.com"/>

                <description>

eXist is an Open Source native XML database featuring efficient,
                        index-based XQuery processing, automatic indexing, extensions for full-text
                        search,  XUpdate  support and tight integration with existing XML development tools.

                </description>

                <implementation-defined-items>

                </implementation-defined-items>

                <features>
                        <feature name="Minimum Conformance" supported="true"/>
                </features>

                <context-properties>

                </context-properties>

        </implementation>

        <syntax>XQuery</syntax>

        <test-run dateRun="{fn:current-dateTime()}">
                <test-suite version="0.1"/>
                <transformation>

No unusual transformations.

Really!

                </transformation>
                <comparison>

No unusual comparisons

                </comparison>
                <otherComments>

The test run took 100 seconds.

                </otherComments>
        </test-run>

{
    for $case in fn:collection(  '/db/XQTS' )//test-case
    return <test-case name="{$case/@name}" result="{$case/@result}"  />
 }
</test-suite-result>


[bugfix] Tests should not assume order of entries in a XDM map()

xquery version "3.1";

module namespace facet="http://exist-db.org/xquery/lucene/test/facets";

declare namespace test="http://exist-db.org/xquery/xqsuite";

declare variable $facet:XML :=
    <letters>
        <letter>
            <from>Hans</from>
            <to>Egon</to>
            <place>Berlin</place>
            <date>2019-03-14</date>
            <time>14:22:19.329+01:00</time>
            <likes>9</likes>
            <score>6.0</score>
            <subject>art</subject>
            <subject>math</subject>
        </letter>
        <letter>
            <from>Rudi</from>
            <to>Egon</to>
            <place>Berlin</place>
            <date>2017-03-13</date>
            <time>15:22:19.329+01:00</time>
            <likes>19</likes>
            <score>8.25</score>
            <subject>history</subject>
        </letter>
        <letter>
            <from>Susi</from>
            <to>Hans</to>
            <place>Hamburg</place>
            <date>2019-04-01</date>
            <likes>29</likes>
            <score>16.5</score>
            <subject>engineering</subject>
            <subject>history</subject>
        </letter>
        <letter>
            <from>Heinz</from>
            <to>Babsi Müller</to>
            <place></place>
            <date>2017-03-11</date>
            <likes>1</likes>
            <score>14.25</score>
            <subject>history</subject>
        </letter>
        <letter>
            <from>Heinz</from>
            <to>Basia Müller</to>
            <place>Wrocław</place>
            <date>2015-06-22</date>
            <likes>5</likes>
            <score>29.50</score>
            <subject>history</subject>
        </letter>
        <letter>
            <from>Heinz</from>
            <to>Basia Kowalska</to>
            <place>Wrocław</place>
            <date>2013-06-22</date>
            <likes>3</likes>
            <subject>history</subject>
            <score>16.0</score>
        </letter>
    </letters>;

declare variable $facet:TAXONOMY :=
    <places>
        <place name="Germany">
            <place name="Berlin"/>
            <place name="Hamburg"/>
        </place>
        <place name="Poland">
            <place name="Wrocław"/>
            <place name="Kraków"/>
        </place>
    </places>;

declare variable $facet:SUBJECT :=
    <subject>
        <subject name="science">
            <subject name="math"/>
            <subject name="engineering"/>
        </subject>
        <subject name="humanities">
            <subject name="art"/>
            <subject name="sociology"/>
            <subject name="history"/>
        </subject>
    </subject>;

declare variable $facet:DOCUMENTS :=
    <documents>
        <document id="D-37/2">
            <title>Ruhe im Wald</title>
            <abstract>Es zwitschern die Vögel im Walde</abstract>
            <abstract>Über dem Walde weht ein Wind</abstract>
            <category>nature</category>
        </document>
        <document id="Z-49/2">
            <title>Streiten und Hoffen</title>
            <abstract>Da nun einmal der Himmel zerrissen und die Götter sich streiten</abstract>
            <category>philosophy</category>
        </document>
    </documents>;

declare variable $facet:MULTI_LANGUAGE :=
    <text>

Es zwitschern<note>singen</note> die Vögel

And the birds are singing

    </text>;

declare variable $facet:RECORDS := 10;

declare variable $facet:CITIES :=
    <cities>{
        for $id in 1 to $facet:RECORDS
        return
            <city>
                <id>{$id}</id>
                City {$id}
            </city>
    }</cities>;

declare variable $facet:PERSONS :=
    for $id in 1 to $facet:RECORDS
    return
        <person>
            <id>{$id}</id>
            <city-id>{ $facet:RECORDS - $id + 1 }</city-id>
        </person>;

declare variable $facet:MODULE :=
    ``[xquery version "3.1";

module namespace idx="http://exist-db.org/lucene/test/";

declare function idx:place-hierarchy($key as xs:string?) {
    if (exists($key)) then
        doc('/db/lucenetest/places.xml')//place[@name=$key]/ancestor-or-self::place/@name
    else
        ()
};

declare function idx:subject-hierarchy($key as xs:string*) {
    if (exists($key)) then
         array:for-each(array {$key}, function($k) {
             doc('/db/lucenetest/subjects.xml')//subject[@name=$k]/ancestor-or-self::subject/@name
         })
    else
        ()
};

declare function idx:city-id-to-label($id as xs:string) {
    let $city := doc('/db/lucenetest/cities.xml')//city[id eq $id]
    return
        if (exists($city)) then
            $city/label/string()
        else
            "unknown"
};

declare function idx:people-from-city($city-id as xs:string) {
    let $people := collection('/db/lucenetest')//person[city-id eq $city-id]/id/string()
    return
        if (exists($people)) then
            $people
        else
            "none"
};
]``;

declare variable $facet:XCONF1 :=
    <collection xmlns="http://exist-db.org/collection-config/1.0">
        <index xmlns:xs="http://www.w3.org/2001/XMLSchema">
            <lucene>
                <analyzer class="org.exist.indexing.lucene.analyzers.NoDiacriticsStandardAnalyzer" id="nodiacritics"/>
                <analyzer class="org.apache.lucene.analysis.core.KeywordAnalyzer" id="keyword"/>
                <analyzer class="org.apache.lucene.analysis.de.GermanAnalyzer" id="german"/>
                <analyzer class="org.apache.lucene.analysis.en.EnglishAnalyzer" id="english"/>
                <module uri="http://exist-db.org/lucene/test/" prefix="idx" at="module.xql"/>
                <text qname="letter" analyzer="nodiacritics">
                    <facet dimension="place" expression="place"/>
                    <facet dimension="location" expression="idx:place-hierarchy(place)" hierarchical="yes"/>
                    <facet dimension="subject" expression="idx:subject-hierarchy(subject)" hierarchical="yes"/>
                    <facet dimension="from" expression="from"/>
                    <facet dimension="to" expression="to"/>
                    <facet dimension="date" expression="tokenize(date, '-')" hierarchical="yes"/>
                    <field name="place" expression="place" analyzer="nodiacritics"/>
                    <field name="from" expression="from" store="no"/>
                    <field name="to" expression="to"/>
                    <field name="date" expression="date" type="xs:date"/>
                    <field name="likes" expression="likes" type="xs:int"/>
                    <field name="score" expression="score" type="xs:double"/>
                    <field name="time" expression="time" type="xs:time"/>
                </text>
                <text qname="document">
                    <field name="title" expression="title"/>
                    <field name="abstract" expression="abstract" analyzer="german"/>
                    <field name="ident" expression="@id/string()" analyzer="keyword"/>
                    <facet dimension="cat" expression="category"/>
                </text>
                <text match="//document/abstract">
                    <field name="german" analyzer="german"/>
                    <facet dimension="cat" expression="../category"/>
                </text>
                <text match="/text/body/div" index="no">
                    <field name="german" if="ancestor::body[@xml:lang = 'de']" analyzer="german"/>
                    <field name="english" if="ancestor::body[@xml:lang = 'en']" analyzer="english"/>
                    <facet dimension="language" expression="ancestor::body/@xml:lang"/>
                    <ignore qname="note"/>
                </text>
                <text qname="person">
                    <facet dimension="city" expression="idx:city-id-to-label(city-id)"/>
                </text>
                <text qname="city">
                    <facet dimension="person" expression="idx:people-from-city(id)"/>
                </text>
            </lucene>
            <range>
                <create qname="id" type="xs:string"/>
                <create qname="city-id" type="xs:string"/>
            </range>
        </index>
    </collection>;

declare
    %test:setUp
function facet:setup() {
    let $testCol := xmldb:create-collection("/db", "lucenetest")
    let $personsCol := xmldb:create-collection("/db/lucenetest", "persons")
    let $confCol := xmldb:create-collection("/db/system/config/db", "lucenetest")
    return (
        xmldb:store($testCol, "module.xql", $facet:MODULE, "application/xquery"),
        xmldb:store($confCol, "collection.xconf", $facet:XCONF1),
        xmldb:store($testCol, "places.xml", $facet:TAXONOMY),
        xmldb:store($testCol, "subjects.xml", $facet:SUBJECT),
        xmldb:store($testCol, "test.xml", $facet:XML),
        xmldb:store($testCol, "documents.xml", $facet:DOCUMENTS),
        xmldb:store($testCol, "multi-lang.xml", $facet:MULTI_LANGUAGE),
        $facet:PERSONS ! xmldb:store($personsCol, ./id || ".xml", .),
        xmldb:store($testCol, "cities.xml", $facet:CITIES),
        xmldb:reindex($personsCol)
    )
};

declare
    %test:tearDown
function facet:tearDown() {
    xmldb:remove("/db/lucenetest"),
    xmldb:remove("/db/system/config/db/lucenetest")
};

declare
    %private
function facet:map-to-string($map) as xs:string* {
    map:for-each($map, function($k, $v) {
        $k || "=" || $v
    })
};

declare
    %test:assertEquals(2, 1, 1, 2)
function facet:query-all-and-facets() {
    let $result := collection("/db/lucenetest")//letter[ft:query(., ())]
    let $where := ft:facets($result, "place", ())
    let $from := ft:facets($result, "from", ())
    let $to := ft:facets($result, "to", ())
    return (
        $where?Berlin, $where?Hamburg, $from?Susi, $to?Egon
    )
};

declare
    %test:assertEmpty
function facet:query-all-and-non-existing-facet() {
    let $result := collection("/db/lucenetest")//letter[ft:query(., ())]
    let $facet := ft:facets($result, "does-not-exist", ())
    return
        $facet?*
};

declare
    %test:arg("from", "Rudi")
    %test:assertEquals(1)
    %test:arg("from", "Susi")
    %test:assertEquals(0)
    %test:arg("from", "Rudi", "Hans")
    %test:assertEquals(2)
function facet:query-and-drill-down($from as xs:string+) {
    let $options := map {
        "facets": map {
            "from": $from
        }
    }
    return
        count(collection("/db/lucenetest")//letter[ft:query(., "Berlin", $options)])
};

declare
    %test:assertEquals(5)
function facet:query-and-drill-down-hierarchical-sequence() {
    let $options := map {
            "leading-wildcard": "yes",
            "filter-rewrite": "yes",
            "facets": map {
                "subject": ("humanities", "history")
            }
        }
    return
        count(collection("/db/lucenetest")//letter[ft:query(., (), $options)])
};

declare
    %test:assertEquals(5)
function facet:query-and-drill-down-hierarchical-array-single() {
    let $options := map {
            "leading-wildcard": "yes",
            "filter-rewrite": "yes",
            "facets": map {
                "subject": [("humanities", "history")]
            }
        }
    return
        count(collection("/db/lucenetest")//letter[ft:query(., (), $options)])
};

declare
    %test:assertEquals(6)
function facet:query-and-drill-down-hierarchical-array-multi() {
    let $options := map {
            "leading-wildcard": "yes",
            "filter-rewrite": "yes",
            "facets": map {
                "subject": [("humanities", "history"), ("humanities", "art")]
            }
        }
    return
        count(collection("/db/lucenetest")//letter[ft:query(., (), $options)])
};

declare
    %test:assertEquals(2)
function facet:query-and-drill-down-hierarchical-array-multi2() {
    let $options := map {
            "leading-wildcard": "yes",
            "filter-rewrite": "yes",
            "facets": map {
                "subject": [("humanities", "art"), ("science", "engineering")]
            }
        }
    return
        count(collection("/db/lucenetest")//letter[ft:query(., (), $options)])
};

declare
    %test:assertEquals(5)
function facet:query-and-drill-down-hierarchical-array-multi3() {
    let $options := map {
            "leading-wildcard": "yes",
            "filter-rewrite": "yes",
            "facets": map {
                "subject": [("humanities", "history"), ("science", "engineering")]
            }
        }
    return
        count(collection("/db/lucenetest")//letter[ft:query(., (), $options)])
};

declare
    %test:assertEquals(1)
function facet:query-and-drill-down-hierarchical-array-multi4() {
    let $options := map {
            "leading-wildcard": "yes",
            "filter-rewrite": "yes",
            "facets": map {
                "subject": [("humanities", "art"), ("science", "math")]
            }
        }
    return
        count(collection("/db/lucenetest")//letter[ft:query(., (), $options)])
};

(:~
 : The facet 'cat' is defined on both: the index on 'document' and 'document/abstract'.
 : Querying 'document' should return 1 as facet count, while 'abstract' should return 2
 : as there are two abstracts being indexed and each has the facet 'nature'.
 :)
declare
    %test:assertEquals(1, 2)
function facet:multiple-indexes-with-same-facet() {
    let $result := doc("/db/lucenetest/documents.xml")//document[ft:query(., ())]
    return
        ft:facets($result, "cat")?nature,
    let $result := doc("/db/lucenetest/documents.xml")//document/abstract[ft:query(., ())]
    return
        ft:facets($result, "cat")?nature
};

declare
    %test:assertEquals(4, 2)
function facet:store-and-remove() {
    let $stored := xmldb:store("/db/lucenetest", "test2.xml", $facet:XML)
    let $result := collection("/db/lucenetest")//letter[ft:query(., ())]
    let $where := ft:facets($result, "place", 10)
    return
        $where?Berlin,
    xmldb:remove("/db/lucenetest", "test2.xml"),
    let $result := collection("/db/lucenetest")//letter[ft:query(., ())]
    let $where := ft:facets($result, "place", 10)
    return
        $where?Berlin
};

declare
    %test:arg("paths", "2017")
    %test:assertEquals(2)
    %test:arg("paths", "2019")
    %test:assertEquals(2)
    %test:arg("paths", "2019", "03")
    %test:assertEquals(1)
    %test:arg("paths", "2019", "03", "14")
    %test:assertEquals(1)
    %test:arg("paths", "2019", "03", "08")
    %test:assertEquals(0)
function facet:hierarchical-facets-query($paths as xs:string+) {
    let $options := map {
            "facets": map {
                "date": $paths
            }
        }
    let $result := collection("/db/lucenetest")//letter[ft:query(., (), $options)]
    return
        count($result)
};

declare
    %test:arg("paths", "science")
    %test:assertEquals(2)
    %test:arg("paths", "science", "engineering")
    %test:assertEquals(1)
    %test:arg("paths", "humanities", "history")
    %test:assertEquals(5)
    %test:arg("paths", "humanities", "art")
    %test:assertEquals(1)
    %test:arg("paths", "science", "math")
    %test:assertEquals(1)
function facet:hierarchical-facets-query-subjects($paths as xs:string+) {
    let $options := map {
            "facets": map {
                "subject": $paths
            }
        }
    let $result := collection("/db/lucenetest")//letter[ft:query(., (), $options)]
    return
        count($result)
};

declare
    %test:arg("paths", "2017")
    %test:assertEquals('{"03":2}')
    %test:arg("paths", "2019")
    %test:assertEquals('{"03":1,"04":1}')
function facet:hierarchical-facets-retrieve($paths as xs:string*) {
    let $result := collection("/db/lucenetest")//letter[ft:query(., ())]
    let $facets := ft:facets($result, "date", (), $paths)
    return
        serialize($facets, map { "method": "json" })
};

declare
    %test:arg("paths", "science")
    %test:assertEqualsPermutation("math=1", "engineering=1")
function facet:hierarchical-facets-retrieve($paths as xs:string*) {
    let $result := collection("/db/lucenetest")//letter[ft:query(., ())]
    let $facets := ft:facets($result, "subject", (), $paths)
    return
        facet:map-to-string($facets)
};

declare
    %test:assertEqualsPermutation("Berlin=2", "Hamburg=1", "Wrocław=2")
function facet:hierarchical-place() {
    let $result := collection("/db/lucenetest")//letter[ft:query(., ())]
    let $facets := ft:facets($result, "location", 10) (: Returns facet counts for "Germany" and "Poland" :)
    for $country in map:keys($facets)
    order by $country
    return
        facet:map-to-string(
            ft:facets($result, "location", 10, $country) (: Get facet counts for sub-categories :)
        )
};

declare
    %test:assertEqualsPermutation("history=5", "art=1", "math=1", "engineering=1")
function facet:hierarchical-subject() {
    let $result := collection("/db/lucenetest")//letter[ft:query(., ())]
    let $facets := ft:facets($result, "subject", 10) (: Returns facet counts for "science" and "humanities" :)
    for $topic in map:keys($facets)
    order by $topic
    return
        facet:map-to-string(
            ft:facets($result, "subject", 10, $topic) (: Get facet counts for sub-categories :)
        )
};

declare
    %test:assertEquals('{"history":1}','{"engineering":1}')
function facet:hierarchical-multivalue-subject() {
    let $result := collection("/db/lucenetest")//letter[ft:query(., 'from:susi')]
    let $facets := ft:facets($result, "subject", 10) (: Returns facet counts for "science" and "humanities" :)
    for $topic in map:keys($facets)
    order by $topic
    return
        serialize(
            ft:facets($result, "subject", 10, $topic), (: Get facet counts for sub-categories :)
            map { "method": "json" }
        )
};

declare
    %test:args("place:hamburg")
    %test:assertEquals(1)
    %test:args("place:berlin")
    %test:assertEquals(2)
    %test:args("from:rudi AND place:berlin")
    %test:assertEquals(1)
    %test:args("from:susi AND place:berlin")
    %test:assertEquals(0)
    %test:args("basia AND place:wroclaw")
    %test:assertEquals(2)
    %test:args("place:wroclaw")
    %test:assertEquals(2)
    %test:args('to:(ba* müller)')
    %test:assertEquals(2)
    %test:args('foo:berlin')
    %test:assertEquals(0)
function facet:query-field($query as xs:string) {
    count(collection("/db/lucenetest")//letter[ft:query(., $query)])
};

declare
    %test:assertEquals("Babsi Müller", "Basia Kowalska", "Basia Müller")
function facet:query-and-sort() {
    for $letter in collection("/db/lucenetest")//letter[ft:query(., "from:heinz", map { "fields": "to" })]
    order by ft:field($letter, "to")
    return
        $letter/to/text()
};

declare
    %test:assertEquals("Basia Kowalska", "Basia Müller", "Babsi Müller")
function facet:query-and-sort-by-date() {
    for $letter in collection("/db/lucenetest")//letter[ft:query(., "from:heinz", map { "fields": "date" })]
    order by ft:field($letter, $field, "xs:date")
    return
        $letter/to/text()
};

declare
    %test:assertEquals("Hans", "Rudi")
function facet:query-and-sort-by-date() {
    for $letter in collection("/db/lucenetest")//letter[ft:query(., "place:berlin", map { "fields": "time" })]
    order by ft:field($letter, "time", "xs:time")
    return
        $letter/from/text()
};

declare
    %test:args("likes", "xs:int")
    %test:assertEquals(1, 3, 5, 9, 19, 29)
    %test:args("score", "xs:float")
    %test:assertEquals(6, 8.25, 14.25, 16, 16.5, 29.5)
function facet:query-and-sort-by-numeric($field as xs:string, $type as xs:string) {
    for $letter in collection("/db/lucenetest")//letter[ft:query(., (), map { "fields": $field })]
    let $likes := ft:field($letter, $field, $type)
    order by $likes
    return
        $likes
};

declare
    %test:assertEmpty
function facet:retrieve-non-existant-field() {
    for $letter in collection("/db/lucenetest")//letter[ft:query(., (), map { "fields": "foo" })]
    return
        ft:field($letter, "foo")
};

declare
    %test:assertEmpty
function facet:retrieve-not-stored() {
    for $letter in collection("/db/lucenetest")//letter[ft:query(., (), map { "fields": "from" })]
    return
        ft:field($letter, "from")
};

declare
    %test:assertEquals("Egon", "Berlin")
function facet:retrieve-multiple-fields() {
    for $letter in collection("/db/lucenetest")//letter[ft:query(., "from:rudi", map { "fields": ("to", "place") })]
    return
        (ft:field($letter, "to"), ft:field($letter, "place"))
};

declare
    %test:assertEquals("2017-03-13", 19, 8.25)
function facet:test-field-type() {
    let $letter := collection("/db/lucenetest")//letter[ft:query(., "from:rudi", map { "fields": ("date", "likes", "score") })]
    return (
        ft:field($letter, "date", "xs:date"),
        ft:field($letter, "likes", "xs:integer"),
        ft:field($letter, "score", "xs:double")
    )
};

declare
    %test:assertEquals(4)
function facet:index-keys() {
    count(collection("/db/lucenetest")/ft:index-keys-for-field("from", (), function($key, $count) { $key }, 10))
};

(:~
 : Check if option index="no" is applied properly: no default index, but the fields should be queryable.
 :)
declare
    %test:args("birds")
    %test:assertEmpty
    %test:args("english:birds")
    %test:assertEquals("
And the birds are singing
")
function facet:query-no-index($query as xs:string) {
    doc("/db/lucenetest/multi-lang.xml")//div[ft:query(., $query)]/p
};

declare
    %test:args("müller")
    %test:assertEquals(2)
function facet:default-analyzer-no-diacritics($query as xs:string) {
    count(collection("/db/lucenetest")//letter[ft:query(., $query)])
};

(:~
 : The configuration defines no default index on 'div', but two fields: 'english' and 'german'.
 : It also configures a facet containing the language. Querying the div with field 'english' set
 : should result in a facet count of 1 for language 'en'.
 :)
declare
    %test:assertEquals(1)
function facet:query-no-default-index-but-facet() {
    let $result := doc("/db/lucenetest/multi-lang.xml")//div[ft:query(., "english:*", map { "leading-wildcard": "yes" })]
    return
        ft:facets($result, "language")?en
};

declare
    (: German stemming :)
    %test:args("abstract:vogel")
    %test:assertEquals("<title>Ruhe im Wald</title>")
    %test:args("abstract:wald")
    %test:assertEquals("<title>Ruhe im Wald</title>")
    %test:args("abstract:streit")
    %test:assertEquals("<title>Streiten und Hoffen</title>")
    %test:args("title:streiten AND abstract:streit")
    %test:assertEquals("<title>Streiten und Hoffen</title>")
    (: No stemming on title :)
    %test:args("title:streit AND abstract:streit")
    %test:assertEmpty
function facet:query-field-with-analyzer($query as xs:string) {
    doc("/db/lucenetest/documents.xml")//document[ft:query(., $query)]/title
};

(: Index on 'abstract' uses default analyzer but has a field
 : with German analyzer indexing same content.
 :)
declare
    %test:args("german:vogel")
    %test:assertEquals("<title>Ruhe im Wald</title>")
    %test:args("vogel")
    %test:assertEmpty
    %test:args("vögel")
    %test:assertEquals("<title>Ruhe im Wald</title>")
    %test:args("german:streit")
    %test:assertEquals("<title>Streiten und Hoffen</title>")
    %test:args("german:streit AND götter")
    %test:assertEquals("<title>Streiten und Hoffen</title>")
    %test:args("german:streit AND gott")
    %test:assertEmpty
    %test:args("german:(streit AND gott)")
    %test:assertEquals("<title>Streiten und Hoffen</title>")
function facet:query-field-no-expression($query as xs:string) {
    doc("/db/lucenetest/documents.xml")//abstract[ft:query(., $query)]/../title
};

declare
    %test:args('<query><term field="ident">Z-49/2</term></query>')
    %test:assertEquals("<title>Streiten und Hoffen</title>")
function facet:query-field-with-keyword-analyzer($query as element()) {
    doc("/db/lucenetest/documents.xml")//document[ft:query(., $query)]/title
};

declare
    %test:args('german:vogel')
    %test:assertEquals(1)
    %test:args('german:vögel')
    %test:assertEquals(1)
    %test:args('german:birds')
    %test:assertEquals(0)
    %test:args('english:bird')
    %test:assertEquals(1)
    %test:args('english:vögel')
    %test:assertEquals(0)
function facet:query-field-with-condition($query as xs:string) {
    count(doc("/db/lucenetest/multi-lang.xml")//div[ft:query(., $query)])
};

declare
    %test:args('german:singen')
    %test:assertEmpty
function facet:field-respects-ignore($query as xs:string) {
    doc("/db/lucenetest/multi-lang.xml")//div[ft:query(., $query)]
};

declare
    %test:args("vögel")
    %test:assertEquals(2)
    %test:args("walde")
    %test:assertEquals(3)
function facet:query-with-union-and-facets($query as xs:string) {
    let $results := doc("/db/lucenetest/documents.xml")//document[ft:query(., $query)] |
        doc("/db/lucenetest/documents.xml")//document[ft:query(abstract, $query)]
    return
        ft:facets($results, "cat")?nature
};

declare
    %test:assertEmpty
function facet:avoid-range-index-conflict-person() {
    let $people := collection("/db/lucenetest")//person[ft:query(., ())]
    let $city-facet := ft:facets($people, "city")
    return
        if ($people and $city-facet?unknown) then
            $city-facet?unknown || " person values were calculated without id range indexes present"
        else
            ()
};

declare
    %test:assertEmpty
function facet:avoid-range-index-conflict-city() {
    let $cities := collection("/db/lucenetest")//city[ft:query(., ())]
    let $persons-facet := ft:facets($cities, "person")
    return
        if ($cities and $persons-facet?none) then
            $persons-facet?none || " city values were calculated without id range indexes present"
        else
            ()
};

xquery version "3.1";

module namespace facet="http://exist-db.org/xquery/lucene/test/facets";

declare namespace test="http://exist-db.org/xquery/xqsuite";

declare variable $facet:XML :=
    <letters>
        <letter>
            <from>Hans</from>
            <to>Egon</to>
            <place>Berlin</place>
            <date>2019-03-14</date>
            <time>14:22:19.329+01:00</time>
            <likes>9</likes>
            <score>6.0</score>
            <subject>art</subject>
            <subject>math</subject>
        </letter>
        <letter>
            <from>Rudi</from>
            <to>Egon</to>
            <place>Berlin</place>
            <date>2017-03-13</date>
            <time>15:22:19.329+01:00</time>
            <likes>19</likes>
            <score>8.25</score>
            <subject>history</subject>
        </letter>
        <letter>
            <from>Susi</from>
            <to>Hans</to>
            <place>Hamburg</place>
            <date>2019-04-01</date>
            <likes>29</likes>
            <score>16.5</score>
            <subject>engineering</subject>
            <subject>history</subject>
        </letter>
        <letter>
            <from>Heinz</from>
            <to>Babsi Müller</to>
            <place></place>
            <date>2017-03-11</date>
            <likes>1</likes>
            <score>14.25</score>
            <subject>history</subject>
        </letter>
        <letter>
            <from>Heinz</from>
            <to>Basia Müller</to>
            <place>Wrocław</place>
            <date>2015-06-22</date>
            <likes>5</likes>
            <score>29.50</score>
            <subject>history</subject>
        </letter>
        <letter>
            <from>Heinz</from>
            <to>Basia Kowalska</to>
            <place>Wrocław</place>
            <date>2013-06-22</date>
            <likes>3</likes>
            <subject>history</subject>
            <score>16.0</score>
        </letter>
    </letters>;

declare variable $facet:TAXONOMY :=
    <places>
        <place name="Germany">
            <place name="Berlin"/>
            <place name="Hamburg"/>
        </place>
        <place name="Poland">
            <place name="Wrocław"/>
            <place name="Kraków"/>
        </place>
    </places>;

declare variable $facet:SUBJECT :=
    <subject>
        <subject name="science">
            <subject name="math"/>
            <subject name="engineering"/>
        </subject>
        <subject name="humanities">
            <subject name="art"/>
            <subject name="sociology"/>
            <subject name="history"/>
        </subject>
    </subject>;

declare variable $facet:DOCUMENTS :=
    <documents>
        <document id="D-37/2">
            <title>Ruhe im Wald</title>
            <abstract>Es zwitschern die Vögel im Walde</abstract>
            <abstract>Über dem Walde weht ein Wind</abstract>
            <category>nature</category>
        </document>
        <document id="Z-49/2">
            <title>Streiten und Hoffen</title>
            <abstract>Da nun einmal der Himmel zerrissen und die Götter sich streiten</abstract>
            <category>philosophy</category>
        </document>
    </documents>;

declare variable $facet:MULTI_LANGUAGE :=
    <text>

Es zwitschern<note>singen</note> die Vögel

And the birds are singing

    </text>;

declare variable $facet:RECORDS := 10;

declare variable $facet:CITIES :=
    <cities>{
        for $id in 1 to $facet:RECORDS
        return
            <city>
                <id>{$id}</id>
                City {$id}
            </city>
    }</cities>;

declare variable $facet:PERSONS :=
    for $id in 1 to $facet:RECORDS
    return
        <person>
            <id>{$id}</id>
            <city-id>{ $facet:RECORDS - $id + 1 }</city-id>
        </person>;

declare variable $facet:MODULE :=
    ``[xquery version "3.1";

module namespace idx="http://exist-db.org/lucene/test/";

declare function idx:place-hierarchy($key as xs:string?) {
    if (exists($key)) then
        doc('/db/lucenetest/places.xml')//place[@name=$key]/ancestor-or-self::place/@name
    else
        ()
};

declare function idx:subject-hierarchy($key as xs:string*) {
    if (exists($key)) then
         array:for-each(array {$key}, function($k) {
             doc('/db/lucenetest/subjects.xml')//subject[@name=$k]/ancestor-or-self::subject/@name
         })
    else
        ()
};

declare function idx:city-id-to-label($id as xs:string) {
    let $city := doc('/db/lucenetest/cities.xml')//city[id eq $id]
    return
        if (exists($city)) then
            $city/label/string()
        else
            "unknown"
};

declare function idx:people-from-city($city-id as xs:string) {
    let $people := collection('/db/lucenetest')//person[city-id eq $city-id]/id/string()
    return
        if (exists($people)) then
            $people
        else
            "none"
};
]``;

declare variable $facet:XCONF1 :=
    <collection xmlns="http://exist-db.org/collection-config/1.0">
        <index xmlns:xs="http://www.w3.org/2001/XMLSchema">
            <lucene>
                <analyzer class="org.exist.indexing.lucene.analyzers.NoDiacriticsStandardAnalyzer" id="nodiacritics"/>
                <analyzer class="org.apache.lucene.analysis.core.KeywordAnalyzer" id="keyword"/>
                <analyzer class="org.apache.lucene.analysis.de.GermanAnalyzer" id="german"/>
                <analyzer class="org.apache.lucene.analysis.en.EnglishAnalyzer" id="english"/>
                <module uri="http://exist-db.org/lucene/test/" prefix="idx" at="module.xql"/>
                <text qname="letter" analyzer="nodiacritics">
                    <facet dimension="place" expression="place"/>
                    <facet dimension="location" expression="idx:place-hierarchy(place)" hierarchical="yes"/>
                    <facet dimension="subject" expression="idx:subject-hierarchy(subject)" hierarchical="yes"/>
                    <facet dimension="from" expression="from"/>
                    <facet dimension="to" expression="to"/>
                    <facet dimension="date" expression="tokenize(date, '-')" hierarchical="yes"/>
                    <field name="place" expression="place" analyzer="nodiacritics"/>
                    <field name="from" expression="from" store="no"/>
                    <field name="to" expression="to"/>
                    <field name="date" expression="date" type="xs:date"/>
                    <field name="likes" expression="likes" type="xs:int"/>
                    <field name="score" expression="score" type="xs:double"/>
                    <field name="time" expression="time" type="xs:time"/>
                </text>
                <text qname="document">
                    <field name="title" expression="title"/>
                    <field name="abstract" expression="abstract" analyzer="german"/>
                    <field name="ident" expression="@id/string()" analyzer="keyword"/>
                    <facet dimension="cat" expression="category"/>
                </text>
                <text match="//document/abstract">
                    <field name="german" analyzer="german"/>
                    <facet dimension="cat" expression="../category"/>
                </text>
                <text match="/text/body/div" index="no">
                    <field name="german" if="ancestor::body[@xml:lang = 'de']" analyzer="german"/>
                    <field name="english" if="ancestor::body[@xml:lang = 'en']" analyzer="english"/>
                    <facet dimension="language" expression="ancestor::body/@xml:lang"/>
                    <ignore qname="note"/>
                </text>
                <text qname="person">
                    <facet dimension="city" expression="idx:city-id-to-label(city-id)"/>
                </text>
                <text qname="city">
                    <facet dimension="person" expression="idx:people-from-city(id)"/>
                </text>
            </lucene>
            <range>
                <create qname="id" type="xs:string"/>
                <create qname="city-id" type="xs:string"/>
            </range>
        </index>
    </collection>;

declare
    %test:setUp
function facet:setup() {
    let $testCol := xmldb:create-collection("/db", "lucenetest")
    let $personsCol := xmldb:create-collection("/db/lucenetest", "persons")
    let $confCol := xmldb:create-collection("/db/system/config/db", "lucenetest")
    return (
        xmldb:store($testCol, "module.xql", $facet:MODULE, "application/xquery"),
        xmldb:store($confCol, "collection.xconf", $facet:XCONF1),
        xmldb:store($testCol, "places.xml", $facet:TAXONOMY),
        xmldb:store($testCol, "subjects.xml", $facet:SUBJECT),
        xmldb:store($testCol, "test.xml", $facet:XML),
        xmldb:store($testCol, "documents.xml", $facet:DOCUMENTS),
        xmldb:store($testCol, "multi-lang.xml", $facet:MULTI_LANGUAGE),
        $facet:PERSONS ! xmldb:store($personsCol, ./id || ".xml", .),
        xmldb:store($testCol, "cities.xml", $facet:CITIES),
        xmldb:reindex($personsCol)
    )
};

declare
    %test:tearDown
function facet:tearDown() {
    xmldb:remove("/db/lucenetest"),
    xmldb:remove("/db/system/config/db/lucenetest")
};

declare
    %test:assertEquals(2, 1, 1, 2)
function facet:query-all-and-facets() {
    let $result := collection("/db/lucenetest")//letter[ft:query(., ())]
    let $where := ft:facets($result, "place", ())
    let $from := ft:facets($result, "from", ())
    let $to := ft:facets($result, "to", ())
    return (
        $where?Berlin, $where?Hamburg, $from?Susi, $to?Egon
    )
};

declare
    %test:assertEmpty
function facet:query-all-and-non-existing-facet() {
    let $result := collection("/db/lucenetest")//letter[ft:query(., ())]
    let $facet := ft:facets($result, "does-not-exist", ())
    return
        $facet?*
};

declare
    %test:arg("from", "Rudi")
    %test:assertEquals(1)
    %test:arg("from", "Susi")
    %test:assertEquals(0)
    %test:arg("from", "Rudi", "Hans")
    %test:assertEquals(2)
function facet:query-and-drill-down($from as xs:string+) {
    let $options := map {
        "facets": map {
            "from": $from
        }
    }
    return
        count(collection("/db/lucenetest")//letter[ft:query(., "Berlin", $options)])
};

declare
    %test:assertEquals(5)
function facet:query-and-drill-down-hierarchical-sequence() {
    let $options := map {
            "leading-wildcard": "yes",
            "filter-rewrite": "yes",
            "facets": map {
                "subject": ("humanities", "history")
            }
        }
    return
        count(collection("/db/lucenetest")//letter[ft:query(., (), $options)])
};

declare
    %test:assertEquals(5)
function facet:query-and-drill-down-hierarchical-array-single() {
    let $options := map {
            "leading-wildcard": "yes",
            "filter-rewrite": "yes",
            "facets": map {
                "subject": [("humanities", "history")]
            }
        }
    return
        count(collection("/db/lucenetest")//letter[ft:query(., (), $options)])
};

declare
    %test:assertEquals(6)
function facet:query-and-drill-down-hierarchical-array-multi() {
    let $options := map {
            "leading-wildcard": "yes",
            "filter-rewrite": "yes",
            "facets": map {
                "subject": [("humanities", "history"), ("humanities", "art")]
            }
        }
    return
        count(collection("/db/lucenetest")//letter[ft:query(., (), $options)])
};

declare
    %test:assertEquals(2)
function facet:query-and-drill-down-hierarchical-array-multi2() {
    let $options := map {
            "leading-wildcard": "yes",
            "filter-rewrite": "yes",
            "facets": map {
                "subject": [("humanities", "art"), ("science", "engineering")]
            }
        }
    return
        count(collection("/db/lucenetest")//letter[ft:query(., (), $options)])
};

declare
    %test:assertEquals(5)
function facet:query-and-drill-down-hierarchical-array-multi3() {
    let $options := map {
            "leading-wildcard": "yes",
            "filter-rewrite": "yes",
            "facets": map {
                "subject": [("humanities", "history"), ("science", "engineering")]
            }
        }
    return
        count(collection("/db/lucenetest")//letter[ft:query(., (), $options)])
};

declare
    %test:assertEquals(1)
function facet:query-and-drill-down-hierarchical-array-multi4() {
    let $options := map {
            "leading-wildcard": "yes",
            "filter-rewrite": "yes",
            "facets": map {
                "subject": [("humanities", "art"), ("science", "math")]
            }
        }
    return
        count(collection("/db/lucenetest")//letter[ft:query(., (), $options)])
};

(:~
 : The facet 'cat' is defined on both: the index on 'document' and 'document/abstract'.
 : Querying 'document' should return 1 as facet count, while 'abstract' should return 2
 : as there are two abstracts being indexed and each has the facet 'nature'.
 :)
declare
    %test:assertEquals(1, 2)
function facet:multiple-indexes-with-same-facet() {
    let $result := doc("/db/lucenetest/documents.xml")//document[ft:query(., ())]
    return
        ft:facets($result, "cat")?nature,
    let $result := doc("/db/lucenetest/documents.xml")//document/abstract[ft:query(., ())]
    return
        ft:facets($result, "cat")?nature
};

declare
    %test:assertEquals(4, 2)
function facet:store-and-remove() {
    let $stored := xmldb:store("/db/lucenetest", "test2.xml", $facet:XML)
    let $result := collection("/db/lucenetest")//letter[ft:query(., ())]
    let $where := ft:facets($result, "place", 10)
    return
        $where?Berlin,
    xmldb:remove("/db/lucenetest", "test2.xml"),
    let $result := collection("/db/lucenetest")//letter[ft:query(., ())]
    let $where := ft:facets($result, "place", 10)
    return
        $where?Berlin
};

declare
    %test:arg("paths", "2017")
    %test:assertEquals(2)
    %test:arg("paths", "2019")
    %test:assertEquals(2)
    %test:arg("paths", "2019", "03")
    %test:assertEquals(1)
    %test:arg("paths", "2019", "03", "14")
    %test:assertEquals(1)
    %test:arg("paths", "2019", "03", "08")
    %test:assertEquals(0)
function facet:hierarchical-facets-query($paths as xs:string+) {
    let $options := map {
            "facets": map {
                "date": $paths
            }
        }
    let $result := collection("/db/lucenetest")//letter[ft:query(., (), $options)]
    return
        count($result)
};

declare
    %test:arg("paths", "science")
    %test:assertEquals(2)
    %test:arg("paths", "science", "engineering")
    %test:assertEquals(1)
    %test:arg("paths", "humanities", "history")
    %test:assertEquals(5)
    %test:arg("paths", "humanities", "art")
    %test:assertEquals(1)
    %test:arg("paths", "science", "math")
    %test:assertEquals(1)
function facet:hierarchical-facets-query-subjects($paths as xs:string+) {
    let $options := map {
            "facets": map {
                "subject": $paths
            }
        }
    let $result := collection("/db/lucenetest")//letter[ft:query(., (), $options)]
    return
        count($result)
};

declare
    %test:arg("paths", "2017")
    %test:assertEquals('{"03":2}')
    %test:arg("paths", "2019")
    %test:assertEquals('{"03":1,"04":1}')
function facet:hierarchical-facets-retrieve($paths as xs:string*) {
    let $result := collection("/db/lucenetest")//letter[ft:query(., ())]
    let $facets := ft:facets($result, "date", (), $paths)
    return
        serialize($facets, map { "method": "json" })
};

declare
    %test:arg("paths", "science")
    %test:assertEquals('{"math":1,"engineering":1}')
function facet:hierarchical-facets-retrieve($paths as xs:string*) {
    let $result := collection("/db/lucenetest")//letter[ft:query(., ())]
    let $facets := ft:facets($result, "subject", (), $paths)
    return
        serialize($facets, map { "method": "json" })
};

declare
    %test:assertEquals('{"Berlin":2,"Hamburg":1}','{"Wrocław":2}')
function facet:hierarchical-place() {
    let $result := collection("/db/lucenetest")//letter[ft:query(., ())]
    let $facets := ft:facets($result, "location", 10) (: Returns facet counts for "Germany" and "Poland" :)
    for $country in map:keys($facets)
    order by $country
    return
        serialize(
            ft:facets($result, "location", 10, $country), (: Get facet counts for sub-categories :)
            map { "method": "json" }
        )
};

declare
    %test:assertEqualsPermutation('{"history":5,"art":1}','{"math":1,"engineering":1}')
function facet:hierarchical-subject() {
    let $result := collection("/db/lucenetest")//letter[ft:query(., ())]
    let $facets := ft:facets($result, "subject", 10) (: Returns facet counts for "science" and "humanities" :)
    for $topic in map:keys($facets)
    order by $topic
    return
        serialize(
            ft:facets($result, "subject", 10, $topic), (: Get facet counts for sub-categories :)
            map { "method": "json" }
        )
};

declare
    %test:assertEquals('{"history":1}','{"engineering":1}')
function facet:hierarchical-multivalue-subject() {
    let $result := collection("/db/lucenetest")//letter[ft:query(., 'from:susi')]
    let $facets := ft:facets($result, "subject", 10) (: Returns facet counts for "science" and "humanities" :)
    for $topic in map:keys($facets)
    order by $topic
    return
        serialize(
            ft:facets($result, "subject", 10, $topic), (: Get facet counts for sub-categories :)
            map { "method": "json" }
        )
};

declare
    %test:args("place:hamburg")
    %test:assertEquals(1)
    %test:args("place:berlin")
    %test:assertEquals(2)
    %test:args("from:rudi AND place:berlin")
    %test:assertEquals(1)
    %test:args("from:susi AND place:berlin")
    %test:assertEquals(0)
    %test:args("basia AND place:wroclaw")
    %test:assertEquals(2)
    %test:args("place:wroclaw")
    %test:assertEquals(2)
    %test:args('to:(ba* müller)')
    %test:assertEquals(2)
    %test:args('foo:berlin')
    %test:assertEquals(0)
function facet:query-field($query as xs:string) {
    count(collection("/db/lucenetest")//letter[ft:query(., $query)])
};

declare
    %test:assertEquals("Babsi Müller", "Basia Kowalska", "Basia Müller")
function facet:query-and-sort() {
    for $letter in collection("/db/lucenetest")//letter[ft:query(., "from:heinz", map { "fields": "to" })]
    order by ft:field($letter, "to")
    return
        $letter/to/text()
};

declare
    %test:assertEquals("Basia Kowalska", "Basia Müller", "Babsi Müller")
function facet:query-and-sort-by-date() {
    for $letter in collection("/db/lucenetest")//letter[ft:query(., "from:heinz", map { "fields": "date" })]
    order by ft:field($letter, $field, "xs:date")
    return
        $letter/to/text()
};

declare
    %test:assertEquals("Hans", "Rudi")
function facet:query-and-sort-by-date() {
    for $letter in collection("/db/lucenetest")//letter[ft:query(., "place:berlin", map { "fields": "time" })]
    order by ft:field($letter, "time", "xs:time")
    return
        $letter/from/text()
};

declare
    %test:args("likes", "xs:int")
    %test:assertEquals(1, 3, 5, 9, 19, 29)
    %test:args("score", "xs:float")
    %test:assertEquals(6, 8.25, 14.25, 16, 16.5, 29.5)
function facet:query-and-sort-by-numeric($field as xs:string, $type as xs:string) {
    for $letter in collection("/db/lucenetest")//letter[ft:query(., (), map { "fields": $field })]
    let $likes := ft:field($letter, $field, $type)
    order by $likes
    return
        $likes
};

declare
    %test:assertEmpty
function facet:retrieve-non-existant-field() {
    for $letter in collection("/db/lucenetest")//letter[ft:query(., (), map { "fields": "foo" })]
    return
        ft:field($letter, "foo")
};

declare
    %test:assertEmpty
function facet:retrieve-not-stored() {
    for $letter in collection("/db/lucenetest")//letter[ft:query(., (), map { "fields": "from" })]
    return
        ft:field($letter, "from")
};

declare
    %test:assertEquals("Egon", "Berlin")
function facet:retrieve-multiple-fields() {
    for $letter in collection("/db/lucenetest")//letter[ft:query(., "from:rudi", map { "fields": ("to", "place") })]
    return
        (ft:field($letter, "to"), ft:field($letter, "place"))
};

declare
    %test:assertEquals("2017-03-13", 19, 8.25)
function facet:test-field-type() {
    let $letter := collection("/db/lucenetest")//letter[ft:query(., "from:rudi", map { "fields": ("date", "likes", "score") })]
    return (
        ft:field($letter, "date", "xs:date"),
        ft:field($letter, "likes", "xs:integer"),
        ft:field($letter, "score", "xs:double")
    )
};

declare
    %test:assertEquals(4)
function facet:index-keys() {
    count(collection("/db/lucenetest")/ft:index-keys-for-field("from", (), function($key, $count) { $key }, 10))
};

(:~
 : Check if option index="no" is applied properly: no default index, but the fields should be queryable.
 :)
declare
    %test:args("birds")
    %test:assertEmpty
    %test:args("english:birds")
    %test:assertEquals("
And the birds are singing
")
function facet:query-no-index($query as xs:string) {
    doc("/db/lucenetest/multi-lang.xml")//div[ft:query(., $query)]/p
};

declare
    %test:args("müller")
    %test:assertEquals(2)
function facet:default-analyzer-no-diacritics($query as xs:string) {
    count(collection("/db/lucenetest")//letter[ft:query(., $query)])
};

(:~
 : The configuration defines no default index on 'div', but two fields: 'english' and 'german'.
 : It also configures a facet containing the language. Querying the div with field 'english' set
 : should result in a facet count of 1 for language 'en'.
 :)
declare
    %test:assertEquals(1)
function facet:query-no-default-index-but-facet() {
    let $result := doc("/db/lucenetest/multi-lang.xml")//div[ft:query(., "english:*", map { "leading-wildcard": "yes" })]
    return
        ft:facets($result, "language")?en
};

declare
    (: German stemming :)
    %test:args("abstract:vogel")
    %test:assertEquals("<title>Ruhe im Wald</title>")
    %test:args("abstract:wald")
    %test:assertEquals("<title>Ruhe im Wald</title>")
    %test:args("abstract:streit")
    %test:assertEquals("<title>Streiten und Hoffen</title>")
    %test:args("title:streiten AND abstract:streit")
    %test:assertEquals("<title>Streiten und Hoffen</title>")
    (: No stemming on title :)
    %test:args("title:streit AND abstract:streit")
    %test:assertEmpty
function facet:query-field-with-analyzer($query as xs:string) {
    doc("/db/lucenetest/documents.xml")//document[ft:query(., $query)]/title
};

(: Index on 'abstract' uses default analyzer but has a field
 : with German analyzer indexing same content.
 :)
declare
    %test:args("german:vogel")
    %test:assertEquals("<title>Ruhe im Wald</title>")
    %test:args("vogel")
    %test:assertEmpty
    %test:args("vögel")
    %test:assertEquals("<title>Ruhe im Wald</title>")
    %test:args("german:streit")
    %test:assertEquals("<title>Streiten und Hoffen</title>")
    %test:args("german:streit AND götter")
    %test:assertEquals("<title>Streiten und Hoffen</title>")
    %test:args("german:streit AND gott")
    %test:assertEmpty
    %test:args("german:(streit AND gott)")
    %test:assertEquals("<title>Streiten und Hoffen</title>")
function facet:query-field-no-expression($query as xs:string) {
    doc("/db/lucenetest/documents.xml")//abstract[ft:query(., $query)]/../title
};

declare
    %test:args('<query><term field="ident">Z-49/2</term></query>')
    %test:assertEquals("<title>Streiten und Hoffen</title>")
function facet:query-field-with-keyword-analyzer($query as element()) {
    doc("/db/lucenetest/documents.xml")//document[ft:query(., $query)]/title
};

declare
    %test:args('german:vogel')
    %test:assertEquals(1)
    %test:args('german:vögel')
    %test:assertEquals(1)
    %test:args('german:birds')
    %test:assertEquals(0)
    %test:args('english:bird')
    %test:assertEquals(1)
    %test:args('english:vögel')
    %test:assertEquals(0)
function facet:query-field-with-condition($query as xs:string) {
    count(doc("/db/lucenetest/multi-lang.xml")//div[ft:query(., $query)])
};

declare
    %test:args('german:singen')
    %test:assertEmpty
function facet:field-respects-ignore($query as xs:string) {
    doc("/db/lucenetest/multi-lang.xml")//div[ft:query(., $query)]
};

declare
    %test:args("vögel")
    %test:assertEquals(2)
    %test:args("walde")
    %test:assertEquals(3)
function facet:query-with-union-and-facets($query as xs:string) {
    let $results := doc("/db/lucenetest/documents.xml")//document[ft:query(., $query)] |
        doc("/db/lucenetest/documents.xml")//document[ft:query(abstract, $query)]
    return
        ft:facets($results, "cat")?nature
};

declare
    %test:assertEmpty
function facet:avoid-range-index-conflict-person() {
    let $people := collection("/db/lucenetest")//person[ft:query(., ())]
    let $city-facet := ft:facets($people, "city")
    return
        if ($people and $city-facet?unknown) then
            $city-facet?unknown || " person values were calculated without id range indexes present"
        else
            ()
};

declare
    %test:assertEmpty
function facet:avoid-range-index-conflict-city() {
    let $cities := collection("/db/lucenetest")//city[ft:query(., ())]
    let $persons-facet := ft:facets($cities, "person")
    return
        if ($cities and $persons-facet?none) then
            $persons-facet?none || " city values were calculated without id range indexes present"
        else
            ()
};


x-path ALL the things

x-path ALL the things

(:
  Copyright 2012-2018 MarkLogic Corporation

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
:)
xquery version "1.0-ml";

module namespace service = "http://marklogic.com/rest-api/extensions/scaffold-content";

import module namespace consts = "http://marklogic.com/data-hub/consts"
at "/MarkLogic/data-hub-framework/impl/consts.xqy";

import module namespace debug = "http://marklogic.com/data-hub/debug"
at "/MarkLogic/data-hub-framework/impl/debug-lib.xqy";

import module namespace functx = "http://www.functx.com"
at "/MarkLogic/functx/functx-1.0-nodoc-2007-01.xqy";

import module namespace hent = "http://marklogic.com/data-hub/hub-entities"
at "/MarkLogic/data-hub-framework/impl/hub-entities.xqy";

import module namespace perf = "http://marklogic.com/data-hub/perflog-lib"
at "/MarkLogic/data-hub-framework/impl/perflog-lib.xqy";

import module namespace esi = "http://marklogic.com/entity-services-impl"
at "/MarkLogic/entity-services/entity-services-impl.xqy";

declare namespace es = "http://marklogic.com/entity-services";

declare namespace rapi = "http://marklogic.com/rest-api";

declare option xdmp:mapping "false";

declare function service:comment-xqy($comment-text as xs:string*)
{
  "(: " || fn:string-join($comment-text, "
     ") || " :)"
};

declare function service:comment-sjs($comment-text as xs:string*)
{
  "/* " || fn:string-join($comment-text, "
     ") || " */"
};

declare function service:kebob-case($str as xs:string)
{
  fn:lower-case(fn:replace(fn:replace($str, "([A-Z])", "-$1"), "^-", ""))
};

declare function service:camel-case($str as xs:string)
{
  fn:string-join(
    let $tokens := fn:tokenize($str, "[-_]")
    return
      (
        $tokens[1],
        for $x in fn:subsequence($tokens, 2)
        return
          functx:capitalize-first($x)
      ),
    "")
};

declare function service:get-property($root as xs:string, $property as xs:string)
{
  if (fn:contains($property, "-")) then
    $root || "['" || $property || "']"
  else
    $root || "." || $property
};

declare function service:casting-function-name-xqy(
  $datatype as xs:string
) as xs:string
{
  if ($datatype eq "iri")
  then "sem:iri"
  else "xs:" || $datatype
};

declare function service:casting-function-name-sjs(
  $datatype as xs:string
) as xs:string
{
  if ($datatype eq "iri")
  then "sem.iri"
  else "xs." || $datatype
};

declare function service:generate-lets($model as map:map, $entity-type-name, $mapping as map:map?)
{
  fn:string-join(
    let $definitions := map:get($model, "definitions")
    let $entity-type := map:get($definitions, $entity-type-name)
    let $properties := map:get($entity-type, "properties")
    let $required-properties := (
      map:get($entity-type, "primaryKey"),
      map:get($entity-type, "required") ! json:array-values(.)
    )
    for $property-name in map:keys($properties)
    let $is-required := $property-name = $required-properties
    let $property := map:get($properties, $property-name)
    let $is-array := map:get($property, "datatype") eq "array"
    let $property := map:get($properties, $property-name)
    let $property-datatype := esi:resolve-datatype($model, $entity-type-name, $property-name)
    let $casting-function-name :=
      if (map:contains($property, "datatype") and map:get($property, "datatype") ne "array") then
        service:casting-function-name-xqy($property-datatype)
      else ()
    let $wrap-if-array := function($str) {
      if ($is-array) then
        "json:to-array(" || $str || "
  )"
      else
        $str
    }
    let $ref :=
      if ($is-array) then
        let $items as map:map? := map:get($property, "items")
        return
          $items ! map:get(., "$ref")
      else
        map:get($property, "$ref")
    let $path-to-property := fn:concat("$source-node/", $property-name)
    let $property-comment :=
      if (fn:empty($ref)) then ()
      else if (fn:contains($ref, "#/definitions")) then
        service:comment-xqy("The following property is a local reference.")
      else (
          service:comment-xqy((
            'The following property assigment comes from an external reference.',
            'Its generated value probably requires developer attention.'
          ))
        )
    let $ref-name := functx:substring-after-last($ref, "/")
    let $extract-reference-fn :=
      fn:string-join((
        "",
        "    plugin:make-reference-object('" || $ref-name || "', (: put your value here :) '')"
      ), "
")
    let $value :=
      if (empty($ref)) then (
        if(service:mapping-present($mapping, $property-name))
        then (fn:concat("$source",service:map-value($property-name, $mapping)))
        else (fn:concat("$source/", $property-name)))
      else if (contains($ref, "#/definitions")) then
        let $inner-var := "$" || fn:lower-case($ref-name) || "s"
        return
          fn:string-join((
            "",
            "let " || $inner-var || " :=",
            "  (: create a sequence of " || $ref-name || " instances from your data :)",
            "  for $sub-entity in ()",
            "  return",
            "    plugin:extract-instance-" || $ref-name || "($sub-entity)",
            "return",
            if ($is-required) then
              "  json:to-array(" || $inner-var || ")"
            else (
              "  if (fn:exists(" || $inner-var || ")) then",
              "    json:to-array(" || $inner-var || ")",
              "  else ()"
            )
          ), "
    ") || "
"
      else
        $wrap-if-array($extract-reference-fn)

    return (
      $property-comment ! ("", .),

      let $name := service:kebob-case($property-name)
      let $valueWrap := fn:concat(fn:string-join(
        if ($casting-function-name) then
          $casting-function-name else ())
        , fn:concat(
          "(", $value,")"))
      return
        "let $" || $name || " := " || $valueWrap
    )
    , "
  ")
(: end code generation block :)
};

declare function service:generate-xqy($entity as xs:string, $flow-type as xs:string, $model as map:map, $mapping as map:map?)
{
  let $root-name :=
    if ($flow-type eq $consts:INPUT_FLOW) then "$raw-content"
    else "$doc"
  return
    document {
<module>xquery version "1.0-ml";

module namespace plugin = "http://marklogic.com/data-hub/plugins";

import module namespace es = "http://marklogic.com/entity-services"
at "/MarkLogic/entity-services/entity-services.xqy";

declare option xdmp:mapping "false";

(:~
: Create Content Plugin
:
: @param $id          - the identifier returned by the collector
: @param $options     - a map containing options. Options are sent from Java
:
: @return - your transformed content
:)
declare function plugin:create-content(
  $id as xs:string,
  {
    if ($flow-type eq $consts:INPUT_FLOW) then
      "$raw-content as node()?,
  "
    else ()
  }$options as map:map) as map:map
{{
  {
    if ($flow-type eq $consts:HARMONIZE_FLOW) then
      "let $doc := fn:doc($id)
  "
    else ()
  }let $source := {$root-name}
  return
  {
    "plugin:extract-instance-" || $entity || "($source)"
  }
}};
  {
for $entity-type-name in map:keys(map:get($model, "definitions"))
return
<extract-instance>
(:~
: Creates a map:map instance from some source document.
: @param $source-node  A document or node that contains
:   data for populating a {$entity-type-name}
: @return A map:map instance with extracted data and
:   metadata about the instance.
:)
declare function plugin:extract-instance-{$entity-type-name}(
$source as node()?
) as map:map
{{
{
  if ($entity-type-name eq $entity) then
  <txt>
  (: the original source documents :)
  let $attachments := $source
  let $source      :=
    if ($source/*:envelope) then
      $source/*:envelope/*:instance
    else if ($source/instance) then
      $source/instance
    else
      $source
  </txt>/text()
  else ()
}
{
  if (fn:empty($mapping) eq fn:false() and fn:empty(map:get($mapping, "name")) eq fn:false()) then
  <txt>(: These mappings were generated using mapping: {map:get($mapping, "name")}, version: {map:get($mapping, "version")} on {fn:current-dateTime()}. :)
  </txt>/text()
  else ()
}
{
service:generate-lets($model, $entity-type-name, $mapping)
}

  (: return the in-memory instance :)
  (: using the XQuery 3.0 syntax... :)
  let $model := json:object()
  let $_ := (
  {
    if ($entity-type-name eq $entity) then
      "  map:put($model, '$attachments', $attachments),"
    else ()
  }
    map:put($model, '$type', '{ $entity-type-name }'),
    map:put($model, '$version', '{ map:get(map:get($model, "info"), "version") }'){
  let $definitions := map:get($model, "definitions")
  let $entity-type := map:get($definitions, $entity-type-name)
  let $properties := map:get($entity-type, "properties")
  let $property-keys := map:keys($properties)
  where fn:count($property-keys) > 0
  return
    ",
" ||
    fn:string-join(
      (: Begin code generation block :)
      let $required-properties := (
        map:get($entity-type, "primaryKey"),
        map:get($entity-type, "required") ! json:array-values(.)
      )
      for $property-name in $property-keys
      let $is-required := $property-name = $required-properties
      return
        "    map:put($model, '" || $property-name || "', $" || service:kebob-case($property-name) || ")"
      , ",
")
  (: end code generation block :)
  }
  )

  (: if you prefer the xquery 3.1 version with the => operator....
  https://www.w3.org/TR/xquery-31/#id-arrow-operator
  let $model :=
  json:object()
  {
  if ($entity-type-name eq $entity) then
    "  =>map:with('$attachments', $attachments)"
  else ()
  }
    =>map:with('$type', '{ $entity-type-name }')
    =>map:with('$version', '{ map:get(map:get($model, "info"), "version") }')
  {
  fn:string-join(
    (: Begin code generation block :)
    let $definitions := map:get($model, "definitions")
    let $entity-type := map:get($definitions, $entity-type-name)
    let $properties := map:get($entity-type, "properties")
    let $required-properties := (
      map:get($entity-type, "primaryKey"),
      json:array-values(map:get($entity-type, "required"))
    )
    for $property-name in map:keys($properties)
    let $is-required := $property-name = $required-properties
    return
      if ($is-required) then
        "  =>map:with('" || $property-name || "', $" || service:kebob-case($property-name) || ")"
      else
        "    =>es:optional('" || $property-name || "', $" || service:kebob-case($property-name) || ")"
    , "
")
  (: end code generation block :)
  }
  :)
  return $model
}};
</extract-instance>/text()
  }
declare function plugin:make-reference-object(
$type as xs:string,
$ref as xs:string)
{{
  let $o := json:object()
  let $_ := (
    map:put($o, '$type', $type),
    map:put($o, '$ref', $ref)
  )
  return
  $o
}};</module>/text()
}
};

declare function service:generate-vars($model as map:map, $entity-type-name, $mapping as map:map?)
{
  fn:string-join(
    let $definitions := map:get($model, "definitions")
    let $entity-type := map:get($definitions, $entity-type-name)
    let $properties := map:get($entity-type, "properties")
    let $required-properties := (
      map:get($entity-type, "primaryKey"),
      map:get($entity-type, "required") ! json:array-values(.)
    )
    for $property-name in map:keys($properties)
    let $is-required := $property-name = $required-properties

    let $property := map:get($properties, $property-name)
    let $is-array := map:get($property, "datatype") eq "array"
    let $property-datatype := esi:resolve-datatype($model, $entity-type-name, $property-name)
    let $casting-function-name := service:casting-function-name-sjs($property-datatype)
    let $wrap-if-array := function($str) {
      if ($is-array) then
        "json:to-array(" || $str || ")"
      else
        $str
    }
    let $ref :=
      if ($is-array) then
        let $items as map:map? := map:get($property, "items")
        return
          $items ! map:get(., "$ref")
      else
        map:get($property, "$ref")
    let $path-to-property :=
      if (service:mapping-present($mapping, $property-name)) then (fn:concat("source.xpath('",service:map-value($property-name, $mapping), "')"))  else (fn:concat("source.xpath('/", $property-name, "')"))
    let $property-comment :=
      if (fn:empty($ref)) then ()
      else if (fn:contains($ref, "#/definitions")) then
        service:comment-sjs("The following property is a local reference.")
      else (
          service:comment-sjs((
            'The following property assigment comes from an external reference.',
            'Its generated value probably requires developer attention.'
          ))
        )
    let $ref-name := functx:substring-after-last($ref, "/")
    (:    let $extract-reference-fn :=
      fn:string-join((
        " {",
        "    '$type': '" || $ref-name || "',",
        "    '$ref': " || service:get-property($path-to-property, $ref-name),
        "  }"
      ), "
"):)
    let $value :=
      if (empty($ref)) then
        "!fn.empty(" || $path-to-property || ") ? " || (
        if($is-array) then
          $path-to-property || ": []"
        else
        $casting-function-name || "("
        || "fn.head(" ||
        $path-to-property ||
        (
          if ($property-datatype eq "iri") then
            ".toString()"
          else
            ()
        ) ||
        ")) : null"
        )

      else if (contains($ref, "#/definitions")) then
        if ($is-array) then
          fn:string-join((
            "[];",
            "if(" || $path-to-property || ") {",
            "  for(const item of Sequence.from(source.xpath('/" || $property-name || "'))) {",
            "    // either return an instance of a " || $ref-name,
            "    "   || service:camel-case($property-name) || ".push(" || service:camel-case("extractInstance-" || $ref-name) || "(item));",
            "    // or a reference to a " || $ref-name,
            "    " || service:camel-case($property-name) || ".push(makeReferenceObject('" || $ref-name || "', item));",
            "  }",
            "}"
          ), "
  ")
        else
          fn:string-join((
            "null;",
            "if(" || $path-to-property || ") {",
            "  // either return an instance of a " || $ref-name,
            "  " || service:camel-case($property-name) || " = " || service:camel-case("extractInstance-" || $ref-name) || "(source.xpath('/" || $property-name || "'));",
            "",
            "  // or a reference to a " || $ref-name,
            "  // " || service:camel-case($property-name) || " = makeReferenceObject('" || $ref-name || "', source.xpath('/" || $property-name || "'));",
            "}"
          ), "
  ")
      else
        if ($is-array) then
          fn:string-join((
            " null;",
            "  if (" || service:get-property($path-to-property, $ref-name) || ") {",
            "    " || service:camel-case($property-name) || " = " || service:get-property($path-to-property, $ref-name) || ".map(function(item) {",
            "      return makeReferenceObject('" || $ref-name || "', source.xpath('/"|| $property-name || "'));",
            "    });",
            "  }"
          ), "
")
        else
          " makeReferenceObject('" || $ref-name || "', " || service:get-property($path-to-property, $ref-name) || ")"
    (:          fn:string-join((
            " {",
            "    '$type': '" || $ref-name || "',",
            "    '$ref': " || service:get-property($path-to-property, $ref-name),
            "  }"
          ), "
"):)

    return (
      $property-comment ! ("", .),
      "let " || service:camel-case($property-name) || " = " || $value || ";"
    )
    , "
  ")
(: end code generation block :)
};

declare function service:generate-sjs($entity as xs:string, $flow-type as xs:string, $model as map:map, $mapping as map:map?)
{
  let $root-name :=
    if ($flow-type eq $consts:INPUT_FLOW) then "rawContent"
    else "doc"
  return
    document {<module>'use strict'

/*
* Create Content Plugin
*
* @param id         - the identifier returned by the collector
* @param options    - an object containing options. Options are sent from Java
*
* @return - your content
*/
function createContent(id, {
  if ($flow-type eq $consts:INPUT_FLOW) then
    "rawContent, "
  else ()
}options) {{
  {
    if ($flow-type eq $consts:HARMONIZE_FLOW) then
      "let doc = cts.doc(id);"
    else ()
  }

  let source;

  // for xml we need to use xpath
  if({$root-name} && xdmp.nodeKind({$root-name}) === 'element' && {$root-name} instanceof XMLDocument) {{
    source = {$root-name}
  }}
  // for json we need to return the instance
  else if({$root-name} && {$root-name} instanceof Document) {{
    source = fn.head({$root-name}.root);
  }}
  // for everything else
  else {{
    source = {if ($flow-type eq $consts:INPUT_FLOW) then $root-name else "doc"};
  }}

  return {service:camel-case("extractInstance-" || $entity) || "(source)"};
}}
  {
for $entity-type-name in map:keys(map:get($model, "definitions"))
return <extract-instance>
/**
* Creates an object instance from some source document.
* @param source  A document or node that contains
*   data for populating a {$entity-type-name}
* @return An object with extracted data and
*   metadata about the instance.
*/
function {service:camel-case("extractInstance-" || $entity-type-name)}(source) {{
  {
    if ($entity eq $entity-type-name) then
  "// the original source documents
  let attachments = source;
  // now check to see if we have XML or json, then create a node clone from the root of the instance
  if (source instanceof Element || source instanceof ObjectNode) {
    source = new NodeBuilder().addNode(fn.head(source.xpath('/*:envelope/*:instance'))).toNode();
  }
  "
    else ()
  }
  {
    if (fn:empty($mapping) eq fn:false() and fn:empty(map:get($mapping, "name")) eq fn:false()) then
      <txt>/* These mappings were generated using mapping: {map:get($mapping, "name")}, version: {map:get($mapping, "version")} on {fn:current-dateTime()}. */
  </txt>/text()
    else ()
  }
  {
    service:generate-vars($model, $entity-type-name, $mapping)
  }

  // return the instance object
  return {{
  {
    if ($entity eq $entity-type-name) then
      "  '$attachments': attachments,"
    else ()
  }
    '$type': '{ $entity-type-name }',
    '$version': '{ map:get(map:get($model, "info"), "version") }'{
  let $definitions := map:get($model, "definitions")
  let $entity-type := map:get($definitions, $entity-type-name)
  let $properties := map:get($entity-type, "properties")
  let $properties-keys := map:keys($properties)
  where fn:count($properties-keys) > 0
  return
    ",
    " ||
    fn:string-join(
      (: Begin code generation block :)
      for $property-name in $properties-keys
      return
        "'" || $property-name || "': " ||  service:camel-case($property-name)
      , ",
    ")
  (: end code generation block :)
  }
  }}
}};
</extract-instance>/text()
}

function makeReferenceObject(type, ref) {{
  return {{
    '$type': type,
    '$ref': ref
  }};
}}

module.exports = {{
  createContent: createContent
}};

</module>/text()
    }
};

declare function service:map-value($key as xs:string, $mapping as map:map?) {
  let $properties := map:get($mapping, "properties")
  let $property := map:get($properties, $key)
  let $source := map:get($mapping, "sourceContext")
  return fn:concat($source, map:get($property, "sourcedFrom"))
};

declare function service:get-mapping($mapping-name) {
  let $mapping-base-name := functx:substring-before-last($mapping-name, "-")
  let $mapping-uri := fn:concat("/mappings/", $mapping-base-name, "/", $mapping-name, ".mapping.json")
  let $mapping-doc := fn:doc($mapping-uri)
  return
    if(fn:exists($mapping-doc)) then (map:new(xdmp:from-json($mapping-doc)))
    else ()
};

declare function service:mapping-present($mapping as map:map?, $property-name as xs:string?) {
  (fn:empty($mapping) eq fn:false() and fn:empty(map:get(map:get($mapping, "properties"), $property-name)) eq fn:false())
};

declare function service:get(
  $context as map:map,
  $params  as map:map
) as document-node()*
{
  debug:dump-env(),

  perf:log('/v1/resources/validate:get', function() {
    let $entity as xs:string := map:get($params, "entity")
    let $code-format as xs:string := map:get($params, "codeFormat")
    let $flow-type as xs:string := map:get($params, "flowType")
    let $mapping-name as xs:string? := map:get($params, "mapping")
    let $mapping as map:map? := if(fn:empty($mapping-name) eq fn:false()) then (service:get-mapping($mapping-name)) else ()
    let $model as map:map? := hent:get-model($entity)
    return
      if (fn:exists($model)) then
        if((fn:empty($mapping) eq fn:false() and fn:empty($mapping-name) eq fn:false()) or (fn:empty($mapping-name) eq fn:true())) then
          if ($code-format eq "xqy") then
            service:generate-xqy($entity, $flow-type, $model, $mapping)
          else
            service:generate-sjs($entity, $flow-type, $model, $mapping)
        else
          fn:error((),"RESTAPI-SRVEXERR", (404, "Not Found", "The requested mapping: " || $mapping-name || " was not found"))
      else
        fn:error((),"RESTAPI-SRVEXERR", (404, "Not Found", "The requested entity: " || $entity || " was not found"))
  })
};

(:
  Copyright 2012-2018 MarkLogic Corporation

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
:)
xquery version "1.0-ml";

module namespace service = "http://marklogic.com/rest-api/extensions/scaffold-content";

import module namespace consts = "http://marklogic.com/data-hub/consts"
at "/MarkLogic/data-hub-framework/impl/consts.xqy";

import module namespace debug = "http://marklogic.com/data-hub/debug"
at "/MarkLogic/data-hub-framework/impl/debug-lib.xqy";

import module namespace functx = "http://www.functx.com"
at "/MarkLogic/functx/functx-1.0-nodoc-2007-01.xqy";

import module namespace hent = "http://marklogic.com/data-hub/hub-entities"
at "/MarkLogic/data-hub-framework/impl/hub-entities.xqy";

import module namespace perf = "http://marklogic.com/data-hub/perflog-lib"
at "/MarkLogic/data-hub-framework/impl/perflog-lib.xqy";

import module namespace esi = "http://marklogic.com/entity-services-impl"
at "/MarkLogic/entity-services/entity-services-impl.xqy";

declare namespace es = "http://marklogic.com/entity-services";

declare namespace rapi = "http://marklogic.com/rest-api";

declare option xdmp:mapping "false";

declare function service:comment-xqy($comment-text as xs:string*)
{
  "(: " || fn:string-join($comment-text, "
     ") || " :)"
};

declare function service:comment-sjs($comment-text as xs:string*)
{
  "/* " || fn:string-join($comment-text, "
     ") || " */"
};

declare function service:kebob-case($str as xs:string)
{
  fn:lower-case(fn:replace(fn:replace($str, "([A-Z])", "-$1"), "^-", ""))
};

declare function service:camel-case($str as xs:string)
{
  fn:string-join(
    let $tokens := fn:tokenize($str, "[-_]")
    return
      (
        $tokens[1],
        for $x in fn:subsequence($tokens, 2)
        return
          functx:capitalize-first($x)
      ),
    "")
};

declare function service:get-property($root as xs:string, $property as xs:string)
{
  if (fn:contains($property, "-")) then
    $root || "['" || $property || "']"
  else
    $root || "." || $property
};

declare function service:casting-function-name-xqy(
  $datatype as xs:string
) as xs:string
{
  if ($datatype eq "iri")
  then "sem:iri"
  else "xs:" || $datatype
};

declare function service:casting-function-name-sjs(
  $datatype as xs:string
) as xs:string
{
  if ($datatype eq "iri")
  then "sem.iri"
  else "xs." || $datatype
};

declare function service:generate-lets($model as map:map, $entity-type-name, $mapping as map:map?)
{
  fn:string-join(
    let $definitions := map:get($model, "definitions")
    let $entity-type := map:get($definitions, $entity-type-name)
    let $properties := map:get($entity-type, "properties")
    let $required-properties := (
      map:get($entity-type, "primaryKey"),
      map:get($entity-type, "required") ! json:array-values(.)
    )
    for $property-name in map:keys($properties)
    let $is-required := $property-name = $required-properties
    let $property := map:get($properties, $property-name)
    let $is-array := map:get($property, "datatype") eq "array"
    let $property := map:get($properties, $property-name)
    let $property-datatype := esi:resolve-datatype($model, $entity-type-name, $property-name)
    let $casting-function-name :=
      if (map:contains($property, "datatype") and map:get($property, "datatype") ne "array") then
        service:casting-function-name-xqy($property-datatype)
      else ()
    let $wrap-if-array := function($str) {
      if ($is-array) then
        "json:to-array(" || $str || "
  )"
      else
        $str
    }
    let $ref :=
      if ($is-array) then
        let $items as map:map? := map:get($property, "items")
        return
          $items ! map:get(., "$ref")
      else
        map:get($property, "$ref")
    let $path-to-property := fn:concat("$source-node/", $property-name)
    let $property-comment :=
      if (fn:empty($ref)) then ()
      else if (fn:contains($ref, "#/definitions")) then
        service:comment-xqy("The following property is a local reference.")
      else (
          service:comment-xqy((
            'The following property assigment comes from an external reference.',
            'Its generated value probably requires developer attention.'
          ))
        )
    let $ref-name := functx:substring-after-last($ref, "/")
    let $extract-reference-fn :=
      fn:string-join((
        "",
        "    plugin:make-reference-object('" || $ref-name || "', (: put your value here :) '')"
      ), "
")
    let $value :=
      if (empty($ref)) then (
        if(service:mapping-present($mapping, $property-name))
        then (fn:concat("$source",service:map-value($property-name, $mapping)))
        else (fn:concat("$source/", $property-name)))
      else if (contains($ref, "#/definitions")) then
        let $inner-var := "$" || fn:lower-case($ref-name) || "s"
        return
          fn:string-join((
            "",
            "let " || $inner-var || " :=",
            "  (: create a sequence of " || $ref-name || " instances from your data :)",
            "  for $sub-entity in ()",
            "  return",
            "    plugin:extract-instance-" || $ref-name || "($sub-entity)",
            "return",
            if ($is-required) then
              "  json:to-array(" || $inner-var || ")"
            else (
              "  if (fn:exists(" || $inner-var || ")) then",
              "    json:to-array(" || $inner-var || ")",
              "  else ()"
            )
          ), "
    ") || "
"
      else
        $wrap-if-array($extract-reference-fn)

    return (
      $property-comment ! ("", .),

      let $name := service:kebob-case($property-name)
      let $valueWrap := fn:concat(fn:string-join(
        if ($casting-function-name) then
          $casting-function-name else ())
        , fn:concat(
          "(", $value,")"))
      return
        "let $" || $name || " := " || $valueWrap
    )
    , "
  ")
(: end code generation block :)
};

declare function service:generate-xqy($entity as xs:string, $flow-type as xs:string, $model as map:map, $mapping as map:map?)
{
  let $root-name :=
    if ($flow-type eq $consts:INPUT_FLOW) then "$raw-content"
    else "$doc"
  return
    document {
<module>xquery version "1.0-ml";

module namespace plugin = "http://marklogic.com/data-hub/plugins";

import module namespace es = "http://marklogic.com/entity-services"
at "/MarkLogic/entity-services/entity-services.xqy";

declare option xdmp:mapping "false";

(:~
: Create Content Plugin
:
: @param $id          - the identifier returned by the collector
: @param $options     - a map containing options. Options are sent from Java
:
: @return - your transformed content
:)
declare function plugin:create-content(
  $id as xs:string,
  {
    if ($flow-type eq $consts:INPUT_FLOW) then
      "$raw-content as node()?,
  "
    else ()
  }$options as map:map) as map:map
{{
  {
    if ($flow-type eq $consts:HARMONIZE_FLOW) then
      "let $doc := fn:doc($id)
  "
    else ()
  }let $source := {$root-name}
  return
  {
    "plugin:extract-instance-" || $entity || "($source)"
  }
}};
  {
for $entity-type-name in map:keys(map:get($model, "definitions"))
return
<extract-instance>
(:~
: Creates a map:map instance from some source document.
: @param $source-node  A document or node that contains
:   data for populating a {$entity-type-name}
: @return A map:map instance with extracted data and
:   metadata about the instance.
:)
declare function plugin:extract-instance-{$entity-type-name}(
$source as node()?
) as map:map
{{
{
  if ($entity-type-name eq $entity) then
  <txt>
  (: the original source documents :)
  let $attachments := $source
  let $source      :=
    if ($source/*:envelope) then
      $source/*:envelope/*:instance
    else if ($source/instance) then
      $source/instance
    else
      $source
  </txt>/text()
  else ()
}
{
  if (fn:empty($mapping) eq fn:false() and fn:empty(map:get($mapping, "name")) eq fn:false()) then
  <txt>(: These mappings were generated using mapping: {map:get($mapping, "name")}, version: {map:get($mapping, "version")} on {fn:current-dateTime()}. :)
  </txt>/text()
  else ()
}
{
service:generate-lets($model, $entity-type-name, $mapping)
}

  (: return the in-memory instance :)
  (: using the XQuery 3.0 syntax... :)
  let $model := json:object()
  let $_ := (
  {
    if ($entity-type-name eq $entity) then
      "  map:put($model, '$attachments', $attachments),"
    else ()
  }
    map:put($model, '$type', '{ $entity-type-name }'),
    map:put($model, '$version', '{ map:get(map:get($model, "info"), "version") }'){
  let $definitions := map:get($model, "definitions")
  let $entity-type := map:get($definitions, $entity-type-name)
  let $properties := map:get($entity-type, "properties")
  let $property-keys := map:keys($properties)
  where fn:count($property-keys) > 0
  return
    ",
" ||
    fn:string-join(
      (: Begin code generation block :)
      let $required-properties := (
        map:get($entity-type, "primaryKey"),
        map:get($entity-type, "required") ! json:array-values(.)
      )
      for $property-name in $property-keys
      let $is-required := $property-name = $required-properties
      return
        "    map:put($model, '" || $property-name || "', $" || service:kebob-case($property-name) || ")"
      , ",
")
  (: end code generation block :)
  }
  )

  (: if you prefer the xquery 3.1 version with the => operator....
  https://www.w3.org/TR/xquery-31/#id-arrow-operator
  let $model :=
  json:object()
  {
  if ($entity-type-name eq $entity) then
    "  =>map:with('$attachments', $attachments)"
  else ()
  }
    =>map:with('$type', '{ $entity-type-name }')
    =>map:with('$version', '{ map:get(map:get($model, "info"), "version") }')
  {
  fn:string-join(
    (: Begin code generation block :)
    let $definitions := map:get($model, "definitions")
    let $entity-type := map:get($definitions, $entity-type-name)
    let $properties := map:get($entity-type, "properties")
    let $required-properties := (
      map:get($entity-type, "primaryKey"),
      json:array-values(map:get($entity-type, "required"))
    )
    for $property-name in map:keys($properties)
    let $is-required := $property-name = $required-properties
    return
      if ($is-required) then
        "  =>map:with('" || $property-name || "', $" || service:kebob-case($property-name) || ")"
      else
        "    =>es:optional('" || $property-name || "', $" || service:kebob-case($property-name) || ")"
    , "
")
  (: end code generation block :)
  }
  :)
  return $model
}};
</extract-instance>/text()
  }
declare function plugin:make-reference-object(
$type as xs:string,
$ref as xs:string)
{{
  let $o := json:object()
  let $_ := (
    map:put($o, '$type', $type),
    map:put($o, '$ref', $ref)
  )
  return
  $o
}};</module>/text()
}
};

declare function service:generate-vars($model as map:map, $entity-type-name, $mapping as map:map?)
{
  fn:string-join(
    let $definitions := map:get($model, "definitions")
    let $entity-type := map:get($definitions, $entity-type-name)
    let $properties := map:get($entity-type, "properties")
    let $required-properties := (
      map:get($entity-type, "primaryKey"),
      map:get($entity-type, "required") ! json:array-values(.)
    )
    for $property-name in map:keys($properties)
    let $is-required := $property-name = $required-properties

    let $property := map:get($properties, $property-name)
    let $is-array := map:get($property, "datatype") eq "array"
    let $property-datatype := esi:resolve-datatype($model, $entity-type-name, $property-name)
    let $casting-function-name := service:casting-function-name-sjs($property-datatype)
    let $wrap-if-array := function($str) {
      if ($is-array) then
        "json:to-array(" || $str || ")"
      else
        $str
    }
    let $ref :=
      if ($is-array) then
        let $items as map:map? := map:get($property, "items")
        return
          $items ! map:get(., "$ref")
      else
        map:get($property, "$ref")
    let $path-to-property :=
      if (service:mapping-present($mapping, $property-name)) then (fn:concat("source.xpath('",service:map-value($property-name, $mapping), "')"))  else (service:get-property("source", $property-name))
    let $property-comment :=
      if (fn:empty($ref)) then ()
      else if (fn:contains($ref, "#/definitions")) then
        service:comment-sjs("The following property is a local reference.")
      else (
          service:comment-sjs((
            'The following property assigment comes from an external reference.',
            'Its generated value probably requires developer attention.'
          ))
        )
    let $ref-name := functx:substring-after-last($ref, "/")
    (:    let $extract-reference-fn :=
      fn:string-join((
        " {",
        "    '$type': '" || $ref-name || "',",
        "    '$ref': " || service:get-property($path-to-property, $ref-name),
        "  }"
      ), "
"):)
    let $value :=
      if (empty($ref)) then
        "!fn.empty(" || $path-to-property || ") ? " || (
        if($is-array) then
          $path-to-property || ": []"
        else
        $casting-function-name || "("
        || "fn.head(" ||
        $path-to-property ||
        (
          if ($property-datatype eq "iri") then
            ".toString()"
          else
            ()
        ) ||
        ")) : null"
        )

      else if (contains($ref, "#/definitions")) then
        if ($is-array) then
          fn:string-join((
            "[];",
            "if(" || $path-to-property || ") {",
            "  for(const item of Sequence.from(source." || $property-name || ")) {",
            "    // either return an instance of a " || $ref-name,
            "  "   || service:camel-case($property-name) || ".push(" || service:camel-case("extractInstance-" || $ref-name) || "(item));",
            "    // or a reference to a " || $ref-name,
            "    // " || service:camel-case($property-name) || ".push(makeReferenceObject('" || $ref-name || "', item));",
            "  }",
            "}"
          ), "
  ")
        else
          fn:string-join((
            "null;",
            "if(" || $path-to-property || ") {",
            "  // either return an instance of a " || $ref-name,
            "  " || service:camel-case($property-name) || " = " || service:camel-case("extractInstance-" || $ref-name) || "(source." || $property-name || ");",
            "",
            "  // or a reference to a " || $ref-name,
            "  // " || service:camel-case($property-name) || " = makeReferenceObject('" || $ref-name || "', source." || $property-name || ");",
            "}"
          ), "
  ")
      else
        if ($is-array) then
          fn:string-join((
            " null;",
            "  if (" || service:get-property($path-to-property, $ref-name) || ") {",
            "    " || service:camel-case($property-name) || " = " || service:get-property($path-to-property, $ref-name) || ".map(function(item) {",
            "      return makeReferenceObject('" || $ref-name || "', source."|| $property-name || ");",
            "    });",
            "  }"
          ), "
")
        else
          " makeReferenceObject('" || $ref-name || "', " || service:get-property($path-to-property, $ref-name) || ")"
    (:          fn:string-join((
            " {",
            "    '$type': '" || $ref-name || "',",
            "    '$ref': " || service:get-property($path-to-property, $ref-name),
            "  }"
          ), "
"):)

    return (
      $property-comment ! ("", .),
      "let " || service:camel-case($property-name) || " = " || $value || ";"
    )
    , "
  ")
(: end code generation block :)
};

declare function service:generate-sjs($entity as xs:string, $flow-type as xs:string, $model as map:map, $mapping as map:map?)
{
  let $root-name :=
    if ($flow-type eq $consts:INPUT_FLOW) then "rawContent"
    else "doc"
  return
    document {<module>'use strict'

/*
* Create Content Plugin
*
* @param id         - the identifier returned by the collector
* @param options    - an object containing options. Options are sent from Java
*
* @return - your content
*/
function createContent(id, {
  if ($flow-type eq $consts:INPUT_FLOW) then
    "rawContent, "
  else ()
}options) {{
  {
    if ($flow-type eq $consts:HARMONIZE_FLOW) then
      "let doc = cts.doc(id);"
    else ()
  }

  let source;

  // for xml we need to use xpath
  if({$root-name} && xdmp.nodeKind({$root-name}) === 'element' && {$root-name} instanceof XMLDocument) {{
    source = {$root-name}
  }}
  // for json we need to return the instance
  else if({$root-name} && {$root-name} instanceof Document) {{
    source = fn.head({$root-name}.root);
  }}
  // for everything else
  else {{
    source = {if ($flow-type eq $consts:INPUT_FLOW) then $root-name else "doc"};
  }}

  return {service:camel-case("extractInstance-" || $entity) || "(source)"};
}}
  {
for $entity-type-name in map:keys(map:get($model, "definitions"))
return <extract-instance>
/**
* Creates an object instance from some source document.
* @param source  A document or node that contains
*   data for populating a {$entity-type-name}
* @return An object with extracted data and
*   metadata about the instance.
*/
function {service:camel-case("extractInstance-" || $entity-type-name)}(source) {{
  // the original source documents
  let attachments = source;
  // now check to see if we have XML or json, then just go to the instance
  if(source instanceof Element) {{
    source = fn.head(source.xpath('/*:envelope/*:instance'))
  }} else if(source instanceof ObjectNode) {{
    source = source.envelope.instance;
  }}
  {
    if (fn:empty($mapping) eq fn:false() and fn:empty(map:get($mapping, "name")) eq fn:false()) then
      <txt>/* These mappings were generated using mapping: {map:get($mapping, "name")}, version: {map:get($mapping, "version")} on {fn:current-dateTime()}. */
  </txt>/text()
    else ()
  }
  {
    service:generate-vars($model, $entity-type-name, $mapping)
  }

  // return the instance object
  return {{
  {
    if ($entity eq $entity-type-name) then
      "  '$attachments': attachments,"
    else ()
  }
    '$type': '{ $entity-type-name }',
    '$version': '{ map:get(map:get($model, "info"), "version") }'{
  let $definitions := map:get($model, "definitions")
  let $entity-type := map:get($definitions, $entity-type-name)
  let $properties := map:get($entity-type, "properties")
  let $properties-keys := map:keys($properties)
  where fn:count($properties-keys) > 0
  return
    ",
    " ||
    fn:string-join(
      (: Begin code generation block :)
      for $property-name in $properties-keys
      return
        "'" || $property-name || "': " ||  service:camel-case($property-name)
      , ",
    ")
  (: end code generation block :)
  }
  }}
}};
</extract-instance>/text()
}

function makeReferenceObject(type, ref) {{
  return {{
    '$type': type,
    '$ref': ref
  }};
}}

module.exports = {{
  createContent: createContent
}};

</module>/text()
    }
};

declare function service:map-value($key as xs:string, $mapping as map:map?) {
  let $properties := map:get($mapping, "properties")
  let $property := map:get($properties, $key)
  let $source := map:get($mapping, "sourceContext")
  return fn:concat($source, map:get($property, "sourcedFrom"))
};

declare function service:get-mapping($mapping-name) {
  let $mapping-base-name := functx:substring-before-last($mapping-name, "-")
  let $mapping-uri := fn:concat("/mappings/", $mapping-base-name, "/", $mapping-name, ".mapping.json")
  let $mapping-doc := fn:doc($mapping-uri)
  return
    if(fn:exists($mapping-doc)) then (map:new(xdmp:from-json($mapping-doc)))
    else ()
};

declare function service:mapping-present($mapping as map:map?, $property-name as xs:string?) {
  (fn:empty($mapping) eq fn:false() and fn:empty(map:get(map:get($mapping, "properties"), $property-name)) eq fn:false())
};

declare function service:get(
  $context as map:map,
  $params  as map:map
) as document-node()*
{
  debug:dump-env(),

  perf:log('/v1/resources/validate:get', function() {
    let $entity as xs:string := map:get($params, "entity")
    let $code-format as xs:string := map:get($params, "codeFormat")
    let $flow-type as xs:string := map:get($params, "flowType")
    let $mapping-name as xs:string? := map:get($params, "mapping")
    let $mapping as map:map? := if(fn:empty($mapping-name) eq fn:false()) then (service:get-mapping($mapping-name)) else ()
    let $model as map:map? := hent:get-model($entity)
    return
      if (fn:exists($model)) then
        if((fn:empty($mapping) eq fn:false() and fn:empty($mapping-name) eq fn:false()) or (fn:empty($mapping-name) eq fn:true())) then
          if ($code-format eq "xqy") then
            service:generate-xqy($entity, $flow-type, $model, $mapping)
          else
            service:generate-sjs($entity, $flow-type, $model, $mapping)
        else
          fn:error((),"RESTAPI-SRVEXERR", (404, "Not Found", "The requested mapping: " || $mapping-name || " was not found"))
      else
        fn:error((),"RESTAPI-SRVEXERR", (404, "Not Found", "The requested entity: " || $entity || " was not found"))
  })
};