added a 2nd option for populating the database with index data - the function documentation

svn path=/trunk/eXist/; revision=13968

xquery version "1.0";

(:
    Module: Browse Indexes: See an overview of .xconf files stored in /db/system/config,
    and browse the associated index keys.
:)

module namespace indexes="http://exist-db.org/xquery/admin-interface/indexes";

declare namespace cc="http://exist-db.org/collection-config/1.0";

declare option exist:serialize "method=xhtml media-type=text/html omit-xml-declaration=no enforce-xhtml=yes";

(:
    Global variables - derived from URL parameters
:)
declare variable $indexes:start-value := request:get-parameter('start-value', '');
declare variable $indexes:callback := util:function(xs:QName("indexes:term-callback"), 2);
declare variable $indexes:max-number-returned := xs:integer(request:get-parameter('max', 100));
declare variable $indexes:index := request:get-parameter('index', '');
declare variable $indexes:sortby := request:get-parameter('sortby', 'term');
declare variable $indexes:sortorder := request:get-parameter('sortorder', 'ascending');
declare variable $indexes:node-name := request:get-parameter('node-name', '');
declare variable $indexes:match := request:get-parameter('match', '');
declare variable $indexes:collection := request:get-parameter('collection', '');
declare variable $indexes:node-set :=
    if ($indexes:node-name ne '' and $indexes:collection ne '') then
        indexes:get-nodeset-from-qname($indexes:collection, $indexes:node-name)
    else if ($indexes:match ne '' and $indexes:collection ne '') then
        indexes:get-nodeset-from-match($indexes:collection, $indexes:match)
    else
        ();
declare variable $indexes:qname :=
    if ($indexes:node-name ne '') then
        if (not(matches($indexes:node-name, 'xml:')) and contains($indexes:node-name, ':')) then
            QName(indexes:get-namespace-uri-from-node-name($indexes:node-name, $indexes:collection), $indexes:node-name)
        else
            xs:QName($indexes:node-name)
    else ();
declare variable $indexes:show-keys-by := request:get-parameter('show-keys-by', '');
declare variable $indexes:index-names :=

        <name>Index Types</name>
        <description>This is the list of known index types and their labels.</description>
        <items>
            <item>
                Lucene
                <value>lucene-index</value>
            </item>
            <item>
                NGram
                <value>ngram-index</value>
            </item>
            <item>
                Range
                <value>range-index</value>
            </item>
            <item>
                Legacy Fulltext
                <value>legacy-fulltext-index</value>
            </item>
        </items>
    ;

(:
    Main function: outputs the page.
:)
declare function indexes:main() {
    let $xconfs :=
        for $xconf in collection('/db/system/config/')/cc:collection
        order by util:collection-name($xconf)
        return $xconf
    return
        if (empty($xconfs)) then

Browse Indexes

No Index Configurations were found in the /db/system/config collection.
                    Consider installing eXist's example files or
                    documentation and then returning here
                    to browse the indexes.

        else if (empty($indexes:node-set)) then

            indexes:list-indexes($xconfs)

        else

            indexes:show-index-keys()
};

(:
    Lists indexes.
:)
declare function indexes:list-indexes($xconfs) {

Browse Indexes

This is an overview of the index definitions applied in the database,
            as well as the "index keys" (the actual items stored in the index).

Browsing {count($xconfs)} indexes:

        { indexes:xconf-to-table($xconfs) }

};

(:
    Transforms an index definition into an HTML table.
:)
declare function indexes:xconf-to-table($xconfs as node()*) as item()* {

{
        for $xconf at $count in $xconfs
        let $xconf-collection-name := util:collection-name($xconf)
        let $data-collection-name := substring-after(util:collection-name($xconf), '/db/system/config')
        order by $xconf-collection-name
        return

                {$count}. {$data-collection-name} {if (xmldb:collection-available($data-collection-name)) then () else ' (no data)'}
                    (View xconf file)

                        Item Indexed
                        Index
                        Instances
                        Show Index Keys By

                    {
                    for $entry in ( indexes:analyze-legacy-fulltext-indexes($xconf),
                        indexes:analyze-lucene-indexes($xconf),
                        indexes:analyze-range-indexes($xconf),
                        indexes:analyze-ngram-indexes($xconf) )
                    let $item := $entry/td[1]
                    let $index := $entry/td[2]
                    (: order by $index, $item :)
                    return $entry
                    }

    }

};

(:
    Shows the index keys on a given nodeset or QName
:)
declare function indexes:show-index-keys() {
    let $query-start-time := util:system-time()

    let $keys :=
        (: legacy fulltext index use the text:index-terms() function :)
        if ($indexes:index eq 'legacy-fulltext-index') then
            text:index-terms($indexes:node-set, $indexes:start-value, $indexes:callback, $indexes:max-number-returned)
        (: range indexes can use util:index-keys() without specifying the index :)
        else if ($indexes:index eq 'range-index') then
            if (util:index-type($indexes:node-set) eq 'xs:string') then
                util:index-keys($indexes:node-set, $indexes:start-value, $indexes:callback, $indexes:max-number-returned)
            else
                let $start-value :=
                    if (util:index-type($indexes:node-set) eq 'xs:double') then
                        if ($indexes:start-value castable as xs:double) then xs:double($indexes:start-value) else xs:double(0)
                    else if (util:index-type($indexes:node-set) eq 'xs:float') then
                        if ($indexes:start-value castable as xs:float) then xs:float($indexes:start-value) else xs:float(0)
                    else if (util:index-type($indexes:node-set) eq 'xs:boolean') then
                        if ($indexes:start-value castable as xs:boolean) then xs:boolean($indexes:start-value) else xs:boolean(0)
                    else if (util:index-type($indexes:node-set) eq 'xs:dateTime') then
                        if ($indexes:start-value castable as xs:dateTime) then xs:dateTime($indexes:start-value) else xs:dateTime('0001-01-01T00:00:00-00:00')
                    else if (util:index-type($indexes:node-set) eq 'xs:date') then
                        if ($indexes:start-value castable as xs:date) then xs:date($indexes:start-value) else xs:date('0001-01-01')
                    else
                        if ($indexes:start-value castable as xs:integer) then xs:integer($indexes:start-value)
                    else 0
                return util:index-keys($indexes:node-set, $start-value, $indexes:callback, $indexes:max-number-returned)
        (: all other indexes need to specify the index in the 5th parameter of util:index-keys() :)
        else
            if ($indexes:show-keys-by eq 'node') then
                util:index-keys($indexes:node-set, $indexes:start-value, $indexes:callback, $indexes:max-number-returned, $indexes:index)
            else
                util:index-keys-by-qname($indexes:qname, $indexes:start-value, $indexes:callback, $indexes:max-number-returned, $indexes:index)

    let $log := util:log("DEBUG", concat("INDEXES index type:    ", $indexes:index))
    let $log := util:log("DEBUG", concat("INDEXES qname     :    ", $indexes:qname))

    (:  Reminder for sorting:
           term = $key/td[1]
           frequency := xs:integer($key/td[2])
           documents := xs:integer($key/td[3])
           position := xs:integer($key/td[4])
    :)

    let $sorted-keys :=
        if ($indexes:sortby eq 'term') then
            if ($indexes:sortorder eq 'ascending') then
                for $key in $keys order by $key/td[1] ascending return $key
            else
                for $key in $keys order by $key/td[1] descending return $key
        else if ($indexes:sortby eq 'frequency') then
            if ($indexes:sortorder eq 'ascending') then
                for $key in $keys order by xs:integer($key/td[2]) ascending, $key/td[1] ascending return $key
            else
                for $key in $keys order by xs:integer($key/td[2]) descending, $key/td[1] ascending return $key
        else if ($indexes:sortby eq 'documents') then
            if ($indexes:sortorder eq 'ascending') then
                for $key in $keys order by xs:integer($key/td[3]) ascending, $key/td[1] ascending return $key
            else
                for $key in $keys order by xs:integer($key/td[3]) descending, $key/td[1] ascending return $key
        else if ($indexes:sortby eq 'position') then
            if ($indexes:sortorder eq 'ascending') then
                for $key in $keys order by xs:integer($key/td[4]) ascending return $key
            else
                for $key in $keys order by xs:integer($key/td[4]) descending return $key
        else $keys

    let $query-end-time := util:system-time()
    let $query-duration := ($query-end-time - $query-start-time) div xs:dayTimeDuration('PT1S')

    return

View Index Keys

Select a different index

{count($keys)} keys returned in {$query-duration}s

Keys for the {indexes:index-name-to-label($indexes:index)} index defined on "{string-join(($indexes:node-name, $indexes:match), '')}" in the {$indexes:collection} collection, by {$indexes:show-keys-by}.

            <form method="get" action="{indexes:remove-parameter-names('start-value')}">

                    Max number returned: <select name="max">{
                        for $number in (10, 100, 1000, 10000)
                        return
                            <option value="{$number}">{if ($number eq $indexes:max-number-returned) then attribute selected {'selected'} else ()}{$number}</option>
                    }</select>

                    Search for terms starting with: <input name="start-value" type="text" value="{$indexes:start-value}"/>
                    {if ($indexes:start-value ne '') then Clear search else ()}
                    {
                    for $param in request:get-parameter-names()[not(. = ('max', 'start-value'))]
                    return
                        <input type="hidden" name="{$param}" value="{request:get-parameter($param, '')}"/>
                    }

                    <input type="submit"/>

            </form>

{
                    for $column in ('term', 'frequency', 'documents', 'position')
                    return
                        {$column} {indexes:sort-direction-indicator($column)}
                }

                { $sorted-keys }

};

(:
    Helper function: Callback function called used in indexes:show-index-keys() for util:index-keys()
:)
declare function indexes:term-callback($term as xs:string, $data as xs:int+) as element() {

        {$term}
        {$data[1]}
        {$data[2]}
        {$data[3]}

};

(:
    Analyzes the Lucene indexes in an index definition
:)
declare function indexes:analyze-lucene-indexes($xconf) {
    let $index-label := indexes:index-name-to-label('lucene-index')
    let $lucene := $xconf/cc:index/cc:lucene
    return if (not($lucene) or not($lucene/cc:text)) then () else
        (
        (: TODO: complete report of default/other Lucene analyzers :)
        (:let $analyzers := $lucene/cc:analyzer
        return
            if ($analyzers) then
                (

Lucene Analyzers

                ,
                for $node in $analyzers
                return
                    if (not($node/@id)) then

default: {concat(' ', $node/@class/string())}

                    else

{$node/@id/string()}: {concat(' ', $node/@class/string())}

                )
            else ()
        ,:)
        let $texts := $lucene//cc:text
        return
            (
            for $text in $texts
            let $qname := if ($text/@qname) then $text/@qname/string() else ()
            let $match := if ($text/@match) then $text/@match/string() else ()
            let $analyzer := if ($text/@analyzer) then $text/@analyzer/string() else ()
            let $collection := substring-after(util:collection-name($text), '/db/system/config')
            let $nodeset := if ($qname) then indexes:get-nodeset-from-qname($collection, $qname) else indexes:get-nodeset-from-match($collection, $match)
            return

                        {if ($qname) then $qname else $match}
                        {if ($text/@boost) then concat(' (boost: ', $text/@boost/string(), ')') else ()}
                        {if ($text/cc:ignore) then (
, concat('(ignore: ', string-join(for $ignore in $text/cc:ignore return $ignore/@qname/string(), ', '), ')')) else ()}
                    {$index-label} {if ($qname) then ' QName' else ' Match'} {if ($analyzer) then concat(' (', $analyzer, ')') else ' (default analyzer)' (: TODO: complete report of default/other Lucene analyzers :)}
                    {count($nodeset)}
                    {
                        if (empty($nodeset)) then ()
                        else
                            (
                            QName
                            ,
                            ', '
                            ,
                            Node
                            )
                    }

            )
        )
};

(:
    Analyzes the legacy fulltext indexes in an index definition
:)
declare function indexes:analyze-legacy-fulltext-indexes($xconf) {
    let $collection := substring-after(util:collection-name($xconf), '/db/system/config')
    let $index := 'legacy-fulltext-index'
    let $index-label := indexes:index-name-to-label($index)
    let $fulltext := $xconf/cc:index/cc:fulltext
    let $creates := $fulltext/cc:create
    let $default-none := $fulltext/@default eq 'none'
    let $attributes-none := $fulltext/@attributes eq 'false'

    let $no-fulltext := if ( not($fulltext) or ($default-none and $attributes-none and not($creates)) ) then '(disabled)' else ()
    return
        if ($no-fulltext) then ()
        else if (not($default-none) and not($attributes-none)) then

                All Elements and Attributes!
                {$index-label}
                {count(collection($collection)//*)} elements and {count(collection($collection)//@*)} attributes
                (Too many to display)

(: is it feasible to display all qnames/nodes here? :)
        else (: if ($creates) then :)
            (
            for $create in $creates
            let $qname := $create/@qname/string()
            let $mixed := $create/@mixed/string()
            return

                    {$qname}
                    {$index-label} {if ($mixed) then '(mixed)' else ()}
                    {count(util:eval(concat('collection(', $collection, ')//', $qname)))}
                    Node

            ,
            let $only-elements-disabled := if ($fulltext/@default eq 'none' and $fulltext/@attributes ne 'no') then '(elements disabled)' else ()
            let $only-attribs-disabled := if ($fulltext/@default ne 'none' and $fulltext/@attributes eq 'no') then '(attributes disabled)' else ()
            return
            if ($only-elements-disabled) then

                    All Attributes!
                    {$index-label}
                    {count(collection($collection)//@*)} attributes
                    (Too many to display)

(: is it feasible to display all qnames/nodes here? :)
            else if ($only-attribs-disabled) then

                    All Elements!
                    {$index-label}
                    {count(collection($collection)//*)} elements
                    (Too many to display)

(: is it feasible to display all qnames/nodes here? :)
            else ()
            )
};

(:
    Analyzes the range indexes in an index definition
:)
declare function indexes:analyze-range-indexes($xconf) {
    let $index-label := indexes:index-name-to-label('range-index')
    let $ranges := $xconf/cc:index/cc:create
    return if (not($ranges)) then () else
        for $range in $ranges
        let $qname := $range/@qname/string()
        let $match := $range/@path/string()
        let $type := $range/@type/string()
        let $collection := substring-after(util:collection-name($range), '/db/system/config')
        let $nodeset := if ($qname) then indexes:get-nodeset-from-qname($collection, $qname) else indexes:get-nodeset-from-match($collection, $match)
        return

                {if ($qname) then $qname else $match}
                {$index-label} {if ($qname) then ' QName' else ' Path'} ({$type})
                {count($nodeset)}
                {
                    if (empty($nodeset)) then ()
                    else
                        Node
                }

};

(:
    Analyzes the NGram indexes in an index definition
:)
declare function indexes:analyze-ngram-indexes($xconf) {
    let $index-label := indexes:index-name-to-label('ngram-index')
    let $ngrams := $xconf/cc:index/cc:ngram
    return if (not($ngrams)) then () else
        for $ngram in $ngrams
        let $qname := $ngram/@qname/string()
        let $collection := substring-after(util:collection-name($ngram), '/db/system/config')
        let $nodeset := indexes:get-nodeset-from-qname($collection, $qname)
        return

                {$qname}
                {$index-label} QName
                {count($nodeset)}
                {
                    if (not(empty($nodeset))) then
                        (
                        QName,
                        ', '
                        ,
                        Node
                        )
                     else ()
                 }

};

(:
    Helper function: Returns a nodeset of instances of a node-name in a collection
:)
declare function indexes:get-nodeset-from-qname($collection as xs:string, $node-name as xs:string) as node()* {
    let $nodeset-expression :=
        concat(
            indexes:get-namespace-decaration-from-node-name($node-name, $collection)
            ,
            'collection("', $collection, '")//', $node-name
        )
    let $log := util:log("DEBUG", concat("INDEXES get-nodeset:          ", $nodeset-expression))
    return
        util:eval($nodeset-expression)
};

(:
    Helper function: Returns a nodeset of instances of a match expression in a collection
:)
declare function indexes:get-nodeset-from-match($collection as xs:string, $match as xs:string) as node()* {
    let $nodeset-expression :=
        concat(
            string-join(
                let $node-names := tokenize(replace($match, '//', '/'), '/')
                return
                    for $node-name in $node-names
                    return
                        indexes:get-namespace-decaration-from-node-name($node-name, $collection)
                , ' ')
            ,
            'collection("', $collection, '")', $match, if (contains($match, '@')) then () else ()
        )
    return
        util:eval($nodeset-expression)
};

(:
    Helper function: Returns the index definition for a given collection
:)
declare function indexes:get-xconf($collection as xs:string) as document-node() {
    let $config-root := '/db/system/config'
    let $xconf-collection := concat($config-root, $collection)
    let $xconf-filename := xmldb:get-child-resources($xconf-collection)[ends-with(., '.xconf')]
    let $xconf := doc(concat($xconf-collection, '/', $xconf-filename))
    return $xconf
};

(:
    Helper function: Looks in the collection.xconf's collection and index elements for namespace URIs for a given node name
:)
declare function indexes:get-namespace-uri-from-node-name($node-name, $collection) {
    let $name := substring-before($node-name, ':')
    let $xconf := indexes:get-xconf($collection)
    let $uri := (namespace-uri-for-prefix($name, $xconf/cc:collection), namespace-uri-for-prefix($name, $xconf//cc:index))
    return
        $uri
};

(:
    Helper function: Constructs a namespace declaration for use in util:eval()
:)
declare function indexes:get-namespace-decaration-from-node-name($node-name as xs:string, $collection as xs:string) as xs:string* {
    if (not(matches($node-name, 'xml:')) and contains($node-name, ':')) then
        let $name := substring-before($node-name, ':')
        let $uri := indexes:get-namespace-uri-from-node-name($node-name, $collection)
        return
            concat('declare namespace ', $name, '="', $uri, '"; ')
    else ()
};

(:
    Helper function: gets the label for a given index-name
:)
declare function indexes:index-name-to-label($index-name as xs:string) as xs:string {
    $indexes:index-names//item[value eq $index-name]/label/text()
};

(:
    ====
    Helper functions for modifying the sort order used in indexes:show-index-keys()
    ====
:)
declare function indexes:toggle-sortorder($current-sortorder) {
    indexes:toggle-sortorder($current-sortorder, ())
};

declare function indexes:toggle-sortorder($current-sortorder, $other-new-parameters) {
    let $neworder :=
        if ($current-sortorder eq 'ascending') then
            'sortorder=descending'
        else
            'sortorder=ascending'
    let $new-parameters := ($neworder, $other-new-parameters)
    return
        indexes:replace-parameters($new-parameters)
};

declare function indexes:set-sortorder($current-sortorder, $current-sortby, $new-sortby) {
    if ($current-sortby eq $new-sortby) then
        indexes:toggle-sortorder($current-sortorder)
    else
        indexes:strip-param-from-param-string(indexes:replace-parameters(concat('sortby=', $new-sortby)), 'sortorder')
};

declare function indexes:set-sortorder($new-sortby) {
    indexes:set-sortorder($indexes:sortorder, $indexes:sortby, $new-sortby)
};

declare function indexes:sort-direction-indicator($sortby as xs:string) {
    if ($sortby eq $indexes:sortby) then
        if ($indexes:sortorder eq 'ascending') then
            ' ↓'
        else
            ' ↑'
    else ()
};

(:
    ====
    Helper functions for handling parameters
    ====
:)
declare function indexes:remove-parameter-names($parameter-names-to-remove) {
    let $current-parameter-names := request:get-parameter-names()
    let $remaining-parameters :=
        indexes:remove-parameter-names(
            for $current-parameter-name in $current-parameter-names
            return
                concat($current-parameter-name, '=', request:get-parameter( $current-parameter-name, () ))
            ,
            $parameter-names-to-remove
            )
    return
        if (exists($remaining-parameters)) then
            concat('?', string-join($remaining-parameters, '&'))
        else
            '?'
};

declare function indexes:remove-parameter-names($current-parameters, $parameter-names-to-remove) {
    for $current-parameter in $current-parameters
    return
        if (substring-before($current-parameter, '=') = $parameter-names-to-remove) then
            ()
        else
            $current-parameter
};

declare function indexes:remove-parameter-names-except($parameter-names-to-keep) {
    let $current-parameter-names := request:get-parameter-names()
    return
        indexes:remove-parameter-names($current-parameter-names[not(. = $parameter-names-to-keep)])
};

declare function indexes:replace-parameters($new-parameters) {
    let $current-parameter-names := request:get-parameter-names()
    let $current-parameters :=
        for $name in $current-parameter-names
        return concat($name, '=', request:get-parameter($name, ()))
    return
        indexes:replace-parameters($current-parameters, $new-parameters)
};

declare function indexes:replace-parameters($current-parameters, $new-parameters) {
    let $new-parameter-names := for $new-parameter in $new-parameters return substring-before($new-parameter, '=')
    let $remaining-parameters := indexes:remove-parameter-names($current-parameters, $new-parameter-names)
    let $result-parameters := for $param in ($remaining-parameters, $new-parameters) order by $param return $param
    return
        concat('?', string-join($result-parameters, '&'))
};

declare function indexes:strip-param-from-param-string($param-string, $param) {
    replace($param-string, concat('&?', $param, '=[^&]*?&?.*$'), '')
};

xquery version "1.0";

(:
    Module: Browse Indexes: See an overview of .xconf files stored in /db/system/config,
    and browse the associated index keys.
:)

module namespace indexes="http://exist-db.org/xquery/admin-interface/indexes";

declare namespace cc="http://exist-db.org/collection-config/1.0";

declare option exist:serialize "method=xhtml media-type=text/html omit-xml-declaration=no enforce-xhtml=yes";

(:
    Global variables - derived from URL parameters
:)
declare variable $indexes:start-value := request:get-parameter('start-value', '');
declare variable $indexes:callback := util:function(xs:QName("indexes:term-callback"), 2);
declare variable $indexes:max-number-returned := xs:integer(request:get-parameter('max', 100));
declare variable $indexes:index := request:get-parameter('index', '');
declare variable $indexes:sortby := request:get-parameter('sortby', 'term');
declare variable $indexes:sortorder := request:get-parameter('sortorder', 'ascending');
declare variable $indexes:node-name := request:get-parameter('node-name', '');
declare variable $indexes:match := request:get-parameter('match', '');
declare variable $indexes:collection := request:get-parameter('collection', '');
declare variable $indexes:node-set :=
    if ($indexes:node-name ne '' and $indexes:collection ne '') then
        indexes:get-nodeset-from-qname($indexes:collection, $indexes:node-name)
    else if ($indexes:match ne '' and $indexes:collection ne '') then
        indexes:get-nodeset-from-match($indexes:collection, $indexes:match)
    else
        ();
declare variable $indexes:qname :=
    if ($indexes:node-name ne '') then
        if (not(matches($indexes:node-name, 'xml:')) and contains($indexes:node-name, ':')) then
            QName(indexes:get-namespace-uri-from-node-name($indexes:node-name, $indexes:collection), $indexes:node-name)
        else
            xs:QName($indexes:node-name)
    else ();
declare variable $indexes:show-keys-by := request:get-parameter('show-keys-by', '');
declare variable $indexes:index-names :=

        <name>Index Types</name>
        <description>This is the list of known index types and their labels.</description>
        <items>
            <item>
                Lucene
                <value>lucene-index</value>
            </item>
            <item>
                NGram
                <value>ngram-index</value>
            </item>
            <item>
                Range
                <value>range-index</value>
            </item>
            <item>
                Legacy Fulltext
                <value>legacy-fulltext-index</value>
            </item>
        </items>
    ;

(:
    Main function: outputs the page.
:)
declare function indexes:main() {
    let $xconfs :=
        for $xconf in collection('/db/system/config/')/cc:collection
        order by util:collection-name($xconf)
        return $xconf
    return
        if (empty($xconfs)) then

Browse Indexes

No Index Configurations were found in the /db/system/config collection.
                    Consider loading the example files and returning
                    to browse the indexes created.

        else if (empty($indexes:node-set)) then

            indexes:list-indexes($xconfs)

        else

            indexes:show-index-keys()
};

(:
    Lists indexes.
:)
declare function indexes:list-indexes($xconfs) {

Browse Indexes

This is an overview of the index definitions applied in the database,
            as well as the "index keys" (the actual items stored in the index).

Browsing {count($xconfs)} indexes:

        { indexes:xconf-to-table($xconfs) }

};

(:
    Transforms an index definition into an HTML table.
:)
declare function indexes:xconf-to-table($xconfs as node()*) as item()* {

{
        for $xconf at $count in $xconfs
        let $xconf-collection-name := util:collection-name($xconf)
        let $data-collection-name := substring-after(util:collection-name($xconf), '/db/system/config')
        order by $xconf-collection-name
        return

                {$count}. {$data-collection-name} {if (xmldb:collection-available($data-collection-name)) then () else ' (no data)'}
                    (View xconf file)

                        Item Indexed
                        Index
                        Instances
                        Show Index Keys By

                    {
                    for $entry in ( indexes:analyze-legacy-fulltext-indexes($xconf),
                        indexes:analyze-lucene-indexes($xconf),
                        indexes:analyze-range-indexes($xconf),
                        indexes:analyze-ngram-indexes($xconf) )
                    let $item := $entry/td[1]
                    let $index := $entry/td[2]
                    (: order by $index, $item :)
                    return $entry
                    }

    }

};

(:
    Shows the index keys on a given nodeset or QName
:)
declare function indexes:show-index-keys() {
    let $query-start-time := util:system-time()

    let $keys :=
        (: legacy fulltext index use the text:index-terms() function :)
        if ($indexes:index eq 'legacy-fulltext-index') then
            text:index-terms($indexes:node-set, $indexes:start-value, $indexes:callback, $indexes:max-number-returned)
        (: range indexes can use util:index-keys() without specifying the index :)
        else if ($indexes:index eq 'range-index') then
            if (util:index-type($indexes:node-set) eq 'xs:string') then
                util:index-keys($indexes:node-set, $indexes:start-value, $indexes:callback, $indexes:max-number-returned)
            else
                let $start-value :=
                    if (util:index-type($indexes:node-set) eq 'xs:double') then
                        if ($indexes:start-value castable as xs:double) then xs:double($indexes:start-value) else xs:double(0)
                    else if (util:index-type($indexes:node-set) eq 'xs:float') then
                        if ($indexes:start-value castable as xs:float) then xs:float($indexes:start-value) else xs:float(0)
                    else if (util:index-type($indexes:node-set) eq 'xs:boolean') then
                        if ($indexes:start-value castable as xs:boolean) then xs:boolean($indexes:start-value) else xs:boolean(0)
                    else if (util:index-type($indexes:node-set) eq 'xs:dateTime') then
                        if ($indexes:start-value castable as xs:dateTime) then xs:dateTime($indexes:start-value) else xs:dateTime('0001-01-01T00:00:00-00:00')
                    else if (util:index-type($indexes:node-set) eq 'xs:date') then
                        if ($indexes:start-value castable as xs:date) then xs:date($indexes:start-value) else xs:date('0001-01-01')
                    else
                        if ($indexes:start-value castable as xs:integer) then xs:integer($indexes:start-value)
                    else 0
                return util:index-keys($indexes:node-set, $start-value, $indexes:callback, $indexes:max-number-returned)
        (: all other indexes need to specify the index in the 5th parameter of util:index-keys() :)
        else
            if ($indexes:show-keys-by eq 'node') then
                util:index-keys($indexes:node-set, $indexes:start-value, $indexes:callback, $indexes:max-number-returned, $indexes:index)
            else
                util:index-keys-by-qname($indexes:qname, $indexes:start-value, $indexes:callback, $indexes:max-number-returned, $indexes:index)

    let $log := util:log("DEBUG", concat("INDEXES index type:    ", $indexes:index))
    let $log := util:log("DEBUG", concat("INDEXES qname     :    ", $indexes:qname))

    (:  Reminder for sorting:
           term = $key/td[1]
           frequency := xs:integer($key/td[2])
           documents := xs:integer($key/td[3])
           position := xs:integer($key/td[4])
    :)

    let $sorted-keys :=
        if ($indexes:sortby eq 'term') then
            if ($indexes:sortorder eq 'ascending') then
                for $key in $keys order by $key/td[1] ascending return $key
            else
                for $key in $keys order by $key/td[1] descending return $key
        else if ($indexes:sortby eq 'frequency') then
            if ($indexes:sortorder eq 'ascending') then
                for $key in $keys order by xs:integer($key/td[2]) ascending, $key/td[1] ascending return $key
            else
                for $key in $keys order by xs:integer($key/td[2]) descending, $key/td[1] ascending return $key
        else if ($indexes:sortby eq 'documents') then
            if ($indexes:sortorder eq 'ascending') then
                for $key in $keys order by xs:integer($key/td[3]) ascending, $key/td[1] ascending return $key
            else
                for $key in $keys order by xs:integer($key/td[3]) descending, $key/td[1] ascending return $key
        else if ($indexes:sortby eq 'position') then
            if ($indexes:sortorder eq 'ascending') then
                for $key in $keys order by xs:integer($key/td[4]) ascending return $key
            else
                for $key in $keys order by xs:integer($key/td[4]) descending return $key
        else $keys

    let $query-end-time := util:system-time()
    let $query-duration := ($query-end-time - $query-start-time) div xs:dayTimeDuration('PT1S')

    return

View Index Keys

Select a different index

{count($keys)} keys returned in {$query-duration}s

Keys for the {indexes:index-name-to-label($indexes:index)} index defined on "{string-join(($indexes:node-name, $indexes:match), '')}" in the {$indexes:collection} collection, by {$indexes:show-keys-by}.

            <form method="get" action="{indexes:remove-parameter-names('start-value')}">

                    Max number returned: <select name="max">{
                        for $number in (10, 100, 1000, 10000)
                        return
                            <option value="{$number}">{if ($number eq $indexes:max-number-returned) then attribute selected {'selected'} else ()}{$number}</option>
                    }</select>

                    Search for terms starting with: <input name="start-value" type="text" value="{$indexes:start-value}"/>
                    {if ($indexes:start-value ne '') then Clear search else ()}
                    {
                    for $param in request:get-parameter-names()[not(. = ('max', 'start-value'))]
                    return
                        <input type="hidden" name="{$param}" value="{request:get-parameter($param, '')}"/>
                    }

                    <input type="submit"/>

            </form>

{
                    for $column in ('term', 'frequency', 'documents', 'position')
                    return
                        {$column} {indexes:sort-direction-indicator($column)}
                }

                { $sorted-keys }

};

(:
    Helper function: Callback function called used in indexes:show-index-keys() for util:index-keys()
:)
declare function indexes:term-callback($term as xs:string, $data as xs:int+) as element() {

        {$term}
        {$data[1]}
        {$data[2]}
        {$data[3]}

};

(:
    Analyzes the Lucene indexes in an index definition
:)
declare function indexes:analyze-lucene-indexes($xconf) {
    let $index-label := indexes:index-name-to-label('lucene-index')
    let $lucene := $xconf/cc:index/cc:lucene
    return if (not($lucene) or not($lucene/cc:text)) then () else
        (
        (: TODO: complete report of default/other Lucene analyzers :)
        (:let $analyzers := $lucene/cc:analyzer
        return
            if ($analyzers) then
                (

Lucene Analyzers

                ,
                for $node in $analyzers
                return
                    if (not($node/@id)) then

default: {concat(' ', $node/@class/string())}

                    else

{$node/@id/string()}: {concat(' ', $node/@class/string())}

                )
            else ()
        ,:)
        let $texts := $lucene//cc:text
        return
            (
            for $text in $texts
            let $qname := if ($text/@qname) then $text/@qname/string() else ()
            let $match := if ($text/@match) then $text/@match/string() else ()
            let $analyzer := if ($text/@analyzer) then $text/@analyzer/string() else ()
            let $collection := substring-after(util:collection-name($text), '/db/system/config')
            let $nodeset := if ($qname) then indexes:get-nodeset-from-qname($collection, $qname) else indexes:get-nodeset-from-match($collection, $match)
            return

                        {if ($qname) then $qname else $match}
                        {if ($text/@boost) then concat(' (boost: ', $text/@boost/string(), ')') else ()}
                        {if ($text/cc:ignore) then (
, concat('(ignore: ', string-join(for $ignore in $text/cc:ignore return $ignore/@qname/string(), ', '), ')')) else ()}
                    {$index-label} {if ($qname) then ' QName' else ' Match'} {if ($analyzer) then concat(' (', $analyzer, ')') else ' (default analyzer)' (: TODO: complete report of default/other Lucene analyzers :)}
                    {count($nodeset)}
                    {
                        if (empty($nodeset)) then ()
                        else
                            (
                            QName
                            ,
                            ', '
                            ,
                            Node
                            )
                    }

            )
        )
};

(:
    Analyzes the legacy fulltext indexes in an index definition
:)
declare function indexes:analyze-legacy-fulltext-indexes($xconf) {
    let $collection := substring-after(util:collection-name($xconf), '/db/system/config')
    let $index := 'legacy-fulltext-index'
    let $index-label := indexes:index-name-to-label($index)
    let $fulltext := $xconf/cc:index/cc:fulltext
    let $creates := $fulltext/cc:create
    let $default-none := $fulltext/@default eq 'none'
    let $attributes-none := $fulltext/@attributes eq 'false'

    let $no-fulltext := if ( not($fulltext) or ($default-none and $attributes-none and not($creates)) ) then '(disabled)' else ()
    return
        if ($no-fulltext) then ()
        else if (not($default-none) and not($attributes-none)) then

                All Elements and Attributes!
                {$index-label}
                {count(collection($collection)//*)} elements and {count(collection($collection)//@*)} attributes
                (Too many to display)

(: is it feasible to display all qnames/nodes here? :)
        else (: if ($creates) then :)
            (
            for $create in $creates
            let $qname := $create/@qname/string()
            let $mixed := $create/@mixed/string()
            return

                    {$qname}
                    {$index-label} {if ($mixed) then '(mixed)' else ()}
                    {count(util:eval(concat('collection(', $collection, ')//', $qname)))}
                    Node

            ,
            let $only-elements-disabled := if ($fulltext/@default eq 'none' and $fulltext/@attributes ne 'no') then '(elements disabled)' else ()
            let $only-attribs-disabled := if ($fulltext/@default ne 'none' and $fulltext/@attributes eq 'no') then '(attributes disabled)' else ()
            return
            if ($only-elements-disabled) then

                    All Attributes!
                    {$index-label}
                    {count(collection($collection)//@*)} attributes
                    (Too many to display)

(: is it feasible to display all qnames/nodes here? :)
            else if ($only-attribs-disabled) then

                    All Elements!
                    {$index-label}
                    {count(collection($collection)//*)} elements
                    (Too many to display)

(: is it feasible to display all qnames/nodes here? :)
            else ()
            )
};

(:
    Analyzes the range indexes in an index definition
:)
declare function indexes:analyze-range-indexes($xconf) {
    let $index-label := indexes:index-name-to-label('range-index')
    let $ranges := $xconf/cc:index/cc:create
    return if (not($ranges)) then () else
        for $range in $ranges
        let $qname := $range/@qname/string()
        let $match := $range/@path/string()
        let $type := $range/@type/string()
        let $collection := substring-after(util:collection-name($range), '/db/system/config')
        let $nodeset := if ($qname) then indexes:get-nodeset-from-qname($collection, $qname) else indexes:get-nodeset-from-match($collection, $match)
        return

                {if ($qname) then $qname else $match}
                {$index-label} {if ($qname) then ' QName' else ' Path'} ({$type})
                {count($nodeset)}
                {
                    if (empty($nodeset)) then ()
                    else
                        Node
                }

};

(:
    Analyzes the NGram indexes in an index definition
:)
declare function indexes:analyze-ngram-indexes($xconf) {
    let $index-label := indexes:index-name-to-label('ngram-index')
    let $ngrams := $xconf/cc:index/cc:ngram
    return if (not($ngrams)) then () else
        for $ngram in $ngrams
        let $qname := $ngram/@qname/string()
        let $collection := substring-after(util:collection-name($ngram), '/db/system/config')
        let $nodeset := indexes:get-nodeset-from-qname($collection, $qname)
        return

                {$qname}
                {$index-label} QName
                {count($nodeset)}
                {
                    if (not(empty($nodeset))) then
                        (
                        QName,
                        ', '
                        ,
                        Node
                        )
                     else ()
                 }

};

(:
    Helper function: Returns a nodeset of instances of a node-name in a collection
:)
declare function indexes:get-nodeset-from-qname($collection as xs:string, $node-name as xs:string) as node()* {
    let $nodeset-expression :=
        concat(
            indexes:get-namespace-decaration-from-node-name($node-name, $collection)
            ,
            'collection("', $collection, '")//', $node-name
        )
    let $log := util:log("DEBUG", concat("INDEXES get-nodeset:          ", $nodeset-expression))
    return
        util:eval($nodeset-expression)
};

(:
    Helper function: Returns a nodeset of instances of a match expression in a collection
:)
declare function indexes:get-nodeset-from-match($collection as xs:string, $match as xs:string) as node()* {
    let $nodeset-expression :=
        concat(
            string-join(
                let $node-names := tokenize(replace($match, '//', '/'), '/')
                return
                    for $node-name in $node-names
                    return
                        indexes:get-namespace-decaration-from-node-name($node-name, $collection)
                , ' ')
            ,
            'collection("', $collection, '")', $match, if (contains($match, '@')) then () else ()
        )
    return
        util:eval($nodeset-expression)
};

(:
    Helper function: Returns the index definition for a given collection
:)
declare function indexes:get-xconf($collection as xs:string) as document-node() {
    let $config-root := '/db/system/config'
    let $xconf-collection := concat($config-root, $collection)
    let $xconf-filename := xmldb:get-child-resources($xconf-collection)[ends-with(., '.xconf')]
    let $xconf := doc(concat($xconf-collection, '/', $xconf-filename))
    return $xconf
};

(:
    Helper function: Looks in the collection.xconf's collection and index elements for namespace URIs for a given node name
:)
declare function indexes:get-namespace-uri-from-node-name($node-name, $collection) {
    let $name := substring-before($node-name, ':')
    let $xconf := indexes:get-xconf($collection)
    let $uri := (namespace-uri-for-prefix($name, $xconf/cc:collection), namespace-uri-for-prefix($name, $xconf//cc:index))
    return
        $uri
};

(:
    Helper function: Constructs a namespace declaration for use in util:eval()
:)
declare function indexes:get-namespace-decaration-from-node-name($node-name as xs:string, $collection as xs:string) as xs:string* {
    if (not(matches($node-name, 'xml:')) and contains($node-name, ':')) then
        let $name := substring-before($node-name, ':')
        let $uri := indexes:get-namespace-uri-from-node-name($node-name, $collection)
        return
            concat('declare namespace ', $name, '="', $uri, '"; ')
    else ()
};

(:
    Helper function: gets the label for a given index-name
:)
declare function indexes:index-name-to-label($index-name as xs:string) as xs:string {
    $indexes:index-names//item[value eq $index-name]/label/text()
};

(:
    ====
    Helper functions for modifying the sort order used in indexes:show-index-keys()
    ====
:)
declare function indexes:toggle-sortorder($current-sortorder) {
    indexes:toggle-sortorder($current-sortorder, ())
};

declare function indexes:toggle-sortorder($current-sortorder, $other-new-parameters) {
    let $neworder :=
        if ($current-sortorder eq 'ascending') then
            'sortorder=descending'
        else
            'sortorder=ascending'
    let $new-parameters := ($neworder, $other-new-parameters)
    return
        indexes:replace-parameters($new-parameters)
};

declare function indexes:set-sortorder($current-sortorder, $current-sortby, $new-sortby) {
    if ($current-sortby eq $new-sortby) then
        indexes:toggle-sortorder($current-sortorder)
    else
        indexes:strip-param-from-param-string(indexes:replace-parameters(concat('sortby=', $new-sortby)), 'sortorder')
};

declare function indexes:set-sortorder($new-sortby) {
    indexes:set-sortorder($indexes:sortorder, $indexes:sortby, $new-sortby)
};

declare function indexes:sort-direction-indicator($sortby as xs:string) {
    if ($sortby eq $indexes:sortby) then
        if ($indexes:sortorder eq 'ascending') then
            ' ↓'
        else
            ' ↑'
    else ()
};

(:
    ====
    Helper functions for handling parameters
    ====
:)
declare function indexes:remove-parameter-names($parameter-names-to-remove) {
    let $current-parameter-names := request:get-parameter-names()
    let $remaining-parameters :=
        indexes:remove-parameter-names(
            for $current-parameter-name in $current-parameter-names
            return
                concat($current-parameter-name, '=', request:get-parameter( $current-parameter-name, () ))
            ,
            $parameter-names-to-remove
            )
    return
        if (exists($remaining-parameters)) then
            concat('?', string-join($remaining-parameters, '&'))
        else
            '?'
};

declare function indexes:remove-parameter-names($current-parameters, $parameter-names-to-remove) {
    for $current-parameter in $current-parameters
    return
        if (substring-before($current-parameter, '=') = $parameter-names-to-remove) then
            ()
        else
            $current-parameter
};

declare function indexes:remove-parameter-names-except($parameter-names-to-keep) {
    let $current-parameter-names := request:get-parameter-names()
    return
        indexes:remove-parameter-names($current-parameter-names[not(. = $parameter-names-to-keep)])
};

declare function indexes:replace-parameters($new-parameters) {
    let $current-parameter-names := request:get-parameter-names()
    let $current-parameters :=
        for $name in $current-parameter-names
        return concat($name, '=', request:get-parameter($name, ()))
    return
        indexes:replace-parameters($current-parameters, $new-parameters)
};

declare function indexes:replace-parameters($current-parameters, $new-parameters) {
    let $new-parameter-names := for $new-parameter in $new-parameters return substring-before($new-parameter, '=')
    let $remaining-parameters := indexes:remove-parameter-names($current-parameters, $new-parameter-names)
    let $result-parameters := for $param in ($remaining-parameters, $new-parameters) order by $param return $param
    return
        concat('?', string-join($result-parameters, '&'))
};

declare function indexes:strip-param-from-param-string($param-string, $param) {
    replace($param-string, concat('&?', $param, '=[^&]*?&?.*$'), '')
};


dont return default collection if x-context not empty and not match

module namespace repo-utils = "http://aac.ac.at/content_repository/utils";

import module namespace xdb="http://exist-db.org/xquery/xmldb";
import module namespace diag =  "http://www.loc.gov/zing/srw/diagnostic/" at  "modules/diagnostics/diagnostics.xqm";

(:~ HELPER functions - configuration, caching, data-access
:)

(:
: this all cannot be defined as global variable, because we want different configurations,
: thus the $config has to be sent as param everywhere
: declare variable $repo-utils:config := doc("config.xml");
: declare variable $repo-utils:mappings := doc(repo-utils:config-value('mappings'));
: declare variable $repo-utils:data-collection := collection(repo-utils:config-value('data.path'));
: declare variable $repo-utils:md-collection := collection(repo-utils:config-value('metadata.path'));
: declare variable $repo-utils:cachePath as xs:string := repo-utils:config-value('cache.path');
:)

declare variable $repo-utils:xmlExt as xs:string := ".xml";
declare variable $repo-utils:responseFormatXml as xs:string := "xml";
declare variable $repo-utils:responseFormatJSon as xs:string := "json";
declare variable $repo-utils:responseFormatText as xs:string := "text";
declare variable $repo-utils:responseFormatHTML as xs:string := "html";

declare function repo-utils:config-value($config, $key as xs:string) as xs:string* {
    $config//property[@key=$key]
};

(:~ Get value of a param based on a key, from config or from request-param (precedence) :)
declare function repo-utils:param-value($config, $key as xs:string, $default as xs:string) as xs:string* {

    let $param := request:get-parameter($key, $default)
    return if ($param) then $param else $config//property[@key=$key]
};

(:~ returns db-collection (as nodeset) based on the identifier in x-context, looked up in the mapping or default collection as defined in config

@returns nodeset of given context
 if no x-context match or no context provided returns default collection from the config (data.path )
 (if available, otherwise again empty result)

WATCHME: changed to lax handling of the context (if no match - go to default)
        gives better recall, but may confuse - alternatively: strict: empty result if x-context does not match
TODO: accept $x-context as xs:string*
:)
declare function repo-utils:context-to-collection($x-context as xs:string, $config) as node()* {
(:      let $mappings := doc(repo-utils:config-value($config, 'mappings')):)
      let $dbcoll-path := repo-utils:context-to-collection-path($x-context, $config)
    return if ($dbcoll-path eq "" ) then ()
                else collection($dbcoll-path)
};

(:~ returns path to db-collection (as nodeset) based on the identifier in x-context,
looked up in the mapping or default data.path as defined in config

@returns dbcoll-path of given context
 if no x-context match or no context provided returns default collection from the config (data.path )
 (if available, otherwise again empty result)

WATCHME: changed to lax handling of the context (if no match - go to default)
        gives better recall, but may confuse - alternatively: strict: empty result if x-context does not match
changed again: only give default if x-context is empty string or 'default'-string.
T
:)
declare function repo-utils:context-to-collection-path($x-context as xs:string, $config) as xs:string {
      let $mappings:= doc(repo-utils:config-value($config, 'mappings'))
    return    (: if ($x-context) then :)
            if (exists($mappings//map[xs:string(@key) eq $x-context]/@path)) then
                    $mappings//map[xs:string(@key) eq $x-context]/xs:string(@path)
                (: else "" :)
                  else if (exists(repo-utils:config-value($config, 'data.path')) and
                            ($x-context = ('', 'default'))) then
                        repo-utils:config-value($config, 'data.path')
                  else ""
};

(:~ Get the resource by PID/handle/URL or some known identifier.
  TODO: NOT ADAPTED YET! (taken from CMD)
  TODO: be clear about resource itself and metadata-record!
:)
(:declare function repo-utils:get-resource-by-id($id as xs:string) as node()* {
  let $collection := collection($cr:dataPath)
  return
    if ($id eq "" or $id eq $cr:collectionRoot) then
    $collection//IsPartOf[. = $cr:collectionRoot]/ancestor::CMD
  else
    util:eval(concat("$collection/ft:query(descendant::MdSelfLink, <term>", xdb:decode($id), "</term>)/ancestor::CMD"))
 (\: $collection/descendant::MdSelfLink[. = xdb:decode($id)]/ancestor::CMD :\)
};
:)

(:~ Checks whether the document is available or not.
  generic, currently not used
:)
declare function repo-utils:is-doc-available($collection as xs:string, $doc-name as xs:string) as xs:boolean {
  fn:doc-available(fn:concat($collection, "/", $doc-name))
};

declare function repo-utils:is-in-cache($doc-name as xs:string,$config) as xs:boolean {
  fn:doc-available(fn:concat(repo-utils:config-value($config, 'cache.path'), "/", $doc-name))
};

declare function repo-utils:get-from-cache($doc-name as xs:string,$config) as item()* {
      fn:doc(fn:concat(repo-utils:config-value($config, 'cache.path'), "/", $doc-name))
};

(:~ Store the data in cache. Uses own writer-account
:)
declare function repo-utils:store-in-cache($doc-name as xs:string, $data as node(),$config) as item()* {
  let $clarin-writer := fn:doc("/db/cr/writer.xml"),
  $cache-path := repo-utils:config-value($config, 'cache.path'),
  $dummy := xdb:login($cache-path, $clarin-writer//write-user/text(), $clarin-writer//write-user-cred/text()),
  $store := (: util:catch("org.exist.xquery.XPathException", :) xdb:store($cache-path, $doc-name, $data), (: , ()) :)
  $stored-doc := fn:doc(concat($cache-path, "/", $doc-name))
  return $stored-doc
(:  ():)
};

(:
this tries to create sub-collections in index for
individual data-collection - but it makes the retrieval etc. also more complicated
so rather encoding the data-collection also in the name

declare function repo-utils:store-in-cache($data-collection as item(), $doc-name as xs:string, $data as node()) as item()* {
    let $data-coll-name := collection-name($data-collection)
    let $index-coll-path := concat($repo-utils:cachePath, "/", $data-coll-name )
   let $create-coll := if (not(xmldb:collection-available($index-coll-path ))) then
                           xmldb:create-collection($repo-utils:cachePath, $data-collname) else ()
  let $store-result := repo-utils:store($index-coll-path, $doc-name, $data, true())

return $store-result
};
:)

(:<options><option key="update">yes</option></options>:)
declare function repo-utils:store($collection as xs:string, $doc-name as xs:string, $data as node(), $overwrite as xs:boolean) as item()* {
  let $clarin-writer := fn:doc("/db/cr/writer.xml"),
  $dummy := xdb:login($collection, $clarin-writer//write-user/text(), $clarin-writer//write-user-cred/text())

  let $rem := if ($overwrite and doc-available(concat($collection, $doc-name))) then xdb:remove($collection, $doc-name) else ()

  let $store := (: util:catch("org.exist.xquery.XPathException", :) xdb:store($collection, $doc-name, $data), (: , ()) :)
  $stored-doc := fn:doc(concat($collection, "/", $doc-name))
  return $stored-doc
};

(:~ Create document name with md5-hash for selected collections (or types) for reuse.
:)
declare function repo-utils:gen-cache-id($type-name as xs:string, $keys as xs:string+, $depth as xs:string) as xs:string {
  let $name-prefix := fn:concat($type-name, $depth),
       $sanitized-names := for $key in $keys return repo-utils:sanitize-name($key)
    return
(:    fn:concat($name-prefix, "-", util:hash(string-join($sorted-names, ""), "MD5"), $repo-utils:xmlExt):)
    fn:concat($name-prefix, "-", string-join($sanitized-names, "-"), $repo-utils:xmlExt)
};

(:~ wipes out problematic characters from names
:)
declare function repo-utils:sanitize-name($name as xs:string) as xs:string {
    translate($name, ":",'_')
};

declare function repo-utils:serialise-as($item as node()?, $format as xs:string, $operation as xs:string, $config) as item()? {
    repo-utils:serialise-as($item, $format, $operation, $config, ())
};

(:~ transform result to HTML, JSON, or leave as XML - according to format parameter

@param $parameters optional additional parameters to be passed to xsl
:)
declare function repo-utils:serialise-as($item as node()?, $format as xs:string, $operation as xs:string, $config, $parameters as node()* ) as item()? {
      if ($format eq $repo-utils:responseFormatJSon) then
	       let $option := util:declare-option("exist:serialize", "method=json media-type=application/json")
	       return $item
	    else if (contains($format, $repo-utils:responseFormatHTML)) then
	           let $xslDoc := doc(concat(repo-utils:config-value($config, 'scripts.path'), repo-utils:config-value($config, concat($operation, ".xsl"))) )
	           let $res := transform:transform($item,$xslDoc,
              			<parameters><param name="format" value="{$format}"/>
              			            <param name="x-context" value="{repo-utils:param-value($config, 'x-context', '' )}"/>
              			            <param name="base_url" value="{repo-utils:config-value($config, 'base.url')}"/>
              			            <param name="mappings-file" value="{repo-utils:config-value($config, 'mappings')}"/>
              			            <param name="scripts_url" value="{repo-utils:config-value($config, 'scripts.url')}"/>
              			             <param name="site_name" value="{repo-utils:config-value($config, 'site.name')}"/>
              			             <param name="site_logo" value="{repo-utils:config-value($config, 'site.logo')}"/>
              			             {$parameters/param}
              			</parameters>)
(:               let $option := util:declare-option("exist:serialize", "method=xml media-type=text/html"):)
let $option := util:declare-option("exist:serialize", "method=xhtml media-type=text/html")
	           return $res
	   else
	       let $option := util:declare-option("exist:serialize", "method=xml media-type=application/xml")
	    return $item
};
(: $repo-utils:responseFormatXml, $repo-utils:responseFormatText:)

	    (:let $option :=
	              if (contains($format, $repo-utils:responseFormatHTML)) then
	                  util:declare-option("exist:serialize", "method=xml media-type=text/xhtml")
	              else
    	              util:declare-option("exist:serialize", "method=xml media-type=application/xml")
:)

	          	(: $item :)

module namespace repo-utils = "http://aac.ac.at/content_repository/utils";

import module namespace xdb="http://exist-db.org/xquery/xmldb";
import module namespace diag =  "http://www.loc.gov/zing/srw/diagnostic/" at  "modules/diagnostics/diagnostics.xqm";

(:~ HELPER functions - configuration, caching, data-access
:)

(:
: this all cannot be defined as global variable, because we want different configurations,
: thus the $config has to be sent as param everywhere
: declare variable $repo-utils:config := doc("config.xml");
: declare variable $repo-utils:mappings := doc(repo-utils:config-value('mappings'));
: declare variable $repo-utils:data-collection := collection(repo-utils:config-value('data.path'));
: declare variable $repo-utils:md-collection := collection(repo-utils:config-value('metadata.path'));
: declare variable $repo-utils:cachePath as xs:string := repo-utils:config-value('cache.path');
:)

declare variable $repo-utils:xmlExt as xs:string := ".xml";
declare variable $repo-utils:responseFormatXml as xs:string := "xml";
declare variable $repo-utils:responseFormatJSon as xs:string := "json";
declare variable $repo-utils:responseFormatText as xs:string := "text";
declare variable $repo-utils:responseFormatHTML as xs:string := "html";

declare function repo-utils:config-value($config, $key as xs:string) as xs:string* {
    $config//property[@key=$key]
};

(:~ Get value of a param based on a key, from config or from request-param (precedence) :)
declare function repo-utils:param-value($config, $key as xs:string, $default as xs:string) as xs:string* {

    let $param := request:get-parameter($key, $default)
    return if ($param) then $param else $config//property[@key=$key]
};

(:~ returns db-collection (as nodeset) based on the identifier in x-context, looked up in the mapping or default collection as defined in config

@returns nodeset of given context
 if no x-context match or no context provided returns default collection from the config (data.path )
 (if available, otherwise again empty result)

WATCHME: changed to lax handling of the context (if no match - go to default)
        gives better recall, but may confuse - alternatively: strict: empty result if x-context does not match
TODO: accept $x-context as xs:string*
:)
declare function repo-utils:context-to-collection($x-context as xs:string, $config) as node()* {
(:      let $mappings := doc(repo-utils:config-value($config, 'mappings')):)
      let $dbcoll-path := repo-utils:context-to-collection-path($x-context, $config)
    return if ($dbcoll-path eq "" ) then ()
                else collection($dbcoll-path)
};

(:~ returns path to db-collection (as nodeset) based on the identifier in x-context,
looked up in the mapping or default data.path as defined in config

@returns dbcoll-path of given context
 if no x-context match or no context provided returns default collection from the config (data.path )
 (if available, otherwise again empty result)

WATCHME: changed to lax handling of the context (if no match - go to default)
        gives better recall, but may confuse - alternatively: strict: empty result if x-context does not match
T
:)
declare function repo-utils:context-to-collection-path($x-context as xs:string, $config) as xs:string {
      let $mappings:= doc(repo-utils:config-value($config, 'mappings'))
    return    (: if ($x-context) then :)
            if (exists($mappings//map[xs:string(@key) eq $x-context]/@path)) then
                    $mappings//map[xs:string(@key) eq $x-context]/xs:string(@path)
                (: else "" :)
                  else if (exists(repo-utils:config-value($config, 'data.path'))) then
                        repo-utils:config-value($config, 'data.path')
                  else ""
};

(:~ Get the resource by PID/handle/URL or some known identifier.
  TODO: NOT ADAPTED YET! (taken from CMD)
  TODO: be clear about resource itself and metadata-record!
:)
(:declare function repo-utils:get-resource-by-id($id as xs:string) as node()* {
  let $collection := collection($cr:dataPath)
  return
    if ($id eq "" or $id eq $cr:collectionRoot) then
    $collection//IsPartOf[. = $cr:collectionRoot]/ancestor::CMD
  else
    util:eval(concat("$collection/ft:query(descendant::MdSelfLink, <term>", xdb:decode($id), "</term>)/ancestor::CMD"))
 (\: $collection/descendant::MdSelfLink[. = xdb:decode($id)]/ancestor::CMD :\)
};
:)

(:~ Checks whether the document is available or not.
  generic, currently not used
:)
declare function repo-utils:is-doc-available($collection as xs:string, $doc-name as xs:string) as xs:boolean {
  fn:doc-available(fn:concat($collection, "/", $doc-name))
};

declare function repo-utils:is-in-cache($doc-name as xs:string,$config) as xs:boolean {
  fn:doc-available(fn:concat(repo-utils:config-value($config, 'cache.path'), "/", $doc-name))
};

declare function repo-utils:get-from-cache($doc-name as xs:string,$config) as item()* {
      fn:doc(fn:concat(repo-utils:config-value($config, 'cache.path'), "/", $doc-name))
};

(:~ Store the data in cache. Uses own writer-account
:)
declare function repo-utils:store-in-cache($doc-name as xs:string, $data as node(),$config) as item()* {
  let $clarin-writer := fn:doc("/db/cr/writer.xml"),
  $cache-path := repo-utils:config-value($config, 'cache.path'),
  $dummy := xdb:login($cache-path, $clarin-writer//write-user/text(), $clarin-writer//write-user-cred/text()),
  $store := (: util:catch("org.exist.xquery.XPathException", :) xdb:store($cache-path, $doc-name, $data), (: , ()) :)
  $stored-doc := fn:doc(concat($cache-path, "/", $doc-name))
  return $stored-doc
(:  ():)
};

(:
this tries to create sub-collections in index for
individual data-collection - but it makes the retrieval etc. also more complicated
so rather encoding the data-collection also in the name

declare function repo-utils:store-in-cache($data-collection as item(), $doc-name as xs:string, $data as node()) as item()* {
    let $data-coll-name := collection-name($data-collection)
    let $index-coll-path := concat($repo-utils:cachePath, "/", $data-coll-name )
   let $create-coll := if (not(xmldb:collection-available($index-coll-path ))) then
                           xmldb:create-collection($repo-utils:cachePath, $data-collname) else ()
  let $store-result := repo-utils:store($index-coll-path, $doc-name, $data, true())

return $store-result
};
:)

(:<options><option key="update">yes</option></options>:)
declare function repo-utils:store($collection as xs:string, $doc-name as xs:string, $data as node(), $overwrite as xs:boolean) as item()* {
  let $clarin-writer := fn:doc("/db/cr/writer.xml"),
  $dummy := xdb:login($collection, $clarin-writer//write-user/text(), $clarin-writer//write-user-cred/text())

  let $rem := if ($overwrite and doc-available(concat($collection, $doc-name))) then xdb:remove($collection, $doc-name) else ()

  let $store := (: util:catch("org.exist.xquery.XPathException", :) xdb:store($collection, $doc-name, $data), (: , ()) :)
  $stored-doc := fn:doc(concat($collection, "/", $doc-name))
  return $stored-doc
};

(:~ Create document name with md5-hash for selected collections (or types) for reuse.
:)
declare function repo-utils:gen-cache-id($type-name as xs:string, $keys as xs:string+, $depth as xs:string) as xs:string {
  let $name-prefix := fn:concat($type-name, $depth),
       $sanitized-names := for $key in $keys return repo-utils:sanitize-name($key)
    return
(:    fn:concat($name-prefix, "-", util:hash(string-join($sorted-names, ""), "MD5"), $repo-utils:xmlExt):)
    fn:concat($name-prefix, "-", string-join($sanitized-names, "-"), $repo-utils:xmlExt)
};

(:~ wipes out problematic characters from names
:)
declare function repo-utils:sanitize-name($name as xs:string) as xs:string {
    translate($name, ":",'_')
};

declare function repo-utils:serialise-as($item as node()?, $format as xs:string, $operation as xs:string, $config) as item()? {
    repo-utils:serialise-as($item, $format, $operation, $config, ())
};

(:~ transform result to HTML, JSON, or leave as XML - according to format parameter

@param $parameters optional additional parameters to be passed to xsl
:)
declare function repo-utils:serialise-as($item as node()?, $format as xs:string, $operation as xs:string, $config, $parameters as node()* ) as item()? {
      if ($format eq $repo-utils:responseFormatJSon) then
	       let $option := util:declare-option("exist:serialize", "method=json media-type=application/json")
	       return $item
	    else if (contains($format, $repo-utils:responseFormatHTML)) then
	           let $xslDoc := doc(concat(repo-utils:config-value($config, 'scripts.path'), repo-utils:config-value($config, concat($operation, ".xsl"))) )
	           let $res := transform:transform($item,$xslDoc,
              			<parameters><param name="format" value="{$format}"/>
              			            <param name="x-context" value="{repo-utils:param-value($config, 'x-context', '' )}"/>
              			            <param name="base_url" value="{repo-utils:config-value($config, 'base.url')}"/>
              			            <param name="mappings-file" value="{repo-utils:config-value($config, 'mappings')}"/>
              			            <param name="scripts_url" value="{repo-utils:config-value($config, 'scripts.url')}"/>
              			             <param name="site_name" value="{repo-utils:config-value($config, 'site.name')}"/>
              			             <param name="site_logo" value="{repo-utils:config-value($config, 'site.logo')}"/>
              			             {$parameters/param}
              			</parameters>)
(:               let $option := util:declare-option("exist:serialize", "method=xml media-type=text/html"):)
let $option := util:declare-option("exist:serialize", "method=xhtml media-type=text/html")
	           return $res
	   else
	       let $option := util:declare-option("exist:serialize", "method=xml media-type=application/xml")
	    return $item
};
(: $repo-utils:responseFormatXml, $repo-utils:responseFormatText:)

	    (:let $option :=
	              if (contains($format, $repo-utils:responseFormatHTML)) then
	                  util:declare-option("exist:serialize", "method=xml media-type=text/xhtml")
	              else
    	              util:declare-option("exist:serialize", "method=xml media-type=application/xml")
:)

	          	(: $item :)


Added yet more examples to JSON documentation.

(:
 : Copyright 2006-2009 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

xquery version "3.0";

 (:~
  : Using this module, you can parse JSON data into XML, manipulate it like any
  : other XML data using XQuery, and serialize the result back as JSON.
  :
  : There are many ways to represent JSON data in XML, some loss-less ("round
  : tripable") and some lossy ("one way").  Loss-less representations preserve
  : the JSON data types boolean, number, and null; lossy
  : representations convert all data to strings.
  :

  : For a loss-less representation, Zorba implements that proposed by
  : John Snelson.
  : For example:
  :

  :   {
  :     "firstName" : "John",
  :     "lastName" : "Smith",
  :     "address" : {
  :       "streetAddress" : "21 2nd Street",
  :       "city" : "New York",
  :       "state" : "NY",
  :       "postalCode" : 10021
  :     },
  :     "phoneNumbers" : [ "212 732-1234", "646 123-4567" ]
  :   }
  :

  : would be represented as:
  :

  :   <json type="object">
  :     <pair name="firstName" type="string">John</pair>
  :     <pair name="lastName" type="string">Smith</pair>
  :     <pair name="address" type="object">
  :       <pair name="streetAddress" type="string">21 2nd Street</pair>
  :       <pair name="city" type="string">New York</pair>
  :       <pair name="state" type="string">NY</pair>
  :       <pair name="postalCode" type="number">10021</pair>
  :     </pair>
  :     <pair name="phoneNumbers" type="array">
  :       <item type="string">212 732-1234</item>
  :       <item type="string">646 123-4567</item>
  :     </pair>
  :   </json>
  :

  : For a lossy representation, Zorba implements
  : JsonML (the array form).
  : For example:
  :

  :   [ "person",
  :     { "created" : "2006-11-11T19:23",
  :       "modified" : "2006-12-31T23:59" },
  :     [ "firstName", "Robert" ],
  :     [ "lastName", "Smith" ],
  :     [ "address",
  :       { "type" : "home" },
  :       [ "street", "12345 Sixth Ave" ],
  :       [ "city", "Anytown" ],
  :       [ "state", "CA" ],
  :       [ "postalCode", "98765-4321" ]
  :     ]
  :   ]
  :

  : would be represented as:
  :

  :   <person created="2006-11-11T19:23" modified="2006-12-31T23:59">
  :     <firstName>Robert</firstName>
  :     <lastName>Smith</lastName>
  :     <address type="home">
  :       <street>12345 Sixth Ave</street>
  :       <city>Anytown</city>
  :       <state>CA</state>
  :       <postalCode>98765-4321</postalCode>
  :     </address>
  :   </person>
  :

  :
  : @author Paul J. Lucas
  : @project data processing/data converters
  :)
module namespace json = "http://www.zorba-xquery.com/modules/converters/json";

import module namespace schema = "http://www.zorba-xquery.com/modules/schema";

import schema namespace json-options =
  "http://www.zorba-xquery.com/modules/converters/json-options";

declare namespace ann = "http://www.zorba-xquery.com/annotations";
declare namespace err = "http://www.w3.org/2005/xqt-errors";
declare namespace zerr = "http://www.zorba-xquery.com/errors";

declare namespace ver = "http://www.zorba-xquery.com/options/versioning";
declare option ver:module-version "2.0";

(:~
 : Parses JSON data from a string and returns an XDM instance using one of the
 : representations described above.
 :
 : @param $json The JSON data to parse.
 : @param $options The parsing options, for example:
 :

 : <options xmlns="http://www.zorba-xquery.com/modules/converters/json-options">
 :   <json-format value="JsonML-array"/>
 : </options>
 :

 : @return said XDM instance.
 : @error err:XQDY0027 if $options can not be validated against the
 : json-options schema.
 : @error zerr:ZJPE0001 if $json contains an illegal JSON character.
 : @error zerr:ZJPE0002 if $json contains an illegal Unicode code-point.
 : @error zerr:ZJPE0003 if $json contains an illegal JSON character escape.
 : @error zerr:ZJPE0004 if $json contains an illegal JSON literal.
 : @error zerr:ZJPE0005 if $json contains an illegal JSON number.
 : @error zerr:ZJPE0006 if $json is not a valid JSON string.
 : @error zerr:ZJPE0007 if $json contains an unterminated string.
 : @error zerr:ZJPE0008 if $json contains an illegal QName.
 : @example test/rbkt/Queries/zorba/json/json-jsonml_array-parse-01.xq
 :)
declare function json:parse(
  $json as xs:string?,
  $options as element(json-options:options)
) as element(*,xs:untyped)*
{
  let $validated-options := if ( schema:is-validated( $options ) ) then
                              $options
                            else
                              validate { $options }
  return json:parse-internal( $json, $validated-options )
};

(:~
 : Parses JSON data from a string and returns an XDM instance using the Snelson
 : representation described above.
 :
 : @param $json The JSON data to parse.
 : @return said XDM instance.
 : @error zerr:ZJPE0001 if $json contains an illegal JSON character.
 : @error zerr:ZJPE0002 if $json contains an illegal Unicode code-point.
 : @error zerr:ZJPE0003 if $json contains an illegal JSON character escape.
 : @error zerr:ZJPE0004 if $json contains an illegal JSON literal.
 : @error zerr:ZJPE0005 if $json contains an illegal JSON number.
 : @error zerr:ZJPE0006 if $json is not a valid JSON string.
 : @error zerr:ZJPE0007 if $json contains an unterminated string.
 : @error zerr:ZJPE0008 if $json contains an illegal QName.
 : @example test/rbkt/Queries/zorba/json/json-snelson-parse-array-01.xq
 :)
declare function json:parse(
  $json as xs:string?
) as element(*,xs:untyped)*
{
  json:parse-internal(
    $json,
    validate {
      <options
          xmlns="http://www.zorba-xquery.com/modules/converters/json-options">
        <json-format value="Snelson"/>
      </options>
    }
  )
};

(:~
 : Serializes an XDM into JSON using one of the representations described
 : above.
 :
 : @param $xml The XDM to serialize.
 : @param $options The serializing options, for example:
 :

 : <options xmlns="http://www.zorba-xquery.com/modules/converters/json-options">
 :   <json-format value="JsonML-array"/>
 :   <whitespace value="indent"/>
 : </options>
 :

 : @return a JSON string.
 : @error err:XQDY0027 if $options can not be validated against the
 : json-options schema.
 : @error zerr:ZJSE0001 if $xml is not a document or element node.
 : @error zerr:ZJSE0002 if $xml contains an element that is missing a required
 : attribute.
 : @error zerr:ZJSE0003 if $xml contains an attribute having an illegal value.
 : @error zerr:ZJSE0004 if $xml contains an illegal element.
 : @error zerr:ZJSE0005 if $xml contains an illegal child element for a JSON
 : type.
 : @error zerr:ZJSE0006 if $xml contains an illegal child element.
 : @error zerr:ZJSE0007 if $xml contains an illegal text node.
 : @error zerr:ZJSE0008 if $xml contains an illegal value for a JSON type.
 : @example test/rbkt/Queries/zorba/json/json-jsonml_array-serialize-01.xq
 :)
declare function json:serialize(
  $xml as item()*,
  $options as element(json-options:options)
) as xs:string
{
  let $validated-options := if ( schema:is-validated( $options ) ) then
                              $options
                            else
                              validate { $options }
  return json:serialize-internal( $xml, $validated-options )
};

(:~
 : Serializes an XDM into JSON using one of the representations described
 : above.
 :
 : @param $xml The XDM to serialize.
 : @return a JSON string.
 : @error zerr:ZJSE0001 if $xml is not a document or element node.
 : @error zerr:ZJSE0002 if $xml contains an element that is missing a required
 : attribute.
 : @error zerr:ZJSE0003 if $xml contains an attribute having an illegal value.
 : @error zerr:ZJSE0004 if $xml contains an illegal element.
 : @error zerr:ZJSE0005 if $xml contains an illegal child element for a JSON
 : type.
 : @error zerr:ZJSE0006 if $xml contains an illegal child element.
 : @error zerr:ZJSE0007 if $xml contains an illegal text node.
 : @error zerr:ZJSE0008 if $xml contains an illegal value for a JSON type.
 : @example test/rbkt/Queries/zorba/json/json-snelson-serialize-array-01.xq
 :)
declare function json:serialize(
  $xml as item()*
) as xs:string
{
  json:serialize-internal($xml,
    validate {
      <options
          xmlns="http://www.zorba-xquery.com/modules/converters/json-options">
        <json-format value="Snelson"/>
      </options>
   }
 )
};

(:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::)

declare %private function json:parse-internal(
  $json as xs:string?,
  $options as item()?
) as element()* external;

declare %ann:streamable %private function json:serialize-internal(
  $xml as item()*,
  $options as item()?
) as xs:string external;

(: vim:set et sw=2 ts=2: :)

(:
 : Copyright 2006-2009 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

xquery version "3.0";

 (:~
  : Using this module, you can parse JSON data into XML, manipulate it like any
  : other XML data using XQuery, and serialize the result back as JSON.
  :
  : There are many ways to represent JSON data in XML, some loss-less ("round
  : tripable") and some lossy ("one way").  Loss-less representations preserve
  : the JSON data types boolean, number, and null; lossy
  : representations convert all data to strings.
  :

  : For a loss-less representation, Zorba implements that proposed by
  : John Snelson.
  : For example:
  :

  :   {
  :     "firstName" : "John",
  :     "lastName" : "Smith",
  :     "address" : {
  :       "streetAddress" : "21 2nd Street",
  :       "city" : "New York",
  :       "state" : "NY",
  :       "postalCode" : 10021
  :     },
  :     "phoneNumbers" : [ "212 732-1234", "646 123-4567" ]
  :   }
  :

  : would be represented as:
  :

  :   <json type="object">
  :     <pair name="firstName" type="string">John</pair>
  :     <pair name="lastName" type="string">Smith</pair>
  :     <pair name="address" type="object">
  :       <pair name="streetAddress" type="string">21 2nd Street</pair>
  :       <pair name="city" type="string">New York</pair>
  :       <pair name="state" type="string">NY</pair>
  :       <pair name="postalCode" type="number">10021</pair>
  :     </pair>
  :     <pair name="phoneNumbers" type="array">
  :       <item type="string">212 732-1234</item>
  :       <item type="string">646 123-4567</item>
  :     </pair>
  :   </json>
  :

  : For a lossy representation, Zorba implements
  : JsonML (the array form).
  : For example:
  :

  :   [ "person",
  :     { "created" : "2006-11-11T19:23",
  :       "modified" : "2006-12-31T23:59" },
  :     [ "firstName", "Robert" ],
  :     [ "lastName", "Smith" ],
  :     [ "address",
  :       { "type" : "home" },
  :       [ "street", "12345 Sixth Ave" ],
  :       [ "city", "Anytown" ],
  :       [ "state", "CA" ],
  :       [ "postalCode", "98765-4321" ]
  :     ]
  :   ]
  :

  : would be represented as:
  :

  :   <person created="2006-11-11T19:23" modified="2006-12-31T23:59">
  :     <firstName>Robert</firstName>
  :     <lastName>Smith</lastName>
  :     <address type="home">
  :       <street>12345 Sixth Ave</street>
  :       <city>Anytown</city>
  :       <state>CA</state>
  :       <postalCode>98765-4321</postalCode>
  :     </address>
  :   </person>
  :

  :
  : @author Paul J. Lucas
  : @project data processing/data converters
  :)
module namespace json = "http://www.zorba-xquery.com/modules/converters/json";

import module namespace schema = "http://www.zorba-xquery.com/modules/schema";

import schema namespace json-options =
  "http://www.zorba-xquery.com/modules/converters/json-options";

declare namespace ann = "http://www.zorba-xquery.com/annotations";
declare namespace err = "http://www.w3.org/2005/xqt-errors";
declare namespace zerr = "http://www.zorba-xquery.com/errors";

declare namespace ver = "http://www.zorba-xquery.com/options/versioning";
declare option ver:module-version "2.0";

(:~
 : Parses JSON data from a string and returns an XDM instance using one of the
 : representations described above.
 :
 : @param $json The JSON data to parse.
 : @param $options The parsing options.
 : @return said XDM instance.
 : @error err:XQDY0027 if $options can not be validated against the
 : json-options schema.
 : @error zerr:ZJPE0001 if $json contains an illegal JSON character.
 : @error zerr:ZJPE0002 if $json contains an illegal Unicode code-point.
 : @error zerr:ZJPE0003 if $json contains an illegal JSON character escape.
 : @error zerr:ZJPE0004 if $json contains an illegal JSON literal.
 : @error zerr:ZJPE0005 if $json contains an illegal JSON number.
 : @error zerr:ZJPE0006 if $json is not a valid JSON string.
 : @error zerr:ZJPE0007 if $json contains an unterminated string.
 : @error zerr:ZJPE0008 if $json contains an illegal QName.
 : @example test/rbkt/Queries/zorba/json/json-jsonml_array-parse-01.xq
 :)
declare function json:parse(
  $json as xs:string?,
  $options as element(json-options:options)
) as element(*,xs:untyped)*
{
  let $validated-options := if ( schema:is-validated( $options ) ) then
                              $options
                            else
                              validate { $options }
  return json:parse-internal( $json, $validated-options )
};

(:~
 : Parses JSON data from a string and returns an XDM instance using the Snelson
 : representation described above.
 :
 : @param $json The JSON data to parse.
 : @return said XDM instance.
 : @error zerr:ZJPE0001 if $json contains an illegal JSON character.
 : @error zerr:ZJPE0002 if $json contains an illegal Unicode code-point.
 : @error zerr:ZJPE0003 if $json contains an illegal JSON character escape.
 : @error zerr:ZJPE0004 if $json contains an illegal JSON literal.
 : @error zerr:ZJPE0005 if $json contains an illegal JSON number.
 : @error zerr:ZJPE0006 if $json is not a valid JSON string.
 : @error zerr:ZJPE0007 if $json contains an unterminated string.
 : @error zerr:ZJPE0008 if $json contains an illegal QName.
 : @example test/rbkt/Queries/zorba/json/json-snelson-parse-array-01.xq
 :)
declare function json:parse(
  $json as xs:string?
) as element(*,xs:untyped)*
{
  json:parse-internal(
    $json,
    validate {
      <options xmlns="http://www.zorba-xquery.com/modules/converters/json-options" >
        <json-format value="Snelson"/>
      </options>
    }
  )
};

(:~
 : Serializes an XDM into JSON using one of the representations described
 : above.
 :
 : @param $xml The XDM to serialize.
 : @param $options The serializing options.
 : @return a JSON string.
 : @error err:XQDY0027 if $options can not be validated against the
 : json-options schema.
 : @error zerr:ZJSE0001 if $xml is not a document or element node.
 : @error zerr:ZJSE0002 if $xml contains an element that is missing a required
 : attribute.
 : @error zerr:ZJSE0003 if $xml contains an attribute having an illegal value.
 : @error zerr:ZJSE0004 if $xml contains an illegal element.
 : @error zerr:ZJSE0005 if $xml contains an illegal child element for a JSON
 : type.
 : @error zerr:ZJSE0006 if $xml contains an illegal child element.
 : @error zerr:ZJSE0007 if $xml contains an illegal text node.
 : @error zerr:ZJSE0008 if $xml contains an illegal value for a JSON type.
 : @example test/rbkt/Queries/zorba/json/json-jsonml_array-serialize-01.xq
 :)
declare function json:serialize(
  $xml as item()*,
  $options as element(json-options:options)
) as xs:string
{
  let $validated-options := if ( schema:is-validated( $options ) ) then
                              $options
                            else
                              validate { $options }
  return json:serialize-internal( $xml, $validated-options )
};

(:~
 : Serializes an XDM into JSON using one of the representations described
 : above.
 :
 : @param $xml The XDM to serialize.
 : @return a JSON string.
 : @error zerr:ZJSE0001 if $xml is not a document or element node.
 : @error zerr:ZJSE0002 if $xml contains an element that is missing a required
 : attribute.
 : @error zerr:ZJSE0003 if $xml contains an attribute having an illegal value.
 : @error zerr:ZJSE0004 if $xml contains an illegal element.
 : @error zerr:ZJSE0005 if $xml contains an illegal child element for a JSON
 : type.
 : @error zerr:ZJSE0006 if $xml contains an illegal child element.
 : @error zerr:ZJSE0007 if $xml contains an illegal text node.
 : @error zerr:ZJSE0008 if $xml contains an illegal value for a JSON type.
 : @example test/rbkt/Queries/zorba/json/json-snelson-serialize-array-01.xq
 :)
declare function json:serialize(
  $xml as item()*
) as xs:string
{
  json:serialize-internal($xml,
    validate {
      <options xmlns="http://www.zorba-xquery.com/modules/converters/json-options" >
        <json-format value="Snelson"/>
      </options>
   }
 )
};

(:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::)

declare %private function json:parse-internal(
  $json as xs:string?,
  $options as item()?
) as element()* external;

declare %ann:streamable %private function json:serialize-internal(
  $xml as item()*,
  $options as item()?
) as xs:string external;

(: vim:set et sw=2 ts=2: :)