[ignore] bibliographic demo: was broken after update to latest saxon. Make sure the XHTML namespace is used everywhere.

svn path=/trunk/eXist/; revision=12413

xquery version "1.0";

declare variable $local:CREDENTIALS := ("guest", "guest");

declare function local:logout() {
    let $logout := request:get-parameter("logout", ())
    return
        if ($logout) then
            session:clear()
        else
            ()
};

declare function local:set-user() {
    session:create(),
    local:logout(),
    let $user := request:get-parameter("user", ())
    let $password := request:get-parameter("password", ())
    let $sessionUser := session:get-attribute("biblio.user")
    return
        if ($user) then (
            session:set-attribute("biblio.user", $user),
            session:set-attribute("biblio.password", $password),
            <set-attribute name="xquery.user" value="{$user}"/>,
            <set-attribute name="xquery.password" value="{$password}"/>
        ) else if ($sessionUser != '') then (
            <set-attribute name="xquery.user" value="{$sessionUser}"/>,
            <set-attribute name="xquery.password" value="{session:get-attribute('biblio.password')}"/>
        ) else (
            <set-attribute name="xquery.user" value="{$local:CREDENTIALS[1]}"/>,
            <set-attribute name="xquery.password" value="{$local:CREDENTIALS[2]}"/>
        )
};

if ($exist:path eq '/') then
    <dispatch xmlns="http://exist.sourceforge.net/NS/exist">
		<redirect url="index.xml"/>
	</dispatch>
(:  Main page: index.xml is a template, which is passed through
    search.xql and the db2xhtml stylesheet. search.xql will run
    the actual search and expand the index.xml template.
:)
else if ($exist:resource eq 'index.xml') then
    <dispatch xmlns="http://exist.sourceforge.net/NS/exist">
        { local:set-user() }
        <view>
            <forward url="search.xql">
                <!-- Errors should be passed through instead of terminating the request -->
        		<set-attribute name="xquery.report-errors" value="yes"/>
        		<set-attribute name="xquery.attribute" value="model"/>
            </forward>
            <forward servlet="XSLTServlet">
    			<set-attribute name="xslt.input"
    				value="model"/>
    			<set-attribute name="xslt.stylesheet"
    				value="../stylesheets/db2xhtml.xsl"/>
    			<set-attribute name="xslt.syntax-highlight" value="no"/>
    			<set-attribute name="xslt.output.media-type"
                        value="application/xhtml+xml"/>
                <set-attribute name="xslt.output.doctype-public"
                	    value="-//W3C//DTD XHTML 1.0 Transitional//EN"/>
                <set-attribute name="xslt.output.doctype-system"
                    value="resources/xhtml1-transitional.dtd"/>
                <set-attribute name="xslt.enforce-xhtml"
                    value="yes"/>
    		</forward>
		</view>
	</dispatch>
(:  Retrieve an item from the query results stored in the HTTP session. The
	format of the URL will be /sandbox/results/X, where X is the number of the
	item in the result set :)
else if ($exist:resource eq 'retrieve') then
	<dispatch xmlns="http://exist.sourceforge.net/NS/exist">
	   { local:set-user() }
		<forward url="{$exist:controller}/session.xql">
		</forward>
	</dispatch>
else if (matches($exist:path, '(resources/|styles/syntax|scripts/|logo.jpg|default-style2.css|curvycorners.js)')) then
    let $newPath := replace($exist:path, '^.*((styles/|scripts/|logo).*)$', '/$1')
    return
        <dispatch xmlns="http://exist.sourceforge.net/NS/exist">
    		<forward url="{$newPath}"/>
    		<cache-control cache="yes"/>
    	</dispatch>
else
    (: everything else is passed through :)
    <dispatch xmlns="http://exist.sourceforge.net/NS/exist">
        { local:set-user() }
        <cache-control cache="yes"/>
    </dispatch>

xquery version "1.0";

declare variable $local:CREDENTIALS := ("guest", "guest");

declare function local:logout() {
    let $logout := request:get-parameter("logout", ())
    return
        if ($logout) then
            session:clear()
        else
            ()
};

declare function local:set-user() {
    session:create(),
    local:logout(),
    let $user := request:get-parameter("user", ())
    let $password := request:get-parameter("password", ())
    let $sessionUser := session:get-attribute("biblio.user")
    return
        if ($user) then (
            session:set-attribute("biblio.user", $user),
            session:set-attribute("biblio.password", $password),
            <set-attribute name="xquery.user" value="{$user}"/>,
            <set-attribute name="xquery.password" value="{$password}"/>
        ) else if ($sessionUser != '') then (
            <set-attribute name="xquery.user" value="{$sessionUser}"/>,
            <set-attribute name="xquery.password" value="{session:get-attribute('biblio.password')}"/>
        ) else (
            <set-attribute name="xquery.user" value="{$local:CREDENTIALS[1]}"/>,
            <set-attribute name="xquery.password" value="{$local:CREDENTIALS[2]}"/>
        )
};

if ($exist:path eq '/') then
    <dispatch xmlns="http://exist.sourceforge.net/NS/exist">
		<redirect url="index.xml"/>
	</dispatch>
(:  Main page: index.xml is a template, which is passed through
    search.xql and the db2xhtml stylesheet. search.xql will run
    the actual search and expand the index.xml template.
:)
else if ($exist:resource eq 'index.xml') then
    <dispatch xmlns="http://exist.sourceforge.net/NS/exist">
        { local:set-user() }
        <view>
            <forward url="search.xql">
                <!-- Errors should be passed through instead of terminating the request -->
        		<set-attribute name="xquery.report-errors" value="yes"/>
        		<set-attribute name="xquery.attribute" value="model"/>
            </forward>
            <forward servlet="XSLTServlet">
    			<set-attribute name="xslt.input"
    				value="model"/>
    			<set-attribute name="xslt.stylesheet"
    				value="../stylesheets/db2xhtml.xsl"/>
    			<set-attribute name="xslt.syntax-highlight" value="no"/>
    			<set-attribute name="xslt.output.media-type"
                        value="application/xhtml+xml"/>
                <set-attribute name="xslt.output.doctype-public"
                	    value="-//W3C//DTD XHTML 1.0 Transitional//EN"/>
                <set-attribute name="xslt.output.doctype-system"
                    value="resources/xhtml1-transitional.dtd"/>
    		</forward>
		</view>
	</dispatch>
(:  Retrieve an item from the query results stored in the HTTP session. The
	format of the URL will be /sandbox/results/X, where X is the number of the
	item in the result set :)
else if ($exist:resource eq 'retrieve') then
	<dispatch xmlns="http://exist.sourceforge.net/NS/exist">
	   { local:set-user() }
		<forward url="{$exist:controller}/session.xql">
		</forward>
	</dispatch>
else if (matches($exist:path, '(resources/|styles/syntax|scripts/|logo.jpg|default-style2.css|curvycorners.js)')) then
    let $newPath := replace($exist:path, '^.*((styles/|scripts/|logo).*)$', '/$1')
    return
        <dispatch xmlns="http://exist.sourceforge.net/NS/exist">
    		<forward url="{$newPath}"/>
    		<cache-control cache="yes"/>
    	</dispatch>
else
    (: everything else is passed through :)
    <dispatch xmlns="http://exist.sourceforge.net/NS/exist">
        { local:set-user() }
        <cache-control cache="yes"/>
    </dispatch>


Added Refresh link beside Up link.  Very convenient when debugging XQueries, rather than having to go up and back down again to see if the collection contents changed.

svn path=/trunk/eXist/; revision=12042

xquery version "1.0";
(: $Id$ :)
(:
    Module: display and browse collections.
:)

module namespace browse="http://exist-db.org/xquery/admin-interface/browse";

declare namespace request="http://exist-db.org/xquery/request";
declare namespace xdb="http://exist-db.org/xquery/xmldb";
declare namespace util="http://exist-db.org/xquery/util";
declare namespace v="http://exist-db.org/versioning";

import module namespace date="http://exist-db.org/xquery/admin-interface/date" at "dates.xqm";

(:
    Main function: outputs the page.
:)
declare function browse:main() as element()
{
    let $colName := request:get-parameter("collection", "/db") return

            { browse:process-action($colName) }

Browsing Collection: {xdb:decode-uri(xs:anyURI($colName))}

            <form method="POST" enctype="multipart/form-data">
                {
                    browse:display-collection($colName)
                }

<input type="submit" name="action" value="Remove Selected"/>

                        <input type="submit" name="action" value="Create Collection"/>
                        New collection:

                        <input type="text" name="create" size="40"/>

                        <input type="submit" name="action" value="Upload"/>
                        <input type="file" size="30" name="upload"/>
                        Store as:

                        <input type="text" name="name" size="20"/>

                <input type="hidden" name="collection" value="{$colName}"/>
                <input type="hidden" name="panel" value="browse"/>
            </form>

};

(:
    Process an action.
:)
declare function browse:process-action($colName as xs:string) as element()*
{
    let $action := request:get-parameter("action", ()) return
        util:catch("java.lang.Exception",
            if($action eq "Remove Selected") then
            (
                browse:remove()
            )
            else if($action eq "Create Collection") then
            (
                browse:create-collection($colName)
            )
            else if($action eq "Store") then
            (
                browse:store($colName)
            )
            else if($action eq "Upload") then
            (
                browse:upload($colName)
            )else(),

                An error occurred while processing the action:

                {$util:exception-message}

        )
};

(:
    Store uploaded content.
:)
declare function browse:upload($colName as xs:string) as element()
{
    let $name := request:get-parameter("name", ()),
    $docName := if($name) then $name else request:get-uploaded-file-name("upload"),
    $file := request:get-uploaded-file-data("upload") return

Actions:

Storing uploaded content to: {$docName}

                {
                    xdb:decode-uri(xs:anyURI(xdb:store($colName, xdb:encode-uri($docName), $file)))
                }

};

(:
    Store files from an URI.

    Allowing this opens a security whole as a user can
    upload arbitrary files on the server if the process is running
    as root.
:)
declare function browse:store($colName as xs:string) as element()
{
    let $uri := request:get-parameter("uri", ()),
    $path := if(starts-with($uri, "file:")) then $uri else concat("file:", $uri),
    $docName := request:get-parameter("name", ()) return

Actions:

Storing resources from URI: {$path}

                {
                    xdb:store($colName, $docName, xs:anyURI($path))
                }

};

(:
    Remove a set of resources.
:)
declare function browse:remove() as element()
{
    let $resources := request:get-parameter("resource", ()) return

Actions:

                {
                    for $resource in $resources
                    return
                        browse:remove-resource($resource)
                }

};

(:
    Remove a resource.
:)
declare function browse:remove-resource($resource as xs:string) as element()*
{
    let $isBinary := util:binary-doc-available($resource),
    $doc := if ($isBinary) then $resource else doc($resource) return

        if($doc)then
        (

Removing document: {xdb:decode-uri(xs:anyURI($resource))} ...
,
            xdb:remove(util:collection-name($doc), util:document-name($doc))
        )
        else
        (

Removing collection: {xdb:decode-uri(xs:anyURI($resource))} ...
,
            xdb:remove($resource)
        )
};

(:
    Create a collection.
:)
declare function browse:create-collection($parentColName as xs:string) as element()
{
    let $newcol := request:get-parameter("create", ()) return

Actions:

            {
                if($newcol) then
                (
                    let $col := xdb:create-collection($parentColName, xdb:encode-uri($newcol)) return

Created collection: {xdb:decode-uri(xs:anyURI(util:collection-name($col)))}.

                )
                else
                (

No name specified for new collection!

                )
            }

};

(:
    Display the contents of a collection in a table view.
:)
declare function browse:display-collection($colName as xs:string) as element()
{

            Name
            Permissions
            Owner
            Group
            Created
            Modified
            Size (KB)
			Revision

            Up
            Refresh

        {
            browse:display-child-collections($colName),
            browse:display-child-resources($colName)
        }

};

declare function browse:display-child-collections($colName as xs:string) as element()*
{
    for $child in xdb:get-child-collections($colName)
        let $path := concat($colName, '/', $child),
        $created := xdb:created($path)
    order by $child return

            <input type="checkbox" name="resource" value="{$path}"/>
            {xdb:decode-uri(xs:anyURI($child))}
            {xdb:permissions-to-string(xdb:get-permissions($path))}
            {xdb:get-owner($path)}
            {xdb:get-group($path)}
            {date:format-dateTime($created)}

};

declare function browse:display-child-resources($colName as xs:string) as element()*
{
    for $child in xdb:get-child-resources($colName)
    let $restUri := concat(request:get-context-path(), '/rest')
    order by $child
	return

            <input type="checkbox" name="resource" value="{$colName}/{$child}"/>
            {xdb:decode-uri(xs:anyURI($child))}
            {xdb:permissions-to-string(xdb:get-permissions($colName, $child))}
            {xdb:get-owner($colName, $child)}
            {xdb:get-group($colName, $child)}
            {date:format-dateTime(xdb:created($colName, $child))}
            {date:format-dateTime(xdb:last-modified($colName, $child))}
            {fn:ceiling(xdb:size($colName, $child) div 1024)}

				{browse:last-revision($colName, $child)//v:properties/v:revision/string()}

};

declare function browse:last-revision($collection as xs:string, $resource as xs:string) {
	let $vCollection := concat("/db/system/versions", $collection)
	let $versions :=
		for $version in collection($vCollection)/v:version[
			v:properties[v:document = $resource]
		]
		order by xs:long($version/v:properties/v:revision) descending
		return $version
	return
		$versions[1]
};

(:
    Get the name of the parent collection from a specified collection path.
:)
declare function browse:get-parent-collection($path as xs:string) as xs:string
{
    if($path eq "/db") then
        $path
    else
        replace($path, "/[^/]*$", "")
};

xquery version "1.0";
(: $Id$ :)
(:
    Module: display and browse collections.
:)

module namespace browse="http://exist-db.org/xquery/admin-interface/browse";

declare namespace request="http://exist-db.org/xquery/request";
declare namespace xdb="http://exist-db.org/xquery/xmldb";
declare namespace util="http://exist-db.org/xquery/util";
declare namespace v="http://exist-db.org/versioning";

import module namespace date="http://exist-db.org/xquery/admin-interface/date" at "dates.xqm";

(:
    Main function: outputs the page.
:)
declare function browse:main() as element()
{
    let $colName := request:get-parameter("collection", "/db") return

            { browse:process-action($colName) }

Browsing Collection: {xdb:decode-uri(xs:anyURI($colName))}

            <form method="POST" enctype="multipart/form-data">
                {
                    browse:display-collection($colName)
                }

<input type="submit" name="action" value="Remove Selected"/>

                        <input type="submit" name="action" value="Create Collection"/>
                        New collection:

                        <input type="text" name="create" size="40"/>

                        <input type="submit" name="action" value="Upload"/>
                        <input type="file" size="30" name="upload"/>
                        Store as:

                        <input type="text" name="name" size="20"/>

                <input type="hidden" name="collection" value="{$colName}"/>
                <input type="hidden" name="panel" value="browse"/>
            </form>

};

(:
    Process an action.
:)
declare function browse:process-action($colName as xs:string) as element()*
{
    let $action := request:get-parameter("action", ()) return
        util:catch("java.lang.Exception",
            if($action eq "Remove Selected") then
            (
                browse:remove()
            )
            else if($action eq "Create Collection") then
            (
                browse:create-collection($colName)
            )
            else if($action eq "Store") then
            (
                browse:store($colName)
            )
            else if($action eq "Upload") then
            (
                browse:upload($colName)
            )else(),

                An error occurred while processing the action:

                {$util:exception-message}

        )
};

(:
    Store uploaded content.
:)
declare function browse:upload($colName as xs:string) as element()
{
    let $name := request:get-parameter("name", ()),
    $docName := if($name) then $name else request:get-uploaded-file-name("upload"),
    $file := request:get-uploaded-file-data("upload") return

Actions:

Storing uploaded content to: {$docName}

                {
                    xdb:decode-uri(xs:anyURI(xdb:store($colName, xdb:encode-uri($docName), $file)))
                }

};

(:
    Store files from an URI.

    Allowing this opens a security whole as a user can
    upload arbitrary files on the server if the process is running
    as root.
:)
declare function browse:store($colName as xs:string) as element()
{
    let $uri := request:get-parameter("uri", ()),
    $path := if(starts-with($uri, "file:")) then $uri else concat("file:", $uri),
    $docName := request:get-parameter("name", ()) return

Actions:

Storing resources from URI: {$path}

                {
                    xdb:store($colName, $docName, xs:anyURI($path))
                }

};

(:
    Remove a set of resources.
:)
declare function browse:remove() as element()
{
    let $resources := request:get-parameter("resource", ()) return

Actions:

                {
                    for $resource in $resources
                    return
                        browse:remove-resource($resource)
                }

};

(:
    Remove a resource.
:)
declare function browse:remove-resource($resource as xs:string) as element()*
{
    let $isBinary := util:binary-doc-available($resource),
    $doc := if ($isBinary) then $resource else doc($resource) return

        if($doc)then
        (

Removing document: {xdb:decode-uri(xs:anyURI($resource))} ...
,
            xdb:remove(util:collection-name($doc), util:document-name($doc))
        )
        else
        (

Removing collection: {xdb:decode-uri(xs:anyURI($resource))} ...
,
            xdb:remove($resource)
        )
};

(:
    Create a collection.
:)
declare function browse:create-collection($parentColName as xs:string) as element()
{
    let $newcol := request:get-parameter("create", ()) return

Actions:

            {
                if($newcol) then
                (
                    let $col := xdb:create-collection($parentColName, xdb:encode-uri($newcol)) return

Created collection: {xdb:decode-uri(xs:anyURI(util:collection-name($col)))}.

                )
                else
                (

No name specified for new collection!

                )
            }

};

(:
    Display the contents of a collection in a table view.
:)
declare function browse:display-collection($colName as xs:string) as element()
{

            Name
            Permissions
            Owner
            Group
            Created
            Modified
            Size (KB)
			Revision

            Up

        {
            browse:display-child-collections($colName),
            browse:display-child-resources($colName)
        }

};

declare function browse:display-child-collections($colName as xs:string) as element()*
{
    for $child in xdb:get-child-collections($colName)
        let $path := concat($colName, '/', $child),
        $created := xdb:created($path)
    order by $child return

            <input type="checkbox" name="resource" value="{$path}"/>
            {xdb:decode-uri(xs:anyURI($child))}
            {xdb:permissions-to-string(xdb:get-permissions($path))}
            {xdb:get-owner($path)}
            {xdb:get-group($path)}
            {date:format-dateTime($created)}

};

declare function browse:display-child-resources($colName as xs:string) as element()*
{
    for $child in xdb:get-child-resources($colName)
    let $restUri := concat(request:get-context-path(), '/rest')
    order by $child
	return

            <input type="checkbox" name="resource" value="{$colName}/{$child}"/>
            {xdb:decode-uri(xs:anyURI($child))}
            {xdb:permissions-to-string(xdb:get-permissions($colName, $child))}
            {xdb:get-owner($colName, $child)}
            {xdb:get-group($colName, $child)}
            {date:format-dateTime(xdb:created($colName, $child))}
            {date:format-dateTime(xdb:last-modified($colName, $child))}
            {fn:ceiling(xdb:size($colName, $child) div 1024)}

				{browse:last-revision($colName, $child)//v:properties/v:revision/string()}

};

declare function browse:last-revision($collection as xs:string, $resource as xs:string) {
	let $vCollection := concat("/db/system/versions", $collection)
	let $versions :=
		for $version in collection($vCollection)/v:version[
			v:properties[v:document = $resource]
		]
		order by xs:long($version/v:properties/v:revision) descending
		return $version
	return
		$versions[1]
};

(:
    Get the name of the parent collection from a specified collection path.
:)
declare function browse:get-parent-collection($path as xs:string) as xs:string
{
    if($path eq "/db") then
        $path
    else
        replace($path, "/[^/]*$", "")
};


Missing comma.

Missing comma.


remove orphaned module

remove orphaned module

svn path=/trunk/eXist/; revision=6749

xquery version "1.0";
(: $Id$ :)
(: Library module imported by xquery.xq :)

module namespace utils="http://exist-db.org/xquery/collection-utils";

declare namespace xmldb="http://exist-db.org/xquery/xmldb";

declare function utils:list-collection-names($collection as xs:string, $user as xs:string, $password as xs:string) as xs:string+
{
	replace($collection, "xmldb:exist://", ""),
	let $root := xmldb:collection($collection, $user, $password) return
		for $child in xmldb:get-child-collections($root) return
			utils:recurseChildren(concat(replace($collection, "xmldb:exist://", ""), "/", $child))
};

declare function utils:recurseChildren($collection) as xs:string+
{
	$collection,
	for $child in xmldb:get-child-collections($collection) return
		utils:recurseChildren(concat($collection, "/", $child))
};


operation=explain - regard the correct namespace

xquery version '3.0';

(:
: Module Name: FCS
: Date: 2012-03-01
:
: XQuery
: Specification : XQuery v1.0
: Module Overview: Federated Content Search
:)

(:~ This module provides methods to serve XML-data via the FCS/SRU-interface
: @see http://clarin.eu/fcs
: @author Matej Durco
: @since 2011-11-01
: @version 1.1
:)
module namespace fcs = "http://clarin.eu/fcs/1.0";

declare namespace err = "http://www.w3.org/2005/xqt-errors";
declare namespace sru = "http://www.loc.gov/zing/srw/";
declare namespace zr = "http://explain.z3950.org/dtd/2.0/";
declare namespace tei = "http://www.tei-c.org/ns/1.0";
declare namespace cmd = "http://www.clarin.eu/cmd/";
import module namespace request="http://exist-db.org/xquery/request";
import module namespace diag =  "http://www.loc.gov/zing/srw/diagnostic/" at  "modules/diagnostics/diagnostics.xqm";

import module namespace repo-utils = "http://aac.ac.at/content_repository/utils" at  "repo-utils.xqm";
import module namespace kwic = "http://exist-db.org/xquery/kwic";
import module namespace cmdcoll = "http://clarin.eu/cmd/collections" at  "/db/cr/modules/cmd/cmd-collections.xqm";
import module namespace cmdcheck = "http://clarin.eu/cmd/check" at  "/db/cr/modules/cmd/cmd-check.xqm";
import module namespace cql = "http://exist-db.org/xquery/cql" at "/db/cr/modules/cqlparser/cqlparser.xqm";

declare variable $fcs:explain as xs:string := "explain";
declare variable $fcs:scan  as xs:string := "scan";
declare variable $fcs:searchRetrieve as xs:string := "searchRetrieve";

declare variable $fcs:scanSortText as xs:string := "text";
declare variable $fcs:scanSortSize as xs:string := "size";
declare variable $fcs:indexXsl := doc('index.xsl');
declare variable $fcs:kwicWidth := 30;

(:~ The main entry-point. Processes request-parameters
@returns the result document (in xml, html or json)
:)
declare function fcs:repo($config-file as xs:string) as item()* {
  let
    $config := if (doc-available($config-file)) then doc($config-file)
                        else diag:diagnostics("general-error", concat("config not available: ", $config-file)) ,

        (: accept "q" as synonym to query-param; "query" overrides:)
    $q := request:get-parameter("q", ""),
    $query := request:get-parameter("query", $q),
        (: if query-parameter not present, 'explain' as DEFAULT operation, otherwise 'searchRetrieve' :)
    $operation :=  if ($query eq "") then request:get-parameter("operation", $fcs:explain)
                    else request:get-parameter("operation", $fcs:searchRetrieve),
    $x-format := request:get-parameter("x-format", $repo-utils:responseFormatXml),
    $x-context := request:get-parameter("x-context", ""),
    (:
    $query-collections :=
    if (matches($collection-params, "^root$") or $collection-params eq "") then
      $cr:collectionRoot
    else
		tokenize($collection-params,','),
        :)
(:      $collection-params, :)
  $max-depth as xs:integer := xs:integer(request:get-parameter("maxdepth", 1))

  let $result :=
      (: if ($operation eq $cr:getCollections) then
		cr:get-collections($query-collections, $format, $max-depth)
      else :)
      if ($operation eq $fcs:explain) then
          fcs:explain($x-context, $config)
      else if ($operation eq $fcs:scan) then
		let $scanClause := request:get-parameter("scanClause", ""),
		$start-term := request:get-parameter("startTerm", 1),
		$response-position := request:get-parameter("responsePosition", 1),
		$max-terms := request:get-parameter("maximumTerms", 50),
	    $max-depth := request:get-parameter("x-maximumDepth", 1),
		$sort := request:get-parameter("sort", 'text')
		 return fcs:scan($scanClause, $x-context, $start-term, $max-terms, $response-position, $max-depth, $sort, $config)
        (: return fcs:scan($scanClause, $x-context) :)
	  else if ($operation eq $fcs:searchRetrieve) then
        if ($query eq "") then diag:diagnostics("param-missing", "query")
        else
      	 let $cql-query := $query,
			$start-item := request:get-parameter("startRecord", 1),
			$max-items := request:get-parameter("maximumRecords", 50),
			$x-dataview := request:get-parameter("x-dataview", repo-utils:config-value($config, 'default.dataview'))
            (: return cr:search-retrieve($cql-query, $query-collections, $format, xs:integer($start-item), xs:integer($max-items)) :)
            return fcs:search-retrieve($cql-query, $x-context, xs:integer($start-item), xs:integer($max-items), $x-dataview, $config)
    else
      diag:diagnostics("unsupported-operation", $operation)

   return  repo-utils:serialise-as($result, $x-format, $operation, $config)

};

(:~ handles the explain-operation requests.
: @param $x-context optional, identifies a resource to return the explain-record for. (Accepts both MD-PID or Res-PID (MdSelfLink or ResourceRef/text))
: @returns either the default root explain-record, or - when provided with the $x-context parameter - the explain-record of given resource
:)
declare function fcs:explain($x-context as xs:string*, $config) as item()* {
    let $context := if ($x-context) then $x-context
                    else repo-utils:config-value($config, 'explain')
    let $md-dbcoll := collection(repo-utils:config-value($config,'metadata.path'))
   let $explain := $md-dbcoll//CMD[Header/MdSelfLink/text() eq $context or .//ResourceRef/text() eq $context]//(explain|zr:explain) (: //ResourceRef/text() :)

    return $explain
};

(:
TODO?: only read explicit indexes + create index on demand.
:)
(:
declare function fcs:scan($scanClause as xs:string, $x-context as xs:string*) {

    let $clause-tokens := tokenize($scanClause,'='),
        $index := $clause-tokens[1],
        $term := $clause-tokens[2],
        (\:$map-index := $repo-utils:mappings/map/index[@key=$index], :\)
        $index-file := concat(repo-utils:config-value('index.prefix'),$index,'.xml'),
        h$result := doc($index-file)

    return $result
};
:)

(:~ This function handles the scan-operation requests
:  (derived from cmd:scanIndex function)
: two phases:
:   1. one create full index for given path/element within given collection (for now the collection is stored in the name - not perfect) (and cache)
:	2. select wished subsequence (on second call, only the second step is performed)

: actually wrapping function handling caching of the actual scan result (coming from do-scan-default())
: or fetching the cached result (if available)
: also dispatching to cmd-collections for the scan-clause=cmd.collections
:   there either scanClause-filter or x-context is used as constraint (scanClause-filter is prefered))

:)
declare function fcs:scan($scan-clause  as xs:string, $x-context as xs:string+, $start-item as xs:integer, $max-items as xs:integer, $response-position as xs:integer, $max-depth as xs:integer, $p-sort as xs:string?, $config) as item()? {

  let $scx := tokenize($scan-clause,'='),
	 $index-name := $scx[1],
	 (:$index := fcs:get-mapping($index-name, $x-context, $config ), :)
	 (: if no index-mapping found, dare to use the index-name as xpath :)
(:	 $index-xpath := if ($index/text()) then $index/text() else $index-name, :)
     $index-xpath :=  fcs:index-as-xpath($index-name,$x-context, $config ),
 	 $filter := ($scx[2],'')[1],
	 $sort := if ($p-sort eq $fcs:scanSortText or $p-sort eq $fcs:scanSortSize) then $p-sort else $fcs:scanSortText

	 (: WATCHME: this is quite unreliable, it relies on manual (urn-like) creation of the ids for the resources)
	   it won't work once the id become PIDs
	   this is only used for generating the cache-id to store the resultset :)
	 let $short-xcontext := substring-after($x-context, concat(repo-utils:config-value($config, 'explain'),':'))
	 let $sanitized-xcontext := if ($short-xcontext='') then repo-utils:sanitize-name($x-context) else $short-xcontext
    let $index-doc-name := repo-utils:gen-cache-id("index", ($sanitized-xcontext, $index-name, $sort, $max-depth),""),

  (: get the base-index from cache, or create and cache :)
  $index-scan := if (repo-utils:is-in-cache($index-doc-name, $config)) then
          repo-utils:get-from-cache($index-doc-name, $config)
        else
            let $data :=
                if ($index-name eq $cmdcoll:scan-collection) then
                    let $starting-handle := if ($filter ne '') then $filter else $x-context
                    return cmdcoll:colls($starting-handle, $max-depth, cmdcoll:base-dbcoll($config))
                  (: just a hack for now, handling of special indexes should be put solved in some more easily extensible way :)
                else if ($index-name eq 'cmd.profile') then
                    let $context := repo-utils:context-to-collection($x-context, $config)
                    return  cmdcheck:stat-profiles($context)
                else
                    fcs:do-scan-default($scan-clause, $index-xpath, $x-context, $sort, $config)

        return repo-utils:store-in-cache($index-doc-name , $data, $config)

	(: extract the required subsequence (according to given sort) :)
	let $res-nodeset := transform:transform($index-scan,$fcs:indexXsl,
			<parameters><param name="scan-clause" value="{$scan-clause}"/>
			            <param name="mode" value="subsequence"/>
						<param name="sort" value="{$sort}"/>
						<param name="filter" value="{$filter}"/>
						<param name="start-item" value="{$start-item}"/>
					    <param name="response-position" value="{$response-position}"/>
						<param name="max-items" value="{$max-items}"/>
			</parameters>),
		$count-items := count($res-nodeset/sru:term),
		(: $colls := if (fn:empty($collection)) then '' else fn:string-join($collection, ","), :)
        $colls := string-join( $x-context, ', ') ,
		$created := fn:current-dateTime()
		(: $scan-clause := concat($xpath, '=', $filter) :)
		(: $res := <Terms colls="{$colls}" created="{$created}" count_items="{$count-items}"
					start-item="{$start-item}" max-items="{$max-items}" sort="{$sort}" scanClause="{$scan-clause}"  >{$res-term}</Terms>
					  count_text="{$count-text}" count_distinct_text="{$distinct-text-count}" :)
        (: $res := <sru:scanResponse>
                    <sru:version>1.2</sru:version>
                    {$res-nodeset}
                    <sru:echoedScanRequest>
                        <sru:scanClause>{$scan-clause}</sru:scanClause>
                        <sru:maximumTerms>{ $count-items }</sru:maximumTerms>

                    </sru:echoedScanRequest>
    			</sru:scanResponse>
           :)
(:	let	$result-count := $doc/Term/@count,
    $result-seq := fn:subsequence($doc/Term/v, $start-item, $end-item),
	$result-frag := ($doc/Term, $result-seq),
    $seq-count := fn:count($result-seq) :)

  return $res-nodeset
};

declare function fcs:do-scan-default($scan-clause as xs:string, $index-xpath as xs:string, $x-context as xs:string, $sort as xs:string, $config) as item()* {
  (:        let $getnodes := util:eval(fn:concat("$data-collection/descendant-or-self::", $index-xpath)),:)
    let $data-collection := repo-utils:context-to-collection($x-context, $config)
        let $getnodes := util:eval(fn:concat("$data-collection//", $index-xpath)),
            (: if we collected strings, we have to wrap them in elements
                    to be able to work with them in xsl :)
            $prenodes := if ($getnodes[1] instance of xs:string) then
                                    for $t in $getnodes return <v>{$t}</v>
                            else for $t in $getnodes return <v>{string-join($t//text()," ")}</v>
        let $nodes := <nodes path="{fn:concat('//', $index-xpath)}"  >{$prenodes}</nodes>,
        	(: use XSLT-2.0 for-each-group functionality to aggregate the values of a node - much, much faster, than XQuery :)
   	    $data := transform:transform($nodes,$fcs:indexXsl,
   	                <parameters><param name="scan-clause" value="{$scan-clause}"/>
   	                <param name="sort" value="{$sort}"/></parameters>)

   	  return $data
};

(:~ main search function (handles the searchRetrieve-operation request)
:)
declare function fcs:search-retrieve($query as xs:string, $x-context as xs:string*, $startRecord as xs:integer, $maximumRecords as xs:integer, $x-dataview as xs:string*, $config) as item()* {

    try {
        let $start-time := util:system-dateTime()
        let $data-collection := repo-utils:context-to-collection($x-context, $config)
        (:if ($x-context) then collection($repo-utils:mappings//map[xs:string(@key) eq $x-context]/@path)
                                else $repo-utils:data-collection:)

        let $xpath-query := concat("$data-collection", fcs:transform-query ($query, $x-context, $config))

        let $results := util:eval($xpath-query)

        let	$result-count := fn:count($results),
        $result-seq := fn:subsequence($results, $startRecord, $maximumRecords),
        $seq-count := fn:count($result-seq),
        $end-time := util:system-dateTime(),
        (:<sru:recordSchema>mods</sru:recordSchema>:)
              (:<xQuery><searchClause xmlns="http://www.loc.gov/zing/cql/xcql/">
              <index>dc.title</index>
              <relation>
              <value>=</value>
              </relation>
              <term>dinosaur</term>
              </searchClause>
              </xQuery>:)
        $records :=
          <sru:records>
    	       {for $rec at $pos in $result-seq
    	           let $rec-data := fcs:format-record-data($rec,$x-dataview, $x-context, $config)
    	           return
    	               <sru:record>
    	                   <sru:recordSchema>http://clarin.eu/fcs/1.0/Resource.xsd</sru:recordSchema>
    	                   <sru:recordPacking>xml</sru:recordPacking>
    	                   <sru:recordData>
    	                     {$rec-data}
                            </sru:recordData>
    	                   <sru:recordPosition>{$pos}</sru:recordPosition>
    	                   <sru:recordIdentifier>{xs:string($rec-data/fcs:ResourceFragment/@ref) }</sru:recordIdentifier>
    	                </sru:record>
    	       }
          </sru:records>,
        $end-time2 := util:system-dateTime(),
        $result :=
        <sru:searchRetrieveResponse>
          <sru:numberOfRecords>{$result-count}</sru:numberOfRecords>
          <sru:echoedSearchRetrieveRequest>
              <sru:version>1.2</sru:version>
              <sru:query>{$query}</sru:query>
              <fcs:x-context>{$x-context}</fcs:x-context>
              <fcs:x-dataview>{$x-dataview}</fcs:x-dataview>
              <sru:startRecord>{$startRecord}</sru:startRecord>
              <sru:maximumRecords>{$maximumRecords}</sru:maximumRecords>
              <sru:query>{$query}</sru:query>
              <sru:baseUrl>{repo-utils:config-value($config, "base.url")}</sru:baseUrl>
          </sru:echoedSearchRetrieveRequest>
          <sru:extraResponseData>
          	<fcs:returnedRecords>{$seq-count}</fcs:returnedRecords>
    		<fcs:duration>{($end-time - $start-time, $end-time2 - $end-time) }</fcs:duration>
    		<fcs:transformedQuery>{ $xpath-query }</fcs:transformedQuery>
          </sru:extraResponseData>
          { $records }
        </sru:searchRetrieveResponse>

        return $result

      }
    catch err:XPTY0004
    {
        diag:diagnostics("query-syntax-error", ($err:code , $err:description, $err:value))
    }
    catch *
    { (: FIXME: this could be any error!! :)
       diag:diagnostics("query-syntax-error", $query)
    }

};

declare function fcs:format-record-data($raw-record-data as node(), $data-view as xs:string*, $x-context as xs:string*, $config as node()) as item()*  {

    let $exp-rec := util:expand($raw-record-data, "expand-xincludes=no") (: kwic:summarize($rec,<config width="40"/>) :)

                        (:	      cmdcoll:get-md-collection-name($raw-record-data):)
	let $title := fcs:apply-index ($raw-record-data, "title",$x-context, $config)
	let $resource-pid := fcs:apply-index ($raw-record-data, "resource-pid",$x-context, $config)
	let $resourcefragment-pid := fcs:apply-index ($raw-record-data, "resourcefragment-pid",$x-context, $config)
	(: to repeat current $x-format param-value in the constructed requested :)
	let $x-format := request:get-parameter("x-format", $repo-utils:responseFormatXml)
	let $resourcefragment-ref := concat('?operation=searchRetrieve&query=resourcefragment-pid="', xmldb:encode-uri($resourcefragment-pid), '"&x-dataview=full&x-context=', $x-context)

    let $kwic := if ('kwic' = $data-view) then
                   let $kwic-config := <config width="{$fcs:kwicWidth}"/>
                   let $kwic-html := kwic:summarize($exp-rec, $kwic-config)

                    return if (exists($kwic-html)) then
                                           <fcs:DataView type="kwic">{
                                               for $match in $kwic-html
                                               return (<fcs:c type="left">{$match/span[1]/text()}</fcs:c>,
                                       (: <c type="left">{kwic:truncate-previous($exp-rec, $matches[1], (), 10, (), ())}</c> :)
                                                      <fcs:kw>{$match/span[2]/text()}</fcs:kw>,
                                                      <fcs:c type="right">{$match/span[3]/text()}</fcs:c>)
                                       (: let $summary  := kwic:get-summary($exp-rec, $matches[1], $config) :)
                        (:	                               <fcs:DataView type="kwic-html">{$kwic-html}</fcs:DataView>:)
                                            }</fcs:DataView>
                                     else (: if no kwic-match let's take first 100 characters
                                        There c/should be some more sophisticated way to extract most significant info
                                        e.g. match on the query-field :)
                                       <fcs:DataView type="kwic">{substring($raw-record-data,1,$fcs:kwicWidth)}</fcs:DataView>
                     else ()
    let $title := <fcs:DataView type="title">{$title}</fcs:DataView>

    return if ($data-view = 'raw') then $raw-record-data
            else <fcs:Resource pid="{$resource-pid}" >
                       <fcs:ResourceFragment pid="{$resourcefragment-pid}" ref="{$resourcefragment-ref}" >
                         {($title, $kwic,
                         if ('full' = $data-view or not(exists($kwic))) then <fcs:DataView type="{$data-view}">{$exp-rec}</fcs:DataView>
                             else ()
                           )}
                           </fcs:ResourceFragment>
                       </fcs:Resource>

};

(:~ This expects a CQL-query that it translates to XPath

It relies on the external cqlparser-module, that delivers the query in XCQL-format (parse-tree of the query as XML)
and then applies a stylesheet

@params $x-context identifier of a resource/collection
@returns XPath version of the CQL-query
:)
declare function fcs:transform-query($cql-query as xs:string, $x-context as xs:string, $config) as xs:string {
    let $mappings := repo-utils:config-value($config, 'mappings')
    return cql:cql2xpath ($cql-query, $x-context, $mappings)

 };

(: old version, "manually" parsing the cql-string
it accepted/understood:
term
index=term
index relation term
:)
declare function fcs:transform-query-old($cql-query as xs:string, $x-context as xs:string, $type as xs:string, $config ) as xs:string {

    let $query-constituents := if (contains($cql-query,'=')) then
                                        tokenize($cql-query, "=")
                                   else tokenize($cql-query, " ")
	let $index := if ($type eq 'scan' or count($query-constituents)>1 )  then $query-constituents[1]
	                else "cql.serverChoice"
	let $searchTerm := if (count($query-constituents)=1) then
							$cql-query
						else if (count($query-constituents)=2) then (: tokenized with '=' :)
						    normalize-space($query-constituents[2])
						else
							$query-constituents[3]

(: try to get a mapping specific to given context, else take the default :)
    let $context-map := fcs:get-mapping("",$x-context, $config),

(: TODO: for every index in $xcql :)
            (: try  to get a) a mapping for given index within the context-map,
                    b) in any of the mapping (if not found in the context-map) , - potentially dangerous!!
                    c) or else take the index itself :)
        $mappings := doc(repo-utils:config-value($config, 'mappings')),
        $index-map := $context-map/index[xs:string(@key) eq $index],
        $resolved-index := if (exists($index-map)) then $index-map/text()
                           else if (exists($mappings//index[xs:string(@key) eq $index])) then
                                $mappings//index[xs:string(@key) eq $index]
                            else $index
            ,
            (: get either a) the specific base-element for the index,
                  b) the default for given map,
                  c) the index itself :)
        $base-elem := if (exists($index-map/@base_elem)) then xs:string($index-map/@base_elem)
                        else if (exists($context-map/@base_elem)) then xs:string($context-map/@base_elem)
                        else $index,
            (: <index status="indexed"> - flag to know if ft-indexes can be used.
            TODO?: should be checked against the actual index-configuration :)
        $indexed := (xs:string($index-map/@status) eq 'indexed'),
        $match-on := if (exists($index-map/@use) ) then xs:string($index-map/@use) else '.'

	let $res := if ($type eq 'scan') then
	                   concat("//", $resolved-index, if ($match-on ne '.') then concat("/", $match-on) else '')
	               else
	                   concat("//", $resolved-index, "[",
	                                       if ($indexed) then
	                                               concat("ft:query(", $match-on, ",<term>",translate($searchTerm,'"',''), "</term>)")
	                                           else
	                                               concat("contains(", $match-on, ",'", translate($searchTerm,'"',''), "')")
	                                        , "]",
								"/ancestor-or-self::", $base-elem)

	return $res

};

(:~ gets the mapping-entry for the index

first tries a mapping within given context, then tries defaults.

if $index-param = "" return the map-element,
    else - if found - return the index-element
:)
declare function fcs:get-mapping($index as xs:string, $x-context as xs:string+, $config) as node()* {
    let $mappings := doc(repo-utils:config-value($config, 'mappings')),

    $context-map := if (exists($mappings//map[xs:string(@key) eq $x-context])) then
                                $mappings//map[xs:string(@key) eq $x-context]
                            else $mappings//map[xs:string(@key) eq 'default'],
    $context-index := $context-map/index[xs:string(@key) eq $index]

    return  if ($index eq '') then
                $context-map
             else if (exists($context-index)) then
                        $context-index
                else (: if no contextual index, dare to take any index - may be dangerous!  :)
                    let $any-index := $mappings//index[xs:string(@key) eq $index]
                    return $any-index
};

declare function fcs:apply-index($data, $index as xs:string, $x-context as xs:string+, $config) as item()* {

  let $index-map := fcs:get-mapping($index,$x-context, $config),
  $match-on := if (exists($index-map/@use) ) then concat('/', xs:string($index-map/@use)) else ''

  return util:eval(concat("$data//", $index-map/path/text(), $match-on))

};

(:~ gets the mapping for the index and creates an xpath (UNION)

@param $index index-key as known to mappings

@returns xpath-equivalent of given index as defined in mappings; multiple xpaths are translated to a UNION,
           value of @use-attribute is also attached;
         if no mapping found, returns the input-index unchanged
:)
declare function fcs:index-as-xpath($index as xs:string, $x-context as xs:string+, $config) as xs:string {
    let $index-map := fcs:get-mapping($index, $x-context, $config )
     return if (exists($index-map)) then
                        let $match-on := if (exists($index-map/@use) ) then concat('/', xs:string($index-map/@use)) else ''
                        return translate(concat('(', string-join($index-map/path,'|'),')', $match-on),'.','/')
                    else $index

};

xquery version "1.0";

(:
: Module Name: FCS
: Date: 2012-03-01
:
: XQuery
: Specification : XQuery v1.0
: Module Overview: Federated Content Search
:)

(:~ This module provides methods to serve XML-data via the FCS/SRU-interface
: @see http://clarin.eu/fcs
: @author Matej Durco
: @since 2011-11-01
: @version 1.1
:)
module namespace fcs = "http://clarin.eu/fcs/1.0";

declare namespace sru = "http://www.loc.gov/zing/srw/";
declare namespace tei = "http://www.tei-c.org/ns/1.0";
declare namespace cmd = "http://www.clarin.eu/cmd/";
import module namespace request="http://exist-db.org/xquery/request";
import module namespace diag =  "http://www.loc.gov/zing/srw/diagnostic/" at  "modules/diagnostics/diagnostics.xqm";
import module namespace repo-utils = "http://aac.ac.at/content_repository/utils" at  "repo-utils.xqm";
import module namespace kwic = "http://exist-db.org/xquery/kwic";
import module namespace cmdcoll = "http://clarin.eu/cmd/collections" at  "/db/cr/modules/cmd/cmd-collections.xqm";
import module namespace cmdcheck = "http://clarin.eu/cmd/check" at  "/db/cr/modules/cmd/cmd-check.xqm";
import module namespace cql = "http://exist-db.org/xquery/cql" at "/db/cr/modules/cqlparser/cqlparser.xqm";

declare variable $fcs:explain as xs:string := "explain";
declare variable $fcs:scan  as xs:string := "scan";
declare variable $fcs:searchRetrieve as xs:string := "searchRetrieve";

declare variable $fcs:scanSortText as xs:string := "text";
declare variable $fcs:scanSortSize as xs:string := "size";
declare variable $fcs:indexXsl := doc('index.xsl');
declare variable $fcs:kwicWidth := 80;

(:~ The main entry-point. Processes request-parameters
@returns the result document (in xml, html or json)
:)
declare function fcs:repo($config-file as xs:string) as item()* {
  let
    $config := if (doc-available($config-file)) then doc($config-file)
                        else diag:diagnostics("general-error", concat("config not available: ", $config-file)) ,

        (: accept "q" as synonym to query-param; "query" overrides:)
    $q := request:get-parameter("q", ""),
    $query := request:get-parameter("query", $q),
        (: if query-parameter not present, 'explain' as DEFAULT operation, otherwise 'searchRetrieve' :)
    $operation :=  if ($query eq "") then request:get-parameter("operation", $fcs:explain)
                    else request:get-parameter("operation", $fcs:searchRetrieve),
    $x-format := request:get-parameter("x-format", $repo-utils:responseFormatXml),
    $x-context := request:get-parameter("x-context", ""),
    (:
    $query-collections :=
    if (matches($collection-params, "^root$") or $collection-params eq "") then
      $cr:collectionRoot
    else
		tokenize($collection-params,','),
        :)
(:      $collection-params, :)
  $max-depth as xs:integer := xs:integer(request:get-parameter("maxdepth", 1))

  let $result :=
      (: if ($operation eq $cr:getCollections) then
		cr:get-collections($query-collections, $format, $max-depth)
      else :)
      if ($operation eq $fcs:explain) then
          fcs:explain($x-context, $config)
      else if ($operation eq $fcs:scan) then
		let $scanClause := request:get-parameter("scanClause", ""),
		$start-term := request:get-parameter("startTerm", 1),
		$response-position := request:get-parameter("responsePosition", 1),
		$max-terms := request:get-parameter("maximumTerms", 50),
	    $max-depth := request:get-parameter("x-maximumDepth", 1),
		$sort := request:get-parameter("sort", 'text')
		 return fcs:scan($scanClause, $x-context, $start-term, $max-terms, $response-position, $max-depth, $sort, $config)
        (: return fcs:scan($scanClause, $x-context) :)
	  else if ($operation eq $fcs:searchRetrieve) then
        if ($query eq "") then diag:diagnostics("param-missing", "query")
        else
      	 let $cql-query := $query,
			$start-item := request:get-parameter("startRecord", 1),
			$max-items := request:get-parameter("maximumRecords", 50),
			$x-dataview := request:get-parameter("x-dataview", repo-utils:config-value($config, 'default.dataview'))
            (: return cr:search-retrieve($cql-query, $query-collections, $format, xs:integer($start-item), xs:integer($max-items)) :)
            return fcs:search-retrieve($cql-query, $x-context, xs:integer($start-item), xs:integer($max-items), $x-dataview, $config)
    else
      diag:diagnostics("unsupported-operation", $operation)

   return  repo-utils:serialise-as($result, $x-format, $operation, $config)

};

(:~ handles the explain-operation requests.
: @param $x-context optional, identifies a resource to return the explain-record for. (Accepts both MD-PID or Res-PID (MdSelfLink or ResourceRef/text))
: @returns either the default root explain-record, or - when provided with the $x-context parameter - the explain-record of given resource
:)
declare function fcs:explain($x-context as xs:string*, $config) as item()* {
    let $context := if ($x-context) then $x-context
                    else repo-utils:config-value($config, 'explain')
    let $md-dbcoll := collection(repo-utils:config-value($config,'metadata.path'))
   let $explain := $md-dbcoll//CMD[Header/MdSelfLink/text() eq $context or .//ResourceRef/text() eq $context]//explain (: //ResourceRef/text() :)

    return $explain
};

(:
TODO?: only read explicit indexes + create index on demand.
:)
(:
declare function fcs:scan($scanClause as xs:string, $x-context as xs:string*) {

    let $clause-tokens := tokenize($scanClause,'='),
        $index := $clause-tokens[1],
        $term := $clause-tokens[2],
        (\:$map-index := $repo-utils:mappings/map/index[@key=$index], :\)
        $index-file := concat(repo-utils:config-value('index.prefix'),$index,'.xml'),
        h$result := doc($index-file)

    return $result
};
:)

(:~ This function handles the scan-operation requests
:  (derived from cmd:scanIndex function)
: two phases:
:   1. one create full index for given path/element within given collection (for now the collection is stored in the name - not perfect) (and cache)
:	2. select wished subsequence (on second call, only the second step is performed)

: actually wrapping function handling caching of the actual scan result (coming from do-scan-default())
: or fetching the cached result (if available)
: also dispatching to cmd-collections for the scan-clause=cmd.collections
:   there either scanClause-filter or x-context is used as constraint (scanClause-filter is prefered))

:)
declare function fcs:scan($scan-clause  as xs:string, $x-context as xs:string+, $start-item as xs:integer, $max-items as xs:integer, $response-position as xs:integer, $max-depth as xs:integer, $p-sort as xs:string?, $config) as item()? {

  let $scx := tokenize($scan-clause,'='),
	 $index-name := $scx[1],
	 (:$index := fcs:get-mapping($index-name, $x-context, $config ), :)
	 (: if no index-mapping found, dare to use the index-name as xpath :)
(:	 $index-xpath := if ($index/text()) then $index/text() else $index-name, :)
     $index-xpath :=  fcs:index-as-xpath($index-name,$x-context, $config ),
 	 $filter := ($scx[2],'')[1],
	 $sort := if ($p-sort eq $fcs:scanSortText or $p-sort eq $fcs:scanSortSize) then $p-sort else $fcs:scanSortText

	 (: WATCHME: this is quite unreliable, it relies on manual (urn-like) creation of the ids for the resources)
	   it won't work once the id become PIDs
	   this is only used for generating the cache-id to store the resultset :)
	 let $short-xcontext := substring-after($x-context, concat(repo-utils:config-value($config, 'explain'),':'))
	 let $sanitized-xcontext := if ($short-xcontext='') then repo-utils:sanitize-name($x-context) else $short-xcontext
    let $index-doc-name := repo-utils:gen-cache-id("index", ($sanitized-xcontext, $index-name, $sort, $max-depth),""),

  (: get the base-index from cache, or create and cache :)
  $index-scan := if (repo-utils:is-in-cache($index-doc-name, $config)) then
          repo-utils:get-from-cache($index-doc-name, $config)
        else
            let $data :=
                if ($index-name eq $cmdcoll:scan-collection) then
                    let $starting-handle := if ($filter ne '') then $filter else $x-context
                    return cmdcoll:colls($starting-handle, $max-depth, cmdcoll:base-dbcoll($config))
                  (: just a hack for now, handling of special indexes should be put solved in some more easily extensible way :)
                else if ($index-name eq 'cmd.profile') then
                    let $context := repo-utils:context-to-collection($x-context, $config)
                    return  cmdcheck:stat-profiles($context)
                else
                    fcs:do-scan-default($scan-clause, $index-xpath, $x-context, $sort, $config)

        return repo-utils:store-in-cache($index-doc-name , $data, $config)

	(: extract the required subsequence (according to given sort) :)
	let $res-nodeset := transform:transform($index-scan,$fcs:indexXsl,
			<parameters><param name="scan-clause" value="{$scan-clause}"/>
			            <param name="mode" value="subsequence"/>
						<param name="sort" value="{$sort}"/>
						<param name="filter" value="{$filter}"/>
						<param name="start-item" value="{$start-item}"/>
					    <param name="response-position" value="{$response-position}"/>
						<param name="max-items" value="{$max-items}"/>
			</parameters>),
		$count-items := count($res-nodeset/sru:term),
		(: $colls := if (fn:empty($collection)) then '' else fn:string-join($collection, ","), :)
        $colls := string-join( $x-context, ', ') ,
		$created := fn:current-dateTime()
		(: $scan-clause := concat($xpath, '=', $filter) :)
		(: $res := <Terms colls="{$colls}" created="{$created}" count_items="{$count-items}"
					start-item="{$start-item}" max-items="{$max-items}" sort="{$sort}" scanClause="{$scan-clause}"  >{$res-term}</Terms>
					  count_text="{$count-text}" count_distinct_text="{$distinct-text-count}" :)
        (: $res := <sru:scanResponse>
                    <sru:version>1.2</sru:version>
                    {$res-nodeset}
                    <sru:echoedScanRequest>
                        <sru:scanClause>{$scan-clause}</sru:scanClause>
                        <sru:maximumTerms>{ $count-items }</sru:maximumTerms>

                    </sru:echoedScanRequest>
    			</sru:scanResponse>
           :)
(:	let	$result-count := $doc/Term/@count,
    $result-seq := fn:subsequence($doc/Term/v, $start-item, $end-item),
	$result-frag := ($doc/Term, $result-seq),
    $seq-count := fn:count($result-seq) :)

  return $res-nodeset
};

declare function fcs:do-scan-default($scan-clause as xs:string, $index-xpath as xs:string, $x-context as xs:string, $sort as xs:string, $config) as item()* {
  (:        let $getnodes := util:eval(fn:concat("$data-collection/descendant-or-self::", $index-xpath)),:)
    let $data-collection := repo-utils:context-to-collection($x-context, $config)
        let $getnodes := util:eval(fn:concat("$data-collection//", $index-xpath)),
            (: if we collected strings, we have to wrap them in elements
                    to be able to work with them in xsl :)
            $prenodes := if ($getnodes[1] instance of xs:string) then
                                    for $t in $getnodes return <v>{$t}</v>
                            else for $t in $getnodes return <v>{string-join($t//text()," ")}</v>
        let $nodes := <nodes path="{fn:concat('//', $index-xpath)}"  >{$prenodes}</nodes>,
        	(: use XSLT-2.0 for-each-group functionality to aggregate the values of a node - much, much faster, than XQuery :)
   	    $data := transform:transform($nodes,$fcs:indexXsl,
   	                <parameters><param name="scan-clause" value="{$scan-clause}"/>
   	                <param name="sort" value="{$sort}"/></parameters>)

   	  return $data
};

(:~ main search function (handles the searchRetrieve-operation request)
:)
declare function fcs:search-retrieve($query as xs:string, $x-context as xs:string*, $startRecord as xs:integer, $maximumRecords as xs:integer, $x-dataview as xs:string*, $config) as item()* {
    let $start-time := util:system-dateTime()
    let $data-collection := repo-utils:context-to-collection($x-context, $config)
    (:if ($x-context) then collection($repo-utils:mappings//map[xs:string(@key) eq $x-context]/@path)
                            else $repo-utils:data-collection:)
    let $xpath-query := concat("$data-collection", fcs:transform-query ($query, $x-context, $config))

    let $results := util:eval($xpath-query)

    let	$result-count := fn:count($results),
    $result-seq := fn:subsequence($results, $startRecord, $maximumRecords),
    $seq-count := fn:count($result-seq),
    $end-time := util:system-dateTime(),

    (:<sru:recordSchema>mods</sru:recordSchema>:)
          (:<xQuery><searchClause xmlns="http://www.loc.gov/zing/cql/xcql/">
          <index>dc.title</index>
          <relation>
          <value>=</value>
          </relation>
          <term>dinosaur</term>
          </searchClause>
          </xQuery>:)
    $result :=
    <sru:searchRetrieveResponse>
      <sru:numberOfRecords>{$result-count}</sru:numberOfRecords>
      <sru:echoedSearchRetrieveRequest>
          <sru:version>1.2</sru:version>
          <sru:query>{$query}</sru:query>
          <fcs:x-context>{$x-context}</fcs:x-context>
          <fcs:x-dataview>{$x-dataview}</fcs:x-dataview>
          <sru:startRecord>{$startRecord}</sru:startRecord>
          <sru:maximumRecords>{$maximumRecords}</sru:maximumRecords>
          <sru:query>{$query}</sru:query>
          <sru:baseUrl>{repo-utils:config-value($config, "base.url")}</sru:baseUrl>
      </sru:echoedSearchRetrieveRequest>
      <sru:extraResponseData>
      	<fcs:returnedRecords>{$seq-count}</fcs:returnedRecords>
		<fcs:duration>{$end-time - $start-time}</fcs:duration>
		<fcs:transformedQuery>{ $xpath-query }</fcs:transformedQuery>
      </sru:extraResponseData>
      <sru:records>
	       {for $rec at $pos in $result-seq
	           let $rec-data := fcs:format-record-data($rec,$x-dataview, $config)
	           return
	               <sru:record>
	                   <sru:recordSchema>http://clarin.eu/fcs/1.0/Resource.xsd</sru:recordSchema>
	                   <sru:recordPacking>xml</sru:recordPacking>
	                   <sru:recordData>
	                     {$rec-data}
                        </sru:recordData>
	                   <sru:recordPosition>{$pos}</sru:recordPosition>
	                   <sru:recordIdentifier>{$rec/@xml:id}</sru:recordIdentifier>
	                </sru:record>
	       }
      </sru:records>
    </sru:searchRetrieveResponse>

    return $result

};

declare function fcs:format-record-data($raw-record-data as node(), $data-view as xs:string*, $config as node()) as item()*  {

    let $exp-rec := util:expand($raw-record-data, "expand-xincludes=no") (: kwic:summarize($rec,<config width="40"/>) :)

    let $kwic := if ('kwic' = $data-view) then
                   let $kwic-config := <config width="{$fcs:kwicWidth}"/>
                   let $kwic-html := kwic:summarize($exp-rec, $kwic-config)

                    return if (exists($kwic-html)) then
                                           <fcs:DataView type="kwic">{
                                               for $match in $kwic-html
                                               return (<fcs:c type="left">{$match/span[1]/text()}</fcs:c>,
                                       (: <c type="left">{kwic:truncate-previous($exp-rec, $matches[1], (), 10, (), ())}</c> :)
                                                      <fcs:kw>{$match/span[2]/text()}</fcs:kw>,
                                                      <fcs:c type="right">{$match/span[3]/text()}</fcs:c>)
                                       (: let $summary  := kwic:get-summary($exp-rec, $matches[1], $config) :)
                        (:	                               <fcs:DataView type="kwic-html">{$kwic-html}</fcs:DataView>:)
                                            }</fcs:DataView>
                                     else (: if no kwic-match let's take first 100 characters
                                        There c/should be some more sophisticated way to extract most significant info
                                        e.g. match on the query-field :)
                                       <fcs:DataView type="kwic">{substring($raw-record-data,1,100)}</fcs:DataView>
                     else ()
    let $title := <fcs:DataView type="title">{cmdcoll:get-md-collection-name($raw-record-data)}</fcs:DataView>
    return if ($data-view = 'raw') then $raw-record-data
            else <fcs:Resource>
                       <fcs:ResourceFragment>
                         {($title, $kwic,
                         if ('full' = $data-view or not(exists($kwic))) then <fcs:DataView type="{$data-view}">{$exp-rec}</fcs:DataView>
                             else ()
                           )}
                           </fcs:ResourceFragment>
                       </fcs:Resource>

};

(:~ This expects a CQL-query that it translates to XPath

It relies on the external cqlparser-module, that delivers the query in XCQL-format (parse-tree of the query as XML)
and then applies a stylesheet

@params $x-context identifier of a resource/collection
@returns XPath version of the CQL-query
:)
declare function fcs:transform-query($cql-query as xs:string, $x-context as xs:string, $config) as xs:string {
    let $mappings := repo-utils:config-value($config, 'mappings')
    return cql:cql2xpath ($cql-query, $x-context, $mappings)

 };

(: old version, "manually" parsing the cql-string
it accepted/understood:
term
index=term
index relation term
:)
declare function fcs:transform-query-old($cql-query as xs:string, $x-context as xs:string, $type as xs:string, $config ) as xs:string {

    let $query-constituents := if (contains($cql-query,'=')) then
                                        tokenize($cql-query, "=")
                                   else tokenize($cql-query, " ")
	let $index := if ($type eq 'scan' or count($query-constituents)>1 )  then $query-constituents[1]
	                else "cql.serverChoice"
	let $searchTerm := if (count($query-constituents)=1) then
							$cql-query
						else if (count($query-constituents)=2) then (: tokenized with '=' :)
						    normalize-space($query-constituents[2])
						else
							$query-constituents[3]

(: try to get a mapping specific to given context, else take the default :)
    let $context-map := fcs:get-mapping("",$x-context, $config),

(: TODO: for every index in $xcql :)
            (: try  to get a) a mapping for given index within the context-map,
                    b) in any of the mapping (if not found in the context-map) , - potentially dangerous!!
                    c) or else take the index itself :)
        $mappings := doc(repo-utils:config-value($config, 'mappings')),
        $index-map := $context-map/index[xs:string(@key) eq $index],
        $resolved-index := if (exists($index-map)) then $index-map/text()
                           else if (exists($mappings//index[xs:string(@key) eq $index])) then
                                $mappings//index[xs:string(@key) eq $index]
                            else $index
            ,
            (: get either a) the specific base-element for the index,
                  b) the default for given map,
                  c) the index itself :)
        $base-elem := if (exists($index-map/@base_elem)) then xs:string($index-map/@base_elem)
                        else if (exists($context-map/@base_elem)) then xs:string($context-map/@base_elem)
                        else $index,
            (: <index status="indexed"> - flag to know if ft-indexes can be used.
            TODO?: should be checked against the actual index-configuration :)
        $indexed := (xs:string($index-map/@status) eq 'indexed'),
        $match-on := if (exists($index-map/@use) ) then xs:string($index-map/@use) else '.'

	let $res := if ($type eq 'scan') then
	                   concat("//", $resolved-index, if ($match-on ne '.') then concat("/", $match-on) else '')
	               else
	                   concat("//", $resolved-index, "[",
	                                       if ($indexed) then
	                                               concat("ft:query(", $match-on, ",<term>",translate($searchTerm,'"',''), "</term>)")
	                                           else
	                                               concat("contains(", $match-on, ",'", translate($searchTerm,'"',''), "')")
	                                        , "]",
								"/ancestor-or-self::", $base-elem)

	return $res

};

(:~ gets the mapping-entry for the index

first tries a mapping within given context, then tries defaults.

if $index-param = "" return the map-element,
    else - if found - return the index-element
:)
declare function fcs:get-mapping($index as xs:string, $x-context as xs:string+, $config) as node()* {
    let $mappings := doc(repo-utils:config-value($config, 'mappings')),

    $context-map := if (exists($mappings//map[xs:string(@key) eq $x-context])) then
                                $mappings//map[xs:string(@key) eq $x-context]
                            else $mappings//map[xs:string(@key) eq 'default'],
    $context-index := $context-map/index[xs:string(@key) eq $index]

    return  if ($index eq '') then
                $context-map
             else if (exists($context-index)) then
                        $context-index
                else (: if no contextual index, dare to take any index - may be dangerous!  :)
                    let $any-index := $mappings//index[xs:string(@key) eq $index]
                    return $any-index
};

(:~ gets the mapping for the index and creates an xpath (UNION)

@param $index index-key as known to mappings

@returns xpath-equivalent of given index as defined in mappings; multiple xpaths are translated to a UNION;
         if no mapping found, returns the input-index unchanged
:)
declare function fcs:index-as-xpath($index as xs:string, $x-context as xs:string+, $config) as xs:string {
    let $index-map := fcs:get-mapping($index, $x-context, $config )
     return if (exists($index-map)) then translate(concat('(', string-join($index-map/path,'|'),')'),'.','/')
                    else $index
};


Switch to TcgaTargetGtex_rsem_isoform_tpm.

; transcriptExpression
(fn [transcripts studyA subtypeA studyB subtypeB]
  (let [tcgaTypes
        (map :value (query {:select [:value]
                            :from [:code]
                            :join [:field [:= :field_id :field.id]
                                   :dataset [:= :dataset_id :dataset.id]]
                            :where [:and [:= :dataset.name "TcgaTargetGTEX_phenotype.txt"]
                                         [:= :field.name "_sample_type"]
                                         [:not= :value ["Control Analyte" "Solid Tissue Normal"]]]}))
        fetchSamples {
          "tcga" (fn [subtype]
                  ((xena-query {:select ["sampleID"]
                                :from ["TcgaTargetGTEX_phenotype.txt"]
                                :where [:and [:in "_study" ["TCGA"]]
                                             [:in "_sample_type" tcgaTypes]
                                             [:in "detailed_category" [subtype]]]})  "sampleID"))
          "gtex" (fn [subtype]
                  ((xena-query {:select ["sampleID"]
                                :from ["TcgaTargetGTEX_phenotype.txt"]
                                :where [:and [:in "_study" ["GTEX"]]
                                             [:in "_sample_type" ["Normal Tissue"]]
                                             [:in "_primary_site" [subtype]]]}) "sampleID"))}]
    [(fetch [{:table "TcgaTargetGtex_rsem_isoform_tpm"
              :columns transcripts
              :samples ((fetchSamples studyA) subtypeA)}])
     (fetch [{:table "TcgaTargetGtex_rsem_isoform_tpm"
              :columns transcripts
              :samples ((fetchSamples studyB) subtypeB)}])]))

; transcriptExpression
(fn [transcripts studyA subtypeA studyB subtypeB]
  (let [tcgaTypes
        (map :value (query {:select [:value]
                            :from [:code]
                            :join [:field [:= :field_id :field.id]
                                   :dataset [:= :dataset_id :dataset.id]]
                            :where [:and [:= :dataset.name "TcgaTargetGTEX_phenotype.txt"]
                                         [:= :field.name "_sample_type"]
                                         [:not= :value ["Control Analyte" "Solid Tissue Normal"]]]}))
        fetchSamples {
          "tcga" (fn [subtype]
                  ((xena-query {:select ["sampleID"]
                                :from ["TcgaTargetGTEX_phenotype.txt"]
                                :where [:and [:in "_study" ["TCGA"]]
                                             [:in "_sample_type" tcgaTypes]
                                             [:in "detailed_category" [subtype]]]})  "sampleID"))
          "gtex" (fn [subtype]
                  ((xena-query {:select ["sampleID"]
                                :from ["TcgaTargetGTEX_phenotype.txt"]
                                :where [:and [:in "_study" ["GTEX"]]
                                             [:in "_sample_type" ["Normal Tissue"]]
                                             [:in "_primary_site" [subtype]]]}) "sampleID"))}]
    [(fetch [{:table "TcgaTargetGtex_expected_count"
              :columns transcripts
              :samples ((fetchSamples studyA) subtypeA)}])
     (fetch [{:table "TcgaTargetGtex_expected_count"
              :columns transcripts
              :samples ((fetchSamples studyB) subtypeB)}])]))