Changed XQTS app according to RFE http://www.w3.org/Bugs/Public/show_bug.cgi?id=5217

svn path=/trunk/eXist/; revision=6905

xquery version "1.0";
(: $Id$ :)
declare namespace response="http://exist-db.org/xquery/response";
declare namespace catalog="http://www.w3.org/2005/02/query-test-XQTSCatalog";
declare namespace hack="http://exist-db.org/xquery/xqts/hacked-test-cases";
declare option exist:serialize "method=xml indent=yes";
declare option exist:output-size-limit "-1";

let $product-version := util:system-property("product-version")
let $product-build := util:system-property("product-build")
let $revision := system:get-revision()
let $test-suite := doc('/db/XQTS/XQTSCatalog.xml')/catalog:test-suite/@version
let $dummy := response:set-header("Content-Disposition", concat(
	"attachment; filename="results_XQTS-" , $test-suite , "_eXist-" ,
	$product-version , '_' , $product-build , ".xml"" ) )

let $cases := fn:collection(  '/db/XQTS' )//test-case
return
<test-suite-result xmlns="http://www.w3.org/2005/02/query-test-XQTSResult">
    <implementation name="eXist-db" version="{$product-version} (revision {$revision} {$product-build})" anonymous-result-column="false">

        <organization name="eXist Open Source Native XML database"  website="http://www.exist-db.org" anonymous="false"/>

        <submittor name="eXist XQTS testsuite runner" email="eXistAdmin@gmail.com"/>

        <description>

eXist is an Open Source native XML database featuring efficient,
                index-based XQuery processing, automatic indexing, extensions for full-text
                search,  XUpdate  support and tight integration with existing XML development tools.

        </description>

        <implementation-defined-items>

        </implementation-defined-items>

        <features>

        <feature name="Minimal Conformance" supported="true"/>
        <feature name="Schema Import" supported="false"/>
        <feature name="Schema Validation" supported="false"/>
        <feature name="Static Typing" supported="false"/>
        <feature name="Static Typing Extensions" supported="false"/>
        <feature name="Full Axis" supported="true"/>
        <feature name="Module" supported="true"/>
        <feature name="Serialization" supported="true"/>
        <feature name="Trivial XML Embedding" supported="false"/>
				  </features>

        <context-properties>
      <context-property name="Statically known namespaces" context-type="static"
      value="XQuery default augmented with xmlns:exist='http://exist.sourceforge.net/NS/exist' (unbound when the XQTS is run)"/>
      <context-property name="Default element/type namespace" context-type="static" value="XQuery default"/>
      <context-property name="Default function namespace" context-type="static"
      value="'http://www.w3.org/2005/xpath-functions' by default"/>
      <context-property name="In-scope schema types" context-type="static" value="N/A"/>
      <context-property name="In-scope element declarations" context-type="static" value="N/A"/>
      <context-property name="In-scope attribute declarations" context-type="static" value="N/A"/>
      <context-property name="In-scope variables" context-type="static" value="None"/>
      <context-property name="Context item static type" context-type="static" value="item()?"/>
      <context-property name="Function signatures" context-type="static" value="XQuery default"/>
      <context-property name="Statically known collations" context-type="static" value="See http://exist.sourceforge.net/xquery.html#N10676"/>
      <context-property name="Default collation" context-type="static" value="http://www.w3.org/2005/xpath-functions/collation/codepoint"/>
      <context-property name="Construction mode" context-type="static"/>
      <context-property name="Ordering mode" context-type="static"/>
      <context-property name="Default order for empty sequences" context-type="static"/>
      <context-property name="Boundary-space policy" context-type="static"/>
      <context-property name="Copy-namespaces mode" context-type="static"/>
      <context-property name="Base URI" context-type="static" value="'/db' by default"/>
      <context-property name="Statically known documents" context-type="static" value="By default, all the documents stored in the database for which the user has required permissions"/>
      <context-property name="Statically known collections" context-type="static" value="By default, all the collections stored in the database for which the user has required permissions"/>
      <context-property name="Statically known default collection type" context-type="static"/>
      <context-property name="Context item" context-type="dynamic"/>
      <context-property name="Context position" context-type="dynamic"/>
      <context-property name="Context size" context-type="dynamic"/>
      <context-property name="Variable values" context-type="dynamic"/>
      <context-property name="Function implementations" context-type="dynamic"/>
      <context-property name="Current dateTime" context-type="dynamic" value="Settable using an option, a pragma, or API"/>
      <context-property name="Implicit timezone" context-type="dynamic" value="Settable using an option, a pragma, or API"/>
      <context-property name="Available documents" context-type="dynamic" value="By default, all the documents stored in the database for which the user has required permissions + external URLs"/>
      <context-property name="Available collections" context-type="dynamic" value="By default, all the collections stored in the database for which the user has required permissions"/>
      <context-property name="Default collection" context-type="dynamic" value="By default, all the collections stored in the database for which the user has required permissions"/>
      <context-property name="expressionUnicode" context-type="static" value="UTF-8"/>
      <context-property name="collations" context-type="static" value="By default, 'http://www.w3.org/2005/xpath-functions/collation/codepoint'"/>
      <context-property name="implicitTimezone" context-type="static" value="By default, the system's one"/>
      <context-property name="errorsMethod" context-type="static" value="Errors are thrown as an org.exist.xquery.XPathException. The exception's getMessage() method contains the error code."/>
        </context-properties>

    </implementation>

    <syntax>XQuery</syntax>

    <test-run dateRun="{fn:current-dateTime()}">
        <test-suite version="{$test-suite}"/>
        <transformation>
        {
            if (empty(doc('/db/XQTS/hacked-tests.xml')/hack:test-cases/hack:test-case/hack:expected-result)) then

No unusual transformations.

            else

The following tests have been evaluated against results provided by other XQuery processors.
,
                (: this paragraph element is not really wanted but we want it to be processed by the XSLT :)
                (:see http://www.w3.org/Bugs/Public/show_bug.cgi?id=5217 :)

                    Test name	Reference software	Considered result
                    {
                        for $hackedTest in doc('/db/XQTS/hacked-tests.xml')/hack:test-cases/hack:test-case/hack:expected-result
                        return

                                {string($hackedTest/../@name)}
                                {string($hackedTest/@from)}
                                {string($hackedTest)}

                     }

        }
        </transformation>
        <comparison>
        {
            if (empty(doc('/db/XQTS/hacked-tests.xml')/hack:test-cases/hack:test-case/@compare)) then

No unusual comparisons

            else

The following tests have required a comparison method different from the standard one.
,
                (:see http://www.w3.org/Bugs/Public/show_bug.cgi?id=5217 :)

                    Test name	Comparison method
                    {
                        for $hackedTest in doc('/db/XQTS/hacked-tests.xml')/hack:test-cases/hack:test-case[@compare]
                        return

                                {string($hackedTest/@name)}
                                {string($hackedTest/@compare)}

                     }

,

UnnormalizedText : result text nodes are concatenated without introducing (normalized) spaces between them.
,

TextAsXML : result string is evaluated as if it were XML, then an XML comparison is done.

        }
        </comparison>
        <otherComments>

The test run took {let $b := for $a in $cases//@dateRun return xs:time($a)
return (max($b) - min($b)) div xs:dayTimeDuration('PT1S')} seconds.

        </otherComments>
    </test-run>

{
    for $case in $cases
    order by $case/@name
    return <test-case name="{$case/@name}" result="{$case/@result}" />
}
</test-suite-result>

xquery version "1.0";
(: $Id$ :)
declare namespace response="http://exist-db.org/xquery/response";
declare namespace catalog="http://www.w3.org/2005/02/query-test-XQTSCatalog";
declare namespace hack="http://exist-db.org/xquery/xqts/hacked-test-cases";
declare option exist:serialize "method=xml indent=yes";
declare option exist:output-size-limit "-1";

let $product-version := util:system-property("product-version")
let $product-build := util:system-property("product-build")
let $revision := system:get-revision()
let $test-suite := doc('/db/XQTS/XQTSCatalog.xml')/catalog:test-suite/@version
let $dummy := response:set-header("Content-Disposition", concat(
	"attachment; filename="results_XQTS-" , $test-suite , "_eXist-" ,
	$product-version , '_' , $product-build , ".xml"" ) )

let $cases := fn:collection(  '/db/XQTS' )//test-case
return
<test-suite-result xmlns="http://www.w3.org/2005/02/query-test-XQTSResult">
    <implementation name="eXist-db" version="{$product-version} (revision {$revision} {$product-build})" anonymous-result-column="false">

        <organization name="eXist Open Source Native XML database"  website="http://www.exist-db.org" anonymous="false"/>

        <submittor name="eXist XQTS testsuite runner" email="eXistAdmin@gmail.com"/>

        <description>

eXist is an Open Source native XML database featuring efficient,
                index-based XQuery processing, automatic indexing, extensions for full-text
                search,  XUpdate  support and tight integration with existing XML development tools.

        </description>

        <implementation-defined-items>

        </implementation-defined-items>

        <features>

        <feature name="Minimal Conformance" supported="true"/>
        <feature name="Schema Import" supported="false"/>
        <feature name="Schema Validation" supported="false"/>
        <feature name="Static Typing" supported="false"/>
        <feature name="Static Typing Extensions" supported="false"/>
        <feature name="Full Axis" supported="true"/>
        <feature name="Module" supported="true"/>
        <feature name="Serialization" supported="true"/>
        <feature name="Trivial XML Embedding" supported="false"/>
				  </features>

        <context-properties>
      <context-property name="Statically known namespaces" context-type="static"
      value="XQuery default augmented with xmlns:exist='http://exist.sourceforge.net/NS/exist' (unbound when the XQTS is run)"/>
      <context-property name="Default element/type namespace" context-type="static" value="XQuery default"/>
      <context-property name="Default function namespace" context-type="static"
      value="'http://www.w3.org/2005/xpath-functions' by default"/>
      <context-property name="In-scope schema types" context-type="static" value="N/A"/>
      <context-property name="In-scope element declarations" context-type="static" value="N/A"/>
      <context-property name="In-scope attribute declarations" context-type="static" value="N/A"/>
      <context-property name="In-scope variables" context-type="static" value="None"/>
      <context-property name="Context item static type" context-type="static" value="item()?"/>
      <context-property name="Function signatures" context-type="static" value="XQuery default"/>
      <context-property name="Statically known collations" context-type="static" value="See http://exist.sourceforge.net/xquery.html#N10676"/>
      <context-property name="Default collation" context-type="static" value="http://www.w3.org/2005/xpath-functions/collation/codepoint"/>
      <context-property name="Construction mode" context-type="static"/>
      <context-property name="Ordering mode" context-type="static"/>
      <context-property name="Default order for empty sequences" context-type="static"/>
      <context-property name="Boundary-space policy" context-type="static"/>
      <context-property name="Copy-namespaces mode" context-type="static"/>
      <context-property name="Base URI" context-type="static" value="'/db' by default"/>
      <context-property name="Statically known documents" context-type="static" value="By default, all the documents stored in the database for which the user has required permissions"/>
      <context-property name="Statically known collections" context-type="static" value="By default, all the collections stored in the database for which the user has required permissions"/>
      <context-property name="Statically known default collection type" context-type="static"/>
      <context-property name="Context item" context-type="dynamic"/>
      <context-property name="Context position" context-type="dynamic"/>
      <context-property name="Context size" context-type="dynamic"/>
      <context-property name="Variable values" context-type="dynamic"/>
      <context-property name="Function implementations" context-type="dynamic"/>
      <context-property name="Current dateTime" context-type="dynamic" value="Settable using an option, a pragma, or API"/>
      <context-property name="Implicit timezone" context-type="dynamic" value="Settable using an option, a pragma, or API"/>
      <context-property name="Available documents" context-type="dynamic" value="By default, all the documents stored in the database for which the user has required permissions + external URLs"/>
      <context-property name="Available collections" context-type="dynamic" value="By default, all the collections stored in the database for which the user has required permissions"/>
      <context-property name="Default collection" context-type="dynamic" value="By default, all the collections stored in the database for which the user has required permissions"/>
      <context-property name="expressionUnicode" context-type="static" value="UTF-8"/>
      <context-property name="collations" context-type="static" value="By default, 'http://www.w3.org/2005/xpath-functions/collation/codepoint'"/>
      <context-property name="implicitTimezone" context-type="static" value="By default, the system's one"/>
      <context-property name="errorsMethod" context-type="static" value="Errors are thrown as an org.exist.xquery.XPathException. The exception's getMessage() method contains the error code."/>
        </context-properties>

    </implementation>

    <syntax>XQuery</syntax>

    <test-run dateRun="{fn:current-dateTime()}">
        <test-suite version="{$test-suite}"/>
        <transformation>
        {
            if (empty(doc('/db/XQTS/hacked-tests.xml')/hack:test-cases/hack:test-case/hack:expected-result)) then

No unusual transformations.

            else

The following tests have been evaluated against results provided by other XQuery processors.
,
                (: this paragraph element is not really wanted but we want it to be processed by the XSLT :)

                    Test name	Reference software	Considered result
                    {
                        for $hackedTest in doc('/db/XQTS/hacked-tests.xml')/hack:test-cases/hack:test-case/hack:expected-result
                        return

                                {string($hackedTest/../@name)}
                                {string($hackedTest/@from)}
                                {string($hackedTest)}

                     }

        }
        </transformation>
        <comparison>
        {
            if (empty(doc('/db/XQTS/hacked-tests.xml')/hack:test-cases/hack:test-case/@compare)) then

No unusual comparisons

            else

The following tests have required a comparison method different from the standard one.
,
                (: this paragraph element is not really wanted but we want it to be processed by the XSLT :)

                    Test name	Comparison method
                    {
                        for $hackedTest in doc('/db/XQTS/hacked-tests.xml')/hack:test-cases/hack:test-case[@compare]
                        return

                                {string($hackedTest/@name)}
                                {string($hackedTest/@compare)}

                     }

,

UnnormalizedText : result text nodes are concatenated without introducing (normalized) spaces between them.
,

TextAsXML : result string is evaluated as if it were XML, then an XML comparison is done.

        }
        </comparison>
        <otherComments>

The test run took {let $b := for $a in $cases//@dateRun return xs:time($a)
return (max($b) - min($b)) div xs:dayTimeDuration('PT1S')} seconds.

        </otherComments>
    </test-run>

{
    for $case in $cases
    order by $case/@name
    return <test-case name="{$case/@name}" result="{$case/@result}" />
}
</test-suite-result>


Removed unnecessary array module from .

Removed unnecessary array module from .

xquery version "3.1";
(:~
:  BaseX array module functions
:
: @see http://docs.basex.org/wiki/Arry_Module
:)
module namespace array = "http://www.w3.org/2005/xpath-functions/array";

import module namespace a = "http://reecedunn.co.uk/xquery/annotations" at "res://reecedunn.co.uk/xquery/annotations.xqy";

declare %a:since("basex", "8.0") function array:size($input as array(*)) as xs:integer external;
declare %a:since("basex", "8.0") function array:get($array as array(*), $position as xs:integer) as item()* external;
declare %a:since("basex", "8.0") function array:append($array as array(*), $member as item()*) as array(*) external;
declare %a:since("basex", "8.0") function array:subarray($array as array(*), $position as xs:integer) as array(*) external;
declare %a:since("basex", "8.0") function array:subarray($array as array(*), $position as xs:integer, $length as xs:integer) as array(*) external;
declare %a:since("basex", "8.5") function array:put($array as array(*), $position as xs:integer, $member as item()*) as array(*) external;
declare %a:since("basex", "8.0") function array:remove($array as array(*), $positions as xs:integer*) as array(*) external;
declare %a:since("basex", "8.0") function array:insert-before($array as array(*), $position as xs:integer, $member as item()*) as array(*) external;
declare %a:since("basex", "8.0") function array:head($array as array(*)) as item()* external;
declare %a:since("basex", "8.0") function array:tail($array as array(*)) as array(*) external;
declare %a:since("basex", "8.0") function array:reverse($array as array(*)) as array(*) external;
declare %a:since("basex", "8.0") function array:join($arrays as array(*)*) as array(*) external;
declare %a:since("basex", "8.0") function array:flatten($items as item()*) as item()* external;
declare %a:since("basex", "8.0") function array:for-each($array as array(*), $function as function(item()*) as item()*) as array(*) external;
declare %a:since("basex", "8.0") function array:filter($array as array(*), $function as function(item()*) as xs:boolean) as array(*) external;
declare %a:since("basex", "8.0") function array:fold-left($array as array(*), $zero as item()*, $function as function(item()*, item()*) as item()*) as item()* external;
declare %a:since("basex", "8.0") function array:fold-right($array as array(*), $zero as item()*, $function as function(item()*, item()*) as item()*) as item()* external;
declare %a:since("basex", "8.0") function array:for-each-pair($array1 as array(*), $array2 as array(*), $function as function(item()*) as item()*) as array(*) external;
declare %a:since("basex", "8.0") function array:sort($array as array(*)) as array(*) external;
declare %a:since("basex", "8.6") function array:sort($array as array(*), $collation as xs:string?) as array(*) external;
declare %a:since("basex", "8.0") %a:until("basex", "8.6") function array:sort($array as array(*), $key as function(item()*) as xs:anyAtomicType*) as array(*) external;
declare %a:since("basex", "8.6") function array:sort($array as array(*), $collation as xs:string?, $key as function(item()*) as xs:anyAtomicType*) as array(*) external;
declare %a:since("basex", "8.0") %a:until("basex", "8.4") %a:see-also("basex", "8.4", "fn:serialize") function array:serialize($input as map(*)) as xs:string external;


tweaking affiliation query

tweaking affiliation query

declare namespace scholarNET='https://scholarnet.github.io/';

declare variable $search-string-file := doc('../pre-voyant-search-strings.tsv');

declare variable $prefixes := ('ب','ل','ك','ف','و');

(: If turned on will only search for variants that include any shaddas in above strings. :)
declare variable $require-existing-shadda := 1;

(: If turned on will not allow any vowels provided in $search-strings to be absent. If off, will match any specified vowels as well as unvocalized versions, but not alternate vowels. :)
declare variable $require-existing-vowels := 0;

(: If turned on will add a search term with the definite article prefixed to the word. :)
declare variable $add-optional-article := 1;

(: If turned on will add attached prepositions and conjunctions as optional search strings. :)
declare variable $add-optional-prefixes := 1;

declare function scholarNET:process-shadda($search-string as xs:string) as xs:string {

    if ($require-existing-vowels) then
        if ($require-existing-shadda) then
            replace(replace($search-string, '([ً-ِْ-ٖ])','(ّ)?$1(ّ)?'),'((ّ)\(ّ\)\?([ً-ِْ-ٖ])\(ّ\)\?)|(\(ّ\)\?([ً-ِْ-ٖ])\(ّ\)\?(ّ))','(ّ)$3$5') (: optional shadda before / after required vowel, but keeps existing shaddas by replacing optional shaddas with required. :)
            (: NB: Looks for required shadda only before required vowel, since Voyant does not seem to support parentheses. :)
        else
            replace(replace($search-string, '([ً-ِْ-ٖ])','(ّ)?$1(ّ)?'),'((ّ)\(ّ\)\?)|(\(ّ\)\?(ّ))','(ّ)?') (: optional shadda before / after required vowel, ignoring existing shaddas by removing duplicate shaddas :)
    else
        if ($require-existing-shadda) then
            replace(replace($search-string, '([ً-ِْ-ٖ])','(ّ)?$1?(ّ)?'),'((ّ)\(ّ\)\?([ً-ِْ-ٖ]\?)\(ّ\)\?)|(\(ّ\)\?([ً-ِْ-ٖ]\?)\(ّ\)\?(ّ))','(ّ)$3$5') (: required shadda + optional existing vowel :)
            (: NB: Looks for required shadda only before optional vowel, since Voyant does not seem to support parentheses. :)
        else
            replace(replace($search-string, '([ً-ٖ])','[(ّ)$1]*'),'(ّ)?(\[\(ّ\)[ً-ٖ]\]\*)(ّ)?','$1') (: optional existing vowel + optional shadda, in any order, replacing existing shaddas :)
};

declare function scholarNET:add-vowels($search-string as xs:string) as xs:string {

if ($require-existing-shadda) then
        replace(
            replace(
                replace(
                    $search-string,
                    '([ء-ي])((\(ّ\))?\??)',
                    '$1$2[ً-ِْ-ٖ]*') (: Adds vowel wildcard after all consonants (behind any required/optional shaddas) :)
                ,'\[ً-ِْ-ٖ\]\*([ً-ِْ-ٖ])'
                ,'$1'), (: Then removes wildcard if it is next to an existing vowel. :)
            '([ء-ي])\[ً-ِْ-ٖ\]\*',
            '$1[ً-ٖ]*') (: If there is no shadda normalizes to shadda-agnostic. :)
    else
        replace(
            replace(
                $search-string,
                '([ء-ي])',
                '$1[ً-ٖ]*') (: Adds vowel wildcard after all consonants :)
            ,'\[ً-ٖ\]\*([\(ّ])'
            ,'$1') (: Then removes wildcard if it is next to an existing vowel-shadda combo. :)
};

(: Removes final vowel because it confuses Voyant Tools. :)
declare function scholarNET:removeFinalVowel($search-string as xs:string) as xs:string {
    replace(
        replace($search-string,
        '\[ً-ٖ\]\*(\*?)($|\|)',
        '$1$2'),
    '\[ً-ِْ-ٖ\]\*(\*?)($|\|)',
    '$1$2')
};

declare function scholarNET:addArticle($search-string as xs:string) as xs:string {
    if ($add-optional-article) then
         (: Duplicates each segment of the search string, adding the article on the front of one. :)
            replace($search-string,'(.+)','$1|الْ?$1')
        else $search-string
};

declare function scholarNET:addPrefixes($search-string as xs:string) as xs:string {
    if ($add-optional-prefixes) then
            let $prefixes-to-add := concat(string-join($prefixes,'?[ً-ِْ-ٖ]?'),'?[ً-ِْ-ٖ]?')
            (: Breaks string into segments by | character and adds prefix query to each string, then rejoins them. :)
            let $prefixes-added := concat($prefixes-to-add,string-join(tokenize($search-string,'\|'),concat('|',$prefixes-to-add)))
            (: Accounts for elided ا when preposition ل is added producing لل instead of لال.
            Duplicates the form with the article (if present) and creates an additional form with لل in place
            of the article. :)
            return if (string-length(normalize-space($search-string))) then
            replace(
                replace(
                    $prefixes-added,
                    concat('((',$prefixes-to-add,')(الْ\?)(.+?(\||$)))'),
                    '$1|لِ?لْ?$4'),
                '\|\|',
                '|')
            else $search-string
        else $search-string
};

declare function scholarNET:normalizeAYW($search-string as xs:string) as xs:string {
let $normalize-alif := replace($search-string,'ا','[آاإأ]')
let $normalize-ya := replace($normalize-alif,'ى','[يىئ]')
return replace($normalize-ya,'و','[وؤ]')
};

declare function scholarNET:voyantSearchString($search-string as xs:string) as xs:string {
    replace(
        scholarNET:normalizeAYW(
            scholarNET:addPrefixes(
                scholarNET:addArticle(
                    scholarNET:removeFinalVowel(
                        scholarNET:add-vowels(
                            scholarNET:process-shadda($search-string)
                            )
                        )
                    )
                )
            ),
       '\(ّ\)','ّ')
};

let $rows := tokenize($search-string-file/root,'\n')

let $processed-rows :=
    for $row in $rows[position()>1]
        let $cells := tokenize($row,'\t')
        let $label := $cells[1]
        let $search-strings :=
            for $cell in $cells[position()>1]
            return (scholarNET:voyantSearchString($cell))
        let $cluster := string-join($cells[position()>1 and string-length(.)],'|')
        let $search-strings-combined := replace(string-join($search-strings,'|'),'\|+\s*$','')
        let $search-url :=
            concat(
                    'http://voyant-tools.org/?corpus=90e03a27efc07a543ee15f894a2db0a0&view=Contexts&query=',
                    escape-html-uri($search-strings-combined),
                    '&context=50&expand=500')
        return (string-join(($label,$search-strings-combined,$cluster,$search-url),'	'),'
')

return (
    concat(string-join(('Macro-affiliation','Search String','Search Cluster','Link (replace ampersands)'),'	'),'
'),
    $processed-rows
    )

declare namespace scholarNET='https://scholarnet.github.io/';

declare variable $search-string-file := doc('../pre-voyant-search-strings.tsv');

declare variable $prefixes := ('ب','ل','ك','ف','و');

(: If turned on will only search for variants that include any shaddas in above strings. :)
declare variable $require-existing-shadda := 1;

(: If turned on will not allow any vowels provided in $search-strings to be absent. If off, will match any specified vowels as well as unvocalized versions, but not alternate vowels. :)
declare variable $require-existing-vowels := 0;

(: If turned on will add a search term with the definite article prefixed to the word. :)
declare variable $add-optional-article := 1;

(: If turned on will add attached prepositions and conjunctions as optional search strings. :)
declare variable $add-optional-prefixes := 1;

declare function scholarNET:process-shadda($search-string as xs:string) as xs:string {

    if ($require-existing-vowels) then
        if ($require-existing-shadda) then
            replace(replace($search-string, '([ً-ِْ-ٖ])','(ّ)?$1(ّ)?'),'((ّ)\(ّ\)\?([ً-ِْ-ٖ])\(ّ\)\?)|(\(ّ\)\?([ً-ِْ-ٖ])\(ّ\)\?(ّ))','((ّ)$3$5|$3$5(ّ))') (: optional shadda before \ after required vowel, but keeps existing shaddas by replacing optional shaddas with required :)
        else
            replace(replace($search-string, '([ً-ِْ-ٖ])','(ّ)?$1(ّ)?'),'((ّ)\(ّ\)\?)|(\(ّ\)\?(ّ))','(ّ)?') (: optional shadda before / after required vowel, ignoring existing shaddas by removing duplicate shaddas :)
    else
        if ($require-existing-shadda) then
            replace(replace($search-string, '([ً-ِْ-ٖ])','(ّ)?$1?(ّ)?'),'((ّ)\(ّ\)\?([ً-ِْ-ٖ]\?)\(ّ\)\?)|(\(ّ\)\?([ً-ِْ-ٖ]\?)\(ّ\)\?(ّ))','($3$5(ّ)|(ّ)$3$5)') (: optional existing vowel + required shadda on either side :)
        else
            replace(replace($search-string, '([ً-ٖ])','[(ّ)$1]*'),'(ّ)?(\[\(ّ\)[ً-ٖ]\]\*)(ّ)?','$1') (: optional existing vowel + optional shadda, in any order, replacing existing shaddas :)
};

declare function scholarNET:add-vowels($search-string as xs:string) as xs:string {

if ($require-existing-shadda) then
        replace(
            replace(
                replace(
                    $search-string,
                    '([ء-ي])((\(ّ\))?\??)',
                    '$1$2[ً-ِْ-ٖ]*') (: Adds vowel wildcard after all consonants (behind any required/optional shaddas) :)
                ,'\[ً-ِْ-ٖ\]\*([ً-ِْ-ٖ])'
                ,'$1'), (: Then removes wildcard if it is next to an existing vowel. :)
            '([ء-ي])\[ً-ِْ-ٖ\]\*',
            '$1[ً-ٖ]*') (: If there is no shadda normalizes to shadda-agnostic. :)
    else
        replace(
            replace(
                $search-string,
                '([ء-ي])',
                '$1[ً-ٖ]*') (: Adds vowel wildcard after all consonants :)
            ,'\[ً-ٖ\]\*([\(ّ])'
            ,'$1') (: Then removes wildcard if it is next to an existing vowel-shadda combo. :)
};

(: Removes final vowel because it confuses Voyant Tools. :)
declare function scholarNET:removeFinalVowel($search-string as xs:string) as xs:string {
    replace(
        replace($search-string,
        '\[ً-ٖ\]\*(\*?)($|\|)',
        '$1$2'),
    '\[ً-ِْ-ٖ\]\*(\*?)($|\|)',
    '$1$2')
};

declare function scholarNET:addArticle($search-string as xs:string) as xs:string {
    if ($add-optional-article) then
         (: Duplicates each segment of the search string, adding the article on the front of one. :)
            replace($search-string,'(.+)','$1|الْ?$1')
        else $search-string
};

declare function scholarNET:addPrefixes($search-string as xs:string) as xs:string {
    if ($add-optional-prefixes) then
            let $prefixes-to-add := concat(string-join($prefixes,'?[ً-ِْ-ٖ]?'),'?[ً-ِْ-ٖ]?')
            (: Breaks string into segments by | character and adds prefix query to each string, then rejoins them. :)
            let $prefixes-added := concat($prefixes-to-add,string-join(tokenize($search-string,'\|'),concat('|',$prefixes-to-add)))
            (: Accounts for elided ا when preposition ل is added producing لل instead of لال.
            Duplicates the form with the article (if present) and creates an additional form with لل in place
            of the article. :)
            return if (string-length(normalize-space($search-string))) then
            replace(
                replace(
                    $prefixes-added,
                    concat('((',$prefixes-to-add,')(الْ\?)(.+?(\||$)))'),
                    '$1|لِ?لْ?$4'),
                '\|\|',
                '|')
            else $search-string
        else $search-string
};

declare function scholarNET:normalizeAYW($search-string as xs:string) as xs:string {
let $normalize-alif := replace($search-string,'ا','[آاإأ]')
let $normalize-ya := replace($normalize-alif,'ى','[يىئ]')
return replace($normalize-ya,'و','[وؤ]')
};

declare function scholarNET:voyantSearchString($search-string as xs:string) as xs:string {
    replace(
        scholarNET:normalizeAYW(
            scholarNET:addPrefixes(
                scholarNET:addArticle(
                    scholarNET:removeFinalVowel(
                        scholarNET:add-vowels(
                            scholarNET:process-shadda($search-string)
                            )
                        )
                    )
                )
            ),
       '\(ّ\)','ّ')
};

let $rows := tokenize($search-string-file/root,'\n')

let $processed-rows :=
    for $row in $rows[position()>1]
        let $cells := tokenize($row,'\t')
        let $label := $cells[1]
        let $search-strings :=
            for $cell in $cells[position()>1]
            return (scholarNET:voyantSearchString($cell))
        let $cluster := string-join($cells[position()>1 and string-length(.)],'|')
        let $search-strings-combined := replace(string-join($search-strings,'|'),'\|\s*$','')
        let $search-url :=
            concat(
                    'http://voyant-tools.org/?corpus=90e03a27efc07a543ee15f894a2db0a0&view=Contexts&query=',
                    escape-html-uri($search-strings-combined),
                    '&context=50&expand=500')
        return (string-join(($label,$search-strings-combined,$cluster,$search-url),'	'),'
')

return (
    concat(string-join(('Macro-affiliation','Search String','Cluster','Link (replace ampersands)'),'	'),'
'),
    $processed-rows
    )


minor improvements + comments + cosmetic in fots_driver/util.xq

(:
 : Copyright 2006-2011 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

(:~
 : Zorba FOTS driver util
 : @author Sorin Nasoi
 :)

module namespace util =
  "http://www.zorba-xquery.com/fots-driver/util";

import schema namespace output =
  "http://www.w3.org/2010/xslt-xquery-serialization";

import module namespace file  =
  "http://expath.org/ns/file";

import module namespace fots-err =
  "http://www.zorba-xquery.com/fots-driver/errors" at "errors.xq";

declare namespace err =
  "http://www.w3.org/2005/xqt-errors";

declare namespace fots =
  "http://www.w3.org/2010/09/qt-fots-catalog";

declare namespace ann =
  "http://www.zorba-xquery.com/annotations";

(:~
 : The serialization parameters for XML serialization.
 :)
 declare variable $util:serParamXml :=
  <output:serialization-parameters>
    <output:method                value="xml" />
    <output:indent                value="no"  />
    <output:omit-xml-declaration  value="yes" />
  </output:serialization-parameters>;

(:~
 : The serialization parameters for XML serialization.
 :)
 declare variable $util:writeXML :=
  <output:serialization-parameters>
    <output:method                value="xml" />
    <output:indent                value="yes"  />
    <output:omit-xml-declaration  value="no" />
  </output:serialization-parameters>;

(:~
 : Search within a given test-case for all element nodes with a given node name.
 : For each such element node return:
 : (a) if the node has a @file attr, the content of that file as unparsed text,
 : or
 : (b) the typed value of the node (assuming it is promotable to string).
 :
 : @param $case test-case element.
 : @param $node-name
 : @param $envBaseURI
 : @return the query text.
 :)
declare %ann:nondeterministic function util:get-value(
  $case       as element(fots:test-case),
  $envBaseURI as xs:anyURI,
  $node-name  as xs:string
) as xs:string
{
  try
  {
    for $node in $case/descendant-or-self::*
    where (fn:local-name-from-QName(fn:node-name($node)) eq $node-name)
    return
      if ($node/@file)
      then fn:unparsed-text(resolve-uri($node/@file, $envBaseURI))
      else fn:data($node)
  }
  catch *
  {
    fn:error($fots-err:errNA, $err:description)
  }
};

(:~
 :  Returns the parent folder of the given file path.
 :  example: util:parent-folder('/home/user/file.ext') returns '/home/user'.
 :
 : @param $path Path.
 : @return the parent folder of the given file.
 :)
declare function util:parent-folder(
  $path as xs:string
) as xs:anyURI
{
  xs:anyURI(fn:substring-before($path, file:base-name($path)))
};

declare function util:serialize-result(
  $result as item()*
) as xs:string*
{
  util:serialize-result($result, $util:serParamXml)
};

declare function util:serialize-result(
  $result as item()*,
  $SerParams
) as xs:string*
{
  for $res in $result
  return
   if ($res instance of node())
   then fn:serialize($res, $SerParams)
   else fn:string($res)
};

(:
 : Copyright 2006-2011 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

(:~
 : Zorba FOTS driver util
 : @author Sorin Nasoi
 :)

module namespace util =
  "http://www.zorba-xquery.com/fots-driver/util";

import schema namespace output =
  "http://www.w3.org/2010/xslt-xquery-serialization";

import module namespace file  =
  "http://expath.org/ns/file";

import module namespace fots-err =
  "http://www.zorba-xquery.com/fots-driver/errors" at "errors.xq";

declare namespace err =
  "http://www.w3.org/2005/xqt-errors";
declare namespace fots =
  "http://www.w3.org/2010/09/qt-fots-catalog";

declare namespace ann =
  "http://www.zorba-xquery.com/annotations";

(:~
 : The serialization parameters for XML serialization.
 :)
 declare variable $util:serParamXml :=
  <output:serialization-parameters>
    <output:method                value="xml" />
    <output:indent                value="no"  />
    <output:omit-xml-declaration  value="yes" />
  </output:serialization-parameters>;

(:~
 : The serialization parameters for XML serialization.
 :)
 declare variable $util:writeXML :=
  <output:serialization-parameters>
    <output:method                value="xml" />
    <output:indent                value="yes"  />
    <output:omit-xml-declaration  value="no" />
  </output:serialization-parameters>;

(:~
 : Retrieve the value of from the given node that is either given as text node
 : or in a file attribute.
 :
 : @param $case test-case element.
 : @param $path the path of the test-set.
 : @return the query text.
 :)
declare %ann:nondeterministic function util:get-value(
  $case       as element(fots:test-case),
  $envBaseURI as xs:anyURI,
  $node-name  as xs:string
) as xs:string {
  try {
    for $node in $case/descendant-or-self::*
    where (fn:local-name-from-QName(fn:node-name($node)) = $node-name)
    return
      if(exists($node/@file))
      then fn:unparsed-text(resolve-uri($node/@file, $envBaseURI))
      else fn:data($node)
  } catch * {
    fn:error($fots-err:errNA, $err:description)
  }
};

(:~
 :  returns the parent folder of the given file path.
 :   example: util:parent-folder('/home/user/file.ext') returns '/home/user'.
 : @param $path Path.
 : @return the parent folder of the given file.
 :)

declare function util:parent-folder(
  $path   as xs:string
) as xs:anyURI {
  xs:anyURI(fn:substring-before($path,
                                file:base-name($path)))
};

declare function util:serialize-result(
  $result    as item()*
) as xs:string* {
util:serialize-result($result,
                      $util:serParamXml)
};

declare function util:serialize-result(
  $result    as item()*,
  $SerParams
) as xs:string* {
  for $res in $result
  return
   if($res instance of node())
   then fn:serialize($res,
                     $SerParams)
   else fn:string($res)
};


Preparing the future : fixed function types.

svn path=/trunk/eXist-1.0/; revision=4245

xquery version "1.0";

import module namespace xdb="http://exist-db.org/xquery/xmldb";
import module namespace image="http://exist-db.org/xquery/image"
at "java:org.exist.xquery.modules.image.ImageModule";

let $t := (image:thumbnail(xs:anyURI("/db/webtest/images"), xs:anyURI("thumbs"), (), "t1_"),
    image:thumbnail(xs:anyURI("/db/webtest/images"), xs:anyURI("/db/webtest/images/thumbs"), (), "t2_"))
let $thumbs := xdb:get-child-resources("/db/webtest/images/thumbs")
return

        <title>Testing image module functions</title>

Width/Height of image

Height: {xs:string(image:get-height(xs:anyURI("/db/webtest/images/logo.jpg")))}

Width: {xs:string(image:get-width(xs:anyURI("/db/webtest/images/logo.jpg")))}

Generated thumbnails: {count($thumbs)}

            {
                for $r in $thumbs
                return

            }

xquery version "1.0";

import module namespace xdb="http://exist-db.org/xquery/xmldb";
import module namespace image="http://exist-db.org/xquery/image"
at "java:org.exist.xquery.modules.image.ImageModule";

let $t := (image:thumbnail("/db/webtest/images", "thumbs", (), "t1_"),
    image:thumbnail("/db/webtest/images", "/db/webtest/images/thumbs", (), "t2_"))
let $thumbs := xdb:get-child-resources("/db/webtest/images/thumbs")
return

        <title>Testing image module functions</title>

Width/Height of image

Height: {xs:string(image:get-height("/db/webtest/images/logo.jpg"))}

Width: {xs:string(image:get-width("/db/webtest/images/logo.jpg"))}

Generated thumbnails: {count($thumbs)}

            {
                for $r in $thumbs
                return

            }