Added display of SVN Revision to html page output

svn path=/trunk/eXist/; revision=6533

xquery version "1.0";
(: $Id$ :)
(:
    Main module of the database administration interface.
:)

declare namespace admin="http://exist-db.org/xquery/admin-interface";
declare namespace request="http://exist-db.org/xquery/request";
declare namespace session="http://exist-db.org/xquery/session";
declare namespace xdb="http://exist-db.org/xquery/xmldb";

import module namespace status="http://exist-db.org/xquery/admin-interface/status" at "status.xqm";
import module namespace browse="http://exist-db.org/xquery/admin-interface/browse" at "browse.xqm";
import module namespace users="http://exist-db.org/xquery/admin-interface/users" at "users.xqm";
import module namespace shut="http://exist-db.org/xquery/admin-interface/shutdown" at "shutdown.xqm";
import module namespace setup="http://exist-db.org/xquery/admin-interface/setup" at "setup.xqm";

(:
    Display the version, SVN revision and user info in the top right corner
:)
declare function admin:info-header($user as xs:string) as element() {

Version: { util:system-property( "product-version" ) }

SVN Revision: { util:system-property( "svn-revision" ) }

Build: {util:system-property("product-build")}

User: { $user}

};

(:
    Select the page to show. Every page is defined in its own module
:)
declare function admin:panel($user as xs:string, $pass as xs:string?) as element() {
    let $panel := request:get-parameter("panel", "status")
    return
        if($panel eq "browse") then
            browse:main($user, $pass)
        else if($panel eq "users") then
            users:main($user, $pass)
        else if($panel eq "shutdown") then
            shut:main($user, $pass)
        else if($panel eq "setup") then
            setup:main()
        else
            status:main()
};

(:
    Main function: display login form if no credentials have been supplied
    or logout is selected.

    $credentials is either an empty sequence or a pair (user, password).
:)
declare function admin:main($credentials as xs:string*) as element()+ {
    if( empty($credentials) or (count($credentials)!=2) ) then
        admin:display-login-form()
    else
        admin:panel($credentials[1], $credentials[2])
};

(:~
    Display the login form.
:)
declare function admin:display-login-form() as element() {

Login

This is a protected resource. Only registered database users can log
        in. If you have not set up any users, login as "admin" and leave the
        password field empty. For testing purposes, you may also log in as
        "guest" with password "guest".

        <form action="{session:encode-url(request:get-uri())}">

                    Please Login

                    Username:
                    <input name="user" type="text" size="20"/>

                    Password:
                    <input name="pass" type="password" size="20"/>

                    <input type="submit"/>

        </form>

};

(:~  Try to authenticate the user name and password from
    the current HTTP session. Returns a pair of (user, password)
    on success, an empty sequence otherwise.
:)
declare function admin:checkUser() as xs:string* {
    let $user := session:get-attribute("user") ,
        $pass := session:get-attribute("password"),
        $login := xdb:authenticate("xmldb:exist:///db", $user, $pass)
    return
        if($login) then
            ($user, $pass)
        else
            ()
};

(:~
    Process user and password passed from the login form.
    Returns a pair of (user, password) if the credentials are
    valid, an empty sequence if not.
:)
declare function admin:doLogin($user as xs:string) as xs:string* {
    let $pass := request:get-parameter("pass", ""),
        $login := session:set-current-user($user, $pass)
    return
        if($login) then
            ($user, $pass)
        else
            ()
};

(:~
    Authenticate the user
:)
declare function admin:login() as xs:string* {
    let $userParam := request:get-parameter("user", ())
    return
        if($userParam) then
            admin:doLogin($userParam)
        else
            admin:checkUser()
};

session:create(),
let $logout := request:get-parameter("logout", ()),
    $s := if($logout) then session:invalidate() else session:create(),
    $credentials := admin:login(),
    $user := if(exists($credentials)) then $credentials[1] else "not logged in"
return

            <title>eXist Database Administration</title>
            <link type="text/css" href="admin.css" rel="stylesheet"/>

                {admin:info-header($user)}

Select a Page

Home

System Status

Browse Collections

Manage Users

Examples Setup

Shutdown

Logout

                        Logged in as: {$user}

                {admin:main($credentials)}

xquery version "1.0";
(: $Id$ :)
(:
    Main module of the database administration interface.
:)

declare namespace admin="http://exist-db.org/xquery/admin-interface";
declare namespace request="http://exist-db.org/xquery/request";
declare namespace session="http://exist-db.org/xquery/session";
declare namespace xdb="http://exist-db.org/xquery/xmldb";

import module namespace status="http://exist-db.org/xquery/admin-interface/status" at "status.xqm";
import module namespace browse="http://exist-db.org/xquery/admin-interface/browse" at "browse.xqm";
import module namespace users="http://exist-db.org/xquery/admin-interface/users" at "users.xqm";
import module namespace shut="http://exist-db.org/xquery/admin-interface/shutdown" at "shutdown.xqm";
import module namespace setup="http://exist-db.org/xquery/admin-interface/setup" at "setup.xqm";

(:
    Display the version and user info in the top right corner
:)
declare function admin:info-header($user as xs:string) as element() {

Version: {util:system-property("product-version")}

Build: {util:system-property("product-build")}

User: { $user}

};

(:
    Select the page to show. Every page is defined in its own module
:)
declare function admin:panel($user as xs:string, $pass as xs:string?) as element() {
    let $panel := request:get-parameter("panel", "status")
    return
        if($panel eq "browse") then
            browse:main($user, $pass)
        else if($panel eq "users") then
            users:main($user, $pass)
        else if($panel eq "shutdown") then
            shut:main($user, $pass)
        else if($panel eq "setup") then
            setup:main()
        else
            status:main()
};

(:
    Main function: display login form if no credentials have been supplied
    or logout is selected.

    $credentials is either an empty sequence or a pair (user, password).
:)
declare function admin:main($credentials as xs:string*) as element()+ {
    if( empty($credentials) or (count($credentials)!=2) ) then
        admin:display-login-form()
    else
        admin:panel($credentials[1], $credentials[2])
};

(:~
    Display the login form.
:)
declare function admin:display-login-form() as element() {

Login

This is a protected resource. Only registered database users can log
        in. If you have not set up any users, login as "admin" and leave the
        password field empty. For testing purposes, you may also log in as
        "guest" with password "guest".

        <form action="{session:encode-url(request:get-uri())}">

                    Please Login

                    Username:
                    <input name="user" type="text" size="20"/>

                    Password:
                    <input name="pass" type="password" size="20"/>

                    <input type="submit"/>

        </form>

};

(:~  Try to authenticate the user name and password from
    the current HTTP session. Returns a pair of (user, password)
    on success, an empty sequence otherwise.
:)
declare function admin:checkUser() as xs:string* {
    let $user := session:get-attribute("user") ,
        $pass := session:get-attribute("password"),
        $login := xdb:authenticate("xmldb:exist:///db", $user, $pass)
    return
        if($login) then
            ($user, $pass)
        else
            ()
};

(:~
    Process user and password passed from the login form.
    Returns a pair of (user, password) if the credentials are
    valid, an empty sequence if not.
:)
declare function admin:doLogin($user as xs:string) as xs:string* {
    let $pass := request:get-parameter("pass", ""),
        $login := session:set-current-user($user, $pass)
    return
        if($login) then
            ($user, $pass)
        else
            ()
};

(:~
    Authenticate the user
:)
declare function admin:login() as xs:string* {
    let $userParam := request:get-parameter("user", ())
    return
        if($userParam) then
            admin:doLogin($userParam)
        else
            admin:checkUser()
};

session:create(),
let $logout := request:get-parameter("logout", ()),
    $s := if($logout) then session:invalidate() else session:create(),
    $credentials := admin:login(),
    $user := if(exists($credentials)) then $credentials[1] else "not logged in"
return

            <title>eXist Database Administration</title>
            <link type="text/css" href="admin.css" rel="stylesheet"/>

                {admin:info-header($user)}

Select a Page

Home

System Status

Browse Collections

Manage Users

Examples Setup

Shutdown

Logout

                        Logged in as: {$user}

                {admin:main($credentials)}


Fixed wrong path.

Fixed wrong path.

svn path=/trunk/eXist-1.0/; revision=2811

xquery version "1.0";

import module namespace util="http://exist-db.org/xquery/util";
import module namespace request="http://exist-db.org/xquery/request";
import module namespace xdb="http://exist-db.org/xquery/xmldb";

declare namespace xqts="http://exist-db.org/xquery/xqts";
declare namespace catalog="http://www.w3.org/2005/02/query-test-XQTSCatalog";

declare variable $xqts:XML_HIGHLIGHT_STYLE { "/db/XQTS/xml-highlight.xsl" };
declare variable $xqts:CONFIG { xqts:initialize() };
declare variable $xqts:XQTS_HOME { $xqts:CONFIG/basedir/text() };

declare function xqts:initialize() as element() {
    let $collection := xdb:create-collection("/db", "XQTS")
    let $config := doc("/db/XQTS/config.xml")/config
    return
        if ($config) then
            $config
        else
            let $home := util:system-property("exist.home")
            let $path := concat($home, "webapp/xqts")
            let $stored := xdb:store-files-from-pattern("/db/XQTS", $path, "*.xml", "text/xml")
            return
                doc("/db/XQTS/config.xml")/config
};

(:~ The required stylesheets were not found in the db. Try to import them. :)
declare function xqts:import-stylesheet() as xs:string* {
    let $home := util:system-property("exist.home")
    let $pathSep := util:system-property("file.separator")
    let $dir :=
        	if(ends-with($home, "WEB-INF")) then
        	    concat(substring-before($home, "WEB-INF"), "xqts", $pathSep, "stylesheets")
        	else
               concat($home, $pathSep, "webapp", $pathSep, "xqts", $pathSep, "stylesheets")
    let $col := xdb:create-collection("/db", "XQTS")
    return
        concat("Imported: ", xdb:store-files-from-pattern($col, $dir, "*.xsl", "text/xml"))
};

(:~ Check for stylesheets required for the application. Try to import them if they could not be found. :)
declare function xqts:check-paths() as xs:string* {
    if (not(doc-available($xqts:XML_HIGHLIGHT_STYLE))) then
        let $dummy := xqts:import-stylesheet()
        return
            if (not(doc-available($xqts:XML_HIGHLIGHT_STYLE))) then
                concat($xqts:XML_HIGHLIGHT_STYLE, " not found! Please store this file into ",
                    "the database collection or the application will not work properly.")
            else ()
    else ()
};

declare function xqts:path-to-uri($path as xs:string) as xs:string {
        if (starts-with($path, "/")) then
            concat("file://", $path)
        else
            concat("file:///", $path)
};

declare function xqts:get-query($case as element(catalog:test-case)) {
   let $query-name := $case//catalog:query/@name
   let $path := concat( xqts:path-to-uri($xqts:XQTS_HOME), "Queries/XQuery/",
       $case/@FilePath, $query-name, ".xq" )
   let $xq-string := util:file-read($path)
   let $tokenized := tokenize($xq-string, "\n")
   for $token in $tokenized return
       ($token,
)
};

declare function xqts:display-page() as element() {
    util:declare-option("exist:serialize", "media-type=text/html method=xhtml"),

            <title>XQuery Test Suite - Test Reports</title>
            <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
            <link type="text/css" href="styles/report.css" rel="stylesheet"/>

xquery version "1.0";

import module namespace util="http://exist-db.org/xquery/util";
import module namespace request="http://exist-db.org/xquery/request";
import module namespace xdb="http://exist-db.org/xquery/xmldb";

declare namespace xqts="http://exist-db.org/xquery/xqts";
declare namespace catalog="http://www.w3.org/2005/02/query-test-XQTSCatalog";

declare variable $xqts:XML_HIGHLIGHT_STYLE { "/db/XQTS/xml-highlight.xsl" };
declare variable $xqts:CONFIG { xqts:initialize() };
declare variable $xqts:XQTS_HOME { $xqts:CONFIG/basedir/text() };

declare function xqts:initialize() as element() {
    let $collection := xdb:create-collection("/db", "XQTS")
    let $config := doc("/db/XQTS")/config
    return
        if ($config) then
            $config
        else
            let $home := util:system-property("exist.home")
            let $path := concat($home, "/tools/XQTS")
            let $stored := xdb:store-files-from-pattern("/db/XQTS", $path, "*.xml", "text/xml")
            return
                doc("/db/XQTS/config.xml")/config
};

(:~ The required stylesheets were not found in the db. Try to import them. :)
declare function xqts:import-stylesheet() as xs:string* {
    let $home := util:system-property("exist.home")
    let $pathSep := util:system-property("file.separator")
    let $dir :=
        	if(ends-with($home, "WEB-INF")) then
        	    concat(substring-before($home, "WEB-INF"), "xqts", $pathSep, "stylesheets")
        	else
               concat($home, $pathSep, "webapp", $pathSep, "xqts", $pathSep, "stylesheets")
    let $col := xdb:create-collection("/db", "XQTS")
    return
        concat("Imported: ", xdb:store-files-from-pattern($col, $dir, "*.xsl", "text/xml"))
};

(:~ Check for stylesheets required for the application. Try to import them if they could not be found. :)
declare function xqts:check-paths() as xs:string* {
    if (not(doc-available($xqts:XML_HIGHLIGHT_STYLE))) then
        let $dummy := xqts:import-stylesheet()
        return
            if (not(doc-available($xqts:XML_HIGHLIGHT_STYLE))) then
                concat($xqts:XML_HIGHLIGHT_STYLE, " not found! Please store this file into ",
                    "the database collection or the application will not work properly.")
            else ()
    else ()
};

declare function xqts:path-to-uri($path as xs:string) as xs:string {
        if (starts-with($path, "/")) then
            concat("file://", $path)
        else
            concat("file:///", $path)
};

declare function xqts:get-query($case as element(catalog:test-case)) {
   let $query-name := $case//catalog:query/@name
   let $path := concat( xqts:path-to-uri($xqts:XQTS_HOME), "Queries/XQuery/",
       $case/@FilePath, $query-name, ".xq" )
   let $xq-string := util:file-read($path)
   let $tokenized := tokenize($xq-string, "\n")
   for $token in $tokenized return
       ($token,
)
};

declare function xqts:display-page() as element() {
    util:declare-option("exist:serialize", "media-type=text/html method=xhtml"),

            <title>XQuery Test Suite - Test Reports</title>
            <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
            <link type="text/css" href="styles/report.css" rel="stylesheet"/>


[apps biblo]  This commit fixes the problem with XSLTForms not recognizing XML comments.  Very odd.

But my fix is to modify the codes-for-tab service to not return the entire code table file but to write a query that specifically only pulls out the information it needs.  This query correctly omits all XML comments.

There are two other points.  First we could add a paramter to each code table to automatically sort the items based on either the label or the value.  But by default the code tables will be in document order, that is the order that they appear in the original document.

Note that we could also do a distinct-values test to check to see if there are duplicate item values which are techically legal but not encouraged.

- Dan

svn path=/trunk/eXist/; revision=12471

xquery version "1.0";
import module namespace style = "http://exist-db.org/mods-style" at "../../../modules/style.xqm";
declare namespace xf = "http://www.w3.org/2002/xforms";
declare option exist:serialize "method=xml media-type=text/xml indent=yes";

(: codes-for-tab.xq
   This module will load all the code tables for a specific tab of a large multi-part form.
   Created for the MODS form.
   It calculates which tabs are needed by looking through the XForms body files
   in the edit/body collection of the mods editor.

   Input parameter: the tab ID
   Output: a list of all the code tables used by the tab in XML

   Author: Dan McCreary
   Date: Aug. 2010

   :)

let $tab-id := request:get-parameter('tab-id', '')
let $debug := xs:boolean(request:get-parameter('debug', 'false'))

(: TODO check for required tab-id parameter and make sure that the tab is a valid tab ID
let $check-tab := if ( string-length($tab-id) < 1 )
  then
        <error>
           <message>Tab ID is a required parameter.</message>
        </error>
else
:)

let $code-table-collection := concat($style:web-path-to-app, '/code-tables/')
let $xforms-body-collection := concat($style:web-path-to-app, '/edit/body')
let $itemsets := collection($xforms-body-collection)/*[@tab-id=$tab-id]//xf:itemset
let $code-table-names :=
   for $itemset in $itemsets
      let $nodeset := string($itemset/@nodeset)
      let $after := substring-after($nodeset, "code-table-name='")
      let $code-table-name := substring-before($after, "']/items/item")
      order by $code-table-name
      return $code-table-name
let $distinct-code-table-names := distinct-values($code-table-names)
let $itemset-count := count($itemsets)
let $count-distinct := count($distinct-code-table-names)

return
<code-tables>

   { if ($debug)
     then
     <debug>
       <code-table-collection>{$code-table-collection}</code-table-collection>
       <xforms-body-collection>{$xforms-body-collection}</xforms-body-collection>
       <tab-id>{$tab-id}</tab-id>
       <itemset-count>{$itemset-count}</itemset-count>
       <code-table-name-count>{$count-distinct}</code-table-name-count>
       <distinct-code-table-names>{$distinct-code-table-names}</distinct-code-table-names>
     </debug>
     else ()
   }

   {for $code-table-name in $distinct-code-table-names

      let $file-path := concat($code-table-collection, $code-table-name, 's.xml')
      let $code-table := doc($file-path)
      return
         <code-table>
            <code-table-name>{$code-table-name}</code-table-name>
            <items>
            {for $item in $code-table//item
            return
               $item
            }
            </items>
         </code-table>
   }
</code-tables>

xquery version "1.0";
import module namespace style = "http://exist-db.org/mods-style" at "../../../modules/style.xqm";
declare namespace xf = "http://www.w3.org/2002/xforms";
declare option exist:serialize "method=xml media-type=text/xml indent=yes";

(: codes-for-tab.xq
   This module will load all the code tables for a specific tab of a large multi-part form.
   Created for the MODS form.
   It calculates which tabs are needed by looking through the XForms body files
   in the edit/body collection of the mods editor.

   Input parameter: the tab ID
   Output: a list of all the code tables used by the tab in XML

   Author: Dan McCreary
   Date: Aug. 2010

   :)

let $tab-id := request:get-parameter('tab-id', '')

(: TODO check for required tab-id parameter and make sure that the tab is a valid tab ID
let $check-tab := if ( string-length($tab-id) < 1 )
  then
        <error>
           <message>Tab ID is a required parameter.</message>
        </error>
else
:)

let $code-table-collection := concat($style:web-path-to-app, '/code-tables/')
let $xforms-body-collection := concat($style:web-path-to-app, '/edit/body')
let $itemsets := collection($xforms-body-collection)/*[@tab-id=$tab-id]//xf:itemset
let $code-table-names :=
   for $itemset in $itemsets
      let $nodeset := string($itemset/@nodeset)
      let $after := substring-after($nodeset, "code-table-name='")
      let $code-table-name := substring-before($after, "']/items/item")
      order by $code-table-name
      return $code-table-name
let $distinct-code-table-names := distinct-values($code-table-names)
let $itemset-count := count($itemsets)
let $count-distinct := count($distinct-code-table-names)

return
<code-tables>
   <code-table-collection>{$code-table-collection}</code-table-collection>
   <xforms-body-collection>{$xforms-body-collection}</xforms-body-collection>
   <tab-id>{$tab-id}</tab-id>
   <itemset-count>{$itemset-count}</itemset-count>
   <code-table-name-count>{$count-distinct}</code-table-name-count>
   <distinct-code-table-names>{$distinct-code-table-names}</distinct-code-table-names>
   {for $code-table-name in $distinct-code-table-names

      let $file-path := concat($code-table-collection, $code-table-name, 's.xml')
      let $code-table := doc($file-path)
      return
         $code-table
   }
</code-tables>


Make left menu customisable in XQDoc XHTML generation.

module namespace doc2html = "http://www.zorba-xquery.com/modules/xqdoc2html";

import module namespace file = "http://www.zorba-xquery.com/modules/file";
import module namespace xqd = "http://www.zorba-xquery.com/modules/xqdoc";

import module namespace xqdg = "http://www.zorba-xquery.com/modules/internal/xqdoc-xhtml" at "xqdoc-xhtml.xq";

import schema namespace xqdoc = "http://www.xqdoc.org/1.0" at "xqdoc-1.0.xsd";

declare copy-namespaces preserve, inherit;

(:~
 : This variable contains the relative paths to directories that contain
 : example code fragments that are included.
 :)
declare variable $doc2html:examplePath as xs:string* := (
    'test/rbkt/Queries/zorba',
    'test/update/Queries',
    'test/rbkt/ExpQueryResults/zorba',
    'test/update/ExpectedTestResults'
);

declare variable $doc2html:table :=
;

declare variable $doc2html:stepBacks as xs:string := "../../../../../../../../../../../../../../../../../../../../";

(:~
 : This variable contains the to the index.html main page
 :)

declare variable $doc2html:indexCollector := <modules/>;

declare function doc2html:getFileName($moduleURI as xs:string) as xs:string
{
    replace(
        replace($moduleURI, "/", "_"),
        "http:__",
        ""
    )
};

declare sequential function doc2html:clearFolder($folderPath as xs:string, $pattern as xs:string) as xs:string*
{
  if(fn:not(file:exists($folderPath))) then
    ""
  else
    for $file in file:files($folderPath, $pattern, fn:true())
    let $filePath := fn:concat($folderPath, file:path-separator(), $file)
    return
      try {
        file:remove($filePath);
        concat("
  SUCCESS: ", $filePath);
      } catch * ($error_code) {
        concat("
  FAILED: ", $filePath)
      }
};

(:~
 : This function generates the XQDoc XML for all the modules found in $modulesPath
 : and writes the resulting XML documents in $targetPath. The hierarchy is preserved.
 :
 : @param $modulePath where to search for modules recursively.
 : @param $targetPath where to generate the XQDoc XML documents.
 : @return The function returns a string sequence with a status messge for each
 :         module processed.
 :)
declare sequential function doc2html:generateXQDocXml(
  $modulesPath as xs:string, $xqdocXmlPath as xs:string
) as xs:string* {
  for $filedirs in tokenize($modulesPath, ";")
  for $file in file:files($filedirs, "\.xq$", fn:true())
  let $filePath := fn:concat($filedirs, file:path-separator(), $file)
  let $xqdoc := xqd:xqdoc(file:path-to-uri($filePath))
  let $moduleDoc := $xqdoc/xqdoc:module
  let $moduleName := $moduleDoc/xqdoc:name
  let $moduleUri := $moduleDoc/xqdoc:uri
  return
    try {
      file:mkdirs($xqdocXmlPath, false());
      let $x := fn:count(file:files($xqdocXmlPath, "\.xml$", fn:true())) + 1
      let $xqdocFileName := fn:concat($xqdocXmlPath, file:path-separator(), doc2html:getFileName($moduleUri), ".xml")
      return
      file:write($xqdocFileName, $xqdoc, <s method="xml" indent="yes" />/@*);
      concat("
SUCCESS: ", $moduleUri, " (", ()(:$xqdocFileName:), ")");
    } catch * ($error_code) {
      concat("
FAILED: ", $moduleUri, " (", ()(:$xqdocFileName:), ")")
    }
};

(:~
 : This function removes the internal functionality from an XQDoc XML node.
 :
 : @param $xqdoc The XQDdoc XML node to remove the functionality from.
 : @return The xqdoc node without the internal functionality.
 :)
declare sequential function doc2html:removeInternalFunctionality($xqdoc as node()) as node() {
  delete node $xqdoc/xqdoc:functions/xqdoc:function[fn:starts-with(fn:data(xqdoc:comment/xqdoc:description), "Helper function")];
  $xqdoc;
};

(:~
 : This function generates the XQDoc XHTML pages for all the XQDoc XML
 : documents found in $xqdocPath and writes the resulting pages to in
 : $targetPath. The hierarchy is preserved.
 :
 : @param $indexPath path to the XHTML index page.
 :)
declare sequential function doc2html:generateXQDocXhtml(
  $indexPath      as xs:string, $xqdocXmlPath   as xs:string,
  $xqdocXhtmlPath as xs:string, $leftMenu as element(menu),
  $modulesPath    as xs:string
) as xs:string* {
  let $indexHtmlDoc := file:read-xml($indexPath)
  return
  for $file in file:files($xqdocXmlPath, "\.xml$", fn:true())
  let $stepsFromIndex := fn:count(fn:tokenize($file, fn:concat("\", file:path-separator()))) - 2
  let $xmlFilePath := concat($xqdocXmlPath, file:path-separator(), $file)
  let $xhtmlFilePath := concat($xqdocXhtmlPath, file:path-separator(), fn:replace($file, "\.xml$", ".html"))
  let $xhtmlFileDir :=
    let $segments := fn:tokenize($xhtmlFilePath, fn:concat("\", file:path-separator()))
    let $lastSegm := $segments[fn:count($segments)]
    return fn:substring-before($xhtmlFilePath, $lastSegm)
  return
    try {
      let $xqdoc := doc2html:removeInternalFunctionality(file:read-xml($xmlFilePath)/xqdoc:xqdoc)
      let $moduleDoc := $xqdoc/xqdoc:module
      let $moduleName := $moduleDoc/xqdoc:name
      let $moduleUri := $moduleDoc/xqdoc:uri
      let $menu :=
{string($leftMenu/@title)}

      let $menu := doc2html:createLeftMenu($menu, $moduleUri, $leftMenu)
      let $xhtml := xqdg:doc($xqdoc, $menu)
      return block {
        file:mkdirs($xhtmlFileDir, false());

        doc2html:configure-xhtml($xhtml, $stepsFromIndex, $modulesPath);

        file:write($xhtmlFilePath, $xhtml, <s method="xhtml" indent="yes" />/@*);

        concat("
SUCCESS: ", $moduleUri, " (", $xhtmlFilePath, ")");
      }
    } catch * ($error_code) {
      concat("
FAILED: ", $xhtmlFilePath)
    }
};

(:~
 : This function gathers the names of the XQDoc XHTML pages for all the XQDoc XML
 : documents found in $xqdocXmlPath. The hierarchy is preserved.
 :)
declare sequential function doc2html:gatherModules(
  $xqdocXmlPath as xs:string
){
  for $file in file:files($xqdocXmlPath, "\.xml$", fn:true())
  let $xmlFilePath := concat($xqdocXmlPath, file:path-separator(), $file)
  let $xhtmlRelativeFilePath := fn:replace($file, "\.xml$", ".html")
  let $xhtmlRelativeFilePathSlash := fn:replace($xhtmlRelativeFilePath, fn:concat("\", file:path-separator()), "/")
  return
  try {
      let $xqdoc := doc2html:removeInternalFunctionality(file:read-xml($xmlFilePath)/xqdoc:xqdoc)
      let $moduleDoc := $xqdoc/xqdoc:module
      let $moduleUri := $moduleDoc/xqdoc:uri
      return block {
        doc2html:collectModule($moduleDoc, $xhtmlRelativeFilePath);
        concat("
SUCCESS: ", $moduleUri, " (", $xmlFilePath, ")");
      }
    } catch * ($error_code) {
      concat("
FAILED: ", $xmlFilePath)
    }
};

declare sequential function doc2html:collectModule ($module, $relativeFileName as xs:string) {
    let $moduleName := $module/xqdoc:name,
        $moduleUri := $module/xqdoc:uri
    return
        insert node <module uri="{$moduleUri/text()}" file="{$relativeFileName}" /> as last into $doc2html:indexCollector;
    ();
};

(:~
 : This function returns the left menu needed in the index.html and also all the XHTML module pages.
 :
 :)
declare sequential function doc2html:createLeftMenu($menu, $moduleUri, $categories)
{
  for $cat1 in $categories/category
  let $last1 := if(fn:exists($cat1/@last)) then fn:boolean(fn:data($cat1/@last)) else fn:false()
  return block {
    doc2html:createCategory($menu, $moduleUri, $cat1, $last1);
    for $cat2 in $cat1/category
    let $last2 := if(fn:exists($cat2/@last)) then fn:boolean(fn:data($cat2/@last)) else fn:false()
    let $menu2 := $menu/li[fn:last()]/ul
    return block {
      doc2html:createCategory($menu2, $moduleUri, $cat2, $last2);
      for $cat3 in $cat2/category
      let $last3 := if(fn:exists($cat3/@last)) then fn:boolean(fn:data($cat3/@last)) else fn:false()
      let $menu3 := $menu/li[fn:last()]/ul/li[fn:last()]/ul
      return block {
        doc2html:createCategory($menu3, $moduleUri, $cat3, $last3);
      };
    };
  };

  $menu;

};

(:~
 : This is a helper function used to add a category and the modules belonging to it to a node in the menu
 :
 : @param $menu the node in the menu where the new category will be added as the last node
 : @param $moduleUri the name of this module will be highlighted in red as this is the current selected item in the menu
 : @param $category the node in the $leftMenu variable that will be added to the $menu
 : @param $last bool indicating if this is the last item in the $menu node needed in order to set the treeview control
 :)
declare sequential function doc2html:createCategory($menu, $moduleUri, $category, $last as xs:boolean)
{
  block {
  if($last) then
    insert nodes

{' '}

    {data($category/@name)}

    {
     for $module in $doc2html:indexCollector/module
      order by fn:data($module/@uri)
      return block {
        if(fn:starts-with(fn:string($module/@uri),data($category/@uri))
           and fn:not(fn:contains (fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/')),'/'))
           and fn:not($module/@uri = $moduleUri)) then

{fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/'))}

        else if(fn:starts-with(fn:string($module/@uri),data($category/@uri))
                and fn:not(fn:contains (fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/')),'/'))
                and ($module/@uri = $moduleUri)) then

{fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/'))}

        else ()
        };
    }

    as last into $menu
  else
    insert nodes

{' '}

    {data($category/@name)}

    {
      for $module in $doc2html:indexCollector/module
      order by fn:data($module/@uri)
      return block {
        if(fn:starts-with(fn:string($module/@uri),data($category/@uri))
           and fn:not(fn:contains (fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/')),'/'))
           and fn:not($module/@uri = $moduleUri)) then

{fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/'))}

        else if(fn:starts-with(fn:string($module/@uri),data($category/@uri))
                and fn:not(fn:contains (fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/')),'/'))
                and ($module/@uri = $moduleUri)) then

{fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/'))}

        else ()
        };
    }

    as last into $menu
    };

   for $module in $category/module
   let $link :=
     if($module/@uri = $moduleUri) then

{string($module/@name)}

     else

{string($module/@name)}

   return insert node $link as last into $menu//li[@class="expandable" and .//span/text()=$category/@name]/ul;

   $menu;
};

declare sequential function doc2html:getFilePath (
  $filename as xs:string, $modulesPath as xs:string
) as xs:string* {
  let $zorbaPath := replace($modulesPath,"/modules","")
  for $relPath in $doc2html:examplePath
  let $path := concat($zorbaPath,'/',$relPath)
  let $test := file:files($path,concat("^",$filename,"$"),fn:true())[1]
  return
  if (fn:exists($test)) then
    $test
  else
    ()
};

declare function doc2html:get-path-to-index ($stepsBack) as xs:string
{
  (: Error prone: stepsBack > stepBacks :)
  fn:substring($doc2html:stepBacks, 1, 3 * $stepsBack)
};

declare sequential function doc2html:configure-xhtml (
  $xhtml, $stepsFromIndex as xs:integer,
  $modulesPath as xs:string
) {
  let $pathToIndex := doc2html:get-path-to-index($stepsFromIndex)
  return
  (: replace the <test> nodes with eiter links to the actual files or inline the actual files based on the 'type' attribute value :)
  for $file in $xhtml//*:include
  let $type := $file/@type
  let $path := doc2html:getFilePath($file/text(), $modulesPath)
  let $read_text := file:read-text($path)
  return block {
  (: at the moment there is no way to ship the files defined as links with the packages/installers
    if (matches($type, "link")) then
      replace node $file with {$path}
    else :)
    if (matches($type, "inline")) then
      replace node $file with
        if (fn:exists($path)) then

 {$path}:

{$read_text}

        else

No example matching pattern '{$file/text()}' was found. Please update 'examplePath' variable.

        else ()
  };

  (: replace the function type description with images :)
  let $xquSpec := "http://www.w3.org/TR/xquery-update-10/",
      $xqsSpec := "http://www.w3.org/TR/xquery-sx-10/#dt-sequential-function",
      $xq11Spec := "http://www.w3.org/TR/xquery-11/#FunctionDeclns"
  let $pathToIndex := doc2html:get-path-to-index($stepsFromIndex)
  let $imagesPath := concat($pathToIndex, "images/")
  for $typeTd in $xhtml//*:td
  where $typeTd/@class eq "type"
  let $type := $typeTd/text()
  return block {
    if (matches($type, "updating")) then
      replace node $typeTd/text() with

    else if (matches($type, "sequential")) then
      replace node $typeTd/text() with

    else if (matches($type, "nondeterministic")) then
      replace node $typeTd/text() with

    else ();
  };

  $xhtml;
};

declare sequential function doc2html:createModuleTable($leftMenu as element(menu)) {
  for $cat1 in $leftMenu/category
  return block {
    doc2html:createModuleHelper($doc2html:table, $cat1);
    for $cat2 in $cat1/category
    let $table2 := $doc2html:table/li[fn:last()]/ul
    return block {
      doc2html:createModuleHelper($table2, $cat2);
      for $cat3 in $cat2/category
      let $table3 := $doc2html:table/li[fn:last()]/ul/li[fn:last()]/ul
      return block {
        doc2html:createModuleHelper($table3, $cat3);
      };
    };
  };

  $doc2html:table;
};

declare sequential function doc2html:createModuleHelper($table, $category)
{
  block {
    insert nodes
{data($category/@name)}

    {
     for $module in $doc2html:indexCollector/module
      order by fn:data($module/@uri)
      return
        if(fn:starts-with(fn:string($module/@uri),data($category/@uri))
           and fn:not(fn:contains (fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/')),'/'))) then

{fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/'))}

        else ()
    }

    as last into $table
    };

   $table;
};

(:~
 : This function reads, updates and returns the new index.html.
 :
 : @param $indexPath Where to search for XML XQDoc documents recursively.
 : @return The content of the new index.html.
 :)
declare sequential function doc2html:generateIndexHtml($indexPath as xs:string, $menu, $modules) as document-node() {
    let $indexHtmlDoc := file:read-xml($indexPath)
    return block {
        insert nodes $menu
        as last into $indexHtmlDoc/*:html/*:body/*:div[@id='main']/*:div[@id='leftMenu'];

        let $module_list := $indexHtmlDoc/*:html/*:body/*:div[@id='main']/*:div[@id='rightcontent']/*:div[@id='module_list']
        return
          if ($module_list)
          then
            insert node $modules as last into $module_list
          else
            ();

        $indexHtmlDoc;
    }
};

declare sequential function doc2html:main(
  $modulePath as xs:string, $xqdocBuildPath as xs:string,
  $indexHtmlPath as xs:string
) {
  let $leftMenu :=
  <menu title="XQuery Libraries">
  <category name="http://expath.org/ns" uri="http://expath.org/ns" />
  <category name="http://www.zorba-xquery.com/modules" uri="http://www.zorba-xquery.com/modules">
    <category name="email" uri="http://www.zorba-xquery.com/modules/email" />
    <category name="excel" uri="http://www.zorba-xquery.com/modules/excel" />
    <category name="image" uri="http://www.zorba-xquery.com/modules/image" />
    <category name="introspection" uri="http://www.zorba-xquery.com/modules/introspection" />
    <category name="security" uri="http://www.zorba-xquery.com/modules/security" />
    <category name="oauth" uri="http://www.zorba-xquery.com/modules/oauth" />
    <category name="webservices" uri="http://www.zorba-xquery.com/modules/webservices" last="true">
      <category name="google" uri="http://www.zorba-xquery.com/modules/webservices/google" />
      <category name="yahoo" uri="http://www.zorba-xquery.com/modules/webservices/yahoo" last="true" />
    </category>
  </category>
  <category name="http://www.w3.org/2005" uri="http://www.w3.org/2005" last="true"/>
  </menu>
  let $menu :=

      {string($leftMenu/@title)}

  return doc2html:main($menu, $leftMenu, $modulePath, $xqdocBuildPath, $indexHtmlPath)
};

declare sequential function doc2html:main(
  $menu as element(ul),  $leftMenu as element(menu),
  $modulePath as xs:string, $xqdocBuildPath as xs:string,
  $indexHtmlPath as xs:string
) {
  declare $xqdocXmlPath   as xs:string := fn:concat($xqdocBuildPath, file:path-separator(), "xml");
  declare $xqdocXhtmlPath as xs:string := fn:concat($xqdocBuildPath, file:path-separator(), "xhtml");

  (: generate the XQDoc XML for all the modules :)
  if (file:mkdirs($xqdocXmlPath, false())) then
  (
    doc2html:clearFolder($xqdocXmlPath,"\.xml$"),
    doc2html:generateXQDocXml($modulePath, $xqdocXmlPath)
  )
  else
    error()
  ;

  (: generate the XQDoc XHTML for all the modules :)
  let $absoluteXhtmlDir := concat($xqdocBuildPath, "/xhtml")
  return
  if (file:mkdirs($absoluteXhtmlDir, false())) then
  (
    doc2html:clearFolder($xqdocXhtmlPath,"xqdoc\.html$"),
    doc2html:gatherModules($xqdocXmlPath),
    doc2html:generateXQDocXhtml($indexHtmlPath, $xqdocXmlPath, $xqdocXhtmlPath, $leftMenu, $modulePath)
  )
  else
    error()
  ;

  let $menu := doc2html:createLeftMenu($menu, "index.html", $leftMenu)
  let $modules := doc2html:createModuleTable($leftMenu)
  let $doc := doc2html:generateIndexHtml($indexHtmlPath, $menu, $modules)
  return doc2html:configure-xhtml($doc/*:html, 0, $modulePath)
  ;
};

module namespace doc2html = "http://www.zorba-xquery.com/modules/xqdoc2html";

import module namespace file = "http://www.zorba-xquery.com/modules/file";
import module namespace xqd = "http://www.zorba-xquery.com/modules/xqdoc";

import module namespace xqdg = "http://www.zorba-xquery.com/modules/internal/xqdoc-xhtml" at "xqdoc-xhtml.xq";

import schema namespace xqdoc = "http://www.xqdoc.org/1.0" at "xqdoc-1.0.xsd";

declare copy-namespaces preserve, inherit;

(:~
 : This variable contains the relative paths to directories that contain
 : example code fragments that are included.
 :)
declare variable $doc2html:examplePath as xs:string* := (
    'test/rbkt/Queries/zorba',
    'test/update/Queries',
    'test/rbkt/ExpQueryResults/zorba',
    'test/update/ExpectedTestResults'
);

declare variable $doc2html:table :=
;

declare variable $doc2html:stepBacks as xs:string := "../../../../../../../../../../../../../../../../../../../../";

(:~
 : This variable contains the to the index.html main page
 :)

declare variable $doc2html:indexCollector := <modules/>;

declare function doc2html:getFileName($moduleURI as xs:string) as xs:string
{
    replace(
        replace($moduleURI, "/", "_"),
        "http:__",
        ""
    )
};

declare sequential function doc2html:clearFolder($folderPath as xs:string, $pattern as xs:string) as xs:string*
{
  if(fn:not(file:exists($folderPath))) then
    ""
  else
    for $file in file:files($folderPath, $pattern, fn:true())
    let $filePath := fn:concat($folderPath, file:path-separator(), $file)
    return
      try {
        file:remove($filePath);
        concat("
  SUCCESS: ", $filePath);
      } catch * ($error_code) {
        concat("
  FAILED: ", $filePath)
      }
};

(:~
 : This function generates the XQDoc XML for all the modules found in $modulesPath
 : and writes the resulting XML documents in $targetPath. The hierarchy is preserved.
 :
 : @param $modulePath where to search for modules recursively.
 : @param $targetPath where to generate the XQDoc XML documents.
 : @return The function returns a string sequence with a status messge for each
 :         module processed.
 :)
declare sequential function doc2html:generateXQDocXml(
  $modulesPath as xs:string, $xqdocXmlPath as xs:string
) as xs:string* {
  for $filedirs in tokenize($modulesPath, ";")
  for $file in file:files($filedirs, "\.xq$", fn:true())
  let $filePath := fn:concat($filedirs, file:path-separator(), $file)
  let $xqdoc := xqd:xqdoc(file:path-to-uri($filePath))
  let $moduleDoc := $xqdoc/xqdoc:module
  let $moduleName := $moduleDoc/xqdoc:name
  let $moduleUri := $moduleDoc/xqdoc:uri
  return
    try {
      file:mkdirs($xqdocXmlPath, false());
      let $x := fn:count(file:files($xqdocXmlPath, "\.xml$", fn:true())) + 1
      let $xqdocFileName := fn:concat($xqdocXmlPath, file:path-separator(), doc2html:getFileName($moduleUri), ".xml")
      return
      file:write($xqdocFileName, $xqdoc, <s method="xml" indent="yes" />/@*);
      concat("
SUCCESS: ", $moduleUri, " (", ()(:$xqdocFileName:), ")");
    } catch * ($error_code) {
      concat("
FAILED: ", $moduleUri, " (", ()(:$xqdocFileName:), ")")
    }
};

(:~
 : This function removes the internal functionality from an XQDoc XML node.
 :
 : @param $xqdoc The XQDdoc XML node to remove the functionality from.
 : @return The xqdoc node without the internal functionality.
 :)
declare sequential function doc2html:removeInternalFunctionality($xqdoc as node()) as node() {
  delete node $xqdoc/xqdoc:functions/xqdoc:function[fn:starts-with(fn:data(xqdoc:comment/xqdoc:description), "Helper function")];
  $xqdoc;
};

(:~
 : This function generates the XQDoc XHTML pages for all the XQDoc XML
 : documents found in $xqdocPath and writes the resulting pages to in
 : $targetPath. The hierarchy is preserved.
 :
 : @param $indexPath path to the XHTML index page.
 :)
declare sequential function doc2html:generateXQDocXhtml(
  $indexPath      as xs:string, $xqdocXmlPath   as xs:string,
  $xqdocXhtmlPath as xs:string, $leftMenu as element(menu),
  $modulesPath    as xs:string
) as xs:string* {
  let $indexHtmlDoc := file:read-xml($indexPath)
  return
  for $file in file:files($xqdocXmlPath, "\.xml$", fn:true())
  let $stepsFromIndex := fn:count(fn:tokenize($file, fn:concat("\", file:path-separator()))) - 2
  let $xmlFilePath := concat($xqdocXmlPath, file:path-separator(), $file)
  let $xhtmlFilePath := concat($xqdocXhtmlPath, file:path-separator(), fn:replace($file, "\.xml$", ".html"))
  let $xhtmlFileDir :=
    let $segments := fn:tokenize($xhtmlFilePath, fn:concat("\", file:path-separator()))
    let $lastSegm := $segments[fn:count($segments)]
    return fn:substring-before($xhtmlFilePath, $lastSegm)
  return
    try {
      let $xqdoc := doc2html:removeInternalFunctionality(file:read-xml($xmlFilePath)/xqdoc:xqdoc)
      let $moduleDoc := $xqdoc/xqdoc:module
      let $moduleName := $moduleDoc/xqdoc:name
      let $moduleUri := $moduleDoc/xqdoc:uri
      let $menu :=
{string($leftMenu/@title)}

      let $menu := doc2html:createLeftMenu($menu, $moduleUri, $leftMenu)
      let $xhtml := xqdg:doc($xqdoc, $menu)
      return block {
        file:mkdirs($xhtmlFileDir, false());

        doc2html:configure-xhtml($xhtml, $stepsFromIndex, $modulesPath);

        file:write($xhtmlFilePath, $xhtml, <s method="xhtml" indent="yes" />/@*);

        concat("
SUCCESS: ", $moduleUri, " (", $xhtmlFilePath, ")");
      }
    } catch * ($error_code) {
      concat("
FAILED: ", $xhtmlFilePath)
    }
};

(:~
 : This function gathers the names of the XQDoc XHTML pages for all the XQDoc XML
 : documents found in $xqdocXmlPath. The hierarchy is preserved.
 :)
declare sequential function doc2html:gatherModules(
  $xqdocXmlPath as xs:string
){
  for $file in file:files($xqdocXmlPath, "\.xml$", fn:true())
  let $xmlFilePath := concat($xqdocXmlPath, file:path-separator(), $file)
  let $xhtmlRelativeFilePath := fn:replace($file, "\.xml$", ".html")
  let $xhtmlRelativeFilePathSlash := fn:replace($xhtmlRelativeFilePath, fn:concat("\", file:path-separator()), "/")
  return
  try {
      let $xqdoc := doc2html:removeInternalFunctionality(file:read-xml($xmlFilePath)/xqdoc:xqdoc)
      let $moduleDoc := $xqdoc/xqdoc:module
      let $moduleUri := $moduleDoc/xqdoc:uri
      return block {
        doc2html:collectModule($moduleDoc, $xhtmlRelativeFilePath);
        concat("
SUCCESS: ", $moduleUri, " (", $xmlFilePath, ")");
      }
    } catch * ($error_code) {
      concat("
FAILED: ", $xmlFilePath)
    }
};

declare sequential function doc2html:collectModule ($module, $relativeFileName as xs:string) {
    let $moduleName := $module/xqdoc:name,
        $moduleUri := $module/xqdoc:uri
    return
        insert node <module uri="{$moduleUri/text()}" file="{$relativeFileName}" /> as last into $doc2html:indexCollector;
    ();
};

(:~
 : This function returns the left menu needed in the index.html and also all the XHTML module pages.
 :
 :)
declare sequential function doc2html:createLeftMenu($menu, $moduleUri, $categories)
{
  for $cat1 in $categories/category
  let $last1 := if(fn:exists($cat1/@last)) then fn:boolean(fn:data($cat1/@last)) else fn:false()
  return block {
    doc2html:createCategory($menu, $moduleUri, $cat1, $last1);
    for $cat2 in $cat1/category
    let $last2 := if(fn:exists($cat2/@last)) then fn:boolean(fn:data($cat2/@last)) else fn:false()
    let $menu2 := $menu/li[fn:last()]/ul
    return block {
      doc2html:createCategory($menu2, $moduleUri, $cat2, $last2);
      for $cat3 in $cat2/category
      let $last3 := if(fn:exists($cat3/@last)) then fn:boolean(fn:data($cat3/@last)) else fn:false()
      let $menu3 := $menu/li[fn:last()]/ul/li[fn:last()]/ul
      return block {
        doc2html:createCategory($menu3, $moduleUri, $cat3, $last3);
      };
    };
  };

  $menu;

};

(:~
 : This is a helper function used to add a category and the modules belonging to it to a node in the menu
 :
 : @param $menu the node in the menu where the new category will be added as the last node
 : @param $moduleUri the name of this module will be highlighted in red as this is the current selected item in the menu
 : @param $category the node in the $leftMenu variable that will be added to the $menu
 : @param $last bool indicating if this is the last item in the $menu node needed in order to set the treeview control
 :)
declare sequential function doc2html:createCategory($menu, $moduleUri, $category, $last as xs:boolean)
{
  block {
  if($last) then
    insert nodes

{' '}

    {data($category/@name)}

    {
     for $module in $doc2html:indexCollector/module
      order by fn:data($module/@uri)
      return block {
        if(fn:starts-with(fn:string($module/@uri),data($category/@uri))
           and fn:not(fn:contains (fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/')),'/'))
           and fn:not($module/@uri = $moduleUri)) then

{fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/'))}

        else if(fn:starts-with(fn:string($module/@uri),data($category/@uri))
                and fn:not(fn:contains (fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/')),'/'))
                and ($module/@uri = $moduleUri)) then

{fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/'))}

        else ()
        };
    }

    as last into $menu
  else
    insert nodes

{' '}

    {data($category/@name)}

    {
      for $module in $doc2html:indexCollector/module
      order by fn:data($module/@uri)
      return block {
        if(fn:starts-with(fn:string($module/@uri),data($category/@uri))
           and fn:not(fn:contains (fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/')),'/'))
           and fn:not($module/@uri = $moduleUri)) then

{fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/'))}

        else if(fn:starts-with(fn:string($module/@uri),data($category/@uri))
                and fn:not(fn:contains (fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/')),'/'))
                and ($module/@uri = $moduleUri)) then

{fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/'))}

        else ()
        };
    }

    as last into $menu
    };

   $menu;
};

declare sequential function doc2html:getFilePath (
  $filename as xs:string, $modulesPath as xs:string
) as xs:string* {
  let $zorbaPath := replace($modulesPath,"/modules","")
  for $relPath in $doc2html:examplePath
  let $path := concat($zorbaPath,'/',$relPath)
  let $test := file:files($path,concat("^",$filename,"$"),fn:true())[1]
  return
  if (fn:exists($test)) then
    $test
  else
    ()
};

declare function doc2html:get-path-to-index ($stepsBack) as xs:string
{
  (: Error prone: stepsBack > stepBacks :)
  fn:substring($doc2html:stepBacks, 1, 3 * $stepsBack)
};

declare sequential function doc2html:configure-xhtml (
  $xhtml, $stepsFromIndex as xs:integer,
  $modulesPath as xs:string
) {
  let $pathToIndex := doc2html:get-path-to-index($stepsFromIndex)
  return
  (: replace the <test> nodes with eiter links to the actual files or inline the actual files based on the 'type' attribute value :)
  for $file in $xhtml//*:include
  let $type := $file/@type
  let $path := doc2html:getFilePath($file/text(), $modulesPath)
  let $read_text := file:read-text($path)
  return block {
  (: at the moment there is no way to ship the files defined as links with the packages/installers
    if (matches($type, "link")) then
      replace node $file with {$path}
    else :)
    if (matches($type, "inline")) then
      replace node $file with
        if (fn:exists($path)) then

 {$path}:

{$read_text}

        else

No example matching pattern '{$file/text()}' was found. Please update 'examplePath' variable.

        else ()
  };

  (: replace the function type description with images :)
  let $xquSpec := "http://www.w3.org/TR/xquery-update-10/",
      $xqsSpec := "http://www.w3.org/TR/xquery-sx-10/#dt-sequential-function",
      $xq11Spec := "http://www.w3.org/TR/xquery-11/#FunctionDeclns"
  let $pathToIndex := doc2html:get-path-to-index($stepsFromIndex)
  let $imagesPath := concat($pathToIndex, "images/")
  for $typeTd in $xhtml//*:td
  where $typeTd/@class eq "type"
  let $type := $typeTd/text()
  return block {
    if (matches($type, "updating")) then
      replace node $typeTd/text() with

    else if (matches($type, "sequential")) then
      replace node $typeTd/text() with

    else if (matches($type, "nondeterministic")) then
      replace node $typeTd/text() with

    else ();
  };

  $xhtml;
};

declare sequential function doc2html:createModuleTable($leftMenu as element(menu)) {
  for $cat1 in $leftMenu/category
  return block {
    doc2html:createModuleHelper($doc2html:table, $cat1);
    for $cat2 in $cat1/category
    let $table2 := $doc2html:table/li[fn:last()]/ul
    return block {
      doc2html:createModuleHelper($table2, $cat2);
      for $cat3 in $cat2/category
      let $table3 := $doc2html:table/li[fn:last()]/ul/li[fn:last()]/ul
      return block {
        doc2html:createModuleHelper($table3, $cat3);
      };
    };
  };

  $doc2html:table;
};

declare sequential function doc2html:createModuleHelper($table, $category)
{
  block {
    insert nodes
{data($category/@name)}

    {
     for $module in $doc2html:indexCollector/module
      order by fn:data($module/@uri)
      return
        if(fn:starts-with(fn:string($module/@uri),data($category/@uri))
           and fn:not(fn:contains (fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/')),'/'))) then

{fn:substring-after(data($module/@uri),fn:concat(data($category/@uri),'/'))}

        else ()
    }

    as last into $table
    };

   $table;
};

(:~
 : This function reads, updates and returns the new index.html.
 :
 : @param $indexPath Where to search for XML XQDoc documents recursively.
 : @return The content of the new index.html.
 :)
declare sequential function doc2html:generateIndexHtml($indexPath as xs:string, $menu, $modules) as document-node() {
    let $indexHtmlDoc := file:read-xml($indexPath)
    return block {
        insert nodes $menu
        as last into $indexHtmlDoc/*:html/*:body/*:div[@id='main']/*:div[@id='leftMenu'];

        let $module_list := $indexHtmlDoc/*:html/*:body/*:div[@id='main']/*:div[@id='rightcontent']/*:div[@id='module_list']
        return
          if ($module_list)
          then
            insert node $modules as last into $module_list
          else
            ();

        $indexHtmlDoc;
    }
};

declare sequential function doc2html:main(
  $modulePath as xs:string, $xqdocBuildPath as xs:string,
  $indexHtmlPath as xs:string
) {
  let $leftMenu :=
  <menu title="XQuery Libraries">
  <category name="http://expath.org/ns" uri="http://expath.org/ns" />
  <category name="http://www.zorba-xquery.com/modules" uri="http://www.zorba-xquery.com/modules">
    <category name="email" uri="http://www.zorba-xquery.com/modules/email" />
    <category name="excel" uri="http://www.zorba-xquery.com/modules/excel" />
    <category name="image" uri="http://www.zorba-xquery.com/modules/image" />
    <category name="introspection" uri="http://www.zorba-xquery.com/modules/introspection" />
    <category name="security" uri="http://www.zorba-xquery.com/modules/security" />
    <category name="oauth" uri="http://www.zorba-xquery.com/modules/oauth" />
    <category name="webservices" uri="http://www.zorba-xquery.com/modules/webservices" last="true">
      <category name="google" uri="http://www.zorba-xquery.com/modules/webservices/google" />
      <category name="yahoo" uri="http://www.zorba-xquery.com/modules/webservices/yahoo" last="true" />
    </category>
  </category>
  <category name="http://www.w3.org/2005" uri="http://www.w3.org/2005" last="true"/>
  </menu>
  let $menu :=

      {string($leftMenu/@title)}

  return doc2html:main($menu, $leftMenu, $modulePath, $xqdocBuildPath, $indexHtmlPath)
};

declare sequential function doc2html:main(
  $menu as element(ul),  $leftMenu as element(menu),
  $modulePath as xs:string, $xqdocBuildPath as xs:string,
  $indexHtmlPath as xs:string
) {
  declare $xqdocXmlPath   as xs:string := fn:concat($xqdocBuildPath, file:path-separator(), "xml");
  declare $xqdocXhtmlPath as xs:string := fn:concat($xqdocBuildPath, file:path-separator(), "xhtml");

  (: generate the XQDoc XML for all the modules :)
  if (file:mkdirs($xqdocXmlPath, false())) then
  (
    doc2html:clearFolder($xqdocXmlPath,"\.xml$"),
    doc2html:generateXQDocXml($modulePath, $xqdocXmlPath)
  )
  else
    error()
  ;

  (: generate the XQDoc XHTML for all the modules :)
  let $absoluteXhtmlDir := concat($xqdocBuildPath, "/xhtml")
  return
  if (file:mkdirs($absoluteXhtmlDir, false())) then
  (
    doc2html:clearFolder($xqdocXhtmlPath,"xqdoc\.html$"),
    doc2html:gatherModules($xqdocXmlPath),
    doc2html:generateXQDocXhtml($indexHtmlPath, $xqdocXmlPath, $xqdocXhtmlPath, $leftMenu, $modulePath)
  )
  else
    error()
  ;

  let $menu := doc2html:createLeftMenu($menu, "index.html", $leftMenu)
  let $modules := doc2html:createModuleTable($leftMenu)
  let $doc := doc2html:generateIndexHtml($indexHtmlPath, $menu, $modules)
  return doc2html:configure-xhtml($doc/*:html, 0, $modulePath)
  ;
};


Service: add general error message

Service: add general error message

(: Subtitle Conversion Framework (SCF) - SCF service
 :
 : Copyright 2019 Institut für Rundfunktechnik GmbH, Munich, Germany
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, the subject work
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

module namespace scf = 'http://www.irt.de/irt_restxq/scf_service';

declare variable $scf:modules_path as xs:string := "modules/";

declare variable $scf:subtitle_format :=
    map {
        'stl': 'STL',
        'stlxml': 'STLXML',
        'ebu-tt': 'EBU-TT',
        'ebu-tt-d': 'EBU-TT-D',
        'ebu-tt-d-basic-de': 'EBU-TT-D-Basic-DE'
    };

declare variable $scf:params_all := ('offset_seconds', 'offset_frames', 'offset_start_of_programme', 'separate_tti', 'clear_uda', 'discard_user_data', 'use_line_height_125', 'ignore_manual_offset_for_tcp', 'indent');

(: returns a simple HTML form for a conversion from one format to another :)
declare function scf:form_conversion($format_source, $format_target, $params) {

{concat($scf:subtitle_format($format_source), ' → ', $scf:subtitle_format($format_target))}
,
    <form action="convert" method="post" enctype="multipart/form-data">
        <input type="hidden" name="format_source" value="{$format_source}"/>
        <input type="hidden" name="format_target" value="{$format_target}"/>
        {scf:form_conversion_fields($params)}
    </form>
};

(: returns the necessary form input fields for a conversion :)
declare function scf:form_conversion_fields($params) {
    Input file: <input type="file" name="input" required="required"/>,
    <button type="submit">Convert</button>,

        {if($params = 'offset_seconds') then
<input type="checkbox" onclick="this.nextElementSibling.disabled=!this.checked"/> consider <input type="text" size="10" name="offset_seconds" value="36000" disabled="disabled"/> seconds offset
 else ()}
        {if($params = 'offset_frames') then
<input type="checkbox" onclick="this.nextElementSibling.disabled=!this.checked"/> consider <input type="text" size="10" name="offset_frames" value="10:00:00:00" disabled="disabled"/> frames offset
 else ()}
        {if($params = 'offset_start_of_programme') then
<input type="checkbox" name="offset_start_of_programme" value="1"/> consider Start-of-Programme offset
 else ()}
        {if($params = 'separate_tti') then
<input type="checkbox" name="separate_tti" value="1"/> do not merge multiple text TTI blocks of the same subtitle
 else ()}
        {if($params = 'clear_uda') then
<input type="checkbox" name="clear_uda" value="1"/> clear STL User-Defined Area (UDA)
 else ()}
        {if($params = 'discard_user_data') then
<input type="checkbox" name="discard_user_data" value="1"/> discard STL User Data (EBN 0xFE)
 else ()}
        {if($params = 'use_line_height_125') then
<input type="checkbox" name="use_line_height_125" value="1"/> use line height "125%" in EBU-TT-D
 else ()}
        {if($params = 'ignore_manual_offset_for_tcp') then
<input type="checkbox" name="ignore_manual_offset_for_tcp" value="1"/> ignore manual offset for TCP
 else ()}
        {if($params = 'indent') then
<input type="checkbox" name="indent" value="1" checked="checked"/> indented output
 else () (: enabled by default :)}

,

};

(: web interface homepage :)
declare
  %rest:path("/webif")
  %output:method("xhtml")
  function scf:overview() {

        <title>SCF service</title>

SCF service

Multi step conversions

        {scf:form_conversion('stl', 'ebu-tt', ('offset_seconds', 'offset_frames', 'offset_start_of_programme', 'discard_user_data', 'ignore_manual_offset_for_tcp', 'indent'))}
        {scf:form_conversion('ebu-tt', 'stl', ())}
        {scf:form_conversion('stl', 'ebu-tt-d-basic-de', ('offset_seconds', 'offset_frames', 'offset_start_of_programme', 'discard_user_data', 'use_line_height_125', 'ignore_manual_offset_for_tcp', 'indent'))}

Custom multi step conversion

        Note: For a particular conversion, not every shown option may be supported!

        <form action="convert" method="post" enctype="multipart/form-data">

                Source format:
                <select name="format_source" required="required">
                    <option selected="selected"/>
                    {for $format in map:keys($scf:subtitle_format) order by $format return <option value="{$format}">{$scf:subtitle_format($format)}</option>}
                </select>
                Target format:
                <select name="format_target" required="required">
                    <option selected="selected"/>
                    {for $format in map:keys($scf:subtitle_format) order by $format return <option value="{$format}">{$scf:subtitle_format($format)}</option>}
                </select>

            {scf:form_conversion_fields($scf:params_all)}
        </form>

Single step conversions

        {scf:form_conversion('stl', 'stlxml', ('separate_tti', 'clear_uda', 'discard_user_data', 'indent'))}
        {scf:form_conversion('stlxml', 'stl', ())}
        {scf:form_conversion('stlxml', 'ebu-tt', ('offset_seconds', 'offset_frames', 'offset_start_of_programme', 'ignore_manual_offset_for_tcp', 'indent'))}
        {scf:form_conversion('ebu-tt', 'stlxml', ('indent'))}
        {scf:form_conversion('ebu-tt', 'ebu-tt-d', ('offset_seconds', 'offset_frames', 'offset_start_of_programme', 'use_line_height_125', 'indent'))}
        {scf:form_conversion('ebu-tt-d', 'ebu-tt-d-basic-de', ('indent'))}

  };

(: param->options map: sets a param (key) to the value of a specific status option (value); afterwards clear the option value to indicate that it has been consumed :)

(: generic Python invokation :)
declare function scf:call_python($status as map(*), $script as xs:string, $param_option_mapping as map(*)) as map(*) {
    (: build params sequence with present options - it is sufficient that an option is present; the actual value doesn't matter! :)
    let $params :=
        for $k in map:keys($param_option_mapping)
        let $v := $k => $param_option_mapping() => $status()
        where exists($v)
        return $k

    (: transformation :)
    let $input as xs:base64Binary := $status('result')
    let $input_filename := file:create-temp-file('scf-service-', '')
    let $output_filename := concat($input_filename, '.out')
    let $write_input := file:write-binary($input_filename, $input)
    let $result_call := proc:system('python3', (file:resolve-path(concat($scf:modules_path, $script)), $input_filename, '-x', $output_filename, $params ! concat('-', .)))
    let $result as document-node() := (# basex:non-deterministic #) {doc($output_filename)} (: prevent evaluation after deleted output file :)
    let $delete_input := file:delete($input_filename)
    let $delete_output := file:delete($output_filename)

    (: remove consumed options + replace result :)
    return map:merge((map:entry('result', $result), $params ! map:entry($param_option_mapping(.), ()), $status))
};

declare function scf:call_stl2stlxml($status as map(*)) as map(*) {
    scf:call_python($status, 'STL2STLXML/stl2stlxml.py', map {
        's': 'option_separate_tti',
        'a': 'option_clear_uda',
        'u': 'option_discard_user_data'
    })
};

declare function scf:call_stlxml2stl($status as map(*)) as map(*) {
    let $input as document-node() := $status('result')
    let $result as xs:base64Binary := xs:base64Binary(
        (: also invoke STLXML-SplitBlocks :)
        let $input_splitted := xslt:transform($input, concat('../', $scf:modules_path, 'STLXML-SplitBlocks/STLXML-SplitBlocks.xslt'))
        return xquery:eval(
            concat("import module namespace stlxml2stl = 'stlxml2stl' at '../", $scf:modules_path, "STLXML2STL/stlxml2stl.xqm'; stlxml2stl:encode(.)"),
            map { '': $input_splitted }
        )
    )
    return map:put($status, 'result', $result)
};

(: generic XSLT conversion :)
declare function scf:call_xslt($status as map(*), $xslt as xs:string, $param_option_mapping as map(*)) as map(*) {
    (: build params map with present options :)
    let $params := map:merge(
        for $k in map:keys($param_option_mapping)
        let $v := $k => $param_option_mapping() => $status()
        where exists($v)
        return map:entry($k, $v)
    )

    (: transformation :)
    let $input as document-node() := $status('result')
    let $result as document-node() := xslt:transform($input, concat('../', $scf:modules_path, $xslt), $params)

    (: remove consumed options + replace result :)
    return map:merge((map:entry('result', $result), map:keys($params) ! map:entry($param_option_mapping(.), ()), $status))
};

declare function scf:call_stlxml2ebu-tt($status as map(*)) as map(*) {
    scf:call_xslt($status, 'STLXML2EBU-TT/STLXML2EBU-TT.xslt', map {
        'offsetTCP': 'option_offset_start_of_programme',
        'offsetInSeconds': 'option_offset_seconds',
        'offsetInFrames': 'option_offset_frames',
        'ignoreManualOffsetForTCP': 'option_ignore_manual_offset_for_tcp'
    })
};

declare function scf:call_ebu-tt2stlxml($status as map(*)) as map(*) {
    scf:call_xslt($status, 'EBU-TT2STLXML/EBU-TT2STLXML.xslt', map{})
};

declare function scf:call_ebu-tt2ebu-tt-d($status as map(*)) as map(*) {
    scf:call_xslt($status, 'EBU-TT2EBU-TT-D/EBU-TT2EBU-TT-D.xslt', map {
        'useLineHeight125Percent': 'option_use_line_height_125',
        'offsetStartOfProgramme': 'option_offset_start_of_programme',
        'offsetInSeconds': 'option_offset_seconds',
        'offsetInFrames': 'option_offset_frames'
    })
};

declare function scf:call_ebu-tt-d2ebu-tt-d-basic-de($status as map(*)) as map(*) {
    scf:call_xslt($status, 'EBU-TT-D2EBU-TT-D-Basic-DE/EBU-TT-D2EBU-TT-D-Basic-DE.xslt', map{})
};

(: sets a specified description as status result :)
declare function scf:status_set_error_generic($status as map(*), $description as xs:string) as map(*) {
    map:put($status, 'result',
        <error>
            <steps>
                {$status('steps') ! <step>{$scf:subtitle_format(.)}</step>}
            </steps>

            <description>{$description}</description>
            <value/>
            <module/>
            <line-number/>
            <column-number/>
        </error>
    )
};

(: sets an unsupported option error as status result :)
declare function scf:status_set_error_unsupported_option($status as map(*), $description as xs:string+) as map(*) {
    let $description_merged := concat('This conversion chain does not support the following option(s): ', string-join($description, '; '), '.')
    return scf:status_set_error_generic($status, $description_merged)
};

(: returns whether the status result represents an error :)
declare function scf:is_status_error($status as map(*)) as xs:boolean {
    $status('result') instance of element()
};

(:
items of the status map:
- steps: sequence of the conversion formats the source file has gone through so far (last item = current format)
- option_offset_seconds: if present, offset in seconds that shall be subtracted from all timecodes
- option_offset_frames: if present, offset in frames that shall be subtracted from all timecodes
- option_offset_start_of_programme: if present, subtract Start-of-Programme offset from all timecodes
- option_use_line_height_125: if present, use the value "125%" (instead of the special value "normal") for the line height in EBU-TT-D
- option_ignore_manual_offset_for_tcp: if present, any manual offset (seconds or frames) will *not* be subtracted from the TCP value
- option_separate_tti: if present, do not merge multiple text TTI blocks of the same subtitle
- option_clear_uda: if present, clear STL User-Defined Area (UDA)
- option_discard_user_data: if present, discard STL User Data
- result: subtitle content in the indicated format
:)

(: execute the actual conversion and return a result (or an error) :)
declare
  %rest:path("/convert")
  %rest:POST
  %rest:form-param("input", "{$input}")
  %rest:form-param("format_source", "{$format_source}")
  %rest:form-param("format_target", "{$format_target}")
  %rest:form-param("offset_seconds", "{$offset_seconds}")
  %rest:form-param("offset_frames", "{$offset_frames}")
  %rest:form-param("offset_start_of_programme", "{$offset_start_of_programme}")
  %rest:form-param("use_line_height_125", "{$use_line_height_125}")
  %rest:form-param("ignore_manual_offset_for_tcp", "{$ignore_manual_offset_for_tcp}")
  %rest:form-param("separate_tti", "{$separate_tti}")
  %rest:form-param("clear_uda", "{$clear_uda}")
  %rest:form-param("discard_user_data", "{$discard_user_data}")
  %rest:form-param("indent", "{$indent}")
  function scf:convert(
    $input as map(*),
    $format_source as xs:string,
    $format_target as xs:string,
    $offset_seconds as xs:string?,
    $offset_frames as xs:string?,
    $offset_start_of_programme as xs:string?,
    $use_line_height_125 as xs:string?,
    $ignore_manual_offset_for_tcp as xs:string?,
    $separate_tti as xs:string?,
    $clear_uda as xs:string?,
    $discard_user_data as xs:string?,
    $indent as xs:string?
  ) {
    let $conversion_option_error_messages := map {
        'option_offset_seconds': 'to consider a timecodes seconds offset',
        'option_offset_frames': 'to consider a timecodes frame offset',
        'option_offset_start_of_programme': 'to consider the Start-of-Programme offset',
        'option_use_line_height_125': 'to use the line height value "125%"',
        'option_ignore_manual_offset_for_tcp': 'to ignore any manual offset for TCP',
        'option_separate_tti': 'to disable text TTI block merging',
        'option_clear_uda': 'to clear STL User-Defined Area (UDA)',
        'option_discard_user_data': 'to remove STL User Data'
    }

    let $conversion_format := concat($format_source, '→', $format_target)
    let $source_stl := $format_source eq 'stl'
    let $target_stl := $format_target eq 'stl'

    let $input_filename := map:keys($input)[1]
    let $input_content_raw := $input($input_filename)
    let $input_content := if ($source_stl) then $input_content_raw else parse-xml(bin:decode-string($input_content_raw, 'UTF-8'))

    (: do conversion :)
    let $conversion_status_init := map {
        'steps': $format_source,
        'option_offset_seconds': $offset_seconds,
        'option_offset_frames': $offset_frames,
        'option_offset_start_of_programme': $offset_start_of_programme,
        'option_use_line_height_125': $use_line_height_125,
        'option_ignore_manual_offset_for_tcp': $ignore_manual_offset_for_tcp,
        'option_separate_tti': $separate_tti,
        'option_clear_uda': $clear_uda,
        'option_discard_user_data': $discard_user_data,
        'result': $input_content
    }
    let $conversion_status := scf:convert_step($conversion_status_init, $format_target)
    let $conversion_option_errors := map:keys($conversion_option_error_messages)[exists($conversion_status(.))] ! $conversion_option_error_messages(.)
    let $conversion := if (scf:is_status_error($conversion_status) or empty($conversion_option_errors)) then $conversion_status else scf:status_set_error_unsupported_option($conversion_status, $conversion_option_errors)

    let $rest_response_format := if ($conversion('result') instance of xs:base64Binary) then 'application/octet-stream' else 'application/xml'
    let $rest_response_filename_without_ext := replace($input_filename, '[.]([Ss][Tt][Ll]|[Xx][Mm][Ll])$', '')
    let $rest_response_filename_suffix := if ($target_stl) then '.stl' else concat('_', $format_target, '.xml')
    let $rest_response_filename := concat($rest_response_filename_without_ext, $rest_response_filename_suffix)
    let $rest_response_success := not(scf:is_status_error($conversion))

    return (
        (: MIME type + filename :)
        <rest:response>
            <output:serialization-parameters>
                <output:omit-xml-declaration value='no'/>
                <output:indent value='{if (exists($indent)) then 'yes' else 'no'}'/>
                <output:media-type value='{$rest_response_format}'/>
            </output:serialization-parameters>
            <http:response status="{if ($rest_response_success) then 200 else 400}">
                {if ($rest_response_success) then <http:header name="Content-Disposition" value="attachment; filename=""{$rest_response_filename}"""/> else ()}
                <http:header name="SCF-Service-Debug-Conversion-Steps" value="{string-join($conversion('steps'), ',')}"/>
            </http:response>
        </rest:response>,
        if ($rest_response_success) then () else <?xml-stylesheet href="static/error.xsl" type="text/xsl" ?>,
        $conversion('result')
    )
  };

(: execute a single conversion step :)
declare function scf:convert_step($status as map(*), $format_target as xs:string) as map(*) {
    let $current_format := $status('steps')[last()]
    let $next_format := scf:next_format($current_format, $format_target)
    return
        (: return, if no conversion needed :)
        if (empty($next_format))
        then $status
        else
            let $next_steps := ($status('steps'), $next_format)
            return try {
                (: invoke next conversion step :)
                let $next_conversion := concat($current_format, '→', $next_format)
                let $next_step_status :=
                    switch($next_conversion)
                    case "stl→stlxml" return
                        scf:call_stl2stlxml($status)
                    case "stlxml→stl" return
                        scf:call_stlxml2stl($status)
                    case "stlxml→ebu-tt" return
                        scf:call_stlxml2ebu-tt($status)
                    case "ebu-tt→stlxml" return
                        scf:call_ebu-tt2stlxml($status)
                    case "ebu-tt→ebu-tt-d" return
                        scf:call_ebu-tt2ebu-tt-d($status)
                    case "ebu-tt-d→ebu-tt-d-basic-de" return
                        scf:call_ebu-tt-d2ebu-tt-d-basic-de($status)
                    default return
                        ()
                let $next_status := map:put($next_step_status, 'steps', $next_steps)
                return scf:convert_step($next_status, $format_target)
            } catch * {
                (: abort with error :)
                map { 'steps': $next_steps, 'result':
                    <error>
                        <steps>
                            {$next_steps ! <step>{$scf:subtitle_format(.)}</step>}
                        </steps>
                        {$err:code}
                        <description>{$err:description}</description>
                        <value>{$err:value}</value>
                        <module>{$err:module}</module>
                        <line-number>{$err:line-number}</line-number>
                        <column-number>{$err:column-number}</column-number>
                    </error>
                }
            }
};

(: determine the next format to which the current (intermediate) result has to be converted, towards the target format :)
declare function scf:next_format($format_source as xs:string, $format_target as xs:string) as xs:string? {
    switch($format_source)
    case $format_target return
        () (: no conversion needed :)
    case 'stl' return
        'stlxml'
    case 'stlxml' return
        if($format_target eq 'stl')
        then 'stl'
        else 'ebu-tt'
    default return
        (: TTML source :)
        if($format_target = ('stl', 'stlxml'))
        then 'stlxml'
        else
            (: TTML source/target :)
            switch($format_source)
            case 'ebu-tt' return
                'ebu-tt-d'
            case 'ebu-tt-d' return
                if($format_target eq 'ebu-tt')
                then () (: no conversion needed :)
                else 'ebu-tt-d-basic-de'
            case 'ebu-tt-d-basic-de'
                return () (: no conversion needed :)
            default return
                ''
};

(: Subtitle Conversion Framework (SCF) - SCF service
 :
 : Copyright 2019 Institut für Rundfunktechnik GmbH, Munich, Germany
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, the subject work
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

module namespace scf = 'http://www.irt.de/irt_restxq/scf_service';

declare variable $scf:modules_path as xs:string := "modules/";

declare variable $scf:subtitle_format :=
    map {
        'stl': 'STL',
        'stlxml': 'STLXML',
        'ebu-tt': 'EBU-TT',
        'ebu-tt-d': 'EBU-TT-D',
        'ebu-tt-d-basic-de': 'EBU-TT-D-Basic-DE'
    };

declare variable $scf:params_all := ('offset_seconds', 'offset_frames', 'offset_start_of_programme', 'separate_tti', 'clear_uda', 'discard_user_data', 'use_line_height_125', 'ignore_manual_offset_for_tcp', 'indent');

(: returns a simple HTML form for a conversion from one format to another :)
declare function scf:form_conversion($format_source, $format_target, $params) {

{concat($scf:subtitle_format($format_source), ' → ', $scf:subtitle_format($format_target))}
,
    <form action="convert" method="post" enctype="multipart/form-data">
        <input type="hidden" name="format_source" value="{$format_source}"/>
        <input type="hidden" name="format_target" value="{$format_target}"/>
        {scf:form_conversion_fields($params)}
    </form>
};

(: returns the necessary form input fields for a conversion :)
declare function scf:form_conversion_fields($params) {
    Input file: <input type="file" name="input" required="required"/>,
    <button type="submit">Convert</button>,

        {if($params = 'offset_seconds') then
<input type="checkbox" onclick="this.nextElementSibling.disabled=!this.checked"/> consider <input type="text" size="10" name="offset_seconds" value="36000" disabled="disabled"/> seconds offset
 else ()}
        {if($params = 'offset_frames') then
<input type="checkbox" onclick="this.nextElementSibling.disabled=!this.checked"/> consider <input type="text" size="10" name="offset_frames" value="10:00:00:00" disabled="disabled"/> frames offset
 else ()}
        {if($params = 'offset_start_of_programme') then
<input type="checkbox" name="offset_start_of_programme" value="1"/> consider Start-of-Programme offset
 else ()}
        {if($params = 'separate_tti') then
<input type="checkbox" name="separate_tti" value="1"/> do not merge multiple text TTI blocks of the same subtitle
 else ()}
        {if($params = 'clear_uda') then
<input type="checkbox" name="clear_uda" value="1"/> clear STL User-Defined Area (UDA)
 else ()}
        {if($params = 'discard_user_data') then
<input type="checkbox" name="discard_user_data" value="1"/> discard STL User Data (EBN 0xFE)
 else ()}
        {if($params = 'use_line_height_125') then
<input type="checkbox" name="use_line_height_125" value="1"/> use line height "125%" in EBU-TT-D
 else ()}
        {if($params = 'ignore_manual_offset_for_tcp') then
<input type="checkbox" name="ignore_manual_offset_for_tcp" value="1"/> ignore manual offset for TCP
 else ()}
        {if($params = 'indent') then
<input type="checkbox" name="indent" value="1" checked="checked"/> indented output
 else () (: enabled by default :)}

,

};

(: web interface homepage :)
declare
  %rest:path("/webif")
  %output:method("xhtml")
  function scf:overview() {

        <title>SCF service</title>

SCF service

Multi step conversions

        {scf:form_conversion('stl', 'ebu-tt', ('offset_seconds', 'offset_frames', 'offset_start_of_programme', 'discard_user_data', 'ignore_manual_offset_for_tcp', 'indent'))}
        {scf:form_conversion('ebu-tt', 'stl', ())}
        {scf:form_conversion('stl', 'ebu-tt-d-basic-de', ('offset_seconds', 'offset_frames', 'offset_start_of_programme', 'discard_user_data', 'use_line_height_125', 'ignore_manual_offset_for_tcp', 'indent'))}

Custom multi step conversion

        Note: For a particular conversion, not every shown option may be supported!

        <form action="convert" method="post" enctype="multipart/form-data">

                Source format:
                <select name="format_source" required="required">
                    <option selected="selected"/>
                    {for $format in map:keys($scf:subtitle_format) order by $format return <option value="{$format}">{$scf:subtitle_format($format)}</option>}
                </select>
                Target format:
                <select name="format_target" required="required">
                    <option selected="selected"/>
                    {for $format in map:keys($scf:subtitle_format) order by $format return <option value="{$format}">{$scf:subtitle_format($format)}</option>}
                </select>

            {scf:form_conversion_fields($scf:params_all)}
        </form>

Single step conversions

        {scf:form_conversion('stl', 'stlxml', ('separate_tti', 'clear_uda', 'discard_user_data', 'indent'))}
        {scf:form_conversion('stlxml', 'stl', ())}
        {scf:form_conversion('stlxml', 'ebu-tt', ('offset_seconds', 'offset_frames', 'offset_start_of_programme', 'ignore_manual_offset_for_tcp', 'indent'))}
        {scf:form_conversion('ebu-tt', 'stlxml', ('indent'))}
        {scf:form_conversion('ebu-tt', 'ebu-tt-d', ('offset_seconds', 'offset_frames', 'offset_start_of_programme', 'use_line_height_125', 'indent'))}
        {scf:form_conversion('ebu-tt-d', 'ebu-tt-d-basic-de', ('indent'))}

  };

(: param->options map: sets a param (key) to the value of a specific status option (value); afterwards clear the option value to indicate that it has been consumed :)

(: generic Python invokation :)
declare function scf:call_python($status as map(*), $script as xs:string, $param_option_mapping as map(*)) as map(*) {
    (: build params sequence with present options - it is sufficient that an option is present; the actual value doesn't matter! :)
    let $params :=
        for $k in map:keys($param_option_mapping)
        let $v := $k => $param_option_mapping() => $status()
        where exists($v)
        return $k

    (: transformation :)
    let $input as xs:base64Binary := $status('result')
    let $input_filename := file:create-temp-file('scf-service-', '')
    let $output_filename := concat($input_filename, '.out')
    let $write_input := file:write-binary($input_filename, $input)
    let $result_call := proc:system('python3', (file:resolve-path(concat($scf:modules_path, $script)), $input_filename, '-x', $output_filename, $params ! concat('-', .)))
    let $result as document-node() := (# basex:non-deterministic #) {doc($output_filename)} (: prevent evaluation after deleted output file :)
    let $delete_input := file:delete($input_filename)
    let $delete_output := file:delete($output_filename)

    (: remove consumed options + replace result :)
    return map:merge((map:entry('result', $result), $params ! map:entry($param_option_mapping(.), ()), $status))
};

declare function scf:call_stl2stlxml($status as map(*)) as map(*) {
    scf:call_python($status, 'STL2STLXML/stl2stlxml.py', map {
        's': 'option_separate_tti',
        'a': 'option_clear_uda',
        'u': 'option_discard_user_data'
    })
};

declare function scf:call_stlxml2stl($status as map(*)) as map(*) {
    let $input as document-node() := $status('result')
    let $result as xs:base64Binary := xs:base64Binary(
        (: also invoke STLXML-SplitBlocks :)
        let $input_splitted := xslt:transform($input, concat('../', $scf:modules_path, 'STLXML-SplitBlocks/STLXML-SplitBlocks.xslt'))
        return xquery:eval(
            concat("import module namespace stlxml2stl = 'stlxml2stl' at '../", $scf:modules_path, "STLXML2STL/stlxml2stl.xqm'; stlxml2stl:encode(.)"),
            map { '': $input_splitted }
        )
    )
    return map:put($status, 'result', $result)
};

(: generic XSLT conversion :)
declare function scf:call_xslt($status as map(*), $xslt as xs:string, $param_option_mapping as map(*)) as map(*) {
    (: build params map with present options :)
    let $params := map:merge(
        for $k in map:keys($param_option_mapping)
        let $v := $k => $param_option_mapping() => $status()
        where exists($v)
        return map:entry($k, $v)
    )

    (: transformation :)
    let $input as document-node() := $status('result')
    let $result as document-node() := xslt:transform($input, concat('../', $scf:modules_path, $xslt), $params)

    (: remove consumed options + replace result :)
    return map:merge((map:entry('result', $result), map:keys($params) ! map:entry($param_option_mapping(.), ()), $status))
};

declare function scf:call_stlxml2ebu-tt($status as map(*)) as map(*) {
    scf:call_xslt($status, 'STLXML2EBU-TT/STLXML2EBU-TT.xslt', map {
        'offsetTCP': 'option_offset_start_of_programme',
        'offsetInSeconds': 'option_offset_seconds',
        'offsetInFrames': 'option_offset_frames',
        'ignoreManualOffsetForTCP': 'option_ignore_manual_offset_for_tcp'
    })
};

declare function scf:call_ebu-tt2stlxml($status as map(*)) as map(*) {
    scf:call_xslt($status, 'EBU-TT2STLXML/EBU-TT2STLXML.xslt', map{})
};

declare function scf:call_ebu-tt2ebu-tt-d($status as map(*)) as map(*) {
    scf:call_xslt($status, 'EBU-TT2EBU-TT-D/EBU-TT2EBU-TT-D.xslt', map {
        'useLineHeight125Percent': 'option_use_line_height_125',
        'offsetStartOfProgramme': 'option_offset_start_of_programme',
        'offsetInSeconds': 'option_offset_seconds',
        'offsetInFrames': 'option_offset_frames'
    })
};

declare function scf:call_ebu-tt-d2ebu-tt-d-basic-de($status as map(*)) as map(*) {
    scf:call_xslt($status, 'EBU-TT-D2EBU-TT-D-Basic-DE/EBU-TT-D2EBU-TT-D-Basic-DE.xslt', map{})
};

(: sets an unsupported option error as status result :)
declare function scf:status_set_error($status as map(*), $description as xs:string+) as map(*) {
    map:put($status, 'result',
        <error>
            <steps>
                {$status('steps') ! <step>{$scf:subtitle_format(.)}</step>}
            </steps>

            <description>This conversion chain does not support the following option(s): {string-join($description, '; ')}.</description>
            <value/>
            <module/>
            <line-number/>
            <column-number/>
        </error>
    )
};

(:
items of the status map:
- steps: sequence of the conversion formats the source file has gone through so far (last item = current format)
- option_offset_seconds: if present, offset in seconds that shall be subtracted from all timecodes
- option_offset_frames: if present, offset in frames that shall be subtracted from all timecodes
- option_offset_start_of_programme: if present, subtract Start-of-Programme offset from all timecodes
- option_use_line_height_125: if present, use the value "125%" (instead of the special value "normal") for the line height in EBU-TT-D
- option_ignore_manual_offset_for_tcp: if present, any manual offset (seconds or frames) will *not* be subtracted from the TCP value
- option_separate_tti: if present, do not merge multiple text TTI blocks of the same subtitle
- option_clear_uda: if present, clear STL User-Defined Area (UDA)
- option_discard_user_data: if present, discard STL User Data
- result: subtitle content in the indicated format
:)

(: execute the actual conversion and return a result (or an error) :)
declare
  %rest:path("/convert")
  %rest:POST
  %rest:form-param("input", "{$input}")
  %rest:form-param("format_source", "{$format_source}")
  %rest:form-param("format_target", "{$format_target}")
  %rest:form-param("offset_seconds", "{$offset_seconds}")
  %rest:form-param("offset_frames", "{$offset_frames}")
  %rest:form-param("offset_start_of_programme", "{$offset_start_of_programme}")
  %rest:form-param("use_line_height_125", "{$use_line_height_125}")
  %rest:form-param("ignore_manual_offset_for_tcp", "{$ignore_manual_offset_for_tcp}")
  %rest:form-param("separate_tti", "{$separate_tti}")
  %rest:form-param("clear_uda", "{$clear_uda}")
  %rest:form-param("discard_user_data", "{$discard_user_data}")
  %rest:form-param("indent", "{$indent}")
  function scf:convert(
    $input as map(*),
    $format_source as xs:string,
    $format_target as xs:string,
    $offset_seconds as xs:string?,
    $offset_frames as xs:string?,
    $offset_start_of_programme as xs:string?,
    $use_line_height_125 as xs:string?,
    $ignore_manual_offset_for_tcp as xs:string?,
    $separate_tti as xs:string?,
    $clear_uda as xs:string?,
    $discard_user_data as xs:string?,
    $indent as xs:string?
  ) {
    let $conversion_option_error_messages := map {
        'option_offset_seconds': 'to consider a timecodes seconds offset',
        'option_offset_frames': 'to consider a timecodes frame offset',
        'option_offset_start_of_programme': 'to consider the Start-of-Programme offset',
        'option_use_line_height_125': 'to use the line height value "125%"',
        'option_ignore_manual_offset_for_tcp': 'to ignore any manual offset for TCP',
        'option_separate_tti': 'to disable text TTI block merging',
        'option_clear_uda': 'to clear STL User-Defined Area (UDA)',
        'option_discard_user_data': 'to remove STL User Data'
    }

    let $conversion_format := concat($format_source, '→', $format_target)
    let $source_stl := $format_source eq 'stl'
    let $target_stl := $format_target eq 'stl'

    let $input_filename := map:keys($input)[1]
    let $input_content_raw := $input($input_filename)
    let $input_content := if ($source_stl) then $input_content_raw else parse-xml(bin:decode-string($input_content_raw, 'UTF-8'))

    (: do conversion :)
    let $conversion_status_init := map {
        'steps': $format_source,
        'option_offset_seconds': $offset_seconds,
        'option_offset_frames': $offset_frames,
        'option_offset_start_of_programme': $offset_start_of_programme,
        'option_use_line_height_125': $use_line_height_125,
        'option_ignore_manual_offset_for_tcp': $ignore_manual_offset_for_tcp,
        'option_separate_tti': $separate_tti,
        'option_clear_uda': $clear_uda,
        'option_discard_user_data': $discard_user_data,
        'result': $input_content
    }
    let $conversion_status := scf:convert_step($conversion_status_init, $format_target)
    let $conversion_option_errors := map:keys($conversion_option_error_messages)[exists($conversion_status(.))] ! $conversion_option_error_messages(.)
    let $conversion := if (empty($conversion_option_errors)) then $conversion_status else scf:status_set_error($conversion_status, $conversion_option_errors)

    let $rest_response_format := if ($conversion('result') instance of xs:base64Binary) then 'application/octet-stream' else 'application/xml'
    let $rest_response_filename_without_ext := replace($input_filename, '[.]([Ss][Tt][Ll]|[Xx][Mm][Ll])$', '')
    let $rest_response_filename_suffix := if ($target_stl) then '.stl' else concat('_', $format_target, '.xml')
    let $rest_response_filename := concat($rest_response_filename_without_ext, $rest_response_filename_suffix)
    let $rest_response_success := not($conversion('result') instance of element())

    return (
        (: MIME type + filename :)
        <rest:response>
            <output:serialization-parameters>
                <output:omit-xml-declaration value='no'/>
                <output:indent value='{if (exists($indent)) then 'yes' else 'no'}'/>
                <output:media-type value='{$rest_response_format}'/>
            </output:serialization-parameters>
            <http:response status="{if ($rest_response_success) then 200 else 400}">
                {if ($rest_response_success) then <http:header name="Content-Disposition" value="attachment; filename=""{$rest_response_filename}"""/> else ()}
                <http:header name="SCF-Service-Debug-Conversion-Steps" value="{string-join($conversion('steps'), ',')}"/>
            </http:response>
        </rest:response>,
        if ($rest_response_success) then () else <?xml-stylesheet href="static/error.xsl" type="text/xsl" ?>,
        $conversion('result')
    )
  };

(: execute a single conversion step :)
declare function scf:convert_step($status as map(*), $format_target as xs:string) as map(*) {
    let $current_format := $status('steps')[last()]
    let $next_format := scf:next_format($current_format, $format_target)
    return
        (: return, if no conversion needed :)
        if (empty($next_format))
        then $status
        else
            let $next_steps := ($status('steps'), $next_format)
            return try {
                (: invoke next conversion step :)
                let $next_conversion := concat($current_format, '→', $next_format)
                let $next_step_status :=
                    switch($next_conversion)
                    case "stl→stlxml" return
                        scf:call_stl2stlxml($status)
                    case "stlxml→stl" return
                        scf:call_stlxml2stl($status)
                    case "stlxml→ebu-tt" return
                        scf:call_stlxml2ebu-tt($status)
                    case "ebu-tt→stlxml" return
                        scf:call_ebu-tt2stlxml($status)
                    case "ebu-tt→ebu-tt-d" return
                        scf:call_ebu-tt2ebu-tt-d($status)
                    case "ebu-tt-d→ebu-tt-d-basic-de" return
                        scf:call_ebu-tt-d2ebu-tt-d-basic-de($status)
                    default return
                        ()
                let $next_status := map:put($next_step_status, 'steps', $next_steps)
                return scf:convert_step($next_status, $format_target)
            } catch * {
                (: abort with error :)
                map { 'steps': $next_steps, 'result':
                    <error>
                        <steps>
                            {$next_steps ! <step>{$scf:subtitle_format(.)}</step>}
                        </steps>
                        {$err:code}
                        <description>{$err:description}</description>
                        <value>{$err:value}</value>
                        <module>{$err:module}</module>
                        <line-number>{$err:line-number}</line-number>
                        <column-number>{$err:column-number}</column-number>
                    </error>
                }
            }
};

(: determine the next format to which the current (intermediate) result has to be converted, towards the target format :)
declare function scf:next_format($format_source as xs:string, $format_target as xs:string) as xs:string? {
    switch($format_source)
    case $format_target return
        () (: no conversion needed :)
    case 'stl' return
        'stlxml'
    case 'stlxml' return
        if($format_target eq 'stl')
        then 'stl'
        else 'ebu-tt'
    default return
        (: TTML source :)
        if($format_target = ('stl', 'stlxml'))
        then 'stlxml'
        else
            (: TTML source/target :)
            switch($format_source)
            case 'ebu-tt' return
                'ebu-tt-d'
            case 'ebu-tt-d' return
                if($format_target eq 'ebu-tt')
                then () (: no conversion needed :)
                else 'ebu-tt-d-basic-de'
            case 'ebu-tt-d-basic-de'
                return () (: no conversion needed :)
            default return
                ''
};


use head instead of predicate

use head instead of predicate

xquery version "1.0-ml";
module namespace adapter = "http://marklogic.com/mlcp/transform/RestTransformAdapter.xqy";

import module namespace tformod = "http://marklogic.com/rest-api/models/transform-model"
    at "/MarkLogic/rest-api/models/transform-model.xqy";

declare default function namespace "http://www.w3.org/2005/xpath-functions";
declare option xdmp:mapping "false";

declare function adapter:transform(
  $content as map:map,
  $context as map:map
) as map:map*
{
  let $uri           := map:get($content, "uri")
  let $docIn         := map:get($content, "value")
  let $params        := xdmp:from-json-string(
    head((map:get($context, "transform_param"), '{"ml.transform":"transform"}'))
  )
  let $transform     := map:get($params, "ml.transform")
  let $targetContext := map:entry("uri", $uri)
  let $_             := map:delete($params, "ml.transform")
  let $docOut        := tformod:apply-transform(
    $transform, $targetContext, $params, $docIn
  )
  return (
    map:put($content, "value", $docOut),
    $content
  )
};

xquery version "1.0-ml";
module namespace adapter = "http://marklogic.com/mlcp/transform/RestTransformAdapter.xqy";

import module namespace tformod = "http://marklogic.com/rest-api/models/transform-model"
    at "/MarkLogic/rest-api/models/transform-model.xqy";

declare default function namespace "http://www.w3.org/2005/xpath-functions";
declare option xdmp:mapping "false";

declare function adapter:transform(
  $content as map:map,
  $context as map:map
) as map:map*
{
  let $uri           := map:get($content, "uri")
  let $docIn         := map:get($content, "value")
  let $params        := xdmp:from-json-string(
    (map:get($context, "transform_param"), '{"ml.transform":"transform"}')[1]
  )
  let $transform     := map:get($params, "ml.transform")
  let $targetContext := map:entry("uri", $uri)
  let $_             := map:delete($params, "ml.transform")
  let $docOut        := tformod:apply-transform(
    $transform, $targetContext, $params, $docIn
  )
  return (
    map:put($content, "value", $docOut),
    $content
  )
};