check if profile-id is referenced - issue diagnostics

xquery version "1.0";
module namespace cmdcheck = "http://clarin.eu/cmd/check";
(: checking (trying to ensure) consistency of the IDs in CMD-records (MdSelfLink vs. ResourceProxies vs. IsPartOf)

TODO: check (and/or generate) the inverse links in IsPartOf (vs. ResourceProxies)
:)

(:import module namespace cmd  = "http://clarin.eu/cmd/collections" at "cmd-collections.xqm";:)
import module namespace repo-utils = "http://aac.ac.at/content_repository/utils" at  "/db/cr/repo-utils.xqm";
import module namespace fcs = "http://clarin.eu/fcs/1.0" at "/db/cr/fcs.xqm";
import module namespace crday  = "http://aac.ac.at/content_repository/data-ay" at "/db/cr/crday.xqm";
import module namespace diag =  "http://www.loc.gov/zing/srw/diagnostic/" at  "modules/diagnostics/diagnostics.xqm";

declare namespace sru = "http://www.loc.gov/zing/srw/";
declare namespace cmd = "http://www.clarin.eu/cmd/";

(:~ default namespace: cmd - declared statically, because dynamic ns-declaration did not work
cmd is the default namespace in (not) all the CMD records
:)
declare default element namespace "http://www.clarin.eu/cmd/";
(:~ default namespace - not declared explicitely, it is declared dynamically where necessary (function: addIsPartOf() :)
declare variable $cmdcheck:default-ns := "http://www.clarin.eu/cmd/";
declare variable $cmdcheck:root-coll := "root";

(:~ runs cmd.profile-scan for all contexts defined in the mappings :)

declare function cmdcheck:run-stats($config-path as xs:string) as item()* {

       let $config := doc($config-path),
           $mappings := doc(repo-utils:config-value($config, 'mappings'))

        let $opt := util:declare-option("exist:serialize", "media-type=text/html method=xhtml")

      for $map in $mappings//map[@key]
                let $map-key := $map/xs:string(@key),
                    $map-dbcoll-path := $map/xs:string(@path),
                    $scan-cmd-profile := fcs:scan('cmd.profile', $map-key, 1, 50, 1, 1, "text", $config)
                return $scan-cmd-profile
};

(:~ currently not used -> DEPRECATE?
init-function meant to call individual functions actually doing something.
at least it resolves x-context to a nodeset
:)
declare function cmdcheck:check($x-context as xs:string+, $config as node() ) as item()* {

    let $log-file-name := concat('log_checks_', repo-utils:sanitize-name($x-context), '.xml')
    let $log-path := repo-utils:config-value($config, 'log.path')

    let $start-time := util:system-dateTime()

    let $data-collection := repo-utils:context-to-collection($x-context, $config),
        $stat-profiles := cmdcheck:scan-profiles($data-collection),
(:        $check-linking := cmdcheck:check-linking($data-collection),:)
        $duration := util:system-dateTime() - $start-time

    let $result-data := <checks context="{$x-context}" on="{$start-time}" duration="{$duration}" >
                    <profiles>{$stat-profiles}</profiles>
                    <check-linking>{$check-linking}</check-linking>
                  </checks>
     let $store := xmldb:store($log-path ,  $log-file-name, $result-data)
    return $store
};

(:~ extracts CMD-Profiles from given nodeset

TODO: match with cmd-terms and diagnostics
:)
declare function cmdcheck:scan-profiles($context as node()*) as item()* {
      (: try- to handle namespace problem - primitively :)
    let $ns-uri := namespace-uri($context[1]/*)
            (: dynamically declare a namespace for the next step, if one is defined in current context
       $dummy := if (exists($ns-uri)) then util:declare-namespace("",$ns-uri) else () :)
       (: this is now trying to overcome the default-ns issue, by accepting with and without ns :)

(:        $dummy := util:declare-namespace("",xs:anyURI(""))       :)
(:    let $profiles := util:eval("$context//(MdProfile|cmd:MdProfile)/text()"):)
(: taking :)
    let $profiles := $context//cmd:CMD/concat(cmd:Header/cmd:MdProfile/text(), '#', cmd:Components/*[1]/local-name())
    let $distinct-profiles := distinct-values($profiles)
    let $profiles-summary := for $profile in $distinct-profiles
(:                                let $profile-name := util:eval("$context[.//(MdProfile|cmd:MdProfile)/text() = $profile][1]//(Components|cmd:Components)/*/name()"):)
                                let $profile-name := substring-after($profile, '#')
                                let $profile-id := substring-before($profile, '#')
                                let $cnt := count($profiles[. eq $profile])
                                return <sru:term>
                                           <sru:value>{if ($profile-id ne '') then $profile-id else $profile-name}</sru:value>
                                           <sru:numberOfRecords>{$cnt }</sru:numberOfRecords>
                                           <sru:displayTerm>{$profile-name}</sru:displayTerm>
                                           { if ($profile-id eq '') then
                                                    <sru:extraTermData>
                                                        { diag:diagnostics("profile-missing", $profile-name) }
                                                    </sru:extraTermData>
                                                  else ()
                                                  }
                                         </sru:term>
    let $count-all := count($distinct-profiles)
    return
        <sru:scanResponse xmlns:sru="http://www.loc.gov/zing/srw/" xmlns:fcs="http://clarin.eu/fcs/1.0">
              <sru:version>1.2</sru:version>
              <sru:terms>
                {$profiles-summary }
               </sru:terms>
               <sru:extraResponseData>
                     <fcs:countTerms>{$count-all}</fcs:countTerms>
                 </sru:extraResponseData>
                 <sru:echoedScanRequest>
                      <sru:scanClause>cmd.profile</sru:scanClause>
                  </sru:echoedScanRequest>
           </sru:scanResponse>
};

declare function cmdcheck:display-overview($config-path as xs:string) as item()* {

   let $config := doc($config-path),
       $dummy := util:declare-namespace("",xs:anyURI("")),
       $mappings := doc(repo-utils:config-value($config, 'mappings'))
(:       $baseurl := repo-utils:config-value($config, 'base.url'),:)

   let $opt := util:declare-option("exist:serialize", "media-type=text/html method=xhtml")

   let $overview :=  crday:display-overview($config-path, 'raw')

   let $profiles-overview :=

collection	profiles

           { for $map in util:eval("$mappings//map[@key]")
                    let $map-key := $map/xs:string(@key),
                        $map-dbcoll-path := $map/xs:string(@path),
                        $scan-cmd-profile := fcs:scan('cmd.profile', $map-key, 1, 50, 1, 1, "text", $config),
                        $scan-formatted := repo-utils:serialise-as( $scan-cmd-profile, 'htmldetail', 'scan', $config, ())
                    return

                        {$map-key}
                        { $scan-formatted }

                        }

            (: { for $profile in $scan-cmd-profile//sru:term
                            return ($profile/sru:value, $profile/sru:displayTerm, $profile/sru:numberOfRecords)
                        }:)

        return repo-utils:serialise-as(
{($overview, $profiles-overview)}
, 'htmlpage', 'html', $config, ())

};

(:~	WARNING - CHANGES DATA! adds IsPartOf to the CMDrecords (contained in the db-coll)
- expects CMD-records for collections in specific db-coll (with name of the db-coll matching the name of the collectionfile):
    ./_corpusstructure/collection_{db-coll-name}.cmdi
- logs the progress of the processing in a separate file
- can be applied repeatedly - deletes(!) old IsPartOfList, before inserting new

@param $x-context an identifier of a collection (as defined in mappings)
@param $config config-node - used to get log.path and collection-path

TODO: could also generate the CMD-records for collections (assuming db-colls as collections)
      (now done by the python script: dir2cmdicollection.py)
TODO: currently a hack: the collection-records also are marked as root elements - this has to be optional at least,
      and may even be dangerous (count(IsPartOf[@level=1])>1)!
TODO: not recursive yet!
:)
declare function cmdcheck:addIsPartOf-colls($x-context as xs:string, $config as node()*) as item()* {

let $log-file-name := concat('log_addIsPartOf_', $x-context, '.xml')
    let $log-path := repo-utils:config-value($config, 'log.path')
    let $log-doc-path := xmldb:store($log-path ,  $log-file-name, <result></result>)
    let $log-doc := doc($log-doc-path)

    (:let $root-dbcoll := repo-utils:context-to-collection($x-context, $config),:)
    let $root-dbcoll-path := repo-utils:context-to-collection-path($x-context, $config)

    let $coll-dbcoll := '_corpusstructure'

    (: beware empty path!! would return first-level collections and run over whole database :)
    let $colls := if ($root-dbcoll-path ne "") then xmldb:get-child-collections($root-dbcoll-path) else ()
    let $start_time := fn:current-dateTime()

    (: dynamic ns-declaration does not work for xpath
    let $declare-dummy := util:declare-namespace("",xs:anyURI($cmdcheck:default-ns))
    :)
    let $log-dummy := update insert <edit x-context="{$x-context}" root-dbcoll="{$root-dbcoll-path}"
                        coll-dbcoll="{$coll-dbcoll}" count-colls="{count($colls)}" time="{$start_time}" /> into $log-doc/result
    (:if (exists($colls)) then:)
    for $coll_name in $colls[not(.=$coll-dbcoll)]
    (: let $coll_name := 'HomeFamilyManagement' :)
    let $files := collection(concat($root-dbcoll-path,$coll_name))
    (: let $files := xmldb:xcollection($root_coll) :)
	let $coll_file := concat($root-dbcoll-path, $coll-dbcoll, '/collection_', $coll_name, '.cmdi')
	let $coll_doc := doc($coll_file)
	let $coll_id := $coll_doc//MdSelfLink/text()

(:	let $cmdi_files := $files[ends-with(util:document-name(.),".cmdi")]:)

    let $pre_time := util:system-dateTime()
	let $duration := $pre_time - $start_time
 return	( update insert <edit coll="{$coll_name}" collid="{$coll_id}" coll_file="{$coll_file}" count="{count($files)}" time="{$pre_time}" /> into $log-doc/result,
			update delete $files//IsPartOfList,
			update delete $coll_doc//IsPartOfList,
			update insert <IsPartOfList>
								<IsPartOf level="1">{$x-context}</IsPartOf>
								<IsPartOf level="1">{$cmdcheck:root-coll}</IsPartOf>
						  </IsPartOfList>
			  	into $coll_doc//Resources,
			update insert <IsPartOfList>
							<IsPartOf level="2">{$x-context}</IsPartOf>
							<IsPartOf level="1">{$coll_id}</IsPartOf>
						  </IsPartOfList>
				into $files//Resources,
		update value $log-doc/result/edit[last()] with (util:system-dateTime() - $pre_time)
        )
    (:
    return	($coll_name, count($cmdi_files), $coll_file,  update insert <IsPartOfList>
							<IsPartOf level="1">{$coll_id}</IsPartOf>
						  </IsPartOfList> into $cmdi_files//Resources ):)
};

(:~ recursive addIsPartOf
starts by finding "orphaned" mdrecords = expecting that to be the root records.
and continues  ?
:)
declare function cmdcheck:addIsPartOf($x-context as xs:string, $config as node()*) as item()* {

let $log-file-name := concat('log_addIsPartOf_', $x-context, '.xml')
    let $log-path := repo-utils:config-value($config, 'log.path')
    let $log-doc-path := xmldb:store($log-path ,  $log-file-name, <result></result>)
    let $log-doc := doc($log-doc-path)

    let $root-dbcoll := repo-utils:context-to-collection($x-context, $config),
        $root-dbcoll-path := repo-utils:context-to-collection-path($x-context, $config)

(:    let $coll-dbcoll := '_corpusstructure':)

    (: beware empty path!! would return first-level collections and run over whole database :)
    let $colls := if ($root-dbcoll-path ne "") then xmldb:get-child-collections($root-dbcoll-path) else ()
    let $start_time := fn:current-dateTime()

    let $log-dummy := update insert <edit x-context="{$x-context}" root-dbcoll="{$root-dbcoll-path}"
                         count-colls="{count($colls)}" time="{$start_time}" /> into $log-doc/result

    let $root := cmdcheck:addIsPartOf-root($root-dbcoll, $log-doc)
    let $updated := cmdcheck:addIsPartOf-r($root-dbcoll, $root, 1, $log-doc)

    return $log-doc
 };

 (:~ consider those, whose MdSelfLink isn't referenced anywhere (orphans) as the root-cmd-collections
 @returns the edited records :)
 declare function cmdcheck:addIsPartOf-root($context as node()*, $log-doc as node()*) as item()* {

    let $pre_time := util:system-dateTime()

    let $orphaned := $context//CMD[not(Header/MdSelfLink = $context//ResourceProxy[ResourceType eq 'Metadata']/ResourceRef)]

    let $update := ( update insert <edit collid="root" count="{count($orphaned)}" time="{$pre_time}" /> into $log-doc/result,
			update delete $orphaned//IsPartOfList,
			update insert <IsPartOfList>
			                 <IsPartOf>{$cmdcheck:root-coll}</IsPartOf>
						  </IsPartOfList>
				into $orphaned//Resources,
		  update value $log-doc/result/edit[last()] with (util:system-dateTime() - $pre_time)
        )

    return $orphaned

 };

 declare function cmdcheck:addIsPartOf-r($context as node()*, $parents as node()*, $level as xs:integer, $log-doc as node()*) as item()* {

    let $start_time := util:system-dateTime()
    let $log-dummy := update insert <edit level="{$level}"
                        count-colls="{count($parents)}" time="{$start_time}" /> into $log-doc/result

  let $update := for $cmd-record in $parents
                    let $parent-id := $cmd-record//MdSelfLink/text()
                    let $children := $context//CMD[Header/MdSelfLink = $cmd-record//ResourceRef]
                	let $ispartoflist := <IsPartOfList>{($cmd-record//IsPartOfList/IsPartOf[not(text()=$cmdcheck:root-coll)],
                								<IsPartOf >{$parent-id}</IsPartOf>) }
                						  </IsPartOfList>

                    let $pre_time := util:system-dateTime()
                	let $duration := $pre_time - $start_time
                    return	( update insert <edit collid="{$parent-id}" count="{count($children)}" time="{$pre_time}" /> into $log-doc/result,
                			update delete $children//IsPartOfList,
                			update insert $ispartoflist
                				into $children//Resources,
                		  update value $log-doc/result/edit[last()] with (util:system-dateTime() - $pre_time)
                        )

    (: go next level - we dont have to recurse in the update-loop,
     but we can do whole next level in one call :)
    let $next-level := $context//CMD[Header/MdSelfLink = $parents//ResourceRef]
    let $updated := if (exists($next-level)) then  cmdcheck:addIsPartOf-r($context, $next-level ,($level+1),$log-doc)
                        else ()
    return $update
};

xquery version "1.0";
module namespace cmdcheck = "http://clarin.eu/cmd/check";
(: checking (trying to ensure) consistency of the IDs in CMD-records (MdSelfLink vs. ResourceProxies vs. IsPartOf)

TODO: check (and/or generate) the inverse links in IsPartOf (vs. ResourceProxies)
:)

(:import module namespace cmd  = "http://clarin.eu/cmd/collections" at "cmd-collections.xqm";:)
import module namespace repo-utils = "http://aac.ac.at/content_repository/utils" at  "/db/cr/repo-utils.xqm";
import module namespace fcs = "http://clarin.eu/fcs/1.0" at "/db/cr/fcs.xqm";
import module namespace crday  = "http://aac.ac.at/content_repository/data-ay" at "/db/cr/crday.xqm";

declare namespace sru = "http://www.loc.gov/zing/srw/";
declare namespace cmd = "http://www.clarin.eu/cmd/";

(:~ default namespace: cmd - declared statically, because dynamic ns-declaration did not work
cmd is the default namespace in (not) all the CMD records
:)
declare default element namespace "http://www.clarin.eu/cmd/";
(:~ default namespace - not declared explicitely, it is declared dynamically where necessary (function: addIsPartOf() :)
declare variable $cmdcheck:default-ns := "http://www.clarin.eu/cmd/";
declare variable $cmdcheck:root-coll := "root";

(:~ runs cmd.profile-scan for all contexts defined in the mappings :)

declare function cmdcheck:run-stats($config-path as xs:string) as item()* {

       let $config := doc($config-path),
           $mappings := doc(repo-utils:config-value($config, 'mappings'))

        let $opt := util:declare-option("exist:serialize", "media-type=text/html method=xhtml")

      for $map in $mappings//map[@key]
                let $map-key := $map/xs:string(@key),
                    $map-dbcoll-path := $map/xs:string(@path),
                    $scan-cmd-profile := fcs:scan('cmd.profile', $map-key, 1, 50, 1, 1, "text", $config)
                return $scan-cmd-profile
};

(:~ currently not used -> DEPRECATE?
init-function meant to call individual functions actually doing something.
at least it resolves x-context to a nodeset
:)
declare function cmdcheck:check($x-context as xs:string+, $config as node() ) as item()* {

    let $log-file-name := concat('log_checks_', repo-utils:sanitize-name($x-context), '.xml')
    let $log-path := repo-utils:config-value($config, 'log.path')

    let $start-time := util:system-dateTime()

    let $data-collection := repo-utils:context-to-collection($x-context, $config),
        $stat-profiles := cmdcheck:scan-profiles($data-collection),
(:        $check-linking := cmdcheck:check-linking($data-collection),:)
        $duration := util:system-dateTime() - $start-time

    let $result-data := <checks context="{$x-context}" on="{$start-time}" duration="{$duration}" >
                    <profiles>{$stat-profiles}</profiles>
                    <check-linking>{$check-linking}</check-linking>
                  </checks>
     let $store := xmldb:store($log-path ,  $log-file-name, $result-data)
    return $store
};

(:~ extracts CMD-Profiles from given nodeset

TODO: profile-name deduction not reliable  - needs match with cmd-terms and diagnostics
:)
declare function cmdcheck:scan-profiles($context as node()*) as item()* {
      (: try- to handle namespace problem - primitively :)
    let $ns-uri := namespace-uri($context[1]/*)
            (: dynamically declare a namespace for the next step, if one is defined in current context
       $dummy := if (exists($ns-uri)) then util:declare-namespace("",$ns-uri) else () :)
       (: this is now trying to overcome the default-ns issue, by accepting with and without ns :)

(:        $dummy := util:declare-namespace("",xs:anyURI(""))       :)
(:    let $profiles := util:eval("$context//(MdProfile|cmd:MdProfile)/text()"):)
    let $profiles := $context//cmd:CMD/concat(cmd:Header/cmd:MdProfile/text(), '#', cmd:Components/*/name())
    let $distinct-profiles := distinct-values($profiles)
    let $profiles-summary := for $profile in $distinct-profiles
(:                                let $profile-name := util:eval("$context[.//(MdProfile|cmd:MdProfile)/text() = $profile][1]//(Components|cmd:Components)/*/name()"):)
                                let $profile-name := substring-after($profile, '#')
                                let $profile-id := substring-before($profile, '#')
                                let $cnt := count($profiles[. eq $profile])
                                return <sru:term>
                                           <sru:value>{$profile-id}</sru:value>
                                           <sru:numberOfRecords>{$cnt }</sru:numberOfRecords>
                                           <sru:displayTerm>{$profile-name}</sru:displayTerm>
                                         </sru:term>
    let $count-all := count($context)
    return
        <sru:scanResponse xmlns:sru="http://www.loc.gov/zing/srw/" xmlns:fcs="http://clarin.eu/fcs/1.0">
              <sru:version>1.2</sru:version>
              <sru:terms>
                {$profiles-summary }
               </sru:terms>
               <sru:extraResponseData>
                     <fcs:countTerms>{$count-all}</fcs:countTerms>
                 </sru:extraResponseData>
                 <sru:echoedScanRequest>
                      <sru:scanClause>cmd.profile</sru:scanClause>
                  </sru:echoedScanRequest>
           </sru:scanResponse>
};

declare function cmdcheck:display-overview($config-path as xs:string) as item()* {

   let $config := doc($config-path),
       $dummy := util:declare-namespace("",xs:anyURI("")),
       $mappings := doc(repo-utils:config-value($config, 'mappings'))
(:       $baseurl := repo-utils:config-value($config, 'base.url'),:)

   let $opt := util:declare-option("exist:serialize", "media-type=text/html method=xhtml")

   let $overview :=  crday:display-overview($config-path, 'raw')

   let $profiles-overview :=

collection	profiles

           { for $map in util:eval("$mappings//map[@key]")
                    let $map-key := $map/xs:string(@key),
                        $map-dbcoll-path := $map/xs:string(@path),
                        $scan-cmd-profile := fcs:scan('cmd.profile', $map-key, 1, 50, 1, 1, "text", $config),
                        $scan-formatted := repo-utils:serialise-as( $scan-cmd-profile, 'htmldetail', 'scan', $config, ())
                    return

                        {$map-key}
                        { $scan-formatted }

                        }

            (: { for $profile in $scan-cmd-profile//sru:term
                            return ($profile/sru:value, $profile/sru:displayTerm, $profile/sru:numberOfRecords)
                        }:)

        return repo-utils:serialise-as(
{($overview, $profiles-overview)}
, 'htmlpage', 'html', $config, ())

};

(:~	WARNING - CHANGES DATA! adds IsPartOf to the CMDrecords (contained in the db-coll)
- expects CMD-records for collections in specific db-coll (with name of the db-coll matching the name of the collectionfile):
    ./_corpusstructure/collection_{db-coll-name}.cmdi
- logs the progress of the processing in a separate file
- can be applied repeatedly - deletes(!) old IsPartOfList, before inserting new

@param $x-context an identifier of a collection (as defined in mappings)
@param $config config-node - used to get log.path and collection-path

TODO: could also generate the CMD-records for collections (assuming db-colls as collections)
      (now done by the python script: dir2cmdicollection.py)
TODO: currently a hack: the collection-records also are marked as root elements - this has to be optional at least,
      and may even be dangerous (count(IsPartOf[@level=1])>1)!
TODO: not recursive yet!
:)
declare function cmdcheck:addIsPartOf-colls($x-context as xs:string, $config as node()*) as item()* {

let $log-file-name := concat('log_addIsPartOf_', $x-context, '.xml')
    let $log-path := repo-utils:config-value($config, 'log.path')
    let $log-doc-path := xmldb:store($log-path ,  $log-file-name, <result></result>)
    let $log-doc := doc($log-doc-path)

    (:let $root-dbcoll := repo-utils:context-to-collection($x-context, $config),:)
    let $root-dbcoll-path := repo-utils:context-to-collection-path($x-context, $config)

    let $coll-dbcoll := '_corpusstructure'

    (: beware empty path!! would return first-level collections and run over whole database :)
    let $colls := if ($root-dbcoll-path ne "") then xmldb:get-child-collections($root-dbcoll-path) else ()
    let $start_time := fn:current-dateTime()

    (: dynamic ns-declaration does not work for xpath
    let $declare-dummy := util:declare-namespace("",xs:anyURI($cmdcheck:default-ns))
    :)
    let $log-dummy := update insert <edit x-context="{$x-context}" root-dbcoll="{$root-dbcoll-path}"
                        coll-dbcoll="{$coll-dbcoll}" count-colls="{count($colls)}" time="{$start_time}" /> into $log-doc/result
    (:if (exists($colls)) then:)
    for $coll_name in $colls[not(.=$coll-dbcoll)]
    (: let $coll_name := 'HomeFamilyManagement' :)
    let $files := collection(concat($root-dbcoll-path,$coll_name))
    (: let $files := xmldb:xcollection($root_coll) :)
	let $coll_file := concat($root-dbcoll-path, $coll-dbcoll, '/collection_', $coll_name, '.cmdi')
	let $coll_doc := doc($coll_file)
	let $coll_id := $coll_doc//MdSelfLink/text()

(:	let $cmdi_files := $files[ends-with(util:document-name(.),".cmdi")]:)

    let $pre_time := util:system-dateTime()
	let $duration := $pre_time - $start_time
 return	( update insert <edit coll="{$coll_name}" collid="{$coll_id}" coll_file="{$coll_file}" count="{count($files)}" time="{$pre_time}" /> into $log-doc/result,
			update delete $files//IsPartOfList,
			update delete $coll_doc//IsPartOfList,
			update insert <IsPartOfList>
								<IsPartOf level="1">{$x-context}</IsPartOf>
								<IsPartOf level="1">{$cmdcheck:root-coll}</IsPartOf>
						  </IsPartOfList>
			  	into $coll_doc//Resources,
			update insert <IsPartOfList>
							<IsPartOf level="2">{$x-context}</IsPartOf>
							<IsPartOf level="1">{$coll_id}</IsPartOf>
						  </IsPartOfList>
				into $files//Resources,
		update value $log-doc/result/edit[last()] with (util:system-dateTime() - $pre_time)
        )
    (:
    return	($coll_name, count($cmdi_files), $coll_file,  update insert <IsPartOfList>
							<IsPartOf level="1">{$coll_id}</IsPartOf>
						  </IsPartOfList> into $cmdi_files//Resources ):)
};

(:~ recursive addIsPartOf
starts by finding "orphaned" mdrecords = expecting that to be the root records.
and continues  ?
:)
declare function cmdcheck:addIsPartOf($x-context as xs:string, $config as node()*) as item()* {

let $log-file-name := concat('log_addIsPartOf_', $x-context, '.xml')
    let $log-path := repo-utils:config-value($config, 'log.path')
    let $log-doc-path := xmldb:store($log-path ,  $log-file-name, <result></result>)
    let $log-doc := doc($log-doc-path)

    let $root-dbcoll := repo-utils:context-to-collection($x-context, $config),
        $root-dbcoll-path := repo-utils:context-to-collection-path($x-context, $config)

(:    let $coll-dbcoll := '_corpusstructure':)

    (: beware empty path!! would return first-level collections and run over whole database :)
    let $colls := if ($root-dbcoll-path ne "") then xmldb:get-child-collections($root-dbcoll-path) else ()
    let $start_time := fn:current-dateTime()

    let $log-dummy := update insert <edit x-context="{$x-context}" root-dbcoll="{$root-dbcoll-path}"
                         count-colls="{count($colls)}" time="{$start_time}" /> into $log-doc/result

    let $root := cmdcheck:addIsPartOf-root($root-dbcoll, $log-doc)
    let $updated := cmdcheck:addIsPartOf-r($root-dbcoll, $root, 1, $log-doc)

    return $log-doc
 };

 (:~ consider those, whose MdSelfLink isn't referenced anywhere (orphans) as the root-cmd-collections
 @returns the edited records :)
 declare function cmdcheck:addIsPartOf-root($context as node()*, $log-doc as node()*) as item()* {

    let $pre_time := util:system-dateTime()

    let $orphaned := $context//CMD[not(Header/MdSelfLink = $context//ResourceProxy[ResourceType eq 'Metadata']/ResourceRef)]

    let $update := ( update insert <edit collid="root" count="{count($orphaned)}" time="{$pre_time}" /> into $log-doc/result,
			update delete $orphaned//IsPartOfList,
			update insert <IsPartOfList>
			                 <IsPartOf>{$cmdcheck:root-coll}</IsPartOf>
						  </IsPartOfList>
				into $orphaned//Resources,
		  update value $log-doc/result/edit[last()] with (util:system-dateTime() - $pre_time)
        )

    return $orphaned

 };

 declare function cmdcheck:addIsPartOf-r($context as node()*, $parents as node()*, $level as xs:integer, $log-doc as node()*) as item()* {

    let $start_time := util:system-dateTime()
    let $log-dummy := update insert <edit level="{$level}"
                        count-colls="{count($parents)}" time="{$start_time}" /> into $log-doc/result

  let $update := for $cmd-record in $parents
                    let $parent-id := $cmd-record//MdSelfLink/text()
                    let $children := $context//CMD[Header/MdSelfLink = $cmd-record//ResourceRef]
                	let $ispartoflist := <IsPartOfList>{($cmd-record//IsPartOfList/IsPartOf[not(text()=$cmdcheck:root-coll)],
                								<IsPartOf >{$parent-id}</IsPartOf>) }
                						  </IsPartOfList>

                    let $pre_time := util:system-dateTime()
                	let $duration := $pre_time - $start_time
                    return	( update insert <edit collid="{$parent-id}" count="{count($children)}" time="{$pre_time}" /> into $log-doc/result,
                			update delete $children//IsPartOfList,
                			update insert $ispartoflist
                				into $children//Resources,
                		  update value $log-doc/result/edit[last()] with (util:system-dateTime() - $pre_time)
                        )

    (: go next level - we dont have to recurse in the update-loop,
     but we can do whole next level in one call :)
    let $next-level := $context//CMD[Header/MdSelfLink = $parents//ResourceRef]
    let $updated := if (exists($next-level)) then  cmdcheck:addIsPartOf-r($context, $next-level ,($level+1),$log-doc)
                        else ()
    return $update
};


Added additional init overload which accepts an inital expression

module namespace renjin = 'http://github.com/james-jw/xq-renjin';
import module namespace r = 'org.jw.basex.renjin.xq-renjin-module';

(: Provided an R object, returns a map to traverse the object :)
declare function renjin:map($item) {(
    for $name in r:names($item)
    let $value := $item($name)
    return map {
      $name:
        if(r:is-r($value)) then renjin:map($value)
        else if(count($value) > 1) then
          let $rows := r:rownames($item) return
          if(count($rows) > 0) then map:merge(
            for $row at $i in $rows return
            map { $row : $value[$i] }
          )
          else array { $value }
        else $value
    }
  ) => map:merge()
};

(: Initializes an R engine :)
declare function renjin:init() as function(*) as function(*) { r:init() };

(: Initializes an R engine and runs an initial expression :)
declare function renjin:init($exp as xs:string) as function(*) {
    r:init() => r:run($exp)
};

(: Returns whether an object represents an R object or not. :)
declare function renjin:is($item) as xs:boolean { r:is-r($item) };

(: Column names:)
declare function renjin:names($item) as xs:string* { r:names($item) };

(: Row names :)
declare function renjin:rownames($item) as xs:string* { r:rownames($item) };

(: Executes an R expression within the R engine provided
 : @return The R engine provided. Useful in chaining.
 :)
declare function renjin:run($renjin, $expression as xs:string) as function(*) { r:run($renjin, $expression) };

(: Exeutes an R expression within the R engine provided, returning its value
 : @return The result of the R execution as an R object
 :)
declare function renjin:eval($renjin, $expression as xs:string, $map as xs:boolean) as item()* {
  let $out := r:eval($renjin, $expression)
  return if($map and r:is-r($out)) then renjin:map($out) else $out
};

(: Exeutes an R expression within the R engine provided, returning its value
 : @return The result of the R execution as a map
 :)
declare function renjin:eval($renjin, $expression as xs:string) {
  renjin:eval($renjin, $expression, true())
};

(: Adds a variables with the provided name and value to the R engine :)
declare function renjin:put($renjin, $var-name as xs:string, $expression as xs:string) as function(*) {
  r:put($renjin, $var-name, $expression)
};

module namespace renjin = 'http://github.com/james-jw/xq-renjin';
import module namespace r = 'org.jw.basex.renjin.xq-renjin-module';

(: Provided an R object, returns a map to traverse the object :)
declare function renjin:map($item) {(
    for $name in r:names($item)
    let $value := $item($name)
    return map {
      $name:
        if(r:is-r($value)) then renjin:map($value)
        else if(count($value) > 1) then
          let $rows := r:rownames($item) return
          if(count($rows) > 0) then map:merge(
            for $row at $i in $rows return
            map { $row : $value[$i] }
          )
          else array { $value }
        else $value
    }
  ) => map:merge()
};

(: Initializes an R engine :)
declare function renjin:init() as function(*) { r:init() };

(: Returns whether an object represents an R object or not. :)
declare function renjin:is($item) as xs:boolean { r:is-r($item) };

(: Column names:)
declare function renjin:names($item) as xs:string* { r:names($item) };

(: Row names :)
declare function renjin:rownames($item) as xs:string* { r:rownames($item) };

(: Executes an R expression within the R engine provided
 : @return The R engine provided. Useful in chaining.
 :)
declare function renjin:run($renjin, $expression as xs:string) as function(*) { r:run($renjin, $expression) };

(: Exeutes an R expression within the R engine provided, returning its value
 : @return The result of the R execution as an R object
 :)
declare function renjin:eval($renjin, $expression as xs:string, $map as xs:boolean) as item()* {
  let $out := r:eval($renjin, $expression)
  return if($map and r:is-r($out)) then renjin:map($out) else $out
};

(: Exeutes an R expression within the R engine provided, returning its value
 : @return The result of the R execution as a map
 :)
declare function renjin:eval($renjin, $expression as xs:string) {
  renjin:eval($renjin, $expression, true())
};

(: Adds a variables with the provided name and value to the R engine :)
declare function renjin:put($renjin, $var-name as xs:string, $expression as xs:string) as function(*) {
  r:put($renjin, $var-name, $expression)
};


Fixed issue with notes folder not being visible. Created 'defines' for headings, to increase precision and permit future changes. Added logic to derive hash from any node within a notes file. Created simple widget with standard links for a paper and its notes.

module namespace notes="http://cefn.com/readinglog/notes";

import module namespace template="http://cefn.com/readinglog/template" at "/home/cefn/Documents/highwire/participatory firmware design/readinglog/scripts/server/xquery/lib/template.xq";

declare namespace ann = "http://www.zorba-xquery.com/annotations";

declare default element namespace "http://www.w3.org/1999/xhtml";

declare variable $notes:authors := "Authors";
declare variable $notes:concepts := "Concepts";
declare variable $notes:headings := (
	$notes:authors,
	$notes:concepts,
	'Methodology',
	'Field of Study',
	'Quotations',
	'References'
);

declare %ann:nondeterministic function notes:allnotes(){
	template:loadhtml("public/notes/")
};

(: Introspects the noted document's unique id from any note element :)
declare function notes:paperhash($node){
	root($node)//comment()[contains(string(.),"UNIQUE HASH")]/replace(string(.)," UNIQUE HASH ([\w]+) ", "$1")
};

declare %ann:nondeterministic function notes:notesoftype($type){
	notes:allnotes()//*[not(self::h2)][preceding::h2[1]//text()[contains(.,$type)]]
};

declare function notes:followingheader($nodes,$title){
	$nodes//*[preceding-sibling::h2[1][text()[normalize-space(.)=$title]]]
};

declare function notes:paperlinks($node){
	let $hash := notes:paperhash($node)
	return

		[
			notes
			pdf
			load
		]

};

module namespace notes="http://cefn.com/readinglog/notes";

import module namespace template="http://cefn.com/readinglog/template" at "/home/cefn/Documents/highwire/participatory firmware design/readinglog/scripts/server/xquery/lib/template.xq";

declare namespace ann = "http://www.zorba-xquery.com/annotations";

declare default element namespace "http://www.w3.org/1999/xhtml";

declare variable $notes:headings := ('Concepts','Methodology','Field of Study','Quotations','References');

declare %ann:nondeterministic function notes:allnotes(){
	template:loadhtml("resources/notes/")
};

declare function notes:followingheader($nodes,$title){
	$nodes//*[preceding-sibling::h2[1][text()=$title]]
};


Intermediate check-in.

Intermediate check-in.

jsoniq version "1.0";

(:
 : Copyright 2006-2013 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

(:===========================================================================:)

(:~
 : This module provides functions for performing FTP commands.
 :

 : @author Paul J. Lucas
 : @library cURL Library
 : @project Zorba/Input Output/FTP Client
 :)
module namespace ftp = "http://zorba.io/modules/ftp-client";

declare namespace an = "http://zorba.io/annotations";
declare namespace ver = "http://zorba.io/options/versioning";

declare option ver:module-version "1.0";

(:~
 : Attempts to connect to an FTP server specified by the given URI.
 :
 : @param $uri The address of the FTP server to connect to.
 : It may either be simple host-name
 : (ftp.example.com)
 : or a URI using the ftp
 : or ftps (FTP over SSL/TLS)
 : schemes
 : (ftp://ftp.example.com).
 : If the latter,
 : the URI may also contain
 : username,
 : password,
 : and
 : port
 : authority subcomponents
 : per RFC 3986.
 : @param $options The options to use:
 :

 :
user

 :

 :        The user to log in as;
 :        default: "ftp" (for anonymous FTP).
 :        If $uri is actually a URI instead of a simple host-name,
 :        the user, if specified, must be part of the URI
 :        and the value of this option is ignored.
 :

 :
password

 :

 :        The password to use to log in;
 :        default: "ftp@example.com" (for anonymous FTP).
 :        If $uri is actually a URI instead of a simple host-name,
 :        the password, if specified, must be part of the URI
 :        and the value of this option is ignored.
 :

 :
port

 :

 :        The port number to use;
 :        default: whatever the default for the protocol is.
 :        If $uri is actually a URI instead of a simple host-name,
 :        the port, if specified, must be part of the URI
 :        and the value of this option is ignored.
 :

 :
protocol

 :

 :        The protocol to use,
 :        either "ftp' or "ftps";
 :        default: "ftp".
 :        If $uri is actually a URI instead of a simple host-name,
 :        the protocol, if specified, must be part of the URI
 :        and the value of this option is ignored.
 :

 :
SSL-communication

 :

 :        Whether to use SSL/TLS, one of:
 :

 :
"none"

 :
Don't use SSL.

 :
"try"

 :
Try using SSL, but, if unsuccessful, continue anyway.

 :
"control"

 :
Require SSL for the control connection.

 :
"all"

 :
Require SSL for all communication.

 :

 :        default:
 :        "none" for protocol "ftp"
 :        and
 :        "all" for protocol "ftps"
 :        Note that any value other than "none"
 :        for "ftp" implies explicit SSL;
 :        use of "ftps" implies implicit SSL.
 :        See "Methods of invoking security."
 :

 :
SSL-verify

 :

 :        When doing FTP over SSL/TLS,
 :        whether to verify the authenticity of the server's certificate
 :        and that the certificate is for that server;
 :        default: true.
 :        (You should never set this to false
 :        unless you are testing your own FTP server
 :        with a self-signed certificate.)
 :

 :
trace

 :

 :        Whether to emit information to standard error
 :        tracing the communication between the FTP client and server;
 :        default: false.
 :

 :

 : @return an opaque URI that serves as a connection handle to be used with
 : other functions in this module.
 : @error ftp:ALREADY_CONNECTED if $uri is already connected to.
 : @error ftp:INVALID_ARGUMENT if any option is invalid.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:connect( $uri as string, $options as object )
  as anyURI external;

(:~
 : Deletes a file from the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $remote-path The path of the file to delete.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:delete( $conn as string, $remote-path as string )
  external;

(:~
 : Disconnects from an FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : After successful completion of of this function, the handle is no longer
 : valid.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:disconnect( $conn as anyURI )
  external;

(:~
 : Gets a binary file from the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $remote-path The path of the file to get.
 : @return the binary content of $remote-path.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:get-binary( $conn as anyURI, $remote-path as string )
  as base64Binary external;

(:~
 : Gets a text file from the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $remote-path The path of the file to get.  It must not be empty.
 : @param $encoding The character encoding of the file.
 : @return the text content of $remote-path.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty
 : or $encoding is either an invalid or unsupported encoding.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:get-text( $conn as anyURI, $remote-path as string, $encoding as string )
  as string external;

(:~
 : Gets a text file, presumed to be encoded in UTF-8, from the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $remote-path The path of the file to get.  It must not be empty.
 : @return the text content of $remote-path.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:get-text( $conn as anyURI, $remote-path as string )
  as string
{
  ftp:get-text( $conn, $remote-path, "UTF-8" )
};

(:~
 : Gets a listing for a directory on an FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $remote-path The full path of the directory on the FTP server to get
 : the directory listing for.
 : An empty path is equivalent to /.
 : @return Returns a sequence of JSON objects, one per file or subdirectory in
 : the listing.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:list( $conn as anyURI, $remote-path as string )
  as object* external;

(:~
 : Creates a directory on the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $remote-path The path of the new directory to create.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:mkdir( $conn as string, $remote-path as string )
  external;

(:~
 : Uploads binary data to a file to the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $binary The binary data to upload.
 : @param $remote-path The path of the file to upload to. It must not be empty.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:put-binary( $conn as anyURI, $binary as base64Binary,
                $remote-path as string )
  external;

(:~
 : Uploads text to a file to the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $text The text to upload.
 : @param $remote-path The path of the file to upload to. It must not be empty.
 : @param $encoding The character encoding of the file.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty
 : or $encoding is either an invalid or unsupported encoding.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:put-text( $conn as anyURI, $text as string, $remote-path as string,
              $encoding as string )
  external;

(:~
 : Uploads text to a UTF-8 encoded file on the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $text The text to upload.
 : @param $remote-path The path of the file to upload to. It must not be empty.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:put-text( $conn as anyURI, $text as string, $remote-path as string )
{
  ftp:put-text( $conn, $text, $remote-path, "UTF-8" )
};

(:~
 : Renames a file or directory on an FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $remote-from-path The path of the file or directory to rename.
 : @param $remote-to-path The new name.
 : @error ftp:INVALID_ARGUMENT if $remote-from-path or
 : $remote-to-path is empty.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:rename( $conn as string, $remote-from-path as string,
            $remote-to-path as string )
  external;

(:~
 : Removes a directory from the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $remote-path The path of the directory to remove.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:rmdir( $conn as string, $remote-path as string )
  external;

(:===========================================================================:)

(: vim:set syntax=xquery et sw=2 ts=2: :)

jsoniq version "1.0";

(:
 : Copyright 2006-2013 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

(:===========================================================================:)

(:~
 : This module provides functions for performing FTP commands.
 :

 : @author Paul J. Lucas
 : @library cURL Library
 : @project Zorba/Input Output/FTP Client
 :)
module namespace ftp = "http://zorba.io/modules/ftp-client";

declare namespace an = "http://zorba.io/annotations";
declare namespace ver = "http://zorba.io/options/versioning";

declare option ver:module-version "1.0";

(:~
 : Attempts to connect to an FTP server specified by the given URI.
 :
 : @param $uri The address of the FTP server to connect to.
 : It may either be simple host-name
 : (ftp.example.com)
 : or a URI using the ftp
 : or ftps (FTP over SSL/TLS)
 : schemes
 : (ftp://ftp.example.com).
 : If the latter,
 : the URI may also contain
 : username,
 : password,
 : and
 : port
 : authority subcomponents
 : per RFC 3986.
 : @param $options The options to use:
 :

 :
user

 :

 :        The user to log in as;
 :        default: "ftp" (for anonymous FTP).
 :        If $uri is actually a URI instead of a simple host-name,
 :        the user, if specified, must be part of the URI
 :        and the value of this option is ignored.
 :

 :
password

 :

 :        The password to use to log in;
 :        default: "ftp@example.com" (for anonymous FTP).
 :        If $uri is actually a URI instead of a simple host-name,
 :        the password, if specified, must be part of the URI
 :        and the value of this option is ignored.
 :

 :
port

 :

 :        The port number to use;
 :        default: whatever the default for the protocol is.
 :        If $uri is actually a URI instead of a simple host-name,
 :        the port, if specified, must be part of the URI
 :        and the value of this option is ignored.
 :

 :
protocol

 :

 :        The protocol to use,
 :        either "ftp' or "ftps";
 :        default: "ftp".
 :        If $uri is actually a URI instead of a simple host-name,
 :        the protocol, if specified, must be part of the URI
 :        and the value of this option is ignored.
 :

 :
SSL-communication

 :

 :        Whether to use SSL/TLS, one of:
 :

 :
"none"

 :
Don't use SSL.

 :
"try"

 :
Try using SSL, but, if unsuccessful, continue anyway.

 :
"control"

 :
Require SSL for the control connection.

 :
"all"

 :
Require SSL for all communication.

 :

 :        default:
 :        "none" for protocol "ftp"
 :        and
 :        "all" for protocol "ftps"
 :        Note that any value other than "none"
 :        for "ftp" implies explicit SSL;
 :        use of "ftps" implies implicit SSL.
 :        See "Methods of invoking security."
 :

 :
SSL-verify

 :

 :        When doing FTP over SSL/TLS,
 :        whether to verify the authenticity of the server's certificate
 :        and that the certificate is for the host;
 :        default: true.
 :

 :
trace

 :

 :        Whether to emit information to standard error
 :        tracing the communication between the FTP client and server;
 :        default: false.
 :

 :

 : @return an opaque URI that serves as a connection handle to be used with
 : other functions in this module.
 : @error ftp:ALREADY_CONNECTED if $uri is already connected to.
 : @error ftp:INVALID_ARGUMENT if any option is invalid.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:connect( $uri as string, $options as object )
  as anyURI external;

(:~
 : Deletes a file from the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $remote-path The path of the file to delete.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:delete( $conn as string, $remote-path as string )
  external;

(:~
 : Disconnects from an FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : After successful completion of of this function, the handle is no longer
 : valid.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:disconnect( $conn as anyURI )
  external;

(:~
 : Gets a binary file from the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $remote-path The path of the file to get.
 : @return the binary content of $remote-path.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:get-binary( $conn as anyURI, $remote-path as string )
  as base64Binary external;

(:~
 : Gets a text file from the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $remote-path The path of the file to get.  It must not be empty.
 : @param $encoding The character encoding of the file.
 : @return the text content of $remote-path.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty
 : or $encoding is either an invalid or unsupported encoding.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:get-text( $conn as anyURI, $remote-path as string, $encoding as string )
  as string external;

(:~
 : Gets a text file, presumed to be encoded in UTF-8, from the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $remote-path The path of the file to get.  It must not be empty.
 : @return the text content of $remote-path.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:get-text( $conn as anyURI, $remote-path as string )
  as string
{
  ftp:get-text( $conn, $remote-path, "UTF-8" )
};

(:~
 : Gets a listing for a directory on an FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $remote-path The full path of the directory on the FTP server to get
 : the directory listing for.
 : An empty path is equivalent to /.
 : @return Returns a sequence of JSON objects, one per file or subdirectory in
 : the listing.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:list( $conn as anyURI, $remote-path as string )
  as object* external;

(:~
 : Creates a directory on the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $remote-path The path of the new directory to create.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:mkdir( $conn as string, $remote-path as string )
  external;

(:~
 : Uploads binary data to a file to the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $binary The binary data to upload.
 : @param $remote-path The path of the file to upload to. It must not be empty.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:put-binary( $conn as anyURI, $binary as base64Binary,
                $remote-path as string )
  external;

(:~
 : Uploads text to a file to the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $text The text to upload.
 : @param $remote-path The path of the file to upload to. It must not be empty.
 : @param $encoding The character encoding of the file.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty
 : or $encoding is either an invalid or unsupported encoding.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:put-text( $conn as anyURI, $text as string, $remote-path as string,
              $encoding as string )
  external;

(:~
 : Uploads text to a UTF-8 encoded file on the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $text The text to upload.
 : @param $remote-path The path of the file to upload to. It must not be empty.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:put-text( $conn as anyURI, $text as string, $remote-path as string )
{
  ftp:put-text( $conn, $text, $remote-path, "UTF-8" )
};

(:~
 : Renames a file or directory on an FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $remote-from-path The path of the file or directory to rename.
 : @param $remote-to-path The new name.
 : @error ftp:INVALID_ARGUMENT if $remote-from-path or
 : $remote-to-path is empty.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:rename( $conn as string, $remote-from-path as string,
            $remote-to-path as string )
  external;

(:~
 : Removes a directory from the FTP server.
 :
 : @param $conn The opaque URI connection handle previously returned by
 : ftp:connect().
 : @param $remote-path The path of the directory to remove.
 : @error ftp:INVALID_ARGUMENT if $remote-path is empty.
 : @error ftp:NOT_CONNECTED if $conn is either an invalid handle
 : or is no longer a valid handle.
 : @error ftp:FTP_ERROR if there was some other FTP error.
 :)
declare %an:sequential function
ftp:rmdir( $conn as string, $remote-path as string )
  external;

(:===========================================================================:)

(: vim:set syntax=xquery et sw=2 ts=2: :)


Updated csv documentation.

Updated csv documentation.

(:
 : Copyright 2006-2009 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
:)

(:~
 : Function library providing convertors from CSV/TXT to XML and back.
 : The functions are optimized to work with large amounts of data, in a streaming way.
 :
 : @author Daniel Turcanu
 :)
module namespace zorba-csv = "http://www.zorba-xquery.com/modules/csv";

(:~
 : Errors namespace URI.
:)
declare variable $zorba-csv:csvNS as xs:string := "http://www.zorba-xquery.com/modules/csv";

(:~
 : Error code for wrong parameter situations.

 : Possible error messages:

 : * "Options field must be of element options instead of ..."

:)
declare variable $zorba-csv:errWrongParam as xs:QName := fn:QName($zorba-csv:csvNS, "zorba-csv:WrongParam");

(:~
 : Parse a CSV or fixed size text and convert to XML.

 : By default each line iw converted to a <row> element, and each field to a <column> element inside <row>.

 : @param $csv the string containing the csv or fixed size text.
 : @param $options having format

 :    <options>

 :        <csv  [separator="default comma ,"] ?

 :          [quote-char="default double quotes &quote;"]?

 :          [quote-escape="default double double quotes &quote;&quote;"]? />

 :

 :        or

 :        <column-widths>

 :          <column-width>[column fixed width, unsigned int]<column-width>*

 :		  	</column-widths>

 :

 :        or

 :        <column-positions>

 :          <column-position>[column position on line, unsigned int]<column-position>*

 :		  	</column-positions>

 :

 :        <first-row-is-header [line="first_line[-last_line]?"]?/>?

 :        <start-from-row line="first_line[-last_line]?"/>?

 :        <add-last-void-columns/>?

 :        <xml-nodes>

 :          [<row-name>

 :            [<column-name/>]?

 :          </row-name>]?

 :        </xml-nodes>?

 :    </options>

 :

 :    All the parameters are optional.

 :    All the parameters are case sensitive. They can be in any namespace.

 :    All strings must have UTF-8 encoding.

 :    Parameters csv, column-widths, column-positions are mutualy exclusive. If none is specified,
 :    the input string is assumed to be csv.

 :    Description of parameters:
 :

 :
csv

 :
 Specifies the parameters for parsing a csv string.

 :

 :
separator

 :
The character or group of characters used to separating fields in a row.
 :            If it is not specified, it defaults to comma ','.
 :

 :
quote-char

 :
The character or group of characters used for quoting the fields that may contain special characters,
 :             like separator, new line or this quote char. The default value is double quote ".

 :

 :
quote-escape

 :
The group of characters used for escaping the quote char inside a field. The whole quote escape group
 :           is translated to a quote char during parsing. The default value is double double quotes "".

 :

 :

 :

 :

 :
column-widths

 :
Specifies the column widths for fixed size text. It contains multiple column-width child elements
 :        specifying the fixed width of each column, from left to right.

 :        If the line has more fields than specified, they are ignored.
 :

 :
column-positions

 :
This is an alternative to column-widths, and specifies instead the starting position of each column.
 :        Column positions are 1 based, and are specified in order from left to right.
 :        The last column is read until end of line. The first column position can be greater than 1, if you want
 :        to parse only a part of the input text.
 :

 :
first-row-is-header

 :
The presence of this element indicates that the first row is to be treated as the name of the columns.
 :        If it is not present, then each field is enclosed in a <column> element,
 :        or how it is specified in <xml-nodes> parameter.

 :        If the first row is the header, then each field is enclosed in an element with the corresponding name from the header.

 :        For example, the csv:

 :        ID,Name,Occupation

 :        1,John,student

 :

 :        is parsed into

 :        <row>

 :        <ID>1</ID>

 :        <Name>John</Name>

 :        <Occupation>student</Occupation>

 :        </row>

 :

 :        If the header names contain characters that cannot be used in a QName, they are replaced with underscore '_'.

 :        The namespace for the header QNames is taken from the column name specified in xml-nodes parameter, or from
 :        the row name, or if that doesn't exist either then empty namespace is used.

 :        If the header is not the first line in the input string, the starting line can be specified in the line attribute.

 :        If a column does not have a name, a new name is constructed in the form columnN where N is the position of the column,
 :        starting from 1.

 :        Subheaders

 :				If the header consists of more than one line, this can be specified in the line attribute in the form
 :        "first_line - last_line". Having more lines as the header translates into a hierarchy of elements in the xml.

 :        For example, the csv:

 :        ID,Name,,Occupation

 :        ,First Name,Last Name,

 :        1,John,Howard,student

 :

 :        is parsed into

 :        <row>

 :        <ID>1</ID>

 :        <Name>

 :          <First_Name>John</First_Name>

 :          <Last_Name>Howard</Last_Name>

 :        </Name>

 :        <Occupation>student</Occupation>

 :        </row>

 :

 :
start-from-row

 :
If the data does not start from line 1 or imediately after the header,
 :        you can specify the starting line in the line attribute.

 :        Also you can use this attribute in the form "first_line - last_line" to specify also the last line
 :        if you don't want the whole csv to be parsed.
 :

 :
add-last-void-columns

 :
In the case when using headers and some data lines are shorter than the header, by default the excess columns are ignored
 :          for those lines. You can set the add-last-void-columns parameter to make all the columns appear in xml even if they are void.
 :

 :
xml-nodes

 :
With this parameter you can specify the names for the row element and for the column element if there is no header.

 :        The first element child of this element specifies the desired QName of the row element in the output xml.
 :        The name of this element will be used as the name of the row element.

 :        The element child of this row element is the column element, and its name will be used as the name of the column elements
 :        that enclose the fields in the output xml if there is no header.

 :        If the csv has a header, only the namespace is used from the column element.

 :        For example, with parameter:

 :        <xml-nodes>

 :        <r>

 :          <c/>

 :        </r>

 :        </xml-nodes>

 :

 :        the output for each line will look like

 :        <r>

 :          <c>field1</c>

 :          <c>field2</c>

 :          .......

 :        </r>

 :

 :        If this parameter is not specified, the row name is by default "row" and the column name is by default "column".
 :

 :

 : @return a sequence of row elements, one for each line in csv
 : @error zorba-csv:WrongParam if the options parameter doesn't have the name "options".
 : @example csv_parse1.xq
 : @example csv_parse2.xq
 : @example csv_parse3.xq
 : @example csv_parse6.xq
 : @example csv_parse11.xq
 : @example csv_parse_utf8_11.xq
 : @example txt_parse5.xq
 : @example txt_parse8.xq
:)
declare function zorba-csv:parse($csv as xs:string,
                                 $options as element(options)?) as element()* external;

(:~
 : Convert XML into CSV or fixed size text.

 : Note: if you want to serialize out the result, make sure that the serializer method is set to Text.
 :  For example, in zorba app, you have to set the param --serialize-text.
 :  When using file:write function, you have to set the serialization option <method>text</method>.

 :
 : @param $xml a sequence of elements, each element representing a row. The name of each row element is ignored.
 :     The childs of each row are the column fields.
 : @param $options having format

 :    <options>

 :        <csv  [separator="default comma ,"] ?

 :          [quote-char="default double quotes &quote;"]?

 :          [quote-escape="default double double quotes &quote;&quote;"]? />

 :

 :        or

 :        <column-widths [align="left|right"]?>

 :          <column-width [align="left|right"]?>[column fixed width, unsigned int]<column-width>*

 :		  	</column-widths>

 :

 :        or

 :        <column-positions [align="left|right"]?>

 :          <column-position [align="left|right"]?>[column position on line, unsigned int]<column-position>*

 :		  	</column-positions>

 :

 :        <first-row-is-header/>?

 :    </options>
 :

 :    All the parameters are optional.

 :    All the parameters are case sensitive. They can be in any namespace.

 :    All strings must have UTF-8 encoding.

 :    Parameters csv, column-widths, column-positions are mutualy exclusive. If none is specified,
 :    the xml is converted to csv.

 :    Description of parameters:
 :

 :
csv

 :
 Specifies the parameters for converting to csv.

 :

 :
separator

 :
The character or group of characters used to separating fields in a row.
 :            If it is not specified, it defaults to comma ','.
 :

 :
quote-char

 :
The character or group of characters used for quoting the fields that may contain special characters,
 :             like separator, new line or this quote char. The default value is double quote ".

 :

 :
quote-escape

 :
The group of characters used for escaping the quote char inside a field. The whole quote escape group
 :           is translated to a quote char during parsing. The default value is double double quotes "".

 :

 :

 :

 :

 :
column-widths

 :
Specifies the column widths for fixed size text. It contains multiple column-width child elements
 :        specifying the fixed width of each column, from left to right.

 :        With the attribute align you can specify how to align fields that are smaller than the column width.
 :        The default alignment is left.

 :

 :
column-positions

 :
This is an alternative to column-widths, and specifies instead the starting position of each column.
 :        Column positions are 1 based, and are specified in order from left to right.
 :        The last column has a variable length.

 :        With the attribute align you can specify how to align fields that are smaller than the column width.
 :        The default alignment is left. The last column does not need alignment.

 :

 :
first-row-is-header

 :
The presence of this element indicates that the first row will contain the header, that is, the names of
 :        the column elements. Only the column names from the first row element are taken into account.

 :        For example, the row xml:

 :        <row>

 :        <ID>1</ID>

 :        <Name>John</Name>

 :        <Occupation>student</Occupation>

 :        </row>

 :

 :        is converted to

 :        ID,Name,Occupation

 :        1,John,student

 :

 :        The header names are the localnames of the column elements, and the namespace is ignored.

 :        Subheaders

 :        If the row-column hierarchy is more complex, then subheaders are also generated on subsequent lines.
 :        The number of subheaders depends on the depth of the column hierarchy.

 :        When generating the subheaders, the non-whitespace text node are also taken into account,
 :        and a separate column is generated for them too.

 :        For example, the xml row element:

 :        <row>

 :        <ID>1</ID>

 :        <Name>

 :          Mr.

 :          <First_Name>John</First_Name>

 :          <Last_Name>Howard</Last_Name>

 :        </Name>

 :        <Occupation>student</Occupation>

 :        </row>

 :        is converted to

 :        ID,Name,,Occupation

 :        ,,First Name,Last Name,

 :        1,Mr.,John,Howard,student

 :

 :        If first-row-is-header is not specified and the columns have a deeper hierarchy,
 :          only the first layer of columns is processed, and the fields are the string values of each column.

 :

 :

 : @return the csv or fixed size text as string containing all the lines
 : @example csv_serialize1.xq
 : @example csv_serialize2.xq
 : @example csv_serialize3.xq
 : @example csv_serialize5.xq
 : @example csv_serialize6.xq
 : @example csv_parse_serialize6.xq
 : @example txt_serialize4.xq
 : @example txt_serialize6.xq
 : @example txt_parse_serialize6.xq
:)
declare function zorba-csv:serialize($xml as element()*,
									$options as element(options)?) as xs:string external;

(:
 : Copyright 2006-2009 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
:)

(:~
 : Function library providing convertors from CSV/TXT to XML and back.
 : The functions are optimized to work with large amounts of data, in a streaming way.
 :
 : @author Daniel Turcanu
 :)
module namespace zorba-csv = "http://www.zorba-xquery.com/modules/csv";

(:~
 : Errors namespace URI.
:)
declare variable $zorba-csv:csvNS as xs:string := "http://www.zorba-xquery.com/modules/csv";

(:~
 : Error code for wrong parameter situations.

 : Possible error messages:

 : * "Options field must be of element options instead of ..."

:)
declare variable $zorba-csv:errWrongParam as xs:QName := fn:QName($zorba-csv:csvNS, "zorba-csv:WrongParam");

(:~
 : Parse a CSV or fixed size text and convert to XML.

 : By default each line iw converted to a <row> element, and each field to a <column> element inside <row>.

 : @param $csv the string containing the csv or fixed size text.
 : @param $options having format
 :    <options>

 :        <csv  [separator="default comma ,"] ?

 :          [quote-char="default double quotes &quote;"]?

 :          [quote-escape="default double double quotes &quote;&quote;"]? />

 :        ]

 :

 :        <column-widths>

 :          <column-width>[column fixed width, unsigned int]<column-width>*

 :		  	</column-widths>

 :        ]

 :

 :        <column-positions>

 :          <column-position>[column position on line, unsigned int]<column-position>*

 :		  	</column-positions>

 :        ]

 :        <first-row-is-header [line="first_line[-last_line]?"]?/>?

 :        <start-from-row line="first_line[-last_line]?"/>?

 :        <add-last-void-columns/>?

 :        <xml-nodes>

 :          [<row-name>

 :            [<column-name/>]?

 :          </row-name>]?

 :        </xml-nodes>?

 :    </options>

 :

 :    All the parameters are optional.

 :    All the parameters are case sensitive. They can be in any namespace.

 :    All strings must have UTF-8 encoding.

 :    Parameters csv, column-widths, column-positions are mutualy exclusive. If none is specified,
 :    the input string is assumed to be csv.

 :    Description of parameters:
 :

 :
csv

 :
 Specifies the parameters for parsing a csv string.

 :

 :
separator

 :
The character or group of characters used to separating fields in a row.
 :            If it is not specified, it defaults to comma ','.
 :

 :
quote-char

 :
The character or group of characters used for quoting the fields that may contain special characters,
 :             like separator, new line or this quote char. The default value is double quote ".

 :

 :
quote-escape

 :
The group of characters used for escaping the quote char inside a field. The whole quote escape group
 :           is translated to a quote char during parsing. The default value is double double quotes "".

 :

 :

 :

 :

 :
column-widths

 :
Specifies the column widths for fixed size text. It contains multiple column-width child elements
 :        specifying the fixed width of each column, from left to right.

 :        If the line has more fields than specified, they are ignored.
 :

 :
column-positions

 :
This is an alternative to column-widths, and specifies instead the starting position of each column.
 :        Column positions are 1 based, and are specified in order from left to right.
 :        The last column is read until end of line. The first column position can be greater than 1, if you want
 :        to parse only a part of the input text.
 :

 :
first-row-is-header

 :
The presence of this element indicates that the first row is to be treated as the name of the columns.
 :        If it is not present, then each field is enclosed in a <column> element,
 :        or how it is specified in <xml-nodes> parameter.

 :        If the first row is the header, then each field is enclosed in an element with the corresponding name from the header.

 :        For example, the csv:

 :        ID,Name,Occupation

 :        1,John,student

 :

 :        is parsed into
 :        <row>

 :        <ID>1</ID>

 :        <Name>John</Name>

 :        <Occupation>student</Occupation>

 :        </row>

 :

 :        If the header names contain characters that cannot be used in a QName, they are replaced with underscore '_'.

 :        The namespace for the header QNames is taken from the column name specified in xml-nodes parameter, or from
 :        the row name, or if that doesn't exist either then empty namespace is used.

 :        If the header is not the first line in the input string, the starting line can be specified in the line attribute.

 :        If a column does not have a name, a new name is constructed in the form columnN where N is the position of the column,
 :        starting from 1.

 :        Subheaders

 :				If the header consists of more than one line, this can be specified in the line attribute in the form
 :        "first_line - last_line". Having more lines as the header translates into a hierarchy of elements in the xml.

 :        For example, the csv:

 :        ID,Name,,Occupation

 :        ,First Name,Last Name,

 :        1,John,Howard,student

 :

 :        is parsed into

 :        <row>

 :        <ID>1</ID>

 :        <Name>

 :          <First_Name>John</First_Name>

 :          <Last_Name>Howard</Last_Name>

 :        </Name>

 :        <Occupation>student</Occupation>

 :        </row>

 :

 :
start-from-row

 :
If the data does not start from line 1 or imediately after the header,
 :        you can specify the starting line in the line attribute.

 :        Also you can use this attribute in the form "first_line - last_line" to specify also the last line
 :        if you don't want the whole csv to be parsed.
 :

 :
add-last-void-columns

 :
In the case when using headers and some data lines are shorter than the header, by default the excess columns are ignored
 :          for those lines. You can set the add-last-void-columns parameter to make all the columns appear in xml even if they are void.
 :

 :
xml-nodes

 :
With this parameter you can specify the names for the row element and for the column element if there is no header.

 :        The first element child of this element specifies the desired QName of the row element in the output xml.
 :        The name of this element will be used as the name of the row element.

 :        The element child of this row element is the column element, and its name will be used as the name of the column elements
 :        that enclose the fields in the output xml if there is no header.

 :        If the csv has a header, only the namespace is used from the column element.

 :        For example, with parameter:

 :        <xml-nodes>

 :        <r>

 :          <c/>

 :        </r>

 :        </xml-nodes>

 :

 :        the output for each line will look like

 :        <r>
 :          <c>field1</c>
 :          <c>field2</c>
 :          .......
 :        </r>

 :

 :        If this parameter is not specified, the row name is by default "row" and the column name is by default "column".
 :

 :

 : @return a sequence of row elements, one for each line in csv
 : @error zorba-csv:WrongParam if the options parameter doesn't have the name "options".
:)
declare function zorba-csv:parse($csv as xs:string,
                                 $options as element(options)?) as element()* external;

(:~
 : Convert XML into CSV or fixed size text.

 :
 : @param $xml a sequence of elements, each element representing a row. The name of each row element is ignored.
 :     The childs of each row are the column fields.
 : @param $options having format
 :    <options>

 :        <csv  [separator="default comma ,"] ?

 :          [quote-char="default double quotes &quote;"]?

 :          [quote-escape="default double double quotes &quote;&quote;"]? />

 :        ]

 :        or

 :        <column-widths [align="left|right"]?>

 :          <column-width [align="left|right"]?>[column fixed width, unsigned int]<column-width>*

 :		  	</column-widths>

 :        ]

 :        or

 :        <column-positions [align="left|right"]?>

 :          <column-position [align="left|right"]?>[column position on line, unsigned int]<column-position>*

 :		  	</column-positions>

 :        ]

 :        <first-row-is-header/>?

 :    </options>
 :

 :    All the parameters are optional.

 :    All the parameters are case sensitive. They can be in any namespace.

 :    All strings must have UTF-8 encoding.

 :    Parameters csv, column-widths, column-positions are mutualy exclusive. If none is specified,
 :    the xml is converted to csv.

 :    Description of parameters:
 :

 :
csv

 :
 Specifies the parameters for converting to csv.

 :

 :
separator

 :
The character or group of characters used to separating fields in a row.
 :            If it is not specified, it defaults to comma ','.
 :

 :
quote-char

 :
The character or group of characters used for quoting the fields that may contain special characters,
 :             like separator, new line or this quote char. The default value is double quote ".

 :

 :
quote-escape

 :
The group of characters used for escaping the quote char inside a field. The whole quote escape group
 :           is translated to a quote char during parsing. The default value is double double quotes "".

 :

 :

 :

 :

 :
column-widths

 :
Specifies the column widths for fixed size text. It contains multiple column-width child elements
 :        specifying the fixed width of each column, from left to right.

 :        With the attribute align you can specify how to align fields that are smaller than the column width.
 :        The default alignment is left.

 :

 :
column-positions

 :
This is an alternative to column-widths, and specifies instead the starting position of each column.
 :        Column positions are 1 based, and are specified in order from left to right.
 :        The last column has a variable length.

 :        With the attribute align you can specify how to align fields that are smaller than the column width.
 :        The default alignment is left. The last column does not need alignment.

 :

 :
first-row-is-header

 :
The presence of this element indicates that the first row will contain the header, that is, the names of
 :        the column elements. Only the column names from the first row element are taken into account.

 :        For example, the row xml:

 :        <row>

 :        <ID>1</ID>

 :        <Name>John</Name>

 :        <Occupation>student</Occupation>

 :        </row>

 :

 :        is converted to

 :        ID,Name,Occupation

 :        1,John,student

 :

 :        The header names are the localnames of the column elements, and the namespace is ignored.

 :        Subheaders

 :        If the row-column hierarchy is more complex, then subheaders are also generated on subsequent lines.
 :        The number of subheaders depends on the depth of the column hierarchy.

 :        When generating the subheaders, the non-whitespace text node are also taken into account,
 :        and a separate column is generated for them too.

 :        For example, the xml row element:

 :        <row>

 :        <ID>1</ID>

 :        <Name>

 :          Mr.
 :          <First_Name>John</First_Name>

 :          <Last_Name>Howard</Last_Name>

 :        </Name>

 :        <Occupation>student</Occupation>

 :        </row>

 :        is converted to

 :        ID,Name,,Occupation

 :        ,,First Name,Last Name,

 :        1,Mr.,John,Howard,student

 :

 :        If first-row-is-header is not specified, only the first layer of columns is processed, and the fields
 :        are the string values of each column.

 :

 :

 : @return the csv or fixed size text as string
:)
declare function zorba-csv:serialize($xml as element()*,
																		$options as element(options)?) as xs:string external;