Create new organizations

Create new organizations

xquery version "1.0";

(: ~
 : Module Name:             new conceptual domain webpage and XQuery
 :
 : Module overview          Creates a new Organization
 :)

(:~
 :    @author Rakesh Dhaval
 :    @version 0.1
~ :)

  import module namespace
  lib-forms="http://www.cagrid.org/xquery/library/forms"
  at "../edit/m-lib-forms.xquery";

  import module namespace
  lib-util="http://www.cagrid.org/xquery/library/util"
  at "../library/m-lib-util.xquery";

  import module namespace
  lib-rendering="http://www.cagrid.org/xquery/library/rendering"
  at "../web/m-lib-rendering.xquery";

  import module namespace
    lib-make-admin-item="http://www.cagrid.org/xquery/library/make-admin-item"
    at "../edit/m-lib-make-admin-item.xquery";

declare namespace openMDR = "http://www.cagrid.org/schema/openMDR";
declare namespace ISO11179= "http://www.cagrid.org/schema/ISO11179";
declare namespace session="http://exist-db.org/xquery/session";
declare namespace response="http://exist-db.org/xquery/response";
declare namespace exist = "http://exist.sourceforge.net/NS/exist";

declare function local:organisation(
   $reg-auth as xs:string,
   $administrative-note as xs:string,
   $administrative-status as xs:string,
   $administered-by as xs:string,
   $submitted-by as xs:string,
   $registered-by as xs:string,
   $context-ids as xs:string*,
   $country-identifiers as xs:string*,
   $language-identifiers as xs:string*,
   $names as xs:string*,
   $definitions as xs:string*,
   $sources as xs:string*,
   $preferred as xs:string?,
   $organization_name as xs:string?,
   $organization_mail_address as xs:string?,
   $contact_information as xs:string?,
   $contact_name as xs:string?,
   $contact_title as xs:string?

   ) as xs:boolean
{
   let $version := '0.1'
   let $data-identifier := lib-forms:generate-id()
   let $new-identifier := concat($reg-auth, '-', $data-identifier, '-', $version)
   let $doc-name := concat($new-identifier,'.xml')

	 let $organization-identifier := lib-forms:generate-id()
	 let $contact-identifier := lib-forms:generate-id()

   let $content := (
            lib-make-admin-item:administration-record($administrative-note,$administrative-status,'Recorded'),
            lib-make-admin-item:custodians($administered-by,$registered-by,$submitted-by),
            lib-make-admin-item:havings(
                    $context-ids,
                    $country-identifiers,
                    $language-identifiers,
                    $names,
                    $definitions,
                    $preferred,
                    $sources)

    )

   (: compose the document :)
   let $document :=
            element openMDR:Organization {
     										attribute organization_identifier{$organization-identifier},
           							        element openMDR:organization_name{$organization_name},
                							element openMDR:organization_mail_address{$organization_mail_address},
                							element openMDR:Contact {
    							            attribute contact_identifier{$contact-identifier},
    							            element openMDR:contact_name{$contact_name},
    							            element openMDR:contact_title{$contact_title},
    							            element openMDR:contact_information{$contact_information}
    							            }
							             }

   let $collection := 'organisation'
   let $message := lib-forms:store-document($document)
   return
      if ($message='stored')
      then true()
      else response:redirect-to(xs:anyURI(concat("../web/login.xquery?calling_page=newOrganisation.xquery&",$message)))
};

declare function local:input-page(
   $message as xs:string?,
   $reg-auth as xs:string?,
   $administrative-note  as xs:string?,
   $administrative-status  as xs:string?,
   $administered-by  as xs:string?,
   $submitted-by  as xs:string?,
   $registered-by  as xs:string?,
   $context-ids as xs:string*,
   $country-identifiers as xs:string*,
   $language-identifiers as xs:string*,
   $names as xs:string*,
   $definitions as xs:string*,
   $sources as xs:string*,
   $action as xs:string?,
   $preferred as xs:string?,
   $org_name as xs:string?,
   $org_mail_address as xs:string?,
   $contact-name as xs:string?,
   $contact-title as xs:string?,
   $contact-information as xs:string?

   ) {
   let $skip-name := substring-after($action,'delete naming entry')
   let $skip-name-index := if ($skip-name>'') then xs:int($skip-name) else 0

   return

                This form will allow you to create a new Organisation in the metadata repository

          <form name="new_organisation" action="newOrganisation.xquery" method="post" class="cagridForm" enctype="multipart/form-data">

             {lib-forms:edit-admin-item($reg-auth,
                     $administrative-note,
                     $administrative-status,
                     $administered-by,
                     $submitted-by,
                     $registered-by,
                     $context-ids,
                     $country-identifiers,
                     $language-identifiers,
                     $names,
                     $definitions,
                     $sources,
                     $preferred,
                     $action)}

Organization

                {

                  	Organization Name
                    <input type="text" name="org_name"></input>

,

                  	Organization Email Address
                    <input type="text" name="org_mail_address"></input>

              	}

Contact

                {

                  	Name
                    <input type="text" name="contact-name"></input>

,

                  	Title
                    <input type="text" name="contact-title"></input>

,

                  	Information
                    <input type="text" name="contact-information"></input>

              	}

                      	<input type="submit" name="update" value="Store"/>
                      	<input type="submit" name="update" value="Clear"/>

          </form>

{$message}

   };

declare function local:success-page()
{
   let $calling-page := request:get-parameter("calling-page","")
   return

Organization created

Return to maintenance menu

Create another Organization

};

declare option exist:serialize "media-type=text/html method=xhtml doctype-public=-//W3C//DTD XHTML 1.0 Transitional//EN doctype-system=http://www.w3.org/TR/2002/REC-xhtml1-20020801/DTD/xhtml1-transitional.dtd";

   session:create(),
   let $title as xs:string := "Creating a New Organisation"
   let $reg-auth := request:get-parameter('registration-authority','')
   let $administrative-note := request:get-parameter('administrative-note','')
   let $administrative-status := request:get-parameter('administrative-status','')
   let $administered-by := request:get-parameter('administered-by','')
   let $submitted-by := request:get-parameter('submitted-by','')
   let $registered-by := request:get-parameter('registered-by','')
   let $context-ids := request:get-parameter('context-ids',())
   let $country-identifiers := request:get-parameter('country-identifiers',())
   let $language-identifiers := request:get-parameter('language-identifiers',())
   let $names := request:get-parameter('names',())
   let $definitions := request:get-parameter('definitions',())
   let $sources := request:get-parameter('sources',())
   let $preferred := request:get-parameter('preferred','')
   let $action := request:get-parameter('update','')
   let $organization_name :=request:get-parameter('org_name','')
   let $organization_mail_address :=request:get-parameter('org_mail_address','')
   let $contact_information :=request:get-parameter('contact-information','')
   let $contact_name :=request:get-parameter('contact-name','')
   let $contact_title :=request:get-parameter('contact-title','')

   return

      lib-rendering:txfrm-webpage(
      $title,
      if ($action='Store')
      then
         (
         if (
               local:organisation
                  (
                     $reg-auth,
                     $administrative-note,
                     $administrative-status,
                     $administered-by,
                     $submitted-by,
                     $registered-by,
                     $context-ids,
                     $country-identifiers,
                     $language-identifiers,
                     $names,
                     $definitions,
                     $sources,
                     $preferred,
                     $organization_name,
                     $organization_mail_address,
                     $contact_name,
                     $contact_title,
                     $contact_information
                  )
            )
         then local:success-page()
         else (local:input-page(
            'could not store document',
                     $reg-auth,
                     $administrative-note,
                     $administrative-status,
                     $administered-by,
                     $submitted-by,
                     $registered-by,
                     $context-ids,
                     $country-identifiers,
                     $language-identifiers,
                     $names,
                     $definitions,
                     $sources,
                     $action,
                     $preferred,
                     $organization_name,
                     $organization_mail_address,
                     $contact_name,
                     $contact_title,
                     $contact_information
                  )
               )
         )
      else local:input-page
           (
               '',
               $reg-auth,
               $administrative-note,
               $administrative-status,
               $administered-by,
               $submitted-by,
               $registered-by,
               $context-ids,
               $country-identifiers,
               $language-identifiers,
               $names,
               $definitions,
               $sources,
               $action,
               $preferred,
               $organization_name,
               $organization_mail_address,
							 $contact_name,
               $contact_title,
               $contact_information
               )
         )

error: pathspec 'grid-incubation/incubator/projects/mdr/projects/mdrCore/db/mdr/edit/newOrganisation.xquery' did not match any file(s) known to git


Small example to register replication-receiver via xquery function

(:
 : Example: register receiver with additional parameters
 :)
xquery version "3.0";

import module namespace replication="http://exist-db.org/xquery/replication"
                        at "java:org.exist.replication.xquery.ReplicationModule";

(: Configuration for setting-up the JMS connection :)
let $jmsConfiguration :=
    map {
        "java.naming.factory.initial"
            := "org.apache.activemq.jndi.ActiveMQInitialContextFactory",
        "java.naming.provider.url" := "tcp://miniserver.local:61616",
        "destination" := "dynamicTopics/eXistdb-replication-demo",
        "connection-factory" := "ConnectionFactory",
        "subscriber.name" :="SubscriptionId",
        "connection.client-id" :="ClientId"
    }

return
    (: Register the function to the JMS broker :)
    replication:register($jmsConfiguration)

error: pathspec 'web/modules/testing/registerReplicationReceiver.xq' did not match any file(s) known to git


Intial commit. Way faster than using string parsing methods for some reason.

xquery version "1.0-ml";
declare namespace matt = "http://matthewroyal.com/blog";

declare variable $matt as xs:string := "http://matthewroyal.com/blog";

(:~
    Join a sequence of codepoints into a xs:string
:)
declare function local:joinCodes($codepoints) as xs:string {
  fn:string-join( for $c in $codepoints return fn:codepoints-to-string($c) )
};

(:~
    CORRECT CSV PARSER, author: Matthew Royal

    This CSV parser correctly parses CSV files with escaped quotation marks ("")
    and carriage returns in column data. It is probably slower than other parsers,
    because the XQuery language standard does not support several key Regex features
    and relies on parsing the file one character at a time instead.

    If your column data contains these escaped characters, to my knowledge this is
    currently the only XQuery-based CSV-to-XML converter that can parse them properly.
    If your data contains no such escape sequences, you will get much better performance
    using a different parser.

    PERFORMANCE: Using my Macbook Pro: 2.3 GHz i7, 16 GB 1600MHz DDR3. Running MarkLogic 7.0-3:
      Converting a 7.9 MB CSV and inserting the resulting XML file took T00:01:08.90S

    RFC 4180 GRAMMAR:
     file = [header CRLF] record *(CRLF record) [CRLF]
     header = field *(COMMA field)
     record = field *(COMMA field)
     field = (escaped / non-escaped)
     escaped = DQUOTE *(TEXTDATA / COMMA / CR / LF / 2DQUOTE) DQUOTE
     non-escaped = *TEXTDATA
     COMMA = %x2C
     CR = %x0D ;as per section 6.1 of RFC 2234 [2]
     DQUOTE =  %x22 ;as per section 6.1 of RFC 2234 [2]
     LF = %x0A ;as per section 6.1 of RFC 2234 [2]
     CRLF = CR LF ;as per section 6.1 of RFC 2234 [2]
     TEXTDATA =  %x20-21 / %x23-2B / %x2D-7E

    Parameters:
      $file as xs:string -- contents of CSV file to parse
      $header as xs:boolean -- indicates that the first row of the file is the header.
:)
declare function local:parseFile(
  $file as xs:string,
  $header as xs:boolean
) as element()* {

  let $mem := map:new(())
  let $fileLength := fn:string-length($file)
  let $_ := map:put($mem, "row", 1)
  let $_ := map:put($mem, "fieldNum", 1)
  let $_ := map:put($mem, "fields", 1)
  let $_ := map:put($mem, "origin", 1)
  let $_ := map:put($mem, "escaped", ())
  let $inputCodepoints := fn:string-to-codepoints($file)
  let $_ :=
    for $l at $i in $inputCodepoints
    let $letter := fn:codepoints-to-string($l)
    return
      (: Quote mark:)
      if ($letter = '"') then
        if (fn:not(map:get($mem, "escaped")) and (local:joinCodes($inputCodepoints[$i - 1]) = "," or fn:not(local:joinCodes($inputCodepoints[$i - 1])))) then
          map:put($mem, "escaped", '"')
        else if (map:get($mem, "escaped") and local:joinCodes($inputCodepoints[$i - 1]) != '"' and local:joinCodes($inputCodepoints[$i + 1]) != '"') then
          map:put($mem, "escaped", ())
        else ()
      (: Comma; newline, end of row; End of file :)
      else if ( (($letter = ',' or $letter = '
') and not(map:get($mem, "escaped"))) or $i = $fileLength ) then
        let $fieldNum := map:get($mem, "fieldNum")
        (: Keep track of how many fields there are :)
        let $fieldMax := if ($fieldNum > map:get($mem, "fields")) then $fieldNum else map:get($mem, "fields")
        let $row := map:get($mem, "row")
        let $o := map:get($mem, "origin")
        let $fieldValue := local:joinCodes(
          $inputCodepoints[$o to (if ($fileLength = $i) then $i - $o + 1 else $i -$o)]
        )
        let $_ :=
          if ($row = 1 and $header) then
            map:put($mem, "header-" || xs:string($fieldNum), $fieldValue)
          else
            map:put($mem, xs:string($row) || "-" || xs:string($fieldNum), $fieldValue)
        let $_ := map:put($mem, "fieldNum", $fieldNum + 1)
        let $_ := map:put($mem, "fields", $fieldMax)
        let $_ := map:put($mem, "origin", $i + 1)
        return
          if ($letter = '
' and not(map:get($mem, "escaped"))) then
            let $_ := map:put($mem, "row", $row + 1)
            let $_ := map:put($mem, "fieldNum", 1)
            return map:put($mem, "escaped", ())
          else
            map:put($mem, "escaped", ())
      else ()
  return
    let $fields := map:get($mem, "fields")
    let $rows   := map:get($mem, "row")
    for $r in 1 to $rows
    return
      if ($r = 1 and $header) then () else (: Skip the header row :)
      element { fn:QName( $matt, "row" ) }
      {
        if ($header) then
          attribute {xs:QName("num")} {$r - 1}
        else
          attribute {xs:QName("num")} {$r}
        ,
        for $f in 1 to $fields
        return
          if (
              map:get($mem, xs:string($r) || "-" || xs:string($f))
          ) then
            element
            {
              if ($header) then
                fn:QName( $matt, fn:replace(map:get($mem, "header-"||xs:string($f)), '["|]', '') )
              else
                fn:QName($matt, "field" || xs:string($f))
            }
            { map:get($mem, xs:string($r) || "-" || xs:string($f)) }
          else ()
      }
};

let $rFile := '"RELATIONSHIP_ID","RELATIONSHIP_NAME","IS_HIERARCHICAL","DEFINES_ANCESTRY","REVERSE_RELATIONSHIP"|
319,"Multilex ingredient to drug class (OMOP)",0,0,320|
320,"Drug class to Multilex ingredient (OMOP)",0,1,|
347,"Concept replaced by",0,0,348|
348,"Concept replaces",0,0,||
'

let $file := 'pin,Pilatus,"Pilatus ""quoted
 string"" mountain",7/1/1984,7/31/1984,,",a word,
,,and another,,",dcassel'
let $field := '"Pilatus ""quoted string"" mountain"'

return
  element csv { local:parseFile($rFile, fn:true()) }

error: pathspec 'correctCsvParser.xqy' did not match any file(s) known to git


[biblio] added missing all-codes.xq web service that loads all codes into the XForms model.  Thanks to Jens for pointing out this file was missing.

This version is not optimized yet since all codes are loaded into the model of all sub-tabs.  The next version will query the tab-specific body statements and only load the codes needed for that tab.  This should be a big improvement in the time it takes to load the forms.

- Dan

svn path=/trunk/eXist/; revision=12143

xquery version "1.0";
import module namespace style = "http://exist-db.org/mods-style" at "../../../modules/style.xqm";
declare option exist:serialize "method=xml media-type=text/xml indent=yes";

let $code-table-collection := concat($style:web-path-to-app, '/code-tables/')

return
<code-tables>
   <collection>{$code-table-collection}</collection>
   {doc(concat($code-table-collection, 'dublin-code-type-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'marc-genre-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'title-type-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'name-type-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'name-part-type-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'language-2-type-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'language-3-type-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'language-3-authority-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'subject-authority-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'role-short-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'name-title-authority-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'display-label-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'script-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'transliteration-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'date-encoding-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'date-point-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'date-keydate-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'date-qualifier-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'record-identifier-source-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'code-text-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'description-standard-authority-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'type-of-resource-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'yes-no-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'genre-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'genre-authority-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'note-type-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'continent-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'geographic-code-authority-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'temporal-encoding-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'classification-authority-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'identifier-type-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'location-unittype-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'physical-location-type-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'physical-location-authority-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'url-access-type-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'url-usage-type-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'form-type-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'form-authority-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'internet-media-type-codes.xml'))/code-table}
   {doc(concat($code-table-collection, 'digital-origin-type-codes.xml'))/code-table}
</code-tables>

error: pathspec 'webapp/biblio/mods/apps/mods/edit/all-codes.xq' did not match any file(s) known to git


added comment

added comment

(:
  Copyright 2012-2016 MarkLogic Corporation

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
:)
xquery version "1.0-ml";

module namespace flow = "http://marklogic.com/data-hub/flow-lib";

import module namespace debug = "http://marklogic.com/data-hub/debug-lib"
  at "/com.marklogic.hub/lib/debug-lib.xqy";

import module namespace hul = "http://marklogic.com/data-hub/hub-utils-lib"
  at "/com.marklogic.hub/lib/hub-utils-lib.xqy";

import module namespace functx = "http://www.functx.com"
  at "/MarkLogic/functx/functx-1.0-nodoc-2007-01.xqy";

declare namespace hub = "http://marklogic.com/data-hub";

declare namespace envelope = "http://marklogic.com/data-hub/envelope";

declare option xdmp:mapping "false";

(: xml describing the default writer :)
declare variable $DEFAULT-WRITER :=
  element hub:writer {
    attribute type { "xquery" },
    attribute module { "/com.marklogic.hub/writers/default.xqy" }
  };

(: xml describing the default content plugin :)
declare variable $DEFAULT-CONTENT-PLUGIN :=
  element hub:plugin {
    attribute dest { "content" },
    attribute type { "xquery" },
    attribute module { "/com.marklogic.hub/plugins/raw.xqy" }
  };

(: xml describing a no-op plugin. used when no plugin is given :)
declare variable $NO-OP-PLUGIN :=
  element hub:plugin {
    attribute type { "null" }
  };

(: the directory where entities live :)
declare variable $ENTITIES-DIR := "/entities/";

declare variable $PLUGIN-NS := "http://marklogic.com/data-hub/plugins/";

declare variable $TYPE-XQUERY := "xquery";

declare variable $TYPE-JAVASCRIPT := "javascript";

declare variable $TYPE-XSLT := "xslt";

declare variable $TYPE-XML := "xml";

declare variable $TYPE-JSON := "json";

(:
 : Determines the type of flow given a filename
 :
 : @param $filename - name of the module plugin file
 : @return - a type (xquery, xml, sjs, ...)
 :)
declare %private function flow:get-type($filename) as xs:string?
{
  let $ext as xs:string := hul:get-file-extension($filename)
  return
    if ($ext = hul:get-xqy-extensions()) then
      $TYPE-XQUERY
    else if ($ext = hul:get-sjs-extensions()) then
      $TYPE-JAVASCRIPT
    else if ($ext = hul:get-xslt-extensions()) then
      $TYPE-XSLT
    else if ($ext = "xml") then
      $TYPE-XML
    else if ($ext = "json") then
      $TYPE-JSON
    else
      fn:error(xs:QName("INVALID_PLUGIN"), $filename)
};

(:~
 : Returns xml describing a collector
 :)
declare %private function flow:get-collector(
  $flow-name as xs:string,
  $uris as xs:string*) as element(hub:collector)
{
  let $uri := $uris[1]
  let $filename as xs:string := hul:get-file-from-uri($uri)
  let $type := flow:get-type($filename)
  return
    element hub:collector {
      attribute type { $type },
      attribute module { $uri }
    }
};

(:~
 : Returns xml describing a plugin
 :)
declare %private function flow:get-plugin(
  $flow-name as xs:string,
  $uris as xs:string*,
  $destination as xs:string) as element(hub:plugin)*
{
  for $uri in $uris
  let $filename as xs:string := hul:get-file-from-uri($uri)
  let $type := flow:get-type($filename)
  return
    element hub:plugin {
      attribute dest { $destination },
      attribute type { $type },
      attribute module { $uri }
    }
};

(:~
 : Returns xml describing a content plugin
 :)
declare %private function flow:get-content(
  $flow-name as xs:string,
  $uris as xs:string*)
{
  let $plugin := flow:get-plugin($flow-name, $uris, "content")
  return
    if ($plugin) then $plugin
    else
      $DEFAULT-CONTENT-PLUGIN
};

(:~
 : Returns xml describing a headers plugin
 :)
declare %private function flow:get-headers(
  $flow-name as xs:string,
  $uris as xs:string*)
{
  let $plugin := flow:get-plugin($flow-name, $uris, "headers")
  return
    if ($plugin) then $plugin
    else
      $NO-OP-PLUGIN
};

(:~
 : Returns xml describing a triples plugin
 :)
declare %private function flow:get-triples(
  $flow-name as xs:string,
  $uris as xs:string*)
{
  let $plugin := flow:get-plugin($flow-name, $uris, "triples")
  return
    if ($plugin) then $plugin
    else
      $NO-OP-PLUGIN
};

(:~
 : Returns xml describing a writer
 :)
declare %private function flow:get-writer(
  $flow-name as xs:string,
  $uris as xs:string*)
{
  if ($uris) then
    for $uri in $uris
    let $filename as xs:string := hul:get-file-from-uri($uri)
    let $type := flow:get-type($filename)
    let $_ :=
      if (debug:on()) then
        debug:log((
          "$flow-name: " || $flow-name,
          "$uri: " || $uri,
          "$filename: " || $filename,
          "$type: " || $type
        ))
      else ()
    return
      element hub:writer {
        attribute type { $type },
        attribute module { $uri }
      }
  else
    $DEFAULT-WRITER
};

(:~
 : Returns a flow by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity that owns the flow
 : @param $flow-name - name of the flow to retrieve
 : @return - xml describing the flow
 :)
declare function flow:get-flow(
  $entity-name as xs:string,
  $flow-name as xs:string,
  $flow-type as xs:string?) as element(hub:flow)
{
  let $uris :=
    hul:run-in-modules(function() {
      let $type :=
        if ($flow-type) then $flow-type
        else "*"
      return
        cts:uri-match($ENTITIES-DIR || $entity-name || "/" || $type || "/" || $flow-name || "/*")
    })
  return
    flow:get-flow($entity-name, $flow-name, $flow-type, $uris)
};

(:~
 : Returns a flow by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity that owns the flow
 : @param $flow-name - name of the flow to retrieve
 : @param $uris - uris used to build the entity xml
 : @return - xml describing the flow
 :)
declare %private function flow:get-flow(
  $entity-name as xs:string,
  $flow-name as xs:string,
  $flow-type as xs:string?,
  $uris as xs:string*) as element(hub:flow)
{
  let $_ := xdmp:log(("entity: " || $entity-name, "flow: " || $flow-name, "flow-type: " || $flow-type, "uris:", $uris))
  let $real-flow-type := fn:replace($uris[1], $ENTITIES-DIR || $entity-name || "/([^/]+)/" || $flow-name || ".*$", "$1")
  let $map := map:map()
  let $_ :=
    for $dir in $uris
    let $type :=
      if ($flow-type) then $flow-type
      else "[^/]+"
    let $dir-name := fn:replace($dir, $ENTITIES-DIR || $entity-name || "/" || $type || "/" || $flow-name || "/([^/]+)/$", "$1")
    let $child-uris := $uris[fn:matches(., $ENTITIES-DIR || $entity-name || "/" || $type || "/" || $flow-name || "/" || $dir-name || "/([^/]+)$")]
    return
      switch ($dir-name)
        case "collector" return
          map:put($map, "collector", flow:get-collector($flow-name, $child-uris))
        case "content" return
          map:put($map, "content", flow:get-content($flow-name, $child-uris))
        case "headers" return
          map:put($map, "headers", flow:get-headers($flow-name, $child-uris))
        case "triples" return
          map:put($map, "triples", flow:get-triples($flow-name, $child-uris))
        case "writer" return
          map:put($map, "writer", flow:get-writer($flow-name, $child-uris))
        default return
          ()
  let $flow := hul:run-in-modules(function() {
    let $uri := $ENTITIES-DIR || $entity-name || "/" || $real-flow-type || "/" || $flow-name || "/" || $flow-name || ".xml"
    return
      fn:doc($uri)/hub:flow
  })
  return
    <flow xmlns="http://marklogic.com/data-hub">
      <name>{$flow-name}</name>
      <entity>{$entity-name}</entity>
      <complexity>{ ($flow/hub:complexity/fn:data(), "simple")[1] }</complexity>
      <data-format>{ $flow/hub:data-format/fn:data() }</data-format>
      <type>{$real-flow-type}</type>
      {map:get($map, "collector")}
      <plugins>
      {
        if ($flow/hub:plugins/*) then
          $flow/hub:plugins/*
        else
        (
          (map:get($map, "content"), $DEFAULT-CONTENT-PLUGIN)[1],
          (map:get($map, "headers"), $NO-OP-PLUGIN)[1],
          (map:get($map, "triples"), $NO-OP-PLUGIN)[1]
        )
      }
      </plugins>
      {
        (map:get($map, "writer"), $DEFAULT-WRITER)[1]
      }
    </flow>
};

(:~
 : Returns the flows that belong to the given entity in the database
 :
 : @param $entity-name - the name of the entity containing the flows
 : @return - xml describing the flows
 :)
declare function flow:get-flows(
  $entity-name as xs:string) as element(hub:flows)
{
  let $uris := hul:run-in-modules(function() {
    cts:uri-match($ENTITIES-DIR || "*")
  })
  let $flows :=
    for $flow in $uris[fn:matches(., $ENTITIES-DIR || $entity-name || "/(input|conformance)/[^/]+/$")]
    let $name := fn:replace($flow, $ENTITIES-DIR || $entity-name || "/(input|conformance)/([^/]+)/$", "$2")
    return
      flow:get-flow($entity-name, $name, (), $uris[fn:matches(., $ENTITIES-DIR || $entity-name || "/(input|conformance)/" || $name || "/.+")])
  return
    <flows xmlns="http://marklogic.com/data-hub">
    {
      $flows
    }
    </flows>
};

(:~
 : Returns a entity by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity to retrieve
 : @return - xml describing the entity
 :)
declare function flow:get-entity(
  $entity-name as xs:string)
{
  let $uris :=
    hul:run-in-modules(function() {
      cts:uri-match($ENTITIES-DIR || $entity-name || "/*")
    })
  return
    flow:get-entity($entity-name, $uris)
};

(:~
 : Returns a entity by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity to retrieve
 : @param $uris - uris used to build the entity xml
 : @return - xml describing the entity
 :)
declare %private function flow:get-entity(
  $entity-name as xs:string,
  $uris as xs:string*)
  as element(hub:entity)
{
  <entity xmlns="http://marklogic.com/data-hub">
    <name>{$entity-name}</name>
    {
      flow:get-flows($entity-name)
    }
  </entity>
};

(:~
 : Returns the entities in the database
 :
 : @return - xml describing the entities
 :)
declare function flow:get-entities() as element(hub:entities)
{
  let $uris := hul:run-in-modules(function() {
    cts:uri-match($ENTITIES-DIR || "*")
  })
  let $entities :=
    for $flow in $uris[fn:matches(., $ENTITIES-DIR || "[^/]+/$")]
    let $name := fn:replace($flow, $ENTITIES-DIR || "([^/]+)/$", "$1")
    return
      flow:get-entity($name, $uris[fn:matches(., $ENTITIES-DIR || $name || "/.+")])
  return
    <entities xmlns="http://marklogic.com/data-hub">
    {
      $entities
    }
    </entities>
};

(:~
 : Runs a collector
 :
 : @param $module-uri - the uri of the collector module
 : @param $options - a map of options passed in by the client
 : @return - a sequence of strings
 :)
declare function flow:run-collector(
  $module-uri as xs:string,
  $options as map:map) as xs:string*
{
  let $module-name := hul:get-module-name($module-uri)
  let $filename as xs:string := hul:get-file-from-uri($module-uri)
  let $type := flow:get-type($filename)
  let $ns :=
    if ($type eq $TYPE-JAVASCRIPT) then ()
    else
      $PLUGIN-NS || fn:lower-case($module-name)
  let $func := xdmp:function(fn:QName($ns, "collect"), $module-uri)
  return
    $func($options)
};

(:~
 : Runs a given flow
 :
 : @param $flow - xml describing the flow
 : @param $idenifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $options - a map of options passed in by the client
 : @return - nothing
 :)
declare function flow:run-flow(
  $flow as element(hub:flow),
  $identifier as xs:string,
  $options as map:map) as empty-sequence()
{
  flow:run-flow($flow, $identifier, (), $options)
};

declare function flow:run-plugins(
  $flow as element(hub:flow),
  $identifier as xs:string,
  $content as item()?,
  $options as map:map
)
{
  let $content :=
    map:new((
      map:entry("identifier", $identifier),
      if ($content) then
        map:entry("content", $content)
      else ()
    ))
  let $data-format := $flow/hub:data-format
  let $_ :=
    for $plugin in $flow/hub:plugins/hub:plugin
    let $destination := $plugin/@dest
    let $resp :=
      flow:run-plugin(
        $plugin,
        $data-format,
        $identifier,
        map:get($content, "content"),
        map:get($content, "headers"),
        map:get($content, "triple"),
        $options)
    return
      if (fn:empty($destination))
      then ()
      else map:put($content, $destination, $resp)
  return
    flow:make-envelope($content, $data-format)
};

declare function flow:run-flow(
  $flow as element(hub:flow),
  $identifier as xs:string,
  $content as item()?,
  $options as map:map) as empty-sequence()
{
  let $envelope := flow:run-plugins($flow, $identifier, $content, $options)
  let $_ :=
    xdmp:invoke-function(function() {
      for $writer in $flow/hub:writer
      return
        flow:run-writer($writer, $identifier, $envelope, $options)
    },
    map:new((
      map:entry("isolation", "different-transaction"),
      map:entry("database", xdmp:database("data-hub-FINAL")),
      map:entry("transactionMode", "update-auto-commit")
    )))

  return
    ()
};

(:~
 : Construct an envelope
 :
 : @param $map - a map with all the stuff in it
 : @return - the newly constructed envelope
 :)
declare function flow:make-envelope($map as map:map, $data-format as xs:string)
{
  xdmp:log(("map:", $map)),
  if ($data-format eq "application/json") then
    xdmp:to-json($map)/node()
  else
    <envelope xmlns="http://marklogic.com/data-hub/envelope">
      <headers>{map:get($map, "headers")}</headers>
      <triples>{map:get($map, "triples")}</triples>
      <content>{map:get($map, "content")}</content>
    </envelope>
};

(:~
 : Run a given plugin
 :
 : @param $plugin - xml describing the plugin to run
 : @param $idenifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $content - the output of the content plugin
 : @param $headers - the output of the headers plugin
 : @param $triples - the output of the triples plugin
 : @param $options - a map of options passed in by the client
 : @return - the output of the plugin. It varies.
 :)
declare function flow:run-plugin(
  $plugin as element(hub:plugin),
  $data-format as xs:string,
  $identifier as xs:string,
  $content as item()?,
  $headers as item()*,
  $triples as sem:triple*,
  $options as map:map)
{
  let $module-uri := $plugin/@module
  return
    if (fn:empty($module-uri)) then ()
    else
      let $destination := $plugin/@dest
      let $module-name := hul:get-module-name($module-uri)
      let $filename as xs:string := hul:get-file-from-uri($module-uri)
      let $type := flow:get-type($filename)
      let $ns :=
        if ($type eq $TYPE-JAVASCRIPT) then ()
        else
          $PLUGIN-NS || fn:lower-case($module-name)
      let $func-name :=
        if ($type eq $TYPE-JAVASCRIPT) then
          "create" || functx:capitalize-first($destination)
        else
          "create-" || $destination
      let $func := xdmp:function(fn:QName($ns, $func-name), $module-uri)
      let $resp := $func($identifier, $content, $headers, $triples, $options)
      return
        typeswitch($resp)
          case document-node() return
            if (fn:count($resp/node()) > 1) then
              fn:error("Too Many Nodes!. Return just 1 node")
            else
              $resp/node()
          case json:array return
            if ($data-format = 'application/xml') then
              json:array-values($resp)
            else
              $resp
          default return
            $resp
};

(:~
 : Run a given writer
 :
 : @param $writer - xml describing the writer to run
 : @param $idenifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $envelope - the envelope
 : @param $options - a map of options passed in by the client
 : @return - the output of the writer. It varies.
 :)
declare function flow:run-writer(
  $writer as element(hub:writer),
  $identifier as xs:string,
  $envelope as item(),
  $options as map:map)
{
  let $module-uri as xs:string := $writer/@module
  let $module-name := hul:get-module-name($module-uri)
  let $ns := $PLUGIN-NS || fn:lower-case($module-name)
  let $func := xdmp:function(fn:QName($ns, "write"), $module-uri)
  return
    $func($identifier, $envelope, $options)
};

declare function flow:make-error-json($ex) {
  map:new((
    map:entry("msg", $ex/error:format-string/fn:data()),
    let $f := $ex/error:stack/error:frame[1]
    return
      (
        map:entry("uri", $f/error:uri/fn:data()),
        map:entry("line", $f/error:line/fn:data()),
        map:entry("column", $f/error:column/fn:data())
      )
  ))
};

declare function flow:validate-entities()
{
  let $errors := json:array()
  let $options := map:map()
  let $_ :=
    for $entity in flow:get-entities()/hub:entity
    for $flow in $entity/hub:flows/hub:flow
    let $data-format := $flow/hub:data-format
    (: validate collector :)
    let $_ :=
      let $collector := $flow/hub:collector
      return
        if ($collector) then
          try {
            flow:run-collector($collector/@module, $options)
          }
          catch($ex) {
            json:array-push($errors, flow:make-error-json($ex))
          }
        else ()
    (: validate plugins :)
    let $_ :=
      for $plugin in $flow/hub:plugins/hub:plugin
      let $destination := $plugin/@dest
      let $module-uri := $plugin/@module
      let $module-name := hul:get-module-name($module-uri)
      let $filename as xs:string := hul:get-file-from-uri($module-uri)
      let $type := flow:get-type($filename)
      let $ns :=
        if ($type eq $flow:TYPE-JAVASCRIPT) then ()
        else
          $flow:PLUGIN-NS || fn:lower-case($module-name)
      let $func-name :=
        if ($type eq $flow:TYPE-JAVASCRIPT) then
          "create" || functx:capitalize-first($destination)
        else
          "create-" || $destination
      return
        (:
         : Note that we are static checking the files.
         : This is because there is no reasonable way to actually
         : call the plugins and pass in data that will work for all plugins.
         :
         : The disadvantage to static checking is that we will not catch typos
         : like ctsdoc <- (missing period) because Javascript is dynamically
         : typed. Static checking will only catch syntax errors in sjs.
         :)
        try {
          if ($type eq $flow:TYPE-XQUERY) then
            xdmp:eval(
              'import module namespace x = "' || $ns || '" at "' || $module-uri || '"; ' ||
              '()',
              map:new(map:entry("staticCheck", fn:true()))
            )
          else
            xdmp:javascript-eval(
              'var x = require("' || $module-uri || '");',
              map:new(map:entry("staticCheck", fn:true()))
            )
        }
        catch($ex) {
          json:array-push($errors, flow:make-error-json($ex))
        }
    return
      ()
  return
    map:new(
      map:entry("errors", $errors)
    )
};

(:
  Copyright 2012-2016 MarkLogic Corporation

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
:)
xquery version "1.0-ml";

module namespace flow = "http://marklogic.com/data-hub/flow-lib";

import module namespace debug = "http://marklogic.com/data-hub/debug-lib"
  at "/com.marklogic.hub/lib/debug-lib.xqy";

import module namespace hul = "http://marklogic.com/data-hub/hub-utils-lib"
  at "/com.marklogic.hub/lib/hub-utils-lib.xqy";

import module namespace functx = "http://www.functx.com"
  at "/MarkLogic/functx/functx-1.0-nodoc-2007-01.xqy";

declare namespace hub = "http://marklogic.com/data-hub";

declare namespace envelope = "http://marklogic.com/data-hub/envelope";

declare option xdmp:mapping "false";

(: xml describing the default writer :)
declare variable $DEFAULT-WRITER :=
  element hub:writer {
    attribute type { "xquery" },
    attribute module { "/com.marklogic.hub/writers/default.xqy" }
  };

(: xml describing the default content plugin :)
declare variable $DEFAULT-CONTENT-PLUGIN :=
  element hub:plugin {
    attribute dest { "content" },
    attribute type { "xquery" },
    attribute module { "/com.marklogic.hub/plugins/raw.xqy" }
  };

(: xml describing a no-op plugin. used when no plugin is given :)
declare variable $NO-OP-PLUGIN :=
  element hub:plugin {
    attribute type { "null" }
  };

(: the directory where entities live :)
declare variable $ENTITIES-DIR := "/entities/";

declare variable $PLUGIN-NS := "http://marklogic.com/data-hub/plugins/";

declare variable $TYPE-XQUERY := "xquery";

declare variable $TYPE-JAVASCRIPT := "javascript";

declare variable $TYPE-XSLT := "xslt";

declare variable $TYPE-XML := "xml";

declare variable $TYPE-JSON := "json";

(:
 : Determines the type of flow given a filename
 :
 : @param $filename - name of the module plugin file
 : @return - a type (xquery, xml, sjs, ...)
 :)
declare %private function flow:get-type($filename) as xs:string?
{
  let $ext as xs:string := hul:get-file-extension($filename)
  return
    if ($ext = hul:get-xqy-extensions()) then
      $TYPE-XQUERY
    else if ($ext = hul:get-sjs-extensions()) then
      $TYPE-JAVASCRIPT
    else if ($ext = hul:get-xslt-extensions()) then
      $TYPE-XSLT
    else if ($ext = "xml") then
      $TYPE-XML
    else if ($ext = "json") then
      $TYPE-JSON
    else
      fn:error(xs:QName("INVALID_PLUGIN"), $filename)
};

(:~
 : Returns xml describing a collector
 :)
declare %private function flow:get-collector(
  $flow-name as xs:string,
  $uris as xs:string*) as element(hub:collector)
{
  let $uri := $uris[1]
  let $filename as xs:string := hul:get-file-from-uri($uri)
  let $type := flow:get-type($filename)
  return
    element hub:collector {
      attribute type { $type },
      attribute module { $uri }
    }
};

(:~
 : Returns xml describing a plugin
 :)
declare %private function flow:get-plugin(
  $flow-name as xs:string,
  $uris as xs:string*,
  $destination as xs:string) as element(hub:plugin)*
{
  for $uri in $uris
  let $filename as xs:string := hul:get-file-from-uri($uri)
  let $type := flow:get-type($filename)
  return
    element hub:plugin {
      attribute dest { $destination },
      attribute type { $type },
      attribute module { $uri }
    }
};

(:~
 : Returns xml describing a content plugin
 :)
declare %private function flow:get-content(
  $flow-name as xs:string,
  $uris as xs:string*)
{
  let $plugin := flow:get-plugin($flow-name, $uris, "content")
  return
    if ($plugin) then $plugin
    else
      $DEFAULT-CONTENT-PLUGIN
};

(:~
 : Returns xml describing a headers plugin
 :)
declare %private function flow:get-headers(
  $flow-name as xs:string,
  $uris as xs:string*)
{
  let $plugin := flow:get-plugin($flow-name, $uris, "headers")
  return
    if ($plugin) then $plugin
    else
      $NO-OP-PLUGIN
};

(:~
 : Returns xml describing a triples plugin
 :)
declare %private function flow:get-triples(
  $flow-name as xs:string,
  $uris as xs:string*)
{
  let $plugin := flow:get-plugin($flow-name, $uris, "triples")
  return
    if ($plugin) then $plugin
    else
      $NO-OP-PLUGIN
};

(:~
 : Returns xml describing a writer
 :)
declare %private function flow:get-writer(
  $flow-name as xs:string,
  $uris as xs:string*)
{
  if ($uris) then
    for $uri in $uris
    let $filename as xs:string := hul:get-file-from-uri($uri)
    let $type := flow:get-type($filename)
    let $_ :=
      if (debug:on()) then
        debug:log((
          "$flow-name: " || $flow-name,
          "$uri: " || $uri,
          "$filename: " || $filename,
          "$type: " || $type
        ))
      else ()
    return
      element hub:writer {
        attribute type { $type },
        attribute module { $uri }
      }
  else
    $DEFAULT-WRITER
};

(:~
 : Returns a flow by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity that owns the flow
 : @param $flow-name - name of the flow to retrieve
 : @return - xml describing the flow
 :)
declare function flow:get-flow(
  $entity-name as xs:string,
  $flow-name as xs:string,
  $flow-type as xs:string?) as element(hub:flow)
{
  let $uris :=
    hul:run-in-modules(function() {
      let $type :=
        if ($flow-type) then $flow-type
        else "*"
      return
        cts:uri-match($ENTITIES-DIR || $entity-name || "/" || $type || "/" || $flow-name || "/*")
    })
  return
    flow:get-flow($entity-name, $flow-name, $flow-type, $uris)
};

(:~
 : Returns a flow by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity that owns the flow
 : @param $flow-name - name of the flow to retrieve
 : @param $uris - uris used to build the entity xml
 : @return - xml describing the flow
 :)
declare %private function flow:get-flow(
  $entity-name as xs:string,
  $flow-name as xs:string,
  $flow-type as xs:string?,
  $uris as xs:string*) as element(hub:flow)
{
  let $_ := xdmp:log(("entity: " || $entity-name, "flow: " || $flow-name, "flow-type: " || $flow-type, "uris:", $uris))
  let $real-flow-type := fn:replace($uris[1], $ENTITIES-DIR || $entity-name || "/([^/]+)/" || $flow-name || ".*$", "$1")
  let $map := map:map()
  let $_ :=
    for $dir in $uris
    let $type :=
      if ($flow-type) then $flow-type
      else "[^/]+"
    let $dir-name := fn:replace($dir, $ENTITIES-DIR || $entity-name || "/" || $type || "/" || $flow-name || "/([^/]+)/$", "$1")
    let $child-uris := $uris[fn:matches(., $ENTITIES-DIR || $entity-name || "/" || $type || "/" || $flow-name || "/" || $dir-name || "/([^/]+)$")]
    return
      switch ($dir-name)
        case "collector" return
          map:put($map, "collector", flow:get-collector($flow-name, $child-uris))
        case "content" return
          map:put($map, "content", flow:get-content($flow-name, $child-uris))
        case "headers" return
          map:put($map, "headers", flow:get-headers($flow-name, $child-uris))
        case "triples" return
          map:put($map, "triples", flow:get-triples($flow-name, $child-uris))
        case "writer" return
          map:put($map, "writer", flow:get-writer($flow-name, $child-uris))
        default return
          ()
  let $flow := hul:run-in-modules(function() {
    let $uri := $ENTITIES-DIR || $entity-name || "/" || $real-flow-type || "/" || $flow-name || "/" || $flow-name || ".xml"
    return
      fn:doc($uri)/hub:flow
  })
  return
    <flow xmlns="http://marklogic.com/data-hub">
      <name>{$flow-name}</name>
      <entity>{$entity-name}</entity>
      <complexity>{ ($flow/hub:complexity/fn:data(), "simple")[1] }</complexity>
      <data-format>{ $flow/hub:data-format/fn:data() }</data-format>
      <type>{$real-flow-type}</type>
      {map:get($map, "collector")}
      <plugins>
      {
        if ($flow/hub:plugins/*) then
          $flow/hub:plugins/*
        else
        (
          (map:get($map, "content"), $DEFAULT-CONTENT-PLUGIN)[1],
          (map:get($map, "headers"), $NO-OP-PLUGIN)[1],
          (map:get($map, "triples"), $NO-OP-PLUGIN)[1]
        )
      }
      </plugins>
      {
        (map:get($map, "writer"), $DEFAULT-WRITER)[1]
      }
    </flow>
};

(:~
 : Returns the flows that belong to the given entity in the database
 :
 : @param $entity-name - the name of the entity containing the flows
 : @return - xml describing the flows
 :)
declare function flow:get-flows(
  $entity-name as xs:string) as element(hub:flows)
{
  let $uris := hul:run-in-modules(function() {
    cts:uri-match($ENTITIES-DIR || "*")
  })
  let $flows :=
    for $flow in $uris[fn:matches(., $ENTITIES-DIR || $entity-name || "/(input|conformance)/[^/]+/$")]
    let $name := fn:replace($flow, $ENTITIES-DIR || $entity-name || "/(input|conformance)/([^/]+)/$", "$2")
    return
      flow:get-flow($entity-name, $name, (), $uris[fn:matches(., $ENTITIES-DIR || $entity-name || "/(input|conformance)/" || $name || "/.+")])
  return
    <flows xmlns="http://marklogic.com/data-hub">
    {
      $flows
    }
    </flows>
};

(:~
 : Returns a entity by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity to retrieve
 : @return - xml describing the entity
 :)
declare function flow:get-entity(
  $entity-name as xs:string)
{
  let $uris :=
    hul:run-in-modules(function() {
      cts:uri-match($ENTITIES-DIR || $entity-name || "/*")
    })
  return
    flow:get-entity($entity-name, $uris)
};

(:~
 : Returns a entity by name. This xml is dynamically constructed
 : by looking in the modules database.
 :
 : @param $entity-name - name of the entity to retrieve
 : @param $uris - uris used to build the entity xml
 : @return - xml describing the entity
 :)
declare %private function flow:get-entity(
  $entity-name as xs:string,
  $uris as xs:string*)
  as element(hub:entity)
{
  <entity xmlns="http://marklogic.com/data-hub">
    <name>{$entity-name}</name>
    {
      flow:get-flows($entity-name)
    }
  </entity>
};

(:~
 : Returns the entities in the database
 :
 : @return - xml describing the entities
 :)
declare function flow:get-entities() as element(hub:entities)
{
  let $uris := hul:run-in-modules(function() {
    cts:uri-match($ENTITIES-DIR || "*")
  })
  let $entities :=
    for $flow in $uris[fn:matches(., $ENTITIES-DIR || "[^/]+/$")]
    let $name := fn:replace($flow, $ENTITIES-DIR || "([^/]+)/$", "$1")
    return
      flow:get-entity($name, $uris[fn:matches(., $ENTITIES-DIR || $name || "/.+")])
  return
    <entities xmlns="http://marklogic.com/data-hub">
    {
      $entities
    }
    </entities>
};

(:~
 : Runs a collector
 :
 : @param $module-uri - the uri of the collector module
 : @param $options - a map of options passed in by the client
 : @return - a sequence of strings
 :)
declare function flow:run-collector(
  $module-uri as xs:string,
  $options as map:map) as xs:string*
{
  let $module-name := hul:get-module-name($module-uri)
  let $filename as xs:string := hul:get-file-from-uri($module-uri)
  let $type := flow:get-type($filename)
  let $ns :=
    if ($type eq $TYPE-JAVASCRIPT) then ()
    else
      $PLUGIN-NS || fn:lower-case($module-name)
  let $func := xdmp:function(fn:QName($ns, "collect"), $module-uri)
  return
    $func($options)
};

(:~
 : Runs a given flow
 :
 : @param $flow - xml describing the flow
 : @param $idenifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $options - a map of options passed in by the client
 : @return - nothing
 :)
declare function flow:run-flow(
  $flow as element(hub:flow),
  $identifier as xs:string,
  $options as map:map) as empty-sequence()
{
  flow:run-flow($flow, $identifier, (), $options)
};

declare function flow:run-plugins(
  $flow as element(hub:flow),
  $identifier as xs:string,
  $content as item()?,
  $options as map:map
)
{
  let $content :=
    map:new((
      map:entry("identifier", $identifier),
      if ($content) then
        map:entry("content", $content)
      else ()
    ))
  let $data-format := $flow/hub:data-format
  let $_ :=
    for $plugin in $flow/hub:plugins/hub:plugin
    let $destination := $plugin/@dest
    let $resp :=
      flow:run-plugin(
        $plugin,
        $data-format,
        $identifier,
        map:get($content, "content"),
        map:get($content, "headers"),
        map:get($content, "triple"),
        $options)
    return
      if (fn:empty($destination))
      then ()
      else map:put($content, $destination, $resp)
  return
    flow:make-envelope($content, $data-format)
};

declare function flow:run-flow(
  $flow as element(hub:flow),
  $identifier as xs:string,
  $content as item()?,
  $options as map:map) as empty-sequence()
{
  let $envelope := flow:run-plugins($flow, $identifier, $content, $options)
  let $_ :=
    xdmp:invoke-function(function() {
      for $writer in $flow/hub:writer
      return
        flow:run-writer($writer, $identifier, $envelope, $options)
    },
    map:new((
      map:entry("isolation", "different-transaction"),
      map:entry("database", xdmp:database("data-hub-FINAL")),
      map:entry("transactionMode", "update-auto-commit")
    )))

  return
    ()
};

(:~
 : Construct an envelope
 :
 : @param $map - a map with all the stuff in it
 : @return - the newly constructed envelope
 :)
declare function flow:make-envelope($map as map:map, $data-format as xs:string)
{
  xdmp:log(("map:", $map)),
  if ($data-format eq "application/json") then
    xdmp:to-json($map)/node()
  else
    <envelope xmlns="http://marklogic.com/data-hub/envelope">
      <headers>{map:get($map, "headers")}</headers>
      <triples>{map:get($map, "triples")}</triples>
      <content>{map:get($map, "content")}</content>
    </envelope>
};

(:~
 : Run a given plugin
 :
 : @param $plugin - xml describing the plugin to run
 : @param $idenifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $content - the output of the content plugin
 : @param $headers - the output of the headers plugin
 : @param $triples - the output of the triples plugin
 : @param $options - a map of options passed in by the client
 : @return - the output of the plugin. It varies.
 :)
declare function flow:run-plugin(
  $plugin as element(hub:plugin),
  $data-format as xs:string,
  $identifier as xs:string,
  $content as item()?,
  $headers as item()*,
  $triples as sem:triple*,
  $options as map:map)
{
  let $module-uri := $plugin/@module
  return
    if (fn:empty($module-uri)) then ()
    else
      let $destination := $plugin/@dest
      let $module-name := hul:get-module-name($module-uri)
      let $filename as xs:string := hul:get-file-from-uri($module-uri)
      let $type := flow:get-type($filename)
      let $ns :=
        if ($type eq $TYPE-JAVASCRIPT) then ()
        else
          $PLUGIN-NS || fn:lower-case($module-name)
      let $func-name :=
        if ($type eq $TYPE-JAVASCRIPT) then
          "create" || functx:capitalize-first($destination)
        else
          "create-" || $destination
      let $func := xdmp:function(fn:QName($ns, $func-name), $module-uri)
      let $resp := $func($identifier, $content, $headers, $triples, $options)
      return
        typeswitch($resp)
          case document-node() return
            if (fn:count($resp/node()) > 1) then
              fn:error("Too Many Nodes!. Return just 1 node")
            else
              $resp/node()
          case json:array return
            if ($data-format = 'application/xml') then
              json:array-values($resp)
            else
              $resp
          default return
            $resp
};

(:~
 : Run a given writer
 :
 : @param $writer - xml describing the writer to run
 : @param $idenifier - the identifier to send to the flow steps (URI in corb lingo)
 : @param $envelope - the envelope
 : @param $options - a map of options passed in by the client
 : @return - the output of the writer. It varies.
 :)
declare function flow:run-writer(
  $writer as element(hub:writer),
  $identifier as xs:string,
  $envelope as item(),
  $options as map:map)
{
  let $module-uri as xs:string := $writer/@module
  let $module-name := hul:get-module-name($module-uri)
  let $ns := $PLUGIN-NS || fn:lower-case($module-name)
  let $func := xdmp:function(fn:QName($ns, "write"), $module-uri)
  return
    $func($identifier, $envelope, $options)
};

declare function flow:make-error-json($ex) {
  map:new((
    map:entry("msg", $ex/error:format-string/fn:data()),
    let $f := $ex/error:stack/error:frame[1]
    return
      (
        map:entry("uri", $f/error:uri/fn:data()),
        map:entry("line", $f/error:line/fn:data()),
        map:entry("column", $f/error:column/fn:data())
      )
  ))
};

declare function flow:validate-entities()
{
  let $errors := json:array()
  let $options := map:map()
  let $_ :=
    for $entity in flow:get-entities()/hub:entity
    for $flow in $entity/hub:flows/hub:flow
    let $data-format := $flow/hub:data-format
    (: validate collector :)
    let $_ :=
      let $collector := $flow/hub:collector
      return
        if ($collector) then
          try {
            flow:run-collector($collector/@module, $options)
          }
          catch($ex) {
            json:array-push($errors, flow:make-error-json($ex))
          }
        else ()
    (: validate plugins :)
    let $_ :=
      for $plugin in $flow/hub:plugins/hub:plugin
      let $destination := $plugin/@dest
      let $module-uri := $plugin/@module
      let $module-name := hul:get-module-name($module-uri)
      let $filename as xs:string := hul:get-file-from-uri($module-uri)
      let $type := flow:get-type($filename)
      let $ns :=
        if ($type eq $flow:TYPE-JAVASCRIPT) then ()
        else
          $flow:PLUGIN-NS || fn:lower-case($module-name)
      let $func-name :=
        if ($type eq $flow:TYPE-JAVASCRIPT) then
          "create" || functx:capitalize-first($destination)
        else
          "create-" || $destination
      return
        try {
          if ($type eq $flow:TYPE-XQUERY) then
            xdmp:eval(
              'import module namespace x = "' || $ns || '" at "' || $module-uri || '"; ' ||
              '()',
              map:new(map:entry("staticCheck", fn:true()))
            )
          else
            xdmp:javascript-eval(
              'var x = require("' || $module-uri || '");',
              map:new(map:entry("staticCheck", fn:true()))
            )
        }
        catch($ex) {
          json:array-push($errors, flow:make-error-json($ex))
        }
    return
      ()
  return
    map:new(
      map:entry("errors", $errors)
    )
};


surveying collchunks for text chapter vol letter headings

let $collChunks := collection('collationChunks/?select=*.xml') => sort(substring-before(substring-after(tokenize(base-uri(), '/')[last()], '_C'), '.xml'))
(:for $c in $collChunks
let $hi := $c//hi/@sID/string():)
let $head := $collChunks//head[@sID]
(:for $h in $head
let $text := $h/following-sibling::text()[following::head[@eID eq $h/@sID]]:)
(:return concat(string-join($text, ' '), '
'):)
let $milestone := $collChunks//milestone[not(@unit='tei:p')][not(@* = ('tei:lg', 'tei:seg', 'tei:head', 'end'))]
let $mileheads := ($milestone)
for $m in $mileheads
let $u := $m/@unit/string()
let $n := $m/@n/string()
let $Atts := string-join($m/@*[not(name() eq 'type')], ', ')
let $text := string-join($m/following::text()[not(matches(., '^\s+$'))][position() lt 5][following-sibling::*[1] eq $m/following-sibling::*[1]] ! normalize-space(), ' ')
return concat($u, ' ', $n, ' ', substring($text, 1, 50), '
')

(:replace(substring($text, 1, 30), '([ “]?[A-Z])\s*([A-Z])', '$1$2'):)

error: pathspec 'collateXPrep/collationChunksSurvey.xquery' did not match any file(s) known to git


added script for bug 1076161, stable output is missing (will add this later today) implicitly the bug is tested in Q15 of the XPathMark.

error: pathspec 'pathfinder/tests/BugDay_2005-11-09_0.9.3/Tests/self_clause.SF-1076161.xq' did not match any file(s) known to git


adding helper-module for analyzing/summarizing xml

module namespace crday  = "http://aac.ac.at/content_repository/data-ay";

import module namespace repo-utils =  "http://aac.ac.at/content_repository/utils" at  "repo-utils.xqm";

import module namespace xdb="http://exist-db.org/xquery/xmldb";
(:import module namespace diag =  "http://www.loc.gov/zing/srw/diagnostic/" at  "modules/diagnostics/diagnostics.xqm";
import module namespace util="http://exist-db.org/xquery/util";
import module namespace kwic="http://exist-db.org/xquery/kwic";
:)
declare namespace sru = "http://www.loc.gov/zing/srw/";
declare namespace fcs = "http://clarin.eu/fcs/1.0";
(:declare namespace tei = "http://www.tei-c.org/ns/1.0";:)

declare variable $crday:docTypeTerms := "Terms";

(:
declare function cmd:base-dbcoll($config) as item()* {
    collection(repo-utils:config-value($config, 'data.path'))
};
:)

(:~ analyzes the xml-structure - sub-elements and text-nodes
in the context of given collection, starting from given xpath

@param $context nodeset to analyze
calls elem-r for recursive processing

@returns xml-with paths and numbers
:)
declare function crday:ay-xml($context as item()*, $path as xs:string, $depth as xs:integer ) as element() {

  (:let $collection := collection($cr:dataPath),
  if ($collections[1] eq $cr:collectionRoot) then
  util:eval(fn:concat("$collection/descendant::IsPartOf[ft:query(., <query><term>", xdb:decode($coll), "</term></query>)]/ancestor-or-self::CMD/descendant-or-self::", $path))
  :)

  let $path-nodes := util:eval(fn:concat("$context/descendant-or-self::", $path))

  let $entries := crday:elem-r($path-nodes, $path, $depth, $depth),
(:      $coll-names-value := if (fn:empty($collections)) then () else attribute colls {fn:string-join($collections, ",")},:)
	  $result := element {$crday:docTypeTerms} {
(:      		  $coll-names-value,:)
      		  attribute depth {$depth},
      		  attribute created {fn:current-dateTime()},
      		  $entries
		}
    return $result
};

(:~ goes down the xml-structure recursively
:)
declare function crday:elem-r($path-nodes as node()*, $path as xs:string, $max-depth as xs:integer, $depth as xs:integer) as element() {
      let $path-count := count($path-nodes),
	$child-elements := $path-nodes/child::element(),
	$subs := distinct-values($child-elements/name()),
	$nodes-child-terminal := if (empty($child-elements)) then $path-nodes else () (: Maybe some selected elements $child-elements[not(element())] later on :),
	$text-nodes := $nodes-child-terminal/text(),
	$text-count := count($text-nodes),
	$text-count-distinct := count(distinct-values($text-nodes))
	return
(:	<Term path="{fn:concat("//", $path)}" name="{text:groups($path, "/([^/]+)$")[last()]}" count="{$path-count}" count_text="{$text-count}"  count_distinct_text="{$text-count-distinct}">{ :)
	<Term path="{fn:concat("//", $path)}" name="{(text:groups($path, "/([^/]+)$")[last()],$path)[1] }" count="{$path-count}" count_text="{$text-count}"  count_distinct_text="{$text-count-distinct}">{
	  if ($depth > 0) then
	    (for $elname in $subs[. != '']
	    return
	      crday:elem-r(util:eval(concat("$path-nodes/", $elname)), concat($path, '/', $elname), $max-depth, $depth - 1)
			(: values moved to own function: scanIndex
		      if ($max-depth eq 1 and $text-count gt 0) then cr:values($path-nodes) else ()) :)
						)
	  else 'maxdepth'
	}</Term>
};

(:~ analyze xml-structure
  API function queryModel.
:)
(:declare function crday:ay-xml-wrap($init-xpath as xs:string, $collection as xs:string+, $max-depth as xs:integer) as item()? {

  let $name := repo-utils:gen-cache-id("model", ($collection, $cmd-index-path), xs:string($max-depth)),
    $doc :=
    if (repo-utils:is-in-cache($name)) then
      repo-utils:get-from-cache($name)
    else
      let $data := cr:elem($collection, $cmd-index-path, $max-depth)
        return repo-utils:store-in-cache($name, $data)

  return $doc
};
:)

error: pathspec 'fcs/wrapper/cr-xq/crday.xqm' did not match any file(s) known to git


Adding the returning of facets, if there are any

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/lcahlander/mysearch.git/'


xenaQuery to find genes within a genomic coordinates

; refGeneRange
(fn [dataset chr start end]
	(xena-query {:select ["position (2)" "position" "exonCount" "exonStarts" "exonEnds" "name2"]
				:from [dataset]
				:where [:in "position" [[chr start end]]]}))

error: pathspec 'js/queries/refGeneRange.xq' did not match any file(s) known to git


Add BaseX Unit Module function signatures.

xquery version "3.0";
(:~
: BaseX Unit Module functions
:
: @see http://docs.basex.org/wiki/Unit_Module
:)
module namespace unit = "http://basex.org/modules/unit";

import module namespace a = "http://reecedunn.co.uk/xquery/annotations" at "res://reecedunn.co.uk/xquery/annotations.xqy";

(:
BaseX Unit Module annotations
@TODO - do these need to be included?
%unit:test
%unit:test("expected", CODE) %unit:test("expected" as xs:string, xs:QName)???
%unit:before
%unit:before(FUNCTION)
%unit:after
%unit:after(FUNCTION)
%unit:before-module
%unit:after-module
%unit:ignore
%unit:ignore(MESSAGE)
:)

declare %a:since("basex", "7.7") function unit:assert($test as item()*) as empty-sequence() external;
declare %a:since("basex", "7.7") function unit:assert($test as item()*, $info as item()) as empty-sequence() external;
declare %a:since("basex", "7.8") function unit:assert-equals($returned as item()*, $expected as item()*) as empty-sequence() external;
declare %a:since("basex", "7.8") function unit:assert-equals($returned as item()*, $expected as item()*, $info as item()) as empty-sequence() external;
declare %a:since("basex", "8.0") function unit:fail() as empty-sequence() external;
declare %a:since("basex", "7.7") function unit:fail($info as item()) as empty-sequence() external;
declare %a:since("basex", "7.7") %a:until("basex", "7.9") function unit:test() as element(testsuite)* external;
declare %a:since("basex", "7.7") %a:until("basex", "7.9") function unit:test($functions as function(*)*) as element(testsuite)* external;
declare %a:since("basex", "7.7") %a:until("basex", "7.9") function unit:test-uris($uris as xs:string*) as element(testsuites) external;

error: pathspec 'src/main/resources/builtin/basex.org/modules/unit.xqy' did not match any file(s) known to git


Add BaseX Higher-Order Function Module function signatures.

xquery version "3.0";
(:~
: BaseX Higher Order Functions Module
:
: @see http://docs.basex.org/wiki/Higher-Order_Functions_Module
:)
module namespace hof = "http://basex.org/modules/hof";

import module namespace a = "http://reecedunn.co.uk/xquery/annotations" at "res://reecedunn.co.uk/xquery/annotations.xqy";

declare %a:since("basex", "7.0") function hof:fold-left1($seq as item()+, $f as function(item()*, item()) as item()*) as item()* external;
declare %a:since("basex", "7.0") function hof:until($pred as function(item()*) as xs:boolean, $f as function(item()*) as item()*, $start as item()*) as item()* external;
declare %a:since("basex", "8.1") function hof:scan-left($seq as item()*, $start as item()*, $f as function(item()*, item()) as item()*) as item()* external;
declare %a:since("basex", "8.1") function hof:take-while($seq as item()*, $pred as function(item()) as xs:boolean) as item()* external;
declare %a:since("basex", "7.2") function hof:top-k-by($seq as item()*, $sort-key as function(item()) as item(), $k as xs:integer) as item()* external;
declare %a:since("basex", "7.2") function hof:top-k-with($seq as item()*, $lt as function(item(), item()) as xs:boolean, $k as xs:integer) as item()* external;
declare %a:since("basex", "7.0") function hof:id($expr as item()*) as item()* external;
declare %a:since("basex", "7.0") function hof:const($expr as item()*, $ignored as item()*) as item()* external;
declare %a:since("basex", "7.0") %a:until("basex", "7.2") function hof:iterate($f as function(item()*, item()*) as item()*, $seq as item()*) as item()* external;

error: pathspec 'src/main/resources/builtin/basex.org/modules/hof.xqy' did not match any file(s) known to git


[MIN] Unit tests: error.xqm

[MIN] Unit tests: error.xqm

error: pathspec 'basex-core/src/test/resources/error.xqm' did not match any file(s) known to git


Adding updated hfr_facilities_to_csd.xq

Adding updated hfr_facilities_to_csd.xq

import module namespace dxf2csd = "http://dhis2.org/csd/dxf/2.0";
import module namespace csd_webconf =  "https://github.com/openhie/openinfoman/csd_webconf";
import module namespace csd_dm = "https://github.com/openhie/openinfoman/csd_dm";
import module namespace functx = "http://www.functx.com";

declare namespace csd = "urn:ihe:iti:csd:2013";

declare variable $careServicesRequest as item() external;

let $doc_name := string($careServicesRequest/@resource)
let $doc := csd_dm:open_document($csd_webconf:db,$doc_name)
let $org_dir := $doc/csd:CSD/csd:organizationDirectory

let $oid := $careServicesRequest/oid
let $csv_src := $careServicesRequest/csv
let $csv :=    csv:parse($csv_src, map{ 'header': true()})/csv/record

let $namespace_uuid := "7ee93e32-78da-4913-82f8-49eb0a618cfc"

let $generate_UUID_v3 := function($name) {
  concat('urn:uuid:' , dxf2csd:generate_UUID_v3($name,$namespace_uuid))
}

let $zone_csd :=
for $zone_record in distinct-values($csv/Zone)
 let $now := current-dateTime()
 let $zone_urn := $generate_UUID_v3(concat('zone:',$zone_record))
  return
   <csd:organization entityID="{$zone_urn}">
     <csd:codedType code="1" codingScheme="{$oid}"/>
     <csd:primaryName>{$zone_record}</csd:primaryName>
     <csd:record
       created="{$now}"
       updated="{$now}"
       sourceDirectory="http://hfrportal.ehealth.go.tz"/>
   </csd:organization>

let $region_csd :=
for $region_record in distinct-values($csv/Region)
 let $now := current-dateTime()
 let $region_urn := $generate_UUID_v3(concat('region:',$region_record))
 let $parent_urns :=
 for $record in $csv
  let $zone := $record/Zone
  let $region := $record/Region
  return if($region_record=$region)
   then $generate_UUID_v3(concat('zone:',$zone))
   else ()
 return
   <csd:organization entityID="{$region_urn}">
     <csd:codedType code="2" codingScheme="{$oid}"/>
     <csd:primaryName>{$region_record}</csd:primaryName>
     <csd:record
       created="{$now}"
       updated="{$now}"
       sourceDirectory="http://hfrportal.ehealth.go.tz"/>
     <csd:parent entityID="{$parent_urns[last()]}"/>
   </csd:organization>

let $district_csd :=
for $district_record in distinct-values($csv/District)
 let $now := current-dateTime()
 let $district_urn := $generate_UUID_v3(concat('district:',$district_record))
 let $parent_urns :=
 for $record in $csv
  let $region := $record/Region
  let $district := $record/District
  return if($district_record=$district)
   then $generate_UUID_v3(concat('region:',$region))
   else ()
 return
   <csd:organization entityID="{$district_urn}">
     <csd:codedType code="3" codingScheme="{$oid}"/>
     <csd:primaryName>{$district_record}</csd:primaryName>
     <csd:record
       created="{$now}"
       updated="{$now}"
       sourceDirectory="http://hfrportal.ehealth.go.tz"/>
     <csd:parent entityID="{$parent_urns[last()]}"/>
   </csd:organization>

let $council_csd :=
for $council_record in distinct-values($csv/Council)
 let $now := current-dateTime()
 let $council_urn := $generate_UUID_v3(concat('council:',$council_record))
 let $parent_urns :=
 for $record in $csv
  let $district := $record/District
  let $council := $record/Council
  return if($council_record=$council)
   then $generate_UUID_v3(concat('district:',$district))
   else ()
 return
   <csd:organization entityID="{$council_urn}">
     <csd:codedType code="4" codingScheme="{$oid}"/>
     <csd:primaryName>{$council_record}</csd:primaryName>
     <csd:record
       created="{$now}"
       updated="{$now}"
       sourceDirectory="http://hfrportal.ehealth.go.tz"/>
     <csd:parent entityID="{$parent_urns[last()]}"/>
   </csd:organization>

let $ward_csd :=
for $ward_record in distinct-values($csv/Ward)
 let $now := current-dateTime()
 let $ward_urn := $generate_UUID_v3(concat('ward:',$ward_record))
 let $parent_urns :=
 for $record in $csv
  let $council := $record/Council
  let $ward := $record/Ward
  return if($ward_record=$ward)
   then $generate_UUID_v3(concat('council:',$council))
   else ()
 return if (not (contains($ward_record,"Not set"))) then
   <csd:organization entityID="{$ward_urn}">
     <csd:codedType code="5" codingScheme="{$oid}"/>
     <csd:primaryName>{$ward_record}</csd:primaryName>
     <csd:record
       created="{$now}"
       updated="{$now}"
       sourceDirectory="http://hfrportal.ehealth.go.tz"/>
     <csd:parent entityID="{$parent_urns[last()]}"/>
   </csd:organization>
   else ()

let $village_csd :=
for $village_record in distinct-values($csv/VillageStreet)
 let $now := current-dateTime()
 let $village_urn := $generate_UUID_v3(concat('village:',$village_record))
 let $parent_urns :=
 for $record in $csv
  let $ward := if ((contains($record/Ward,"Not set"))) then $record/Council else($record/Ward)
  let $village := $record/VillageStreet
  return if($village_record=$village)
   then $generate_UUID_v3(concat('ward:',$ward))
   else ()
 return if (not (contains($village_record,"Not set"))) then
   <csd:organization entityID="{$village_urn}">
     <csd:codedType code="6" codingScheme="{$oid}"/>
     <csd:primaryName>{$village_record}</csd:primaryName>
     <csd:record
       created="{$now}"
       updated="{$now}"
       sourceDirectory="http://hfrportal.ehealth.go.tz"/>
     <csd:parent entityID="{$parent_urns[last()]}"/>
   </csd:organization>
   else ()

let $facility_csd :=
for $record in $csv
 let $facNum := $record/FacilityNumber
 let $facName  := $record/FacilityName
 let $now := current-dateTime()
 let $facility_urn := $generate_UUID_v3(concat('facility:',$facNum))
 let $parent := if ((contains($record/VillageStreet,"Not set"))) then
                     (if ((contains($record/Ward,"Not set"))) then $record/Council else $record/Ward)
                     else($record/VillageStreet)
 let $parent_string := if ((contains($record/VillageStreet,"Not set"))) then
                     (if ((contains($record/Ward,"Not set"))) then "council:" else "ward:")
                     else("village:")
 let $parent_urn := $generate_UUID_v3(concat($parent_string,$parent))
 return if (not (contains($facName,"Not set"))) then
   <csd:organization entityID="{$facility_urn}">
     <csd:otherID assigningAuthorityName='health_facility_registry' code="{$facNum}"/>
     <csd:codedType code="7" codingScheme="{$oid}"/>
     <csd:primaryName>{$facName/text()}</csd:primaryName>
     <csd:record
       created="{$now}"
       updated="{$now}"
       sourceDirectory="http://hfrportal.ehealth.go.tz"/>
     <csd:parent entityID="{$parent_urn}"/>
   </csd:organization>
   else ()

let $csd :=
  <csd:CSD>
    <csd:organizationDirectory>
      {($zone_csd,$region_csd,$district_csd,$council_csd,$ward_csd,$village_csd,$facility_csd)}
    </csd:organizationDirectory>
    <csd:serviceDirectory/>
    <csd:facilityDirectory/>
    <csd:providerDirectory/>
  </csd:CSD>

return  csd_dm:add($csd_webconf:db,$csd,$doc_name)

error: pathspec 'resources/stored_updating_query_definitions/hfr_facilities_to_csd.xq' did not match any file(s) known to git


Add BaseX HTML Module function signatures.

xquery version "3.0";
(:~
: BaseX HTML Functions Module
:
: @see http://docs.basex.org/wiki/HTML_Module
:)
module namespace html = "http://basex.org/modules/html";

import module namespace a = "http://reecedunn.co.uk/xquery/annotations" at "res://reecedunn.co.uk/xquery/annotations.xqy";

declare %a:since("basex", "7.6") function html:parser() as xs:string external;
declare %a:since("basex", "7.6") function html:parse($input as xs:anyAtomicType) as document-node() external;
declare %a:since("basex", "7.6") function html:parse($input as xs:anyAtomicType, $options as map(*)?) as document-node() external;

error: pathspec 'src/main/resources/builtin/basex.org/modules/html.xqy' did not match any file(s) known to git


Add BaseX Stream Module function signatures.

xquery version "3.0";
(:~
: BaseX Stream Module functions
:
: @see http://docs.basex.org/wiki/Stream_Module
: @see http://docs.basex.org/wiki/Lazy_Module
:)
module namespace stream = "http://basex.org/modules/stream";

import module namespace a = "http://reecedunn.co.uk/xquery/annotations" at "res://reecedunn.co.uk/xquery/annotations.xqy";

declare %a:since("basex", "7.7") function stream:materialize($value as item()*) as item()* external;
declare %a:since("basex", "7.7") function stream:is-streamable($item as item()) as item() external;

error: pathspec 'src/main/resources/builtin/basex.org/modules/stream.xqy' did not match any file(s) known to git


First XQuery datetime test.

First XQuery datetime test.

import module namespace dt = "http://www.zorba-xquery.com/modules/datetime";

dt:parse-date( "1/2/1903", "%D" )

error: pathspec 'test/rbkt/Queries/zorba/datetime/datetime-parse-date-1.xq' did not match any file(s) known to git


Added xq file expressing query 4 using XQuery

Web data management & xml (CSE 5335)/Programming assignment 7/Individual queries/4.xq

/bin/sh: 1: Syntax error: word unexpected (expecting ")")


test: add xqsuite for random-number-generator

xquery version "3.1";

module namespace fn-rng="http://exist-db.org/xquery/test/fnRandomNumberGenerator";

declare namespace test="http://exist-db.org/xquery/xqsuite";

declare variable $fn-rng:long-seed := 123456789;
declare variable $fn-rng:text-seed := 'sample seed';
declare variable $fn-rng:date-seed := xs:date('1970-01-01');
declare variable $fn-rng:dateTime-seed := xs:dateTime('1970-01-01T00:00:00.000Z');

declare
    %test:assertExists
function fn-rng:seed-number () {
    fn:random-number-generator($fn-rng:long-seed)
};

declare
    %test:assertExists
function fn-rng:seed-text () {
    fn:random-number-generator($fn-rng:text-seed)
};

declare
    %test:assertExists
function fn-rng:seed-date () {
    fn:random-number-generator($fn-rng:date-seed)
};

declare
    %test:assertExists
function fn-rng:seed-dateTime () {
    fn:random-number-generator($fn-rng:dateTime-seed)
};

declare
    %test:assertExists
function fn-rng:seed-current-dateTime () {
    fn:random-number-generator(fn:current-dateTime())
};

declare
    %test:assertTrue
function fn-rng:deterministic () {
    fn:random-number-generator($fn-rng:long-seed)?number eq
    fn:random-number-generator($fn-rng:long-seed)?number
};

declare
    %test:assertTrue
function fn-rng:deterministic-next () {
    fn:random-number-generator($fn-rng:long-seed)?next()?number eq
    fn:random-number-generator($fn-rng:long-seed)?next()?number
};

declare
    %private
function fn-rng:get-generator-reference () {
    fn:random-number-generator($fn-rng:long-seed)
};

declare
    %test:assertTrue
function fn-rng:deterministic-reference () {
    let $fn := fn-rng:get-generator-reference()
    return
        $fn?number eq
        $fn?number
};

declare
    %test:assertTrue
function fn-rng:deterministic-reference-next () {
    let $fn := fn-rng:get-generator-reference()
    return
        $fn?next()?number eq
        $fn?next()?number
};

declare variable $fn-rng:generator-reference := fn-rng:get-generator-reference();

declare
    %private
function fn-rng:number-from-generator-reference () {
    $fn-rng:generator-reference?next()?number
};

declare
    %test:assertTrue
function fn-rng:deterministic-side-effect () {
    let $call := fn-rng:number-from-generator-reference()

    return
        fn-rng:number-from-generator-reference() eq
        $fn-rng:generator-reference?next()?number
};

error: pathspec 'exist-core/src/test/xquery/xquery3/fnRandomNumberGenerator.xql' did not match any file(s) known to git


[ignore] Script to rewrite all documentation links so they point to the static HTML pages on exist-db.org.

svn path=/trunk/eXist/; revision=15321

(:
    For the demo server: rewrite all links to documentation pages and
    replace them with the static files on exist-db.org
:)
xquery version "1.0";

declare namespace xhtml="http://www.w3.org/1999/xhtml";

declare option exist:serialize "method=xhtml media-type=text/html";

(:
    XML=`ls -1 *.xml`; for F in $XML; do echo "<map url='$F' replace='$F'/>"; done
:)
declare variable $local:MAP :=
    <mapping>
        <map url='acknowledge.xml' replace='acknowledge.html'/>
        <map url='ant-tasks.xml' replace='ant-tasks.html'/>
        <map url='atompub.xml' replace='atompub.html'/>
        <map url='backup.xml' replace='backup.html'/>
        <map url='beginners-guide-to-xrx-v4.xml' replace='beginners-guide-to-xrx-v4.html'/>
        <map url='building.xml' replace='building.html'/>
        <map url='client.xml' replace='client.html'/>
        <map url='cluster.xml' replace='cluster.html'/>
        <map url='community.xml' replace='community.html'/>
        <map url='config-cache.xml' replace='config-cache.html'/>
        <map url='config-compression.xml' replace='config-compression.html'/>
        <map url='config-context.xml' replace='config-context.html'/>
        <map url='config-counter.xml' replace='config-counter.html'/>
        <map url='config-datetime.xml' replace='config-datetime.html'/>
        <map url='config-example.xml' replace='config-example.html'/>
        <map url='config-file.xml' replace='config-file.html'/>
        <map url='config-fluent.xml' replace='config-fluent.html'/>
        <map url='config-openid.xml' replace='config-openid.html'/>
        <map url='configuration.xml' replace='configuration.html'/>
        <map url='config-versioning.xml' replace='config-versioning.html'/>
        <map url='config-xqdoc.xml' replace='config-xqdoc.html'/>
        <map url='config-xslt.xml' replace='config-xslt.html'/>
        <map url='credits.xml' replace='credits.html'/>
        <map url='debugger.xml' replace='debugger.html'/>
        <map url='deployment.xml' replace='deployment.html'/>
        <map url='devguide_codereview.xml' replace='devguide_codereview.html'/>
        <map url='devguide_indexes.xml' replace='devguide_indexes.html'/>
        <map url='devguide_log4j.xml' replace='devguide_log4j.html'/>
        <map url='devguide_manifesto.xml' replace='devguide_manifesto.html'/>
        <map url='devguide_rest.xml' replace='devguide_rest.html'/>
        <map url='devguide_soap.xml' replace='devguide_soap.html'/>
        <map url='devguide.xml' replace='devguide.html'/>
        <map url='devguide_xmldb.xml' replace='devguide_xmldb.html'/>
        <map url='devguide_xmlrpc.xml' replace='devguide_xmlrpc.html'/>
        <map url='devguide_xquery.xml' replace='devguide_xquery.html'/>
        <map url='documentation.xml' replace='documentation.html'/>
        <map url='download.xml' replace='download.html'/>
        <map url='eclipse.xml' replace='eclipse.html'/>
        <map url='exist-stack.xml' replace='exist-stack.html'/>
        <map url='extensions.xml' replace='extensions.html'/>
        <map url='facts.xml' replace='facts.html'/>
        <map url='ftlegacy.xml' replace='ftlegacy.html'/>
        <map url='function_modules.xml' replace='function_modules.html'/>
        <map url='header.xml' replace='header.html'/>
        <map url='indexing.xml' replace='indexing.html'/>
        <map url='index.xml' replace='index.html'/>
        <map url='installing-exist-on-amazon-ec2.xml' replace='installing-exist-on-amazon-ec2.html'/>
        <map url='jmx.xml' replace='jmx.html'/>
        <map url='journal.xml' replace='journal.html'/>
        <map url='kwic.xml' replace='kwic.html'/>
        <map url='ldap-security.xml' replace='ldap-security.html'/>
        <map url='lucene.xml' replace='lucene.html'/>
        <map url='ngram.xml' replace='ngram.html'/>
        <map url='production_good_practice.xml' replace='production_good_practice.html'/>
        <map url='production_web_proxying.xml' replace='production_web_proxying.html'/>
        <map url='quickstart.xml' replace='quickstart.html'/>
        <map url='readying-centos-for-exist.xml' replace='readying-centos-for-exist.html'/>
        <map url='roadmap.xml' replace='roadmap.html'/>
        <map url='scheduler.xml' replace='scheduler.html'/>
        <map url='security.xml' replace='security.html'/>
        <map url='sidebar.xml' replace='sidebar.html'/>
        <map url='template.xml' replace='template.html'/>
        <map url='triggers.xml' replace='triggers.html'/>
        <map url='tuning.xml' replace='tuning.html'/>
        <map url='ubuntu-server.xml' replace='ubuntu-server.html'/>
        <map url='update_ext.xml' replace='update_ext.html'/>
        <map url='upgrading.xml' replace='upgrading.html'/>
        <map url='urlrewrite.xml' replace='urlrewrite.html'/>
        <map url='validation.xml' replace='validation.html'/>
        <map url='versioning.xml' replace='versioning.html'/>
        <map url='webdav.xml' replace='webdav.html'/>
        <map url='xacml-dev.xml' replace='xacml-dev.html'/>
        <map url='xacml-features.xml' replace='xacml-features.html'/>
        <map url='xacml-intro.xml' replace='xacml-intro.html'/>
        <map url='xacml-usage.xml' replace='xacml-usage.html'/>
        <map url='xacml.xml' replace='xacml.html'/>
        <map url='xinclude.xml' replace='xinclude.html'/>
        <map url='xmlprague06.xml' replace='xmlprague06.html'/>
        <map url='xquery.xml' replace='xquery.html'/>
    </mapping>
;

declare function local:transform($node as node()) {
    typeswitch ($node)
        case document-node() return
            local:transform($node/*)
        case element(xhtml:a) return
            let $link :=
                if (contains($node/@href, "/")) then
                    replace($node/@href, "^.*/([^/]+)$", "$1")
                else
                    $node/@href
            return
                if ($local:MAP/map[@url = $link]) then
                    <xhtml:a href="http://exist-db.org/{$local:MAP/map[@url = $link]/@replace}">
                    {
                        $node/@*[local-name(.) != 'href'],
                        for $child in $node/node() return local:transform($child)
                    }
                    </xhtml:a>
                else
                    $node
        case element() return
            element { node-name($node) } {
                $node/@*,
                for $child in $node/node() return local:transform($child)
            }
        default return
            $node
};

let $input := request:get-data()
let $log := util:log("DEBUG", $input)
return
    local:transform($input)

error: pathspec 'webapp/fixlinks.xql' did not match any file(s) known to git


extruded Wolf's xquery test into library

svn path=/trunk/eXist/; revision=11528

xquery version "1.0";

module namespace t="http://exist-db.org/xquery/testing";

import module namespace xdb="http://exist-db.org/xquery/xmldb";

declare function t:setup-action($action) {
    typeswitch ($action)
        case element(create-collection) return
            xdb:create-collection($action/@parent, $action/@name)
        case element(store) return
			t:store($action)
        case element(remove-collection) return
            xdb:remove($action/@collection)
        default return
            ()
};

declare function t:store($action as element(store)) {
    let $type := if ($action/@type) then $action/@type/string() else "application/xml"
    let $data :=
		if ($action/*) then
			$action/*[1]
		else
			$action/string()
	return
        xdb:store($action/@collection, $action/@name, $data, $type)
};

declare function t:setup($setup as element(setup)) {
    for $action in $setup/*
    return
        t:setup-action($action)
};

declare function t:tearDown($tearDown as element(tearDown)?) {
    for $action in $tearDown/*
    return
        t:setup-action($action)
};

declare function t:run-test($test as element(test), $count as xs:integer) {

	let $context :=
	    string-join(
        	for $var in $test/../variable
        	return
        	    concat("declare variable $", $var/@name, " := ", util:serialize($var/*, ()), ";"),
            ""
        )
    let $output :=
		util:catch("*",
			util:eval(concat($context, $test/code/string())),
			<error>Compilation error: {$util:exception-message}</error>
		)
    let $expected :=
        if ($test/@output eq 'text') then
            data($test/expected)
        else $test/expected/node()
    let $OK :=
        if ($test/xpath) then
            exists(t:xpath($output, $test/xpath))
        else if ($test/@output eq 'text') then
            normalize-space(string-join(for $x in $output return string($x),' ')) eq normalize-space($expected)
        else
            deep-equal($output, $expected)
    return
        <test n="{$count}" pass="{$OK}">
        {
            if (not($OK)) then
                ($test/task, $test/expected, <result>{$output}</result>)
            else ()
        }
        </test>
};

declare function t:xpath($output as item()*, $xpath as element()) {
    for $prefix in in-scope-prefixes($xpath)
    let $nsuri := namespace-uri-for-prefix($prefix, $xpath)
    return
        if ($prefix != 'xml') then
        util:declare-namespace($prefix, $nsuri)
        else (),
    util:eval(concat("$output", $xpath))
};

declare function t:run-testSet($set as element(TestSet)) {
    let $null := t:setup($set/setup)
    let $result :=
        <TestSet>
        {$set/testName}
        {$set/description}
        {
            for $test at $p in $set/test
            return
                t:run-test($test, $p)
        }
        </TestSet>
    let $null := t:tearDown($set/tearDown)
    return $result
};

declare function t:format-testResult($result) {
    let $formatted :=

{$result/testName}

{$result/description/p}

          {for $test in $result//test
           return

                    {$test/@n/.}
                    {$test/@pass/.}

          }

     return
        $formatted
};

error: pathspec 'src/org/exist/xquery/lib/test.xq' did not match any file(s) known to git