Expansion of scoped key names appears to be correct

(: =====================================================

   DITA Link Manager Model

   Model component of Model/View/Controller. Maintains
   the link management data.

   Provides queries to create and update resource use records,
   dependency indexes, etc.

   These queries are called whenever new content is added to
   given database.

   Author: W. Eliot Kimber

   Copyright (c) 2015 DITA For Small Teams
   Licensed under Apache License 2

   ===================================================== :)

module namespace lmm="http://dita-for-small-teams.org/xquery/modules/linkmgr-model";

declare namespace dfst="http://dita-for-small-teams.org";

import module namespace df="http://dita-for-small-teams.org/xquery/modules/dita-utils";
import module namespace bxutil="http://dita-for-small-teams.org/xquery/modules/basex-utils";
import module namespace lmutil="http://dita-for-small-teams.org/xquery/modules/linkmgmt-utils";
import module namespace dfstcnst="http://dita-for-small-teams.org/xquery/modules/dfst-constants";
import module namespace relpath="http://dita-for-small-teams.org/xquery/modules/relpath-utils";

(: Given an element with an @id value, construct the unique resource key for it.
   For a given element the resource ID is guaranteed to be unique within
   a snapshot (git commit).

   @param elem Element to get resource ID for.
   @returns Resource key string.

   The resource key is a combination of the absolute URI of the containing
   document and the element's tree position within the document, producing
   a unique key for any element in any document on a given snapshot (version
   in time). Resource keys are not reliably unique across snapshots as the position of
   the element within its containing document could change from version to
   version.

   The resource key is used to look up the element in where-used records, either
   as the element used or the element doing the use (links).

 :)
declare function lmm:constructResourceKeyForElement($elem as element()) as xs:string {
  let $targetDocHash := hash:md5(document-uri(root($elem)))
  let $treepos := for $anc in ($elem/ancestor-or-self::*)
                      return string(count($anc | $anc/preceding-sibling::*))
  let $key := string-join($treepos, '.')
  return $targetDocHash || '^' || $key

};

(:~
 : Create or update the link management indexes for the specified repository
 :
 :)
declare %updating function lmm:updateLinkManagementIndexes(
                               $contentDbName as xs:string,
                               $metadataDbName as xs:string) {
   (: Query the database to find all links and for each link, record a resource use
      record in the database.

      Every addressible element has a "resource key" which combines the URI of
      the containing document and the element's @id value (only elements with
      @id attributes are addressible.

      The link management indexes are stored in a separate database from
      the content database: every branch-specific database has a corresponding
      metadata database that contains the where-used records, resolved maps,
      key space information, and any other metadata you might want to store.

      For each document used there is a directory whose name is the hash
      of the document's URI (which will be unique within a branch). Within
      that directory are the use records for elements within that document.
      This arrangement has the effect of providing a reference count for
      for each document, making it quick to determine if a given document
      is used by any links. It also allows for narrowing the scope of
      where-used queries to just the entries for the document.

    :)

    (: Need to handle both context-free links (direct URI references) and
       context-specific links (key references).

       For context-specific links have to process all the maps and construct
       resolved maps to serve as the resolution context. This could also
       be modeled as updating the key spaces but I think it makes more
       sense to have the maps be the focus, from which key spaces are derived,
       rather than making the key spaces the primary focus.

     :)

    let $logID := 'linkMgmtIndex' (: ID of the log to log messages to :)

    let $directLinks := lmutil:findAllDirectLinks($contentDbName)

    return (
      try {
          db:delete($metadataDbName, $dfstcnst:where-used-dir)
      } catch * {
          (: FIXME: Log to log doc :)
          db:output(<error>Exception deleting where-used directory "{$dfstcnst:where-used-dir}": {$err:description}</error>)
      },

      (: Now create new resource use records for direct links :)

      lmm:createDirectLinkResourceRecords($metadataDbName, $directLinks, $logID),

      (: Now create resolved maps and key space documents for each of the root maps.
       :)
      lmm:constructKeySpaces(
         $contentDbName,
         $metadataDbName,
         $logID),

      (: Now create resource use records for all the indirect links: :)
      lmm:createIndirectLinkResourceRecords(
                  $metadataDbName,
                  lmutil:findAllIndirectLinks($contentDbName),
                  $logID)
    )

};

declare %updating function lmm:createDirectLinkResourceRecords(
        $metadataDbName as xs:string,
        $directLinks as map(*)*,
        $logID as xs:string) {
    for $linkItem in $directLinks
       let $resolveResult := lmutil:resolveDirectLink($linkItem)
       return lmm:createOrUpdateResourceUseRecord(
                   $metadataDbName,
                   $linkItem,
                   $resolveResult,
                   $logID)

};

declare %updating function lmm:createIndirectLinkResourceRecords(
                               $metadataDbName as xs:string,
                               $indirectLinks,
                               $logID as xs:string) {
    for $linkItem in $indirectLinks
        let $resolveResult as map(*) := lmutil:resolveIndirectLink($linkItem)
        return lmm:createOrUpdateResourceUseRecord(
                   $metadataDbName,
                   $linkItem,
                   $resolveResult,
                   $logID)
};

(: Given a link and the resolved result, creates a resource use record for each
   target the link resolved to.

 :)
declare %updating function lmm:createOrUpdateResourceUseRecord(
                              $metadataDbName,
                              $linkItem as map(*),
                              $resolveResult as map(*),
                              $logID as xs:string) {
   let $resolveResult as map(*) := lmutil:resolveDirectLink($linkItem)
   let $targets := $resolveResult('target')
   for $target in $targets
       return lmm:createOrUpdateResourceUseRecordForLinkTarget(
                    $metadataDbName,
                    $linkItem,
                    $target,
                    $logID)
};

(:
 : Creates a use record reflecting the use of the target element by the document
 : that contains the link.
 :
 : The use records are organized into directories, one directory per used document,
 : where the directory name is the MD5 hash of the document URI (just to make a
 : shorter name that won't have any problematic characters).
 :
 : Each use record document's filename is the MD5 hash of the linking document's
 : URI plus the MD5 hash of the linking element.
 :
 : Potential problem: the same element could, in theory, link to the same target in
 : two different ways: as a link and as a conref, although this case is extremely
 : unlikely. Really need a way to distinguish each abstract link represented by
 : a given DITA link-establishing element but the current implementation doesn't
 : provide that abstraction. It could through a refactor. Keeping it simple for now.
 :)
declare %updating function lmm:createOrUpdateResourceUseRecordForLinkTarget(
           $metadataDbName as xs:string,
           $linkItem as map(*),
           $target as element(),
           $logID as xs:string) {
   let $targetDoc := root($target)
   let $link := $linkItem('link')
   let $containingDir := concat($dfstcnst:where-used-dir, '/',
                                lmm:constructResourceKeyForElement($target),
                                '/')
   let $reskey := lmm:constructResourceKeyForElement($link)
   let $recordFilename := concat('use-record_', $reskey, '.xml')
   let $format := if ($link/@format)
                     then string($link/@format)
                     else 'dita'
   let $scope := if ($link/@scope)
                     then string($link/@scope)
                     else 'local'
   let $linkContext := $linkItem('linkContext')

   (: df:getTitleForLinkElementContainer($link) :)
   let $useRecord :=
     <dfst:useRecord resourceKey="{$reskey}"
                     targetDoc="{document-uri($targetDoc)}"
                     usingDoc="{document-uri(root($link))}"
                     linkType="{df:getBaseLinkType($link)}"
                     linkClass="{string($link/@class)}"
                     linkContext="{$linkContext}"
                     format="{$format}"
                     scope="{$scope}"
     >
       <title>{if (df:class($link, 'map/topicref'))
                  then string((root($link)/*/*[df:class(., 'topic/title')] |
                        root($link)/*/@title)[1])
                  else string($link/ancestor::*[df:class(., 'topic/topic')][1]/*[df:class(., 'topic/title')])}</title>
     </dfst:useRecord>
    let $useRecordUri := relpath:newFile($containingDir, $recordFilename)
    return try {
       (
       if (db:exists($metadataDbName, $useRecordUri))
          then db:delete($metadataDbName, $useRecordUri)
          else(),
       db:replace($metadataDbName,
                  $useRecordUri,
                  $useRecord),
       (: FIXME: Write record to log doc :)
       db:output(<info>Stored use record "{$useRecordUri}"</info>)
       )
    } catch * {
       (: FIXME: Write record to log doc :)
       db:output(<error>Error storing use record to "{$useRecordUri}": {$err:description}</error>)
    }
};

(:~
 : Construct key spaces from root maps.
 :
 : @param contentDbName Name of datbase that contains the content to construct key spaces from
 : @param metadataDbName Name of metadata database to store construct key spaces in.
 :
 : Finds all the root maps (requires that all direct-link use records are up to date).
 :
 : For each root map, constructs and stores the resolved map.
 :
 :)
declare %updating function lmm:constructKeySpaces(
        $contentDbName as xs:string,
        $metadataDbName as xs:string,
        $logID as xs:string) {

  (: General process:

     1. Find all DITA maps

     2. For each map, construct a resolved map that preserves submap
        boundaries and original source map details.

     The resolved maps can then be used to determine the effective
     binding for any key without the need to separately construct
     key spaces. This technique was pioneered and suggested by
     Chris Nitchie, designer of the DITA 1.3 scoped keys mechanism.

     Note that there's no point in trying to determine if a given
     map is a "root map" as any map can, in theory, be used as a root
     map. Likewise, a map that happens to have no local-scope
     topicrefs to it may not be intended to be the root of a publication.

     One alternative to this approach is to only consider maps that have
     peer-scope topicrefs to them to be root maps (per the DITA 1.3
     meaning for peer-scope topicrefs to dita maps). In this case, there needs
     to be a "master map" that serves to simply identify the "top level
     root maps", that is, the maps that are considered by the document
     authors to be The root maps, that is, the maps they consider to
     be the roots of publications. Given this master map the system can
     then consider only peer-scope map references in order to determine
     the root maps and thus the maps for which key space construction
     is relevant.

   :)

   let $maps := collection($contentDbName)/*[df:class(., 'map/map')]
   return for $map in $maps
              return (db:output(<info>Resolving map {document-uri(root($map))}...</info>),
                      lmm:constructResolvedMap(
                                              $map,
                                              $metadataDbName,
                                              $logID)
                     )

};

(:~
 : Construct a resolved map suiteable for doing key resolution. The map
 : is stored in the metadata database.
 :
 : @param map DITA map that is the root of the resolved map.
 : @param metadataDbName Name of the metadata database to store the
 :        resolved map in
 :)
declare %updating function lmm:constructResolvedMap(
                                    $map as element(),
                                    $metadataDbName as xs:string,
                                    $logID as xs:string) {
  let $contentDbName := db:name($map)
  let $resolvedMap := lmm:resolveMap($map, $logID)
  let $resolvedMapURI := lmm:getResolvedMapURIForMap($map)
  return (db:replace($metadataDbName, $resolvedMapURI, $resolvedMap),
          db:output(<info>Stored resolved map "{$resolvedMapURI}"</info>))

};

(:~
 : Takes a root map and returns the resolved version of the map.
 :
 : The resolution process preserves the original structure and
 : adds <topicgroup> elements to record submap boundaries.
 :
 : The resulting resolved map reflects all directly-referenced submaps.
 : Because the resolved map is used for constructing key spaces it is
 : not useful or necessary to resolve key-based map references as those
 : maps cannot contribute to the final set of effective key spaces.
 :)
declare function lmm:resolveMap(
                     $map as element(),
                     $logID as xs:string) as element() {

    (: Array of sequences of key scope names, lowest to highest
       (e.g., as for ancestor:: axis)
       Each sequence is the set of key scope names defined
       on a given map or topicref, establishing a new set
       of key scopes.

       There is always the root (anonymous) key scope
       bound to the root map. The root map can also
       establish additional scope names.
    :)
    let $keyScopes as array(*) :=
       [ (if ($map/@keyscope)
             then tokenize(string($map/@keyscope), ' ')
             else ()
         )
       ]
    let $resolvedMap :=
          element {name($map)}
            {
              attribute origMapURI { document-uri(root($map)) },
              attribute origMapDB { db:name($map) },
              attribute xml:base { document-uri(root($map)) },
              $map/@*,
              for $node in $map/node()
                  return lmm:resolveMapHandleNode($node, $keyScopes, $logID)
            }
    return $resolvedMap
};

(:~
 : Simple identity transform dispatch handler for nodes of any type.
 :)
declare function lmm:resolveMapHandleNode(
                     $node as node(),
                     $keyScopes as array(*),
                     $logID as xs:string) as node()* {
  typeswitch ($node)
    case element() return lmm:resolveMapHandleElement(
                              $node,
                              $keyScopes,
                              $logID)
    case attribute(keys) return lmm:expandKeyNames(
                                    $node,
                                    $keyScopes,
                                    $logID)
    default return $node
};

(:~
 : Apply identity transform to elements of any type.
 :)
declare function lmm:resolveMapHandleElement(
                       $elem as element(),
                       $keyScopes as array(*),
                       $logID as xs:string) as element()* {
   let $result :=
     if (df:class($elem, 'map/topicref'))
        then lmm:resolveMapHandleTopicref($elem, $keyScopes, $logID)
        else lmm:resolveMapCopy($elem, $keyScopes, $logID)
   return $result
};

(:~
 : Copy an element as for xsl:copy instruction.
 :)
declare function lmm:resolveMapCopy(
                    $elem as element(),
                    $keyScopes as array(*),
                    $logID as xs:string) as element()* {
   let $result :=
     element {name($elem)} {
        for $node in ($elem/@*, $elem/node())
            return lmm:resolveMapHandleNode($node, $keyScopes, $logID)
     }
   return $result
};

(:~
 : Handle topicrefs. If the topicref is to local-scope map, then resolve the topicref
 : to the map and include the map in the resolved result, otherwise apply normal
 : copy processing.
 :)
declare function lmm:resolveMapHandleTopicref(
                      $elem as element(),
                      $keyScopes as array(*),
                      $logID as xs:string) as element()* {
   let $result :=
     if ($elem/@format = ('ditamap') and df:getEffectiveScope($elem) = ('local'))
        then lmm:resolveMapHandleMapRef($elem, $keyScopes, $logID)
        else lmm:resolveMapCopy($elem, $keyScopes, $logID)
   return $result
};

(:~
 : Handle local-scope map references.
 :
 : Constructs a <dfst:submap> element that captures the details about the original
 : submap and then applies the identity tranform to the topicref and reltable
 : children of the submap.
 :
 : The dfst:submap element sets the xml:base attribute to the URI of the submap document
 : so that URI references copied from the included map will be correct.
 :
 :)
declare function lmm:resolveMapHandleMapRef(
                        $elem as element(),
                        $keyScopes as array(*),
                        $logID as xs:string) as element()* {
  let $resolutionMap as map(*) := df:resolveTopicRef($elem)
  let $submap := $resolutionMap('target')
  (: FIXME: add resolution messages to log once we get logging infrastructure in place :)
  return
    <dfst:submap
      xml:base="{document-uri(root($submap))}"
      origMapURI="{document-uri(root($submap))}"
      origMapClass="{string($submap/@class)}"
      class="+ map/topicref dfst-d/submap "
    >
      {lmm:constructMergedKeyscopeAtt($elem, $submap)},
      <topicmeta>
        <submap-meta>{
          comment {'submap metadata goes here '}
        }</submap-meta>
      </topicmeta>,
      {for $e in $submap/*[df:class(., 'map/topicref') or df:class(., 'map/reltable')]
          return lmm:resolveMapHandleElement($e, $keyScopes, $logID)
      }
    </dfst:submap>
};

(:~
 : Merge the key scope names from two elements that may both specify @keyscope
 :)
declare function lmm:constructMergedKeyscopeAtt(
                        $elem1 as element(),
                        $elem2 as element()) as attribute()? {
   let $scopeNames as xs:string? := lmm:mergeKeyScopeNames($elem1, $elem2)
   let $result :=
       if ($scopeNames)
          then attribute keyscope {$scopeNames}
          else ()
   return $result
};

(:~
 : Merge the key scope names from two elements that may both specify @keyscope
 :)
declare function lmm:mergeKeyScopeNames($elem1 as element(), $elem2 as element()) as xs:string? {

  let $scopeNames := (tokenize($elem1/@keyscope), tokenize($elem2/@keyscope))
  let $result :=
      if (count($scopeNames) > 0)
         then string-join(distinct-values($scopeNames), " ")
         else ()
  return $result
};

(:~
 : Expand the @keys value to add scope qualifications for all the scopes
 : @return keys attribute node with the expanded values.
 :)
declare function lmm:expandKeyNames($keysAtt as attribute(keys),
                                    $keyScopes as array(*),
                                    $logID as xs:string) as attribute() {

  let $baseKeyNames as xs:string* :=
                    tokenize(string($keysAtt), " ")
  let $expandedKeyNames := for $keyName in $baseKeyNames
                               return lmm:scopeQualifyKeyName($keyName, $keyScopes, ($keyName))
  let $result := attribute {name($keysAtt)} {string-join($expandedKeyNames, ' ')}
  return $result
};

(:~
 : Recursive function to construct list of scope-qualified key names for a given
 : base key.
 :
 : Takes the unqualified key name and adds to it the scope names from the end
 : of the keyScopes array. Adds the result to the incoming list of qualified
 : key names and then calls itself against the result and the cadr of the
 : key scope array.
 :)
declare function lmm:scopeQualifyKeyName(
                     $keyName as xs:string,
                     $keyScopes as array(*)?,
                     $qualfiedKeyNames as xs:string+) as xs:string* {
  let $result :=
    if (not(exists($keyScopes)) or array:size($keyScopes) = 0 or not($keyScopes(1)))
       then $qualfiedKeyNames
       else
          let $scopeNames as xs:string* := $keyScopes?1  (: "?" is the array lookup operator :)
          let $newNames as xs:string* :=
              for $scopeName in $scopeNames
                  return $scopeName || '.' || $keyName
          return
            for $newName in $newNames
                 return lmm:scopeQualifyKeyName(
                           $newNames,
                           $keyScopes?(2 to array:size($keyScopes)),
                           ($qualfiedKeyNames, $newNames)
                           )
  return $result
};

(:~
 : Construct the database URI to use for a resolved map.
 :
 : @param map DITA map element to get the URI for
 : @returns The URI of the resolved map as a string
 :)
declare function lmm:getResolvedMapURIForMap(
                        $map as element()) as xs:string {
  let $mapDocHash := hash:md5(document-uri(root($map)))
  return $dfstcnst:resolved-map-dir ||
         "/" || $mapDocHash || ".ditamap"
};

(: End of Module :)

(: =====================================================

   DITA Link Manager Model

   Model component of Model/View/Controller. Maintains
   the link management data.

   Provides queries to create and update resource use records,
   dependency indexes, etc.

   These queries are called whenever new content is added to
   given database.

   Author: W. Eliot Kimber

   Copyright (c) 2015 DITA For Small Teams
   Licensed under Apache License 2

   ===================================================== :)

module namespace lmm="http://dita-for-small-teams.org/xquery/modules/linkmgr-model";

declare namespace dfst="http://dita-for-small-teams.org";

import module namespace df="http://dita-for-small-teams.org/xquery/modules/dita-utils";
import module namespace bxutil="http://dita-for-small-teams.org/xquery/modules/basex-utils";
import module namespace lmutil="http://dita-for-small-teams.org/xquery/modules/linkmgmt-utils";
import module namespace dfstcnst="http://dita-for-small-teams.org/xquery/modules/dfst-constants";
import module namespace relpath="http://dita-for-small-teams.org/xquery/modules/relpath-utils";

(: Given an element with an @id value, construct the unique resource key for it.
   For a given element the resource ID is guaranteed to be unique within
   a snapshot (git commit).

   @param elem Element to get resource ID for.
   @returns Resource key string.

   The resource key is a combination of the absolute URI of the containing
   document and the element's tree position within the document, producing
   a unique key for any element in any document on a given snapshot (version
   in time). Resource keys are not reliably unique across snapshots as the position of
   the element within its containing document could change from version to
   version.

   The resource key is used to look up the element in where-used records, either
   as the element used or the element doing the use (links).

 :)
declare function lmm:constructResourceKeyForElement($elem as element()) as xs:string {
  let $targetDocHash := hash:md5(document-uri(root($elem)))
  let $treepos := for $anc in ($elem/ancestor-or-self::*)
                      return string(count($anc | $anc/preceding-sibling::*))
  let $key := string-join($treepos, '.')
  return $targetDocHash || '^' || $key

};

(:~
 : Create or update the link management indexes for the specified repository
 :
 :)
declare %updating function lmm:updateLinkManagementIndexes(
                               $contentDbName as xs:string,
                               $metadataDbName as xs:string) {
   (: Query the database to find all links and for each link, record a resource use
      record in the database.

      Every addressible element has a "resource key" which combines the URI of
      the containing document and the element's @id value (only elements with
      @id attributes are addressible.

      The link management indexes are stored in a separate database from
      the content database: every branch-specific database has a corresponding
      metadata database that contains the where-used records, resolved maps,
      key space information, and any other metadata you might want to store.

      For each document used there is a directory whose name is the hash
      of the document's URI (which will be unique within a branch). Within
      that directory are the use records for elements within that document.
      This arrangement has the effect of providing a reference count for
      for each document, making it quick to determine if a given document
      is used by any links. It also allows for narrowing the scope of
      where-used queries to just the entries for the document.

    :)

    (: Need to handle both context-free links (direct URI references) and
       context-specific links (key references).

       For context-specific links have to process all the maps and construct
       resolved maps to serve as the resolution context. This could also
       be modeled as updating the key spaces but I think it makes more
       sense to have the maps be the focus, from which key spaces are derived,
       rather than making the key spaces the primary focus.

     :)

    let $logID := 'linkMgmtIndex' (: ID of the log to log messages to :)

    let $directLinks := lmutil:findAllDirectLinks($contentDbName)

    return (
      try {
          db:delete($metadataDbName, $dfstcnst:where-used-dir)
      } catch * {
          (: FIXME: Log to log doc :)
          db:output(<error>Exception deleting where-used directory "{$dfstcnst:where-used-dir}": {$err:description}</error>)
      },

      (: Now create new resource use records for direct links :)

      lmm:createDirectLinkResourceRecords($metadataDbName, $directLinks, $logID),

      (: Now create resolved maps and key space documents for each of the root maps.
       :)
      lmm:constructKeySpaces(
         $contentDbName,
         $metadataDbName,
         $logID),

      (: Now create resource use records for all the indirect links: :)
      lmm:createIndirectLinkResourceRecords(
                  $metadataDbName,
                  lmutil:findAllIndirectLinks($contentDbName),
                  $logID)
    )

};

declare %updating function lmm:createDirectLinkResourceRecords(
        $metadataDbName as xs:string,
        $directLinks as map(*)*,
        $logID as xs:string) {
    for $linkItem in $directLinks
       let $resolveResult := lmutil:resolveDirectLink($linkItem)
       return lmm:createOrUpdateResourceUseRecord(
                   $metadataDbName,
                   $linkItem,
                   $resolveResult,
                   $logID)

};

declare %updating function lmm:createIndirectLinkResourceRecords(
                               $metadataDbName as xs:string,
                               $indirectLinks,
                               $logID as xs:string) {
    for $linkItem in $indirectLinks
        let $resolveResult as map(*) := lmutil:resolveIndirectLink($linkItem)
        return lmm:createOrUpdateResourceUseRecord(
                   $metadataDbName,
                   $linkItem,
                   $resolveResult,
                   $logID)
};

(: Given a link and the resolved result, creates a resource use record for each
   target the link resolved to.

 :)
declare %updating function lmm:createOrUpdateResourceUseRecord(
                              $metadataDbName,
                              $linkItem as map(*),
                              $resolveResult as map(*),
                              $logID as xs:string) {
   let $resolveResult as map(*) := lmutil:resolveDirectLink($linkItem)
   let $targets := $resolveResult('target')
   for $target in $targets
       return lmm:createOrUpdateResourceUseRecordForLinkTarget(
                    $metadataDbName,
                    $linkItem,
                    $target,
                    $logID)
};

(:
 : Creates a use record reflecting the use of the target element by the document
 : that contains the link.
 :
 : The use records are organized into directories, one directory per used document,
 : where the directory name is the MD5 hash of the document URI (just to make a
 : shorter name that won't have any problematic characters).
 :
 : Each use record document's filename is the MD5 hash of the linking document's
 : URI plus the MD5 hash of the linking element.
 :
 : Potential problem: the same element could, in theory, link to the same target in
 : two different ways: as a link and as a conref, although this case is extremely
 : unlikely. Really need a way to distinguish each abstract link represented by
 : a given DITA link-establishing element but the current implementation doesn't
 : provide that abstraction. It could through a refactor. Keeping it simple for now.
 :)
declare %updating function lmm:createOrUpdateResourceUseRecordForLinkTarget(
           $metadataDbName as xs:string,
           $linkItem as map(*),
           $target as element(),
           $logID as xs:string) {
   let $targetDoc := root($target)
   let $link := $linkItem('link')
   let $containingDir := concat($dfstcnst:where-used-dir, '/',
                                lmm:constructResourceKeyForElement($target),
                                '/')
   let $reskey := lmm:constructResourceKeyForElement($link)
   let $recordFilename := concat('use-record_', $reskey, '.xml')
   let $format := if ($link/@format)
                     then string($link/@format)
                     else 'dita'
   let $scope := if ($link/@scope)
                     then string($link/@scope)
                     else 'local'
   let $linkContext := $linkItem('linkContext')

   (: df:getTitleForLinkElementContainer($link) :)
   let $useRecord :=
     <dfst:useRecord resourceKey="{$reskey}"
                     targetDoc="{document-uri($targetDoc)}"
                     usingDoc="{document-uri(root($link))}"
                     linkType="{df:getBaseLinkType($link)}"
                     linkClass="{string($link/@class)}"
                     linkContext="{$linkContext}"
                     format="{$format}"
                     scope="{$scope}"
     >
       <title>{if (df:class($link, 'map/topicref'))
                  then string((root($link)/*/*[df:class(., 'topic/title')] |
                        root($link)/*/@title)[1])
                  else string($link/ancestor::*[df:class(., 'topic/topic')][1]/*[df:class(., 'topic/title')])}</title>
     </dfst:useRecord>
    let $useRecordUri := relpath:newFile($containingDir, $recordFilename)
    return try {
       (
       if (db:exists($metadataDbName, $useRecordUri))
          then db:delete($metadataDbName, $useRecordUri)
          else(),
       db:replace($metadataDbName,
                  $useRecordUri,
                  $useRecord),
       (: FIXME: Write record to log doc :)
       db:output(<info>Stored use record "{$useRecordUri}"</info>)
       )
    } catch * {
       (: FIXME: Write record to log doc :)
       db:output(<error>Error storing use record to "{$useRecordUri}": {$err:description}</error>)
    }
};

(:~
 : Construct key spaces from root maps.
 :
 : @param contentDbName Name of datbase that contains the content to construct key spaces from
 : @param metadataDbName Name of metadata database to store construct key spaces in.
 :
 : Finds all the root maps (requires that all direct-link use records are up to date).
 :
 : For each root map, constructs and stores the resolved map.
 :
 :)
declare %updating function lmm:constructKeySpaces(
        $contentDbName as xs:string,
        $metadataDbName as xs:string,
        $logID as xs:string) {

  (: General process:

     1. Find all DITA maps

     2. For each map, construct a resolved map that preserves submap
        boundaries and original source map details.

     The resolved maps can then be used to determine the effective
     binding for any key without the need to separately construct
     key spaces. This technique was pioneered and suggested by
     Chris Nitchie, designer of the DITA 1.3 scoped keys mechanism.

     Note that there's no point in trying to determine if a given
     map is a "root map" as any map can, in theory, be used as a root
     map. Likewise, a map that happens to have no local-scope
     topicrefs to it may not be intended to be the root of a publication.

     One alternative to this approach is to only consider maps that have
     peer-scope topicrefs to them to be root maps (per the DITA 1.3
     meaning for peer-scope topicrefs to dita maps). In this case, there needs
     to be a "master map" that serves to simply identify the "top level
     root maps", that is, the maps that are considered by the document
     authors to be The root maps, that is, the maps they consider to
     be the roots of publications. Given this master map the system can
     then consider only peer-scope map references in order to determine
     the root maps and thus the maps for which key space construction
     is relevant.

   :)

   let $maps := collection($contentDbName)/*[df:class(., 'map/map')]
   return for $map in $maps
              return (db:output(<info>Resolving map {document-uri(root($map))}...</info>),
                      lmm:constructResolvedMap(
                                              $map,
                                              $metadataDbName,
                                              $logID)
                     )

};

(:~
 : Construct a resolved map suiteable for doing key resolution. The map
 : is stored in the metadata database.
 :
 : @param map DITA map that is the root of the resolved map.
 : @param metadataDbName Name of the metadata database to store the
 :        resolved map in
 :)
declare %updating function lmm:constructResolvedMap(
                                    $map as element(),
                                    $metadataDbName as xs:string,
                                    $logID as xs:string) {
  let $contentDbName := db:name($map)
  let $resolvedMap := lmm:resolveMap($map, $logID)
  let $resolvedMapURI := lmm:getResolvedMapURIForMap($map)
  return (db:replace($metadataDbName, $resolvedMapURI, $resolvedMap),
          db:output(<info>Stored resolved map "{$resolvedMapURI}"</info>))

};

(:~
 : Takes a root map and returns the resolved version of the map.
 :
 : The resolution process preserves the original structure and
 : adds <topicgroup> elements to record submap boundaries.
 :
 : The resulting resolved map reflects all directly-referenced submaps.
 : Because the resolved map is used for constructing key spaces it is
 : not useful or necessary to resolve key-based map references as those
 : maps cannot contribute to the final set of effective key spaces.
 :)
declare function lmm:resolveMap(
                     $map as element(),
                     $logID as xs:string) as element() {

    (: Array of sequences of key scope names, lowest to highest
       (e.g., as for ancestor:: axis)
       Each sequence is the set of key scope names defined
       on a given map or topicref, establishing a new set
       of key scopes.

       There is always the root (anonymous) key scope
       bound to the root map. The root map can also
       establish additional scope names.
    :)
    let $keyScopes as array(*) :=
       [ (if ($map/@keyscope)
             then tokenize(string($map/@keyscope), ' ')
             else ()
         )
       ]
    let $resolvedMap :=
          element {name($map)}
            {
              attribute origMapURI { document-uri(root($map)) },
              attribute origMapDB { db:name($map) },
              attribute xml:base { document-uri(root($map)) },
              $map/@*,
              for $node in $map/node()
                  return lmm:resolveMapHandleNode($node, $keyScopes, $logID)
            }
    return $resolvedMap
};

(:~
 : Simple identity transform dispatch handler for nodes of any type.
 :)
declare function lmm:resolveMapHandleNode(
                     $node as node(),
                     $keyScopes as array(*),
                     $logID as xs:string) as node()* {
  typeswitch ($node)
    case element() return lmm:resolveMapHandleElement(
                              $node,
                              $keyScopes,
                              $logID)
    case attribute(keys) return lmm:expandKeyNames(
                                    $node,
                                    $keyScopes,
                                    $logID)
    default return $node
};

(:~
 : Apply identity transform to elements of any type.
 :)
declare function lmm:resolveMapHandleElement(
                       $elem as element(),
                       $keyScopes as array(*),
                       $logID as xs:string) as element()* {
   let $result :=
     if (df:class($elem, 'map/topicref'))
        then lmm:resolveMapHandleTopicref($elem, $keyScopes, $logID)
        else lmm:resolveMapCopy($elem, $keyScopes, $logID)
   return $result
};

(:~
 : Copy an element as for xsl:copy instruction.
 :)
declare function lmm:resolveMapCopy(
                    $elem as element(),
                    $keyScopes as array(*),
                    $logID as xs:string) as element()* {
   let $result :=
     element {name($elem)} {
        for $node in ($elem/@*, $elem/node())
            return lmm:resolveMapHandleNode($node, $keyScopes, $logID)
     }
   return $result
};

(:~
 : Handle topicrefs. If the topicref is to local-scope map, then resolve the topicref
 : to the map and include the map in the resolved result, otherwise apply normal
 : copy processing.
 :)
declare function lmm:resolveMapHandleTopicref(
                      $elem as element(),
                      $keyScopes as array(*),
                      $logID as xs:string) as element()* {
   let $result :=
     if ($elem/@format = ('ditamap') and df:getEffectiveScope($elem) = ('local'))
        then lmm:resolveMapHandleMapRef($elem, $keyScopes, $logID)
        else lmm:resolveMapCopy($elem, $keyScopes, $logID)
   return $result
};

(:~
 : Handle local-scope map references.
 :
 : Constructs a <dfst:submap> element that captures the details about the original
 : submap and then applies the identity tranform to the topicref and reltable
 : children of the submap.
 :
 : The dfst:submap element sets the xml:base attribute to the URI of the submap document
 : so that URI references copied from the included map will be correct.
 :
 :)
declare function lmm:resolveMapHandleMapRef(
                        $elem as element(),
                        $keyScopes as array(*),
                        $logID as xs:string) as element()* {
  let $resolutionMap as map(*) := df:resolveTopicRef($elem)
  let $submap := $resolutionMap('target')
  (: FIXME: add resolution messages to log once we get logging infrastructure in place :)
  return
    <dfst:submap
      xml:base="{document-uri(root($submap))}"
      origMapURI="{document-uri(root($submap))}"
      origMapClass="{string($submap/@class)}"
      class="+ map/topicref dfst-d/submap "
    >
      {lmm:constructMergedKeyscopeAtt($elem, $submap)},
      <topicmeta>
        <submap-meta>{
          comment {'submap metadata goes here '}
        }</submap-meta>
      </topicmeta>,
      {for $e in $submap/*[df:class(., 'map/topicref') or df:class(., 'map/reltable')]
          return lmm:resolveMapHandleElement($e, $keyScopes, $logID)
      }
    </dfst:submap>
};

(:~
 : Merge the key scope names from two elements that may both specify @keyscope
 :)
declare function lmm:constructMergedKeyscopeAtt(
                        $elem1 as element(),
                        $elem2 as element()) as attribute()? {
   let $scopeNames as xs:string? := lmm:mergeKeyScopeNames($elem1, $elem2)
   let $result :=
       if ($scopeNames)
          then attribute keyscope {$scopeNames}
          else ()
   return $result
};

(:~
 : Merge the key scope names from two elements that may both specify @keyscope
 :)
declare function lmm:mergeKeyScopeNames($elem1 as element(), $elem2 as element()) as xs:string? {

  let $scopeNames := (tokenize($elem1/@keyscope), tokenize($elem2/@keyscope))
  let $result :=
      if (count($scopeNames) > 0)
         then string-join(distinct-values($scopeNames), " ")
         else ()
  return $result
};

(:~
 : Expand the @keys value to add scope qualifications for all the scopes
 : @return keys attribute node with the expanded values.
 :)
declare function lmm:expandKeyNames($keysAtt as attribute(keys),
                                    $keyScopes as array(*),
                                    $logID as xs:string) as attribute() {

  let $baseKeyNames as xs:string* :=
                    tokenize(string($keysAtt), " ")
  let $expandedKeyNames := for $keyName in $baseKeyNames
                               return lmm:scopeQualifyKeyName($keyName, $keyScopes, ($keyName))
  let $result := attribute {name($keysAtt)} {string-join($expandedKeyNames, ' ')}
  return $result
};

(:~
 : Recursive function to construct list of scope-qualified key names for a given
 : base key.
 :
 : Takes the unqualified key name and adds to it the scope names from the end
 : of the keyScopes array. Adds the result to the incoming list of qualified
 : key names and then calls itself against the result and the cadr of the
 : key scope array.
 :)
declare function lmm:scopeQualifyKeyName(
                     $keyName as xs:string,
                     $keyScopes as array(*)?,
                     $qualfiedKeyNames as xs:string+) as xs:string* {
  let $result :=
    if (count($keyScopes) = 0)
       then $qualfiedKeyNames
       else
          let $scopeNames as xs:string* := $keyScopes?1  (: "?" is the array lookup operator :)
          let $newNames as xs:string* :=
              for $scopeName in $scopeNames
                  return $scopeName || '.' || $keyName
          return
            for $newName in $newNames
                 return lmm:scopeQualifyKeyName(
                           $newNames,
                           $keyScopes?(2 to array:size($keyScopes)),
                           ($qualfiedKeyNames, $newNames)
                           )
  return $result
};

(:~
 : Construct the database URI to use for a resolved map.
 :
 : @param map DITA map element to get the URI for
 : @returns The URI of the resolved map as a string
 :)
declare function lmm:getResolvedMapURIForMap(
                        $map as element()) as xs:string {
  let $mapDocHash := hash:md5(document-uri(root($map)))
  return $dfstcnst:resolved-map-dir ||
         "/" || $mapDocHash || ".ditamap"
};

(: End of Module :)


The script that generates the W3C submission reports sets some of the tests as 'not applicable'. Also sets the test 'namespace-errors-q16' as passing.

(: Schema for results :)
import schema default element namespace "http://www.w3.org/2005/02/query-test-XQTSResult" at "w3c_update_reportingresults/XQTSResult.xsd";

(: Test.xml file from ctest :)
declare variable $ctests as document-node() external;

if(fn:empty($ctests)) then
  fn:error()
else
  validate {
    <test-suite-result>
       <implementation name="Zorba"
         version='1.5 (svn rev. 8827)'
         anonymous-result-column="false">

         <organization
            name='FLWOR Foundation'
            website="http://zorba-xquery.com/"
            anonymous="false" />

         <submittor
            name="Zorba Team"
            email="contact@zorba-xquery.com" />

         <description>

Zorba is a general purpose XQuery processor implementing in C++
            the W3C family of specifications. It is not an XML database.
            The query processor has been designed to be embeddable in a
            variety of environments such as other programming languages
            extended with XML processing capabilities, browsers,
            database servers, XML message dispatchers, or smartphones.
            Its architecture employes a modular design, which allows customizing
            the Zorba query processor to the environment’s needs.
            In particular the architecture of the query processor allows a
            pluggable XML store (e.g. main memory, DOM stores, persistent
            disk-based large stores, S3 stores). Zorba runs on most platforms
            and is available under the Apache license v2. Currently, Zorba
            implements the following W3C specifications: XQuery 1.0, XQuery Update
            Facility 1.0, XQuery Scripting Extension 1.0, and XSLT 2.0 and XQuery 1.0
            Serialization.

         </description>

         <implementation-defined-items>
           <implementation-defined-item
              name="supportedRevalidationModes"
              value="lax, skip, strict" />

          <implementation-defined-item
             name="defaultRevalidationMode"
             value="lax" />

         </implementation-defined-items>

         <features>
            <feature name='Minimal Conformance' supported='true' />
            <feature name='Update Facility Static Typing Feature' supported='false' />
         </features>

         <context-properties>
           <context-property
             name="Revalidation mode"
             value="lax"
             context-type="static"/>
         </context-properties>

       </implementation>

       <syntax>XQueryX</syntax>

       <test-run dateRun="2010-10-16">
          <test-suite version="current"/>
          <transformation>
Standard
</transformation>
          <comparison>
Standard
</comparison>
          <otherComments>
XQUTS version taken from CVS as of 13 Sept.
</otherComments>
       </test-run>

       {
         for $test in $ctests/*:Site/*:Testing/*:Test
         let $testname := fn:tokenize(fn:data($test/*:Name), "/")[last()]
         return
         if(fn:exists(fn:index-of(('revalidation-declaration-01-fail','revalidation-declaration-03-fail','revalidation-declaration-05-fail','revalidate-valtrans-ins-003'
         ,'fn-put-003-fail','fn-put-004-fail','fn-put-005-fail','fn-put-006-fail'),$testname))) then
          <test-case
             name="{$testname}"
             result="not applicable"
             comment="this test was not run by Zorba: please consult the 'Guidelines for Running the XML Query Update Test Suite' for more details."
           />
          else if (fn:contains(fn:data($test),'StaticTypingFeature')) then
          <test-case
             name="{$testname}"
             result="not applicable"
             comment="Zorba does not support the 'Update Facility Static Typing Feature'"
           />
          else if ($testname = 'namespace-errors-q16') then
          <test-case
             name="{$testname}"
             result="pass"
             comment="Zorba reports the correct result. For details please see 'http://www.w3.org/Bugs/Public/show_bug.cgi?id=10432'"
           />
          else
           <test-case
             name="{$testname}"
             result="{fn:substring(fn:data($test/@Status), 0, 5)}"
           />
       }
    </test-suite-result>
  }

(: Schema for results :)
import schema default element namespace "http://www.w3.org/2005/02/query-test-XQTSResult" at "w3c_update_reportingresults/XQTSResult.xsd";

(: Test.xml file from ctest :)
declare variable $ctests as document-node() external;

if(fn:empty($ctests)) then
  fn:error()
else
  validate {
    <test-suite-result>
       <implementation name="Zorba"
         version='1.5 (svn rev. 8737)'
         anonymous-result-column="false">

         <organization
            name='FLWOR Foundation'
            website="http://zorba-xquery.com/"
            anonymous="false" />

         <submittor
            name="Zorba Team"
            email="contact@zorba-xquery.com" />

         <description>

Zorba is a general purpose XQuery processor implementing in C++
            the W3C family of specifications. It is not an XML database.
            The query processor has been designed to be embeddable in a
            variety of environments such as other programming languages
            extended with XML processing capabilities, browsers,
            database servers, XML message dispatchers, or smartphones.
            Its architecture employes a modular design, which allows customizing
            the Zorba query processor to the environment’s needs.
            In particular the architecture of the query processor allows a
            pluggable XML store (e.g. main memory, DOM stores, persistent
            disk-based large stores, S3 stores). Zorba runs on most platforms
            and is available under the Apache license v2. Currently, Zorba
            implements the following W3C specifications: XQuery 1.0, XQuery Update
            Facility 1.0, XQuery Scripting Extension 1.0, and XSLT 2.0 and XQuery 1.0
            Serialization.

         </description>

         <implementation-defined-items>
           <implementation-defined-item
              name="supportedRevalidationModes"
              value="lax, skip, strict" />

          <implementation-defined-item
             name="defaultRevalidationMode"
             value="lax" />

         </implementation-defined-items>

         <features>
            <feature name='Minimal Conformance' supported='true' />
            <feature name='Update Facility Static Typing Feature' supported='false' />
         </features>

         <context-properties>
           <context-property
             name="Revalidation mode"
             value="lax"
             context-type="static"/>
         </context-properties>

       </implementation>

       <syntax>XQuery</syntax>

       <test-run dateRun="2010-08-20">
          <test-suite version="1.0.1"/>
          <transformation>
Standard
</transformation>
          <comparison>
Standard
</comparison>
          <otherComments>
None
</otherComments>
       </test-run>

       {
         for $test in $ctests/*:Site/*:Testing/*:Test
         let $testname := fn:tokenize(fn:data($test/*:Name), "/")[last()]
         return
           <test-case
             name="{$testname}"
             result="{fn:substring(fn:data($test/@Status), 0, 5)}"
           />
       }
    </test-suite-result>
  }


Small change in link_crawler1

Small change in link_crawler1

(:
 : Copyright 2006-2011 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
:)

import module namespace http = "http://www.zorba-xquery.com/modules/http-client";
import module namespace file = "http://expath.org/ns/file";
import module namespace ddl = "http://www.zorba-xquery.com/modules/store/dynamic/collections/ddl";
import module namespace dml = "http://www.zorba-xquery.com/modules/store/dynamic/collections/dml";
import module namespace map = "http://www.zorba-xquery.com/modules/store/data-structures/unordered-map";
(:import module namespace lcc = "http://www.zorba-xquery.com/scripts/link-crawler-collections" at "link_crawler_collections.xq";:)
import module namespace tidy="http://www.zorba-xquery.com/modules/converters/html";
import schema namespace tidy-options="http://www.zorba-xquery.com/modules/converters/html-options";
import schema namespace httpsch = "http://expath.org/ns/http-client";
declare namespace ann = "http://www.zorba-xquery.com/annotations";
declare namespace xhtml="http://www.w3.org/1999/xhtml";
declare namespace output="http://www.w3.org/2010/xslt-xquery-serialization";
declare namespace err="http://www.w3.org/2005/xqt-errors";

(: declare variable $result-file as xs:string external; :)(:PROJECT_SOURCE_DIR:)

declare variable $top-uri  as xs:string := "http://www.zorba-xquery.com/site2/html/index.html"; (: the start page :)
declare variable $uri-host as xs:string := "http://www.zorba-xquery.com"; (:  what differentiates an internal uri :)

declare variable $supported-media-types as xs:string+ := ("text/xml", "application/xml", "text/xml-external-parsed-entity", "application/xml-external-parsed-entity",
                            "application/atom+xml", "text/html");

declare variable $internal-uris as xs:QName := xs:QName("internal-uris");
declare variable $external-uris as xs:QName := xs:QName("external-uris");

declare variable $broken-internal as xs:QName := xs:QName("local:broken-internal");
declare variable $broken-external as xs:QName := xs:QName("local:broken-external");
declare variable $pages-cannot-parse as xs:QName := xs:QName("local:pages-cannot-parse");

declare function local:my-substring-before($s1 as xs:string, $s2 as xs:string) as xs:string
{
  let $sb := fn:substring-before($s1, $s2)
  return
    if($sb = "") then
      $s1
    else
      $sb
};

declare %ann:sequential function local:get-uris-from-page($uri as xs:string,
                                                          $reluri as xs:string,
                                                          $call-from as xs:string)
{
  variable $method;
  if(fn:starts-with($uri, $uri-host)) then
  {
    map:insert($internal-uris, $uri, $uri);
    $method := "GET";
    fn:trace($uri, "internal ");
  }
  else
  {
    map:insert($external-uris, $uri, $uri);
    $method := "HEAD";
    fn:trace($uri, "external ");
  }

  variable $load-result;
  variable $content-string;
  try{
    $load-result := http:send-request(<httpsch:request method="{$method}" href="{$uri}"/>, (), ());
    if($load-result[1]/@status eq 200) then
      if(fn:starts-with($uri, $uri-host)) then
      {
        $content-string := string($load-result[2]);
        let $media-type := local:my-substring-before($load-result[1]/httpsch:header[@name = 'Content-Type'][1]/fn:string(@value), ";")
        return
          if($media-type = "text/html") then
            let $content := tidy:parse($content-string,
                                        <options xmlns="http://www.zorba-xquery.com/modules/converters/html-options" >
                                          <tidyParam name="output-xml" value="yes" />
                                          <tidyParam name="doctype" value="omit" />
                                          <tidyParam name="quote-nbsp" value="no" />
                                          <tidyParam name="char-encoding" value="utf8" />
                                          <tidyParam name="newline" value="LF" />
                                          <tidyParam name="tidy-mark" value="no" />
                                          <tidyParam name="new-inline-tags" value="nav header section article footer xqdoc:custom d c options json-param" />
                                        </options>)
            for $other-uri2 in  ($content//*:a/string(@href),
                                $content//*:link/string(@href),
                                $content//*:script/string(@src),
                                $content//*:img/string(@src),
                                $content//*:area/string(@href)
                               )
            let $other-uri := fn:normalize-space($other-uri2)
            let $absuri := local:my-substring-before(fn:resolve-uri($other-uri, $uri), "#")
            return
              if(fn:not(fn:starts-with($other-uri, "#")) and
                 fn:empty(map:get($internal-uris, $absuri)) and
                 fn:empty(map:get($external-uris, $absuri))) then
                local:get-uris-from-page($absuri, $other-uri, $uri);
              else (:already followed this link:)
                ();
          else(: it's binary :)
            fn:trace((" has binary content ", $media-type), "");
       }
       else(:success loading external link:)
         ();
    else (: broken link :)
      if(fn:starts-with($uri, $uri-host)) then
      {
        dml:insert-nodes-last($broken-internal, <internal-broken-uri>
                                                    <uri>{$reluri}</uri>
                                                    <call-from>{$call-from}</call-from>
                                                    <media-type>{local:my-substring-before($load-result[1]/httpsch:header[@name = 'Content-Type'][1]/fn:string(@value), ";")}</media-type>
                                                    </internal-broken-uri>);
      }
      else
      {
        dml:insert-nodes-last($broken-external, <external-broken-uri>
                                                    <uri>{$uri}</uri>
                                                    <call-from>{$call-from}</call-from>
                                                    <media-type>{local:my-substring-before($load-result[1]/httpsch:header[@name = 'Content-Type'][1]/fn:string(@value), ";")}</media-type>
                                                    </external-broken-uri>);
      }
  }catch ZXQP0003
  {
    dml:insert-nodes-last($pages-cannot-parse, <page-cannot-parse>
                                                 <uri>{$uri}</uri>
                                                 <reluri>{$reluri}</reluri>
                                                 <call-from>{$call-from}</call-from>
                                                 <err-code>{$err:code}</err-code>
                                                 <err-description>{$err:description}</err-description>
                                                 <err-value>{$err:value}</err-value>
                                                 <err-module>{$err:module}</err-module>
                                                 <err-line>{$err:line-number}</err-line>
                                                 </page-cannot-parse>);
    try{ (: tidy failed to parse the html, use regex:)
        let $content := $content-string
        let $search := fn:analyze-string($content, "(<|&lt;|<)(((a|link|area).+?href)|((script|img).+?src))=([""'])(.*?)\7")
        for $other-uri2 in  $search//fn:group[@nr=8]/fn:string()
        let $other-uri := fn:normalize-space($other-uri2)
        let $absuri := local:my-substring-before(fn:resolve-uri($other-uri, $uri), "#")
        return
          if(fn:not(fn:starts-with($other-uri, "#")) and
             fn:empty(map:get($internal-uris, $absuri)) and
             fn:empty(map:get($external-uris, $absuri))) then
            local:get-uris-from-page($absuri, $other-uri, $uri);
          else (:already followed this link:)
            ();
    }catch *
    {
      dml:insert-nodes-last($pages-cannot-parse, <page-cannot-parse>
                                                   <uri>{$uri}</uri>
                                                   <reluri>{$reluri}</reluri>
                                                   <call-from>{$call-from}</call-from>
                                                   <err-code>{$err:code}</err-code>
                                                   <err-description>{$err:description}</err-description>
                                                   <err-value>{$err:value}</err-value>
                                                   <err-module>{$err:module}</err-module>
                                                   <err-line>{$err:line-number}</err-line>
                                                   </page-cannot-parse>);
    }
  }catch *
  {
    dml:insert-nodes-last($pages-cannot-parse, <page-cannot-parse>
                                                 <uri>{$uri}</uri>
                                                 <reluri>{$reluri}</reluri>
                                                 <call-from>{$call-from}</call-from>
                                                 <err-code>{$err:code}</err-code>
                                                 <err-description>{$err:description}</err-description>
                                                 <err-value>{$err:value}</err-value>
                                                 <err-module>{$err:module}</err-module>
                                                 <err-line>{$err:line-number}</err-line>
                                                 </page-cannot-parse>);
  }
};

map:create($internal-uris, xs:QName("xs:string"));
ddl:create($broken-internal);
map:create($external-uris, xs:QName("xs:string"));
ddl:create($broken-external);
ddl:create($pages-cannot-parse);

local:get-uris-from-page($top-uri, $top-uri, "");

(:display results:)
let $full-report :=
<link-crawler website="{$top-uri}">
<internal-broken-uris>
{for $i in dml:collection($broken-internal)
let $u := $i/uri
group by $u
return $i}
</internal-broken-uris>
<pages-cannot-parse>
{for $e in dml:collection($pages-cannot-parse)
let $u := $e/uri
group by $u
return $e}
</pages-cannot-parse>
<external-uris>
{for $e in map:keys($external-uris)
let $v := $e/attribute/@value
group by $v
return <external-uri>{$e}</external-uri>}
</external-uris>
<external-broken-uris>
{for $e in dml:collection($broken-external)
let $u := $e/uri
group by $u
return  $e}
</external-broken-uris>

</link-crawler>

return
file:write(fn:resolve-uri("link_crawler_result.xml"),
            $full-report,
            <output:serialization-parameters>
                <output:indent value="yes"/>
            </output:serialization-parameters>)

(:
 : Copyright 2006-2011 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
:)

import module namespace http = "http://www.zorba-xquery.com/modules/http-client";
import module namespace file = "http://expath.org/ns/file";
import module namespace ddl = "http://www.zorba-xquery.com/modules/store/dynamic/collections/ddl";
import module namespace dml = "http://www.zorba-xquery.com/modules/store/dynamic/collections/dml";
import module namespace map = "http://www.zorba-xquery.com/modules/store/data-structures/unordered-map";
(:import module namespace lcc = "http://www.zorba-xquery.com/scripts/link-crawler-collections" at "link_crawler_collections.xq";:)
import module namespace tidy="http://www.zorba-xquery.com/modules/converters/html";
import schema namespace tidy-options="http://www.zorba-xquery.com/modules/converters/html-options";
import schema namespace httpsch = "http://expath.org/ns/http-client";
declare namespace ann = "http://www.zorba-xquery.com/annotations";
declare namespace xhtml="http://www.w3.org/1999/xhtml";
declare namespace output="http://www.w3.org/2010/xslt-xquery-serialization";
declare namespace err="http://www.w3.org/2005/xqt-errors";

declare variable $result-file as xs:string external; (:PROJECT_SOURCE_DIR:)

declare variable $top-uri  as xs:string := "http://www.zorba-xquery.com/site2/html/index.html"; (: the start page :)
declare variable $uri-host as xs:string := "http://www.zorba-xquery.com/site2/"; (:  what differentiates an internal uri :)

declare variable $supported-media-types as xs:string+ := ("text/xml", "application/xml", "text/xml-external-parsed-entity", "application/xml-external-parsed-entity",
                            "application/atom+xml", "text/html");

declare variable $internal-uris as xs:QName := xs:QName("internal-uris");
declare variable $external-uris as xs:QName := xs:QName("external-uris");

declare variable $broken-internal as xs:QName := xs:QName("local:broken-internal");
declare variable $broken-external as xs:QName := xs:QName("local:broken-external");
declare variable $pages-cannot-parse as xs:QName := xs:QName("local:pages-cannot-parse");

declare function local:my-substring-before($s1 as xs:string, $s2 as xs:string) as xs:string
{
  let $sb := fn:substring-before($s1, $s2)
  return
    if($sb = "") then
      $s1
    else
      $sb
};

declare %ann:sequential function local:get-uris-from-page($uri as xs:string,
                                                          $reluri as xs:string,
                                                          $call-from as xs:string)
{
  variable $method;
  if(fn:starts-with($uri, $uri-host)) then
  {
    map:insert($internal-uris, $uri, $uri);
    $method := "GET";
  }
  else
  {
    map:insert($external-uris, $uri, $uri);
    $method := "HEAD";
  }
  fn:trace($uri, "");

  variable $load-result;
  variable $content-string;
  try{
    $load-result := http:send-request(<httpsch:request method="{$method}" href="{$uri}"/>, (), ());
    if($load-result[1]/@status eq 200) then
      if(fn:starts-with($uri, $uri-host)) then
      {
        $content-string := string($load-result[2]);
        let $media-type := local:my-substring-before($load-result[1]/httpsch:header[@name = 'Content-Type'][1]/fn:string(@value), ";")
        return
          if($media-type = "text/html") then
            let $content := tidy:parse($content-string,
                                        <options xmlns="http://www.zorba-xquery.com/modules/converters/html-options" >
                                          <tidyParam name="output-xml" value="yes" />
                                          <tidyParam name="doctype" value="omit" />
                                          <tidyParam name="quote-nbsp" value="no" />
                                          <tidyParam name="char-encoding" value="utf8" />
                                          <tidyParam name="newline" value="LF" />
                                          <tidyParam name="tidy-mark" value="no" />
                                          <tidyParam name="new-inline-tags" value="nav header section article footer xqdoc:custom d c options json-param" />
                                        </options>)
            for $other-uri2 in  ($content//*:a/string(@href),
                                $content//*:link/string(@href),
                                $content//*:script/string(@src),
                                $content//*:img/string(@src),
                                $content//*:area/string(@href)
                               )
            let $other-uri := fn:normalize-space($other-uri2)
            let $absuri := local:my-substring-before(fn:resolve-uri($other-uri, $uri), "#")
            return
              if(fn:not(fn:starts-with($other-uri, "#")) and
                 fn:empty(map:get($internal-uris, $absuri)) and
                 fn:empty(map:get($external-uris, $absuri))) then
                local:get-uris-from-page($absuri, $other-uri, $uri);
              else (:already followed this link:)
                ();
          else(: it's binary :)
            fn:trace((" has binary content ", $media-type), "");
       }
       else(:success loading external link:)
         ();
    else (: broken link :)
      if(fn:starts-with($uri, $uri-host)) then
      {
        dml:insert-nodes-last($broken-internal, <internal-broken-uri>
                                                    <uri>{$reluri}</uri>
                                                    <call-from>{$call-from}</call-from>
                                                    <media-type>{local:my-substring-before($load-result[1]/httpsch:header[@name = 'Content-Type'][1]/fn:string(@value), ";")}</media-type>
                                                    </internal-broken-uri>);
      }
      else
      {
        dml:insert-nodes-last($broken-external, <external-broken-uri>
                                                    <uri>{$uri}</uri>
                                                    <call-from>{$call-from}</call-from>
                                                    <media-type>{local:my-substring-before($load-result[1]/httpsch:header[@name = 'Content-Type'][1]/fn:string(@value), ";")}</media-type>
                                                    </external-broken-uri>);
      }
  }catch ZXQP0003
  {
    dml:insert-nodes-last($pages-cannot-parse, <page-cannot-parse>
                                                 <uri>{$uri}</uri>
                                                 <reluri>{$reluri}</reluri>
                                                 <call-from>{$call-from}</call-from>
                                                 <err-code>{$err:code}</err-code>
                                                 <err-description>{$err:description}</err-description>
                                                 <err-value>{$err:value}</err-value>
                                                 <err-module>{$err:module}</err-module>
                                                 <err-line>{$err:line-number}</err-line>
                                                 </page-cannot-parse>);
    try{ (: tidy failed to parse the html, use regex:)
        let $content := $content-string
        let $search := fn:analyze-string($content, "(<|&lt;|<)(((a|link|area).+?href)|((script|img).+?src))=([""'])(.*?)\7")
        for $other-uri2 in  $search//fn:group[@nr=8]/fn:string()
        let $other-uri := fn:normalize-space($other-uri2)
        let $absuri := local:my-substring-before(fn:resolve-uri($other-uri, $uri), "#")
        return
          if(fn:not(fn:starts-with($other-uri, "#")) and
             fn:empty(map:get($internal-uris, $absuri)) and
             fn:empty(map:get($external-uris, $absuri))) then
            local:get-uris-from-page($absuri, $other-uri, $uri);
          else (:already followed this link:)
            ();
    }catch *
    {
      dml:insert-nodes-last($pages-cannot-parse, <page-cannot-parse>
                                                   <uri>{$uri}</uri>
                                                   <reluri>{$reluri}</reluri>
                                                   <call-from>{$call-from}</call-from>
                                                   <err-code>{$err:code}</err-code>
                                                   <err-description>{$err:description}</err-description>
                                                   <err-value>{$err:value}</err-value>
                                                   <err-module>{$err:module}</err-module>
                                                   <err-line>{$err:line-number}</err-line>
                                                   </page-cannot-parse>);
    }
  }catch *
  {
    dml:insert-nodes-last($pages-cannot-parse, <page-cannot-parse>
                                                 <uri>{$uri}</uri>
                                                 <reluri>{$reluri}</reluri>
                                                 <call-from>{$call-from}</call-from>
                                                 <err-code>{$err:code}</err-code>
                                                 <err-description>{$err:description}</err-description>
                                                 <err-value>{$err:value}</err-value>
                                                 <err-module>{$err:module}</err-module>
                                                 <err-line>{$err:line-number}</err-line>
                                                 </page-cannot-parse>);
  }
};

map:create($internal-uris, xs:QName("xs:string"));
ddl:create($broken-internal);
map:create($external-uris, xs:QName("xs:string"));
ddl:create($broken-external);
ddl:create($pages-cannot-parse);

local:get-uris-from-page($top-uri, $top-uri, "");

(:display results:)
let $full-report :=
<link-crawler website="{$top-uri}">
<internal-broken-uris>
{for $i in dml:collection($broken-internal)
let $u := $i/uri
group by $u
return $i}
</internal-broken-uris>
<pages-cannot-parse>
{for $e in dml:collection($pages-cannot-parse)
let $u := $e/uri
group by $u
return $e}
</pages-cannot-parse>
<external-uris>
{for $e in map:keys($external-uris)
let $v := $e/attribute/@value
group by $v
return <external-uri>{$e}</external-uri>}
</external-uris>
<external-broken-uris>
{for $e in dml:collection($broken-external)
let $u := $e/uri
group by $u
return  $e}
</external-broken-uris>

</link-crawler>

return
file:write($result-file,
            $full-report,
            <output:serialization-parameters>
                <output:indent value="yes"/>
            </output:serialization-parameters>)


Specify links for task navigation.

Specify links for task navigation.

module namespace logx="http://cefn.com/logx";

declare default element namespace "http://www.w3.org/1999/xhtml";

declare variable $logx:sourceurls as xs:string external;

declare variable $logx:collection := logx:aggregate($logx:sourceurls);

declare function logx:aggregate($filepaths as xs:string){
    for $token in tokenize($filepaths, ',') return doc(resolve-uri($token))
};

declare function logx:editor-link($node){
	logx:editor-link($node, $node//text())
};

declare function logx:editor-link($node, $text){
	{$text}
};

declare function logx:viewer-link($path, $nodes){
	{$nodes}
};

(: Returns all the document nodes of the XHTML note files :)
declare function logx:menu(){
	for $item in (
		logx:viewer-link('xq/index.xq','Home'),
		logx:viewer-link('xq/tasks.xq','Tasks'),
		logx:viewer-link('xq/headers.xq','Browse')
	)
	return ($item,' ')
};

(: templates a webpage, including the nodes specified :)
declare function logx:webpage($nodes){

		    <meta content="text/html; charset=utf-16" http-equiv="content-type"></meta>
			<link href="/style.css" rel="stylesheet" type="text/css"></link>

						{logx:menu()}

						<form action="/search.xq">

									<input type="text" name="text" />
									<input type="submit" value="Search"/>

						</form>

								{$nodes}

};

module namespace logx="http://cefn.com/logx";

declare default element namespace "http://www.w3.org/1999/xhtml";

declare variable $logx:sourceurls as xs:string external;

declare variable $logx:collection := logx:aggregate($logx:sourceurls);

declare function logx:aggregate($filepaths as xs:string){
    for $token in tokenize($filepaths, ',') return doc(resolve-uri($token))
};

declare function logx:editor-link($node){
	logx:editor-link($node, $node//text())
};

declare function logx:editor-link($node, $text){
	{$text}
};

declare function logx:viewer-link($path, $nodes){
	{$nodes}
};

(: Returns all the document nodes of the XHTML note files :)
declare function logx:menu(){

			{logx:viewer-link('xq/index.xq','Home')}

			{logx:viewer-link('xq/untagged.xq','Untagged')}

Item2

};

(: templates a webpage, including the nodes specified :)
declare function logx:webpage($nodes){

		    <meta content="text/html; charset=utf-16" http-equiv="content-type"></meta>
			<link href="/style.css" rel="stylesheet" type="text/css"></link>

						Log

						{logx:menu()}

						<form action="/search.xq">

									<input type="text" name="text" />
									<input type="submit" value="Search"/>

						</form>

								{$nodes}

};