DHFPROD-3719: Improve performance checking for blocked merges

xquery version "1.0-ml";

(:
 : This is an implementation library, not an interface to the Smart Mastering functionality.
 :
 : Match blocks prevent automatic merges from happening. They can be created
 : manually and are automatically created when a merge gets rolled back.
 : Blocks are implemented here as managed RDF triples of the form:
 :   <uri1> $const:PRED-MATCH-BLOCK <uri2>
 : The reverse triple also gets stored:
 :   <uri2> $const:PRED-MATCH-BLOCK <uri1>
 :)

module namespace blocks-impl = "http://marklogic.com/smart-mastering/blocks-impl";

import module namespace const = "http://marklogic.com/smart-mastering/constants"
  at "/com.marklogic.smart-mastering/constants.xqy";
import module namespace sem = "http://marklogic.com/semantics"
  at "/MarkLogic/semantics.xqy";

declare option xdmp:mapping "false";

(:
 : Return a JSON array of any URIs the that input URI is blocked from matching.
 : @param $uri  input URI
 : @return JSON array of URIs
 :)
declare function blocks-impl:get-blocks($uri as xs:string?)
  as array-node()
{
  array-node {
    if (fn:exists($uri)) then
      if (map:contains($cached-blocks-by-uri, $uri)) then
        map:get($cached-blocks-by-uri, $uri)[. ne ""]
      else
        blocks-impl:get-blocks-of-uris($uri)
    else ()
  }
};

declare variable $cached-blocks-by-uri as map:map := map:map();

declare function blocks-impl:get-values(
  $map as map:map,
  $key as xs:string
)
{
  let $values := map:get($map, $key)
  return
    if (fn:exists($values))
    then $values
    else let $n :=  map:map()
    return (map:put($map, $key, $n), $n)
};

declare function blocks-impl:get-blocks-of-uris($uris as xs:string*)
as xs:string*{
  if (fn:exists($uris)) then (
    let $iris := $uris ! sem:iri(.)
    let $elapsed := xdmp:elapsed-time()
    let $solution :=
      sem:sparql(
        "select distinct ?targetURI (?uri as ?blocked) where {
            {
              ?uri ?isBlocked ?target.
              ?target ?isBlocked ?targetURI.
            } UNION {
              ?target ?isBlocked ?uri.
              ?uri ?isBlocked ?targetURI.
            }
            FILTER (?uri != ?targetURI)
          }",
        map:new((
          map:entry("target", $iris),
          map:entry("isBlocked", $const:PRED-MATCH-BLOCK)
        )),
        ("map","optimize=0"),
        cts:triple-range-query((),(), $iris, "=")
      )

    let $elapsed := xs:integer((xdmp:elapsed-time() - $elapsed) div xs:dayTimeDuration("PT0.001S"))
    let $_ := if (xdmp:trace-enabled($const:TRACE-MATCH-RESULTS)) then
                xdmp:trace($const:TRACE-MATCH-RESULTS, "get-blocks-of-uris," || xdmp:request() || ",sparql," || $elapsed ||  ",uris," || fn:string-join($uris, ";"))
              else ()
    let $elapsed := xdmp:elapsed-time()
    let $tmp := map:map()
    let $values :=
      for $triple in $solution
      let $value := fn:string(map:get($triple, "blocked"))
      let $target := fn:string(map:get($triple, "targetURI"))
      let $current-for-target := blocks-impl:get-values ($tmp, $target)
      let $current-for-value := blocks-impl:get-values($tmp, $value)
      return
        (
          map:put($current-for-target,$value, fn:true()),
          map:put($current-for-value,$target, fn:true()),
          $value
        )
    let $_ := for $k in map:keys($tmp)
    return map:put($cached-blocks-by-uri, $k,
      fn:distinct-values((map:get($cached-blocks-by-uri, $k), map:keys(map:get($tmp, $k))))  )
    let $_populate-empty :=
      for $uri in $uris
      where fn:not(map:contains($cached-blocks-by-uri, $uri))
      return
        map:put($cached-blocks-by-uri, $uri, "")
    let $elapsed := xs:integer((xdmp:elapsed-time() - $elapsed) div xs:dayTimeDuration("PT0.001S"))
    let $_ := if (xdmp:trace-enabled($const:TRACE-MATCH-RESULTS)) then
                xdmp:trace($const:TRACE-MATCH-RESULTS, "get-blocks-of-uris," || xdmp:request() || ",map," || $elapsed)
              else ()
    return fn:distinct-values($values)
  ) else ()
};

(:
 : Block all pairs of URIs from matching. This function will start with URI #1
 : and call matcher:block-match with it each of the remaining URIs. It then
 : recurses on the tail, repeating the process of blocking URI #2 from matching
 : with the remaining URIs (3, 4). This stops when there are zero or one URIs
 : remaining.
 : No return type specified to allow tail call optimization.
 :
 : @param uris the sequence of URIs
 : @return empty sequence
 :)
declare function blocks-impl:block-matches($uris as xs:string*)
{
  if (fn:empty($uris) or fn:count($uris) = 1) then
  (: We're done :)
    ()
  else
    let $tail := fn:tail($uris)
    let $_ := $tail ! blocks-impl:block-match(fn:head($uris), .)
    return blocks-impl:block-matches($tail)
};

(:
 : Prevent the two input URIs from being allowed to match. Helper function for block-matches.
 :
 : @param $uri1  First input URI
 : @param $uri2  Second input URI
 : @error will throw xs:QName("SM-CANT-BLOCK") if unable to record the block.
 : @return empty sequence
 :)
declare function blocks-impl:block-match($uri1 as xs:string, $uri2 as xs:string)
as empty-sequence()
{
  let $_ :=
    (: Suppress sem:rdf-insert's return value :)
    sem:rdf-insert(
      (
        sem:triple(sem:iri($uri1), $const:PRED-MATCH-BLOCK, sem:iri($uri2)),
        sem:triple(sem:iri($uri2), $const:PRED-MATCH-BLOCK, sem:iri($uri1))
      )
    )
  return ()
};

(:
 : Clear a match block between the two input URIs.
 :
 : @param $uri1  First input URI
 : @param $uri2  Second input URI
 :
 : @error will throw xs:QName("SM-CANT-UNBLOCK") if a block is present, but it cannot be cleared
 : @return  fn:true if a block was found and cleared; fn:false if no block was found
 :)
declare function blocks-impl:allow-match($uri1 as xs:string, $uri2 as xs:string)
{
  let $nodes := sem:database-nodes((
    cts:triples(sem:iri($uri1), $const:PRED-MATCH-BLOCK, sem:iri($uri2)),
    cts:triples(sem:iri($uri2), $const:PRED-MATCH-BLOCK, sem:iri($uri1))
  ))
  return (
    $nodes ! xdmp:node-delete(.),
    fn:exists($nodes)
  )
};

xquery version "1.0-ml";

(:
 : This is an implementation library, not an interface to the Smart Mastering functionality.
 :
 : Match blocks prevent automatic merges from happening. They can be created
 : manually and are automatically created when a merge gets rolled back.
 : Blocks are implemented here as managed RDF triples of the form:
 :   <uri1> $const:PRED-MATCH-BLOCK <uri2>
 : The reverse triple also gets stored:
 :   <uri2> $const:PRED-MATCH-BLOCK <uri1>
 :)

module namespace blocks-impl = "http://marklogic.com/smart-mastering/blocks-impl";

import module namespace const = "http://marklogic.com/smart-mastering/constants"
  at "/com.marklogic.smart-mastering/constants.xqy";
import module namespace sem = "http://marklogic.com/semantics"
  at "/MarkLogic/semantics.xqy";

declare option xdmp:mapping "false";

(:
 : Return a JSON array of any URIs the that input URI is blocked from matching.
 : @param $uri  input URI
 : @return JSON array of URIs
 :)
declare function blocks-impl:get-blocks($uri as xs:string?)
  as array-node()
{
  array-node {
    if (fn:exists($uri)) then
      if (map:contains($cached-blocks-by-uri, $uri)) then
        map:get($cached-blocks-by-uri, $uri)[. ne ""]
      else
        blocks-impl:get-blocks-of-uris($uri)
    else ()
  }
};

declare variable $cached-blocks-by-uri as map:map := map:map();

declare function blocks-impl:get-blocks-of-uris($uris as xs:string*)
  as xs:string*
{
    if (fn:exists($uris)) then (
      let $iris := $uris ! sem:iri(.)
      let $solution :=
        sem:sparql(
          "select distinct ?targetURI (?uri as ?blocked) where {
            {
              ?uri ?isBlocked ?target.
              ?target ?isBlocked ?targetURI.
            } UNION {
              ?target ?isBlocked ?uri.
              ?uri ?isBlocked ?targetURI.
            }
            FILTER (?uri != ?targetURI)
          }",
          map:new((
            map:entry("target", $iris),
            map:entry("isBlocked", $const:PRED-MATCH-BLOCK)
          )),
          ("map","optimize=0"),
          cts:triple-range-query((),(), $iris, "=")
        )
      let $values :=
        fn:distinct-values(
          for $triple in $solution
          let $value := fn:string(map:get($triple, "blocked"))
          let $target := fn:string(map:get($triple, "targetURI"))
          let $current-for-target := map:get($cached-blocks-by-uri, $target)
          let $current-for-value := map:get($cached-blocks-by-uri, $value)
          return
            (
              map:put($cached-blocks-by-uri, $target, fn:distinct-values(($current-for-target,$value))),
              map:put($cached-blocks-by-uri, $value, fn:distinct-values(($current-for-value,$target))),
              $value
            )
        )
      let $_populate-empty :=
        for $uri in $uris
        where fn:not(map:contains($cached-blocks-by-uri, $uri))
        return
          map:put($cached-blocks-by-uri, $uri, "")
      return $values
    ) else ()
};

(:
 : Block all pairs of URIs from matching. This function will start with URI #1
 : and call matcher:block-match with it each of the remaining URIs. It then
 : recurses on the tail, repeating the process of blocking URI #2 from matching
 : with the remaining URIs (3, 4). This stops when there are zero or one URIs
 : remaining.
 : No return type specified to allow tail call optimization.
 :
 : @param uris the sequence of URIs
 : @return empty sequence
 :)
declare function blocks-impl:block-matches($uris as xs:string*)
{
  if (fn:empty($uris) or fn:count($uris) = 1) then
  (: We're done :)
    ()
  else
    let $tail := fn:tail($uris)
    let $_ := $tail ! blocks-impl:block-match(fn:head($uris), .)
    return blocks-impl:block-matches($tail)
};

(:
 : Prevent the two input URIs from being allowed to match. Helper function for block-matches.
 :
 : @param $uri1  First input URI
 : @param $uri2  Second input URI
 : @error will throw xs:QName("SM-CANT-BLOCK") if unable to record the block.
 : @return empty sequence
 :)
declare function blocks-impl:block-match($uri1 as xs:string, $uri2 as xs:string)
as empty-sequence()
{
  let $_ :=
    (: Suppress sem:rdf-insert's return value :)
    sem:rdf-insert(
      (
        sem:triple(sem:iri($uri1), $const:PRED-MATCH-BLOCK, sem:iri($uri2)),
        sem:triple(sem:iri($uri2), $const:PRED-MATCH-BLOCK, sem:iri($uri1))
      )
    )
  return ()
};

(:
 : Clear a match block between the two input URIs.
 :
 : @param $uri1  First input URI
 : @param $uri2  Second input URI
 :
 : @error will throw xs:QName("SM-CANT-UNBLOCK") if a block is present, but it cannot be cleared
 : @return  fn:true if a block was found and cleared; fn:false if no block was found
 :)
declare function blocks-impl:allow-match($uri1 as xs:string, $uri2 as xs:string)
{
  let $nodes := sem:database-nodes((
    cts:triples(sem:iri($uri1), $const:PRED-MATCH-BLOCK, sem:iri($uri2)),
    cts:triples(sem:iri($uri2), $const:PRED-MATCH-BLOCK, sem:iri($uri1))
  ))
  return (
    $nodes ! xdmp:node-delete(.),
    fn:exists($nodes)
  )
};


*** empty log message ***

*** empty log message ***

svn path=/trunk/eXist-1.0/; revision=721

xquery version "1.0";

declare namespace request="http://exist-db.org/xquery/request";
declare namespace util="http://exist-db.org/xquery/util";

(: Namespace for the local functions in this script :)
declare namespace f="http://exist-db.org/xquery/local-functions";

(:
    Display hits for the current page. The number of hits to print per page
    is read from parameter "howmany". Parameter "start" identifies the
    current offset in the result set.
:)
declare function f:display($hits as item()*, $count as xs:int)
as element()+
{
    if (empty($hits)) then

Nothing found! Back to query form.

    else
        let $howmany := request:request-parameter("howmany", "10") cast as xs:int,
            $start := request:request-parameter("start", "1") cast as xs:int,
            $end := if ($start + $howmany le $count) then $start + $howmany - 1 else $count
        return (

                        <New Query>

                { f:navbar($start, $end, $howmany, $count) }
                {
                    for $p in $start to $end
                    let $current := item-at($hits, $p),
                        $style := if($p mod 2 eq 0) then "high" else "low"
                    return

                            {$p}

                                <xml-source>{$current}</xml-source>

                }
                { f:navbar($start, $end, $howmany, $count) }

                        <New Query>

        )
};

(: Display the navigation bar :)
declare function f:navbar($start as xs:int, $end as xs:int,
$hitsPerPage as xs:int, $count as xs:int) as element()
{
    let $uri := request:request-uri()
    return

                {
                    (: Link to previous page :)
                    if ($start gt 1) then
                        <<
                    else ()
                }

                {
                    (: Create page shortcuts :)
                    let $sections := $count idiv $hitsPerPage - 1,
                        $currentSection := $start idiv $hitsPerPage,
                        $lastSection :=
                            if ($sections lt 0) then 0 else
                                (: show 10 max unless current page > 10 :)
                                if ($sections gt 9) then
                                    if ($currentSection gt 8) then $currentSection + 1
                                    else 9
                                else $sections
                    for $i in 0 to $lastSection
                    let $s := xs:int($i * $hitsPerPage + 1)
                    return
                        if($i ne $currentSection) then

                                [{$i + 1}]

                        else
                            (: current page doesn't get a link :)
                            [{$i + 1}]
                }

                {
                    (: Link to next page :)
                    if ($end lt $count) then

                            >>

                    else ()
                }

};

(: Auxiliary function called in add-to-history()
   to avoid having twice the same query in history
:)
declare function f:string-list-union(
   $list as xs:string*,
   $s as xs:string ) as xs:string*
{
  let $contains :=
    for $ss in $list
      return
        if ( $ss = $s ) then true
        else ""

  return
    if ( $contains ) then
      $list
    else
      ( $list, $s )
};

(:  Add the last query to the query-history. The history is
    stored in the session as an XQuery sequence.
:)
declare function f:add-to-history($query as xs:string) as empty()
{
    let $history := request:get-session-attribute("history")
    return
        request:set-session-attribute( "history",
                                       f:string-list-union($history, $query) )
};

declare function f:handleException() as element()+
{
	,

Defined server limits exceeded!
,
	(
		if($util:exception eq
			"org.exist.xquery.TerminatedException$TimeoutException") then

The execution time of your query exceeded the predefined limits.
			It has been forced to terminate.

		else

The query exceeded the predefined size limits for
			temporary document fragments and has been forced to
			terminate.

	),

Limits can be configured! Please have a look at the
	"watchdog" section in the configuration file (conf.xml).

};

declare function f:eval($query as xs:string) as element()+
{
	let $startTime := current-time(),
		$collection := request:request-parameter("collection", ())
	return
		util:catch("org.exist.xquery.TerminatedException",
			let	$result := util:eval($query, $collection),
				$count := count($result),
				$queryTime := current-time() - $startTime
			return (

Found {$count} hits in
				{seconds-from-duration($queryTime)} seconds.
				{if($count gt 100) then "Max. 100 will be shown." else ()}

,
				request:set-session-attribute(
					"results",
					subsequence($result, 1, 100)
				),
				f:add-to-history($query),
				f:display($result, $count)
			),
			f:handleException()
		)
};

(:  The main function. If a query has been passed in parameter
    "query", execute the query and store the results into the
    session. If "query" is empty, try to retrieve the previous
    result set from the session.
:)
declare function f:main() as element()+
{
    let $query := request:request-parameter("query", ()),
        $previous := request:get-session-attribute("results")
    return
        if ($query) then
           f:eval($query)
        else if ($previous) then
            f:display($previous, count($previous))
        else

Please specify a query! Back to query
            form.

};

<document xmlns:xi="http://www.w3.org/2001/XInclude">

        <logo src="logo.jpg"/>
        <title>Open Source XML Database</title>
	    <author email="wolfgang@exist-db.org">Wolfgang M. Meier</author>

xquery version "1.0";

declare namespace request="http://exist-db.org/xquery/request";
declare namespace util="http://exist-db.org/xquery/util";

(: Namespace for the local functions in this script :)
declare namespace f="http://exist-db.org/xquery/local-functions";

(:
    Display hits for the current page. The number of hits to print per page
    is read from parameter "howmany". Parameter "start" identifies the
    current offset in the result set.
:)
declare function f:display($hits as item()*, $count as xs:int)
as element()+
{
    if (empty($hits)) then

Nothing found! Back to query form.

    else
        let $howmany := request:request-parameter("howmany", "10") cast as xs:int,
            $start := request:request-parameter("start", "1") cast as xs:int,
            $end := if ($start + $howmany le $count) then $start + $howmany - 1 else $count
        return (

                        <New Query>

                { f:navbar($start, $end, $howmany, $count) }
                {
                    for $p in $start to $end
                    let $current := item-at($hits, $p),
                        $style := if($p mod 2 eq 0) then "high" else "low"
                    return

                            {$p}

                                <xml-source>{$current}</xml-source>

                }
                { f:navbar($start, $end, $howmany, $count) }

                        <New Query>

        )
};

(: Display the navigation bar :)
declare function f:navbar($start as xs:int, $end as xs:int,
$hitsPerPage as xs:int, $count as xs:int) as element()
{
    let $uri := request:request-uri()
    return

                {
                    (: Link to previous page :)
                    if ($start gt 1) then
                        <<
                    else ()
                }

                {
                    (: Create page shortcuts :)
                    let $sections := $count idiv $hitsPerPage - 1,
                        $currentSection := $start idiv $hitsPerPage,
                        $lastSection :=
                            if ($sections lt 0) then 0 else
                                (: show 10 max unless current page > 10 :)
                                if ($sections gt 9) then
                                    if ($currentSection gt 8) then $currentSection + 1
                                    else 9
                                else $sections
                    for $i in 0 to $lastSection
                    let $s := xs:int($i * $hitsPerPage + 1)
                    return
                        if($i ne $currentSection) then

                                [{$i + 1}]

                        else
                            (: current page doesn't get a link :)
                            [{$i + 1}]
                }

                {
                    (: Link to next page :)
                    if ($end lt $count) then

                            >>

                    else ()
                }

};

(: Auxiliary function called in add-to-history()
   to avoid having twice the same query in history
:)
declare function f:string-list-union(
   $list as xs:string*,
   $s as xs:string ) as xs:string*
{
  let $contains :=
    for $ss in $list
      return
        if ( $ss = $s ) then true
        else ""

  return
    if ( $contains ) then
      $list
    else
      ( $list, $s )
};

(:  Add the last query to the query-history. The history is
    stored in the session as an XQuery sequence.
:)
declare function f:add-to-history($query as xs:string) as empty()
{
    let $history := request:get-session-attribute("history")
    return
        request:set-session-attribute( "history",
                                       f:string-list-union($history, $query) )
};

declare function f:handleException() as element()+
{
	,

Defined server limits exceeded!
,
	(
		if($util:exception eq
			"org.exist.xquery.TerminatedException$TimeoutException") then

The execution time of your query exceeded the predefined limits.
			It has been forced to terminate.

		else

The query exceeded the predefined size limits for
			temporary document fragments and has been forced to
			terminate.

	),

Limits can be configured! Please have a look at the
	"watchdog" section in the configuration file (conf.xml).

};

declare function f:eval($query as xs:string) as element()+
{
	let $startTime := current-time(),
		$collection := request:request-parameter("collection", ())
	return
		util:catch("org.exist.xquery.TerminatedException",
			let	$result := util:eval($query, $collection),
				$count := count($result),
				$queryTime := current-time() - $startTime
			return (

Found {$count} hits in
				{get-seconds-from-dayTimeDuration($queryTime)} seconds.
				{if($count gt 100) then "Max. 100 will be shown." else ()}

,
				request:set-session-attribute(
					"results",
					subsequence($result, 1, 100)
				),
				f:add-to-history($query),
				f:display($result, $count)
			),
			f:handleException()
		)
};

(:  The main function. If a query has been passed in parameter
    "query", execute the query and store the results into the
    session. If "query" is empty, try to retrieve the previous
    result set from the session.
:)
declare function f:main() as element()+
{
    let $query := request:request-parameter("query", ()),
        $previous := request:get-session-attribute("results")
    return
        if ($query) then
           f:eval($query)
        else if ($previous) then
            f:display($previous, count($previous))
        else

Please specify a query! Back to query
            form.

};

<document xmlns:xi="http://www.w3.org/2001/XInclude">

        <logo src="logo.jpg"/>
        <title>Open Source XML Database</title>
	    <author email="wolfgang@exist-db.org">Wolfgang M. Meier</author>


Adding edit functionality for Value Domain


One more spot with race-reader still

One more spot with race-reader still

xquery version "1.0-ml";
module namespace ingester = "http://marklogic.com/rest-api/transform/ingester";

import module namespace race = "http://grechaw.github.io/entity-types#Race-0.0.1"
    at "/ext/Race-0.0.1.xqy";

(:~
: User: cgreer
: Date: 7/21/16
: Time: 1:34 PM
:)

declare function ingester:transform(
    $context as map:map,
    $params as map:map,
    $body as document-node()
) as document-node()?
{
    let $uri := map:get($context, "uri")
    let $_ := xdmp:log(("Procesing URI " || $uri))
    let $_ :=
        if (fn:matches($uri, "/runners/.*\.json"))
        then () (: "runners" are denormalized into runs, so we don't put them in prod as separate doc. :)
        else if (fn:matches($uri, "/runs/.*\.json"))
        then
            xdmp:document-insert(
                    concat("/runs/", $uri=>fn:substring-after("runs/")=>fn:substring-before(".json"), ".xml"),
                    race:instance-to-envelope(race:extract-instance-Run(doc($uri))),
                    (xdmp:permission("examples-reader", "read"), xdmp:permission("examples-writer", "insert"), xdmp:permission("examples-writer", "update")),
                    "run-envelopes")
        else if (fn:matches($uri, "/races/.*\.json"))
        then
            xdmp:document-insert(
                    "/races/" || $uri=>fn:substring-after("races/")=>substring-before(".json") || ".xml",
                    race:instance-to-envelope(
                    race:extract-instance-Race(doc($uri))),
                    (xdmp:permission("examples-reader", "read"), xdmp:permission("examples-writer", "insert"), xdmp:permission("examples-writer", "update")),
                    "race-envelopes")
        else ()
    return document { " " }
};

xquery version "1.0-ml";
module namespace ingester = "http://marklogic.com/rest-api/transform/ingester";

import module namespace race = "http://grechaw.github.io/entity-types#Race-0.0.1"
    at "/ext/Race-0.0.1.xqy";

(:~
: User: cgreer
: Date: 7/21/16
: Time: 1:34 PM
:)

declare function ingester:transform(
    $context as map:map,
    $params as map:map,
    $body as document-node()
) as document-node()?
{
    let $uri := map:get($context, "uri")
    let $_ := xdmp:log(("Procesing URI " || $uri))
    let $_ :=
        if (fn:matches($uri, "/runners/.*\.json"))
        then () (: "runners" are denormalized into runs, so we don't put them in prod as separate doc. :)
        else if (fn:matches($uri, "/runs/.*\.json"))
        then
            xdmp:document-insert(
                    concat("/runs/", $uri=>fn:substring-after("runs/")=>fn:substring-before(".json"), ".xml"),
                    race:instance-to-envelope(race:extract-instance-Run(doc($uri))),
                    (xdmp:permission("race-reader", "read"), xdmp:permission("race-writer", "insert"), xdmp:permission("race-writer", "update")),
                    "run-envelopes")
        else if (fn:matches($uri, "/races/.*\.json"))
        then
            xdmp:document-insert(
                    "/races/" || $uri=>fn:substring-after("races/")=>substring-before(".json") || ".xml",
                    race:instance-to-envelope(
                    race:extract-instance-Race(doc($uri))),
                    (xdmp:permission("examples-reader", "read"), xdmp:permission("examples-writer", "insert"), xdmp:permission("examples-writer", "update")),
                    "race-envelopes")
        else ()
    return document { " " }
};


Checkpoint for issue #139

Checkpoint for issue #139

(:
 Copyright 2002-2016 MarkLogic Corporation.  All Rights Reserved.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
:)
xquery version "1.0-ml";

(: This module supports code generation for entity services.
 : This module has no public API.  See entity-services.xqy for
 : the public interface.
 :)
module namespace es-codegen = "http://marklogic.com/entity-services-codegen";

import module namespace esi = "http://marklogic.com/entity-services-impl"
    at "entity-services-impl.xqy";

import module namespace functx   = "http://www.functx.com" at "/MarkLogic/functx/functx-1.0-nodoc-2007-01.xqy";

declare namespace es = "http://marklogic.com/entity-services";
declare namespace tde = "http://marklogic.com/xdmp/tde";

declare option xdmp:mapping "false";

declare private function es-codegen:casting-function-name(
    $datatype as xs:string
) as xs:string
{
    if ($datatype eq "iri")
    then "sem:iri"
    else "xs:" || $datatype
};

declare private function es-codegen:comment(
    $comment-text as xs:string
) as xs:string
{
    concat('(:
',
            $comment-text,
           '
:)
')
};

declare function es-codegen:instance-converter-generate(
    $model as map:map
) as document-node()
{
    let $info := map:get($model, "info")
    let $title := map:get($info, "title")
    let $prefix := lower-case(substring($title,1,1)) || substring($title,2)
    let $version:= map:get($info, "version")
    let $base-uri := esi:resolve-base-uri($info)
    return
document {
<module>xquery version "1.0-ml";

(: This module was generated by MarkLogic Entity Services.                          :)
{es-codegen:comment("The source entity type document was " || $title || "-" || $version)}
(:                                                                                :)
(: To use this module, examine how you wish to extract data from sources,           :)
(: and modify the various extract-instance-{{X}} functions to                         :)
(: create the instances you wish.                                                   :)
(:                                                                                :)
(: You may wish to/need to alter                                                    :)
(: 1.  values.  For example, creating duration values from decimal months.          :)
(: 2.  references.  This conversion module assumes you want to denormalize          :)
(:     instances when storing them in documents.  You may choose to remove          :)
(:     code that denormalizes, and just include reference values in your instances  :)
(:     instead.                                                                     :)
(: 3.  Source XPath expressions.  The data coming into the extract-instance-{{X}}     :)
(:     functions will probably not be exactly what this module predicts.            :)
(:                                                                                :)
(: After modifying this file, put it in your project for deployment to the modules  :)
(: database of your application, and check it into your source control system.      :)
(:                                                                                :)
(: Modification History:                                                            :)
{es-codegen:comment("Generated at timestamp: " || fn:current-dateTime())}
(:   Persisted by AUTHOR                                                            :)
(:   Date: DATE                                                                     :)
module namespace {$prefix}
    = "{$base-uri}{$title}-{$version}";

import module namespace es = "http://marklogic.com/entity-services"
    at "/MarkLogic/entity-services/entity-services.xqy";

declare option xdmp:mapping "false";

(:  extract-instance-{{entity-type}} functions                                        :)
(:                                                                                :)
(:  These functions take together take a source document and create a nested        :)
(:  map structure from it.                                                          :)
(:  The resulting map is used by instance-to-canonical-xml to create documents      :)
(:  in the database.                                                                :)
(:                                                                                :)
(:  It is expected that an implementer will edit at least XPath expressions in      :)
(:  the extraction functions.  It is less likely that you will want to edit         :)
(:  the instance-to-canonical-xml or envelope functions.                            :)
{
    for $entity-type-name in map:keys(map:get($model, "definitions"))
    return
    <extract-instance>
(:~
 : Creates a map:map instance from some source document.
 : @param $source-node  A document or node that contains
 :   data for populating a {$entity-type-name}
 : @return A map:map instance with extracted data and
 :   metadata about the instance.
 :)
declare function {$prefix}:extract-instance-{$entity-type-name}(
    $source as node()?
) as map:map
{{
    let $source-node :=
        if ( ($source instance of document-node())
            or (exists($source/{$entity-type-name})))
        then $source/node()
        else $source
    let $instance := json:object()
(: Add the original source document as an attachment.                               :)
        =>map:with('$attachments',
            typeswitch($source-node)
            case object-node() return xdmp:quote($source)
            case array-node() return xdmp:quote($source)
            default return $source)
        =>map:with('$type', '{ $entity-type-name }')
    return
(: if this $source-node is a reference to another instance, then extract its key    :)
    if (empty($source-node/*))
    then $instance=>map:with('$ref', $source-node/text())
(: Otherwise, this source node contains instance data, so populate it.              :)
    else
        $instance
(: The following line identifies the type of this instance.  Do not change it.      :)
(: because this implementation uses an XML envelope.                                :)
(:                                                                                :)
(: The following code populates the properties of the                               :)
{es-codegen:comment("'" || $entity-type-name || "' entity type")}
(: Ensure that all of the property paths are correct for your source data.          :)
(: The general pattern is                                                           :)
(: =>map:with('keyName', casting-function($source-node/path/to/data))               :)
(: but you may also wish to convert values                                          :)
(: =>map:with('dateKeyName',                                                        :)
(:       xdmp:parse-dateTime("[Y0001]-[M01]-[D01]T[h01]:[m01]:[s01].[f1][Z]",       :)
(:       $source-node/path/to/data/in/the/source))                                  :)
(: You can also implement lookup functions,                                         :)
(: =>map:with('lookupKey',                                                          :)
(:       cts:search( collection('customers'),                                       :)
(:           string($source-node/path/to/lookup/key))/id                            :)
(: or populate the instance with constants.                                         :)
(: =>map:with('constantValue', 10)                                                  :)
(: The output of this function should structurally match the output of              :)
(: es:model-get-test-instances($model)                                              :)
(:                                                                                :)
 {
    (: Begin code generation block :)
    let $entity-type := $model=>map:get("definitions")=>map:get($entity-type-name)
    let $properties := map:get($entity-type, "properties")
    for $property-name in map:keys($properties)
    let $required-properties := if (empty(map:get($entity-type, "required")))
        then ()
        else json:array-values( map:get($entity-type, "required") )
    let $is-required := $property-name =
            ( map:get($entity-type, "primaryKey"), $required-properties )
    let $is-array :=
            map:get(map:get($properties, $property-name), "datatype")
            eq "array"
    let $property-datatype := esi:resolve-datatype($model, $entity-type-name, $property-name)
    let $casting-function-name := es-codegen:casting-function-name($property-datatype)
    let $wrap-if-array := function($str, $fn, $arrity-ok) {
            if ($is-array and $is-required)
            then concat("json:to-array(", $str, " ! ", $fn, "(.) )")
            else
            if ($is-array)
            then concat("es:extract-array(", $str, ", ", $fn, if ($arrity-ok) then "#1" else (), ")")
            else concat($fn, "(", $str, ")")
        }
    let $ref :=
        if ($is-array)
        then $properties=>map:get($property-name)=>map:get("items")=>map:get("$ref")
        else $properties=>map:get($property-name)=>map:get("$ref")
    let $path-to-property := concat("$source-node/", $property-name)
    let $property-comment :=
        if (empty($ref))
        then ""
        else if (contains($ref, "#/definitions"))
        then es-codegen:comment("The following property is a local reference.")=>concat("
 ")
        else concat(
        es-codegen:comment('The following property assigment comes from an external reference.'),
        "
 ",
        es-codegen:comment('Its generated value probably requires developer attention.'),
        "
 ")
    let $ref-name := functx:substring-after-last($ref, "/")
    let $extract-reference-fn :=
            concat("function($path) { json:object()",
                      "=>map:with('$type', '", $ref-name, "')",
                      "=>map:with('$ref', $path/", $ref-name, "/text() ) }")
    let $value :=
        if (empty($ref))
        then
            $wrap-if-array($path-to-property, $casting-function-name, true())
        else
            if (contains($ref, "#/definitions"))
            then
                $wrap-if-array($path-to-property, concat($prefix, ":extract-instance-", $ref-name), true())
            else
                $wrap-if-array($path-to-property, $extract-reference-fn, false())
    (: if a property is required, use map:with to force inclusion :)
    let $function-call-string :=
        if ($is-required)
        then "=>   map:with("
        else "=>es:optional("
    return
    concat($property-comment,
           $function-call-string,
           functx:pad-string-to-length("'" || $property-name || "',", " ", max((  (string-length($property-name)+4), 25) )+1 ),
           $value,
           ")
")
    (: end code generation block :)
    }
}};
</extract-instance>/text()
}

(:~
 : Turns an entity instance into an XML structure.
 : This out-of-the box implementation traverses a map structure
 : and turns it deterministically into an XML tree.
 : Using this function as-is should be sufficient for most use
 : cases, and will play well with other generated artifacts.
 : @param $entity-instance A map:map instance returned from one of the extract-instance
 :    functions.
 : @return An XML element that encodes the instance.
 :)
declare function {$prefix}:instance-to-canonical-xml(
    $entity-instance as map:map
) as element()
{{
    (: Construct an element that is named the same as the Entity Type :)
    element {{ map:get($entity-instance, "$type") }}  {{
        if ( map:contains($entity-instance, "$ref") )
        then map:get($entity-instance, "$ref")
        else
            for $key in map:keys($entity-instance)
            let $instance-property := map:get($entity-instance, $key)
            where ($key castable as xs:NCName and $key ne "$type")
            return
                typeswitch ($instance-property)
                (: This branch handles embedded objects.  You can choose to prune
                   an entity's representation of extend it with lookups here. :)
                case json:object+
                    return
                        for $prop in $instance-property
                        return element {{ $key }} {{ {$prefix}:instance-to-canonical-xml($prop) }}
                (: An array can also treated as multiple elements :)
                case json:array
                    return
                        for $val in json:array-values($instance-property)
                        return
                            if ($val instance of json:object)
                            then element {{ $key }} {{
                                attribute datatype {{ "array" }},
                                {$prefix}:instance-to-canonical-xml($val) }}
                            else element {{ $key }} {{
                                attribute datatype {{ "array" }},
                                $val }}
                (: A sequence of values should be simply treated as multiple elements :)
                case item()+
                    return
                        for $val in $instance-property
                        return element {{ $key }} {{ $val }}
                default return element {{ $key }} {{ $instance-property }}
    }}
}};

(:
 : Wraps a canonical instance (returned by instance-to-canonical-xml())
 : within an envelope patterned document, along with the source
 : document, which is stored in an attachments section.
 : @param $entity-instance an instance, as returned by an extract-instance
 : function
 : @return A document which wraps both the canonical instance and source docs.
 :)
declare function {$prefix}:instance-to-envelope(
    $entity-instance as map:map
) as document-node()
{{
    document {{
        element es:envelope {{
            element es:instance {{
                element es:info {{
                    element es:title {{ map:get($entity-instance,'$type') }},
                    element es:version {{ "{$version}" }}
                }},
                {$prefix}:instance-to-canonical-xml($entity-instance)
            }},
            element es:attachments {{
                map:get($entity-instance, "$attachments")
            }}
        }}
    }}
}};

</module>/text()
}

};

declare private function es-codegen:comment-for-conversion(
    $target-property-name as xs:string,
    $target-model as map:map,
    $source-model as map:map
) as xs:string
{
    es-codegen:comment("boo")
};

declare private function es-codegen:value-for-conversion(
    $source-model as map:map,
    $target-model as map:map,
    $target-entity-type-name as xs:string,
    $target-property-name as xs:string,
    $module-prefix
) as xs:string
{
    let $target-entity-type := $target-model
        =>map:get("definitions")
        =>map:get($target-entity-type-name)
    let $target-property := $target-entity-type
        =>map:get("properties")
        =>map:get($target-property-name)
    let $source-entity-type := $source-model
        =>map:get("definitions")
        =>map:get($target-entity-type-name)    (: this function is only called with matching types/props :)
    let $source-properties :=
        if (exists($source-entity-type))
        then $source-entity-type=>map:get("properties")
        else ()
    let $is-missing-source :=
        (empty($source-properties) or not($target-property-name = map:keys($source-properties)))
    let $source-correlate :=
        if (exists($source-properties))
        then map:get($source-properties, $target-property-name)
        else ()
    let $target-is-array := $target-property=>map:get("datatype") eq 'array'
    let $source-is-array :=
        exists($source-correlate) and $source-correlate=>map:get("datatype") eq 'array'
    let $target-ref :=
        if ($target-is-array)
        then $target-property=>map:get("items")=>map:get("$ref")
        else $target-property=>map:get("$ref")
    let $source-ref :=
        if ($source-is-array)
        then $source-correlate=>map:get("items")=>map:get("$ref")
        else
        if (exists($source-correlate))
        then $source-correlate=>map:get("$ref")
        else ()
    let $is-scalar-from-ref := empty($target-ref) and exists($source-ref)
    let $target-is-scalar-array := $target-is-array and empty($target-ref)
         and map:get($target-property, "items")=>map:contains("datatype")
    let $source-is-scalar-array := $source-is-array and empty($source-ref)
        and map:get($source-correlate, "items")=>map:contains("datatype")
    let $properties-correlate := not($target-is-scalar-array) and not($source-is-scalar-array) (: TODO and not ref :)
    let $is-array-from-scalar := $target-is-scalar-array and not($source-is-scalar-array)
    let $is-array-from-array := $target-is-scalar-array and $source-is-scalar-array
    let $truncates-array := not($target-is-array) and $source-is-scalar-array
    let $is-scalar-from-array := empty($target-ref) and $truncates-array

    let $target-datatype := esi:resolve-datatype($target-model, $target-entity-type-name, $target-property-name)
    let $casting-function-name := es-codegen:casting-function-name($target-datatype)
    let $required-properties := if (empty(map:get($target-entity-type, "required")))
        then ()
        else json:array-values( map:get($target-entity-type, "required") )
    let $is-required := $target-property-name =
            ( map:get($target-entity-type, "primaryKey"), $required-properties )
    let $path-to-property := concat("$source-node/", $target-entity-type-name, "/", $target-property-name)
    let $target-ref-name := functx:substring-after-last($target-ref, "/")
    let $source-ref-name := functx:substring-after-last($source-ref, "/")
    (: warning: property path override if source is ref and target is scalar :)
    let $path-to-property := if ($is-scalar-from-ref)
                            then $path-to-property || "/" || $source-ref-name
                            else $path-to-property
    let $wrap-if-array := function($str, $fn, $arrity-ok) {
            if ($target-is-array and $is-required)
            then concat("json:to-array(", $str, " ! ", $fn, "(.) )")
            else
            if ($target-is-array)
            then concat("es:extract-array(", $str, ", ", $fn, if ($arrity-ok) then "#1" else (), ")")
            else concat($fn, "(", $str, ")")
        }
    let $is-reference-from-scalar-or-array :=
        exists($target-ref) and empty($source-ref)

    let $extract-reference-fn :=
            concat("function($path) { json:object()",
                      "=>map:with('$type', '", $target-ref-name, "')",
                      "=>map:with('$ref', $path/", $target-ref-name, "/text() ) }")
    let $extract-scalar-fn :=
            concat("function($path) { json:object()",
                      "=>map:with('$type', '", $target-ref-name, "')",
                      "=>map:with('$ref', ", $casting-function-name, "($path)[1] ) }")

    let $comment :=
        if ($is-missing-source)
        then es-codegen:comment("The following property was missing from the source type")
        else if ($truncates-array)
        then es-codegen:comment("Warning: potential data loss, truncated array.")
        else ()
    let $function-call-string :=
        if ($is-required)
        then "=>   map:with("
        else "=>es:optional("
    let $property-padding :=
        functx:pad-string-to-length("'" || $target-property-name || "',", " ", max((  (string-length($target-property-name)+4), 25) )+1 )
    let $value :=
        (: case one -- missing source :)
        if ($is-missing-source)
        then concat($casting-function-name, "( () )")
        else if ($is-scalar-from-array)
        then concat($casting-function-name, "( fn:head(", $path-to-property, ") )")
        else if (empty($target-ref))
        then $wrap-if-array($path-to-property, $casting-function-name, true())
        else if ($is-reference-from-scalar-or-array)
        then $wrap-if-array($path-to-property, $extract-scalar-fn, true())
        else $wrap-if-array($path-to-property, $extract-reference-fn, false())

    return
        fn:concat($comment,
            $function-call-string,
            $property-padding,
            $value,
            ")
"
          )
};

declare function es-codegen:version-translator-generate(
    $source-model as map:map,
    $target-model as map:map
) as document-node()
{
    let $target-info := map:get($target-model, "info")
    let $target-title := map:get($target-info, "title")
    let $target-prefix := lower-case(substring($target-title,1,1)) || substring($target-title,2)
    let $target-version:= map:get($target-info, "version")
    let $target-definitions := map:get($target-model, "definitions")
    let $target-entity-type-names := map:keys($target-definitions)
    let $target-base-uri := esi:resolve-base-uri($target-info)

    let $source-info := map:get($source-model, "info")
    let $source-title := map:get($source-info, "title")
    let $source-prefix := lower-case(substring($source-title,1,1)) || substring($source-title,2)
    let $source-version:= map:get($source-info, "version")
    let $source-definitions := map:get($source-model, "definitions")
    let $source-base-uri := esi:resolve-base-uri($source-info)

    let $module-prefix := $target-prefix || "-from-" || $source-prefix
    let $module-namespace := concat(
        $target-base-uri,
        $target-title,
        "-" ,
        $target-version,
        "-from-",
        $source-title,
        "-",
        $source-version)
    let $target-info := json:object()

    return document {
<module>xquery version "1.0-ml";
module namespace {$module-prefix}
    = "{$module-namespace}";

import module namespace es = "http://marklogic.com/entity-services"
    at "/MarkLogic/entity-services/entity-services.xqy";

(:
 This module was generated by MarkLogic Entity Services.
 Its purpose is to create instances of entity types
 defined in
 {$target-title || ", version " || $target-version}
 from documents that were persisted according to model
 {$source-title || ", version " || $source-version}

 Modification History:
 Generated at timestamp: {fn:current-dateTime()}
 Persisted by AUTHOR
 Date: DATE
:)

{
    for $entity-type-name in $target-entity-type-names
    let $info-map := json:object()
    let $_ := map:put($target-info, $entity-type-name, $info-map)
    return
    <convert-instance>
(:~
 : Creates a map:map instance representation of the target entity type
 : from a document that contains the source entity instance.
 : @param $source-node  A document or node that contains data conforming to the
 : source entity type
 : @return A map:map instance that holds the data for this entity type.
 :)
declare function {$module-prefix}:convert-instance-{$entity-type-name}(
    $source-node as node()
) as map:map
{{
 json:object()
(:
 The following line identifies the type of this instance.  Do not change it.
:)
 =>map:with('$type', '{ $entity-type-name }')
(:
 The following lines are generated from the '{$entity-type-name}' entity type."
 :)
{
    (: Begin code generation block :)
    let $entity-type := map:get($target-definitions, $entity-type-name)
    let $source-entity-type :=
            if (map:contains($source-definitions, $entity-type-name))
            then map:get($source-definitions, $entity-type-name)
            else map:put($info-map, "missing from source model.", ())
    let $ts-func :=
        function($value) {
            typeswitch($value)
            case json:array return fn:string-join(json:array-values($value), ", ")
            default return $value
        }
    let $compare :=
        function($property) {
            (
            $ts-func(map:get($entity-type, $property)),
            if (exists($source-entity-type) and map:contains($source-entity-type, $property))
            then "( in source: " || $ts-func(map:get($source-entity-type, $property)) || " )"
            else "( in source: None )"
            )
        }
    let $_ :=
            (
                map:put($info-map, "primaryKey: ",
                    if (map:contains($entity-type, "primaryKey"))
                    then $compare("primaryKey")
                    else "None"),
                map:put($info-map, "required: ",
                    if (map:contains($entity-type, "required"))
                    then $compare("required")
                    else "None"),
                map:put($info-map, "range indexes: ",
                    if (map:contains($entity-type, "rangeIndex"))
                    then $compare("rangeIndex")
                    else "None"),
                map:put($info-map, "word lexicons: ",
                    if (map:contains($entity-type, "wordLexicon"))
                    then $compare("wordLexicon")
                    else "None")
            )
    let $properties := map:get($entity-type, "properties")
    for $property-name in map:keys($properties)
    return
        es-codegen:value-for-conversion($source-model, $target-model, $entity-type-name, $property-name, $module-prefix)
    (: end code generation block :)
    }
}};
    </convert-instance>/text(),

    (: Make comments for removed ET types :)
    for $removed-entity-type-name in map:keys($source-definitions)
    where not( $removed-entity-type-name = $target-entity-type-names)
    return
    <removed-type>
(:
 Entity type {$removed-entity-type-name} is in source document
 but not in target document.
 The following XPath expressions should get values from the source
 instances but there is no specified target.
:)
{
map:put($target-info, $removed-entity-type-name, map:entry("", "Removed Type"))
}

(:
 Target Model {$target-title || "-" || $target-version} Info:

{
map:keys($target-info) !
    function($et) {
        let $et-report := map:get($target-info, $et)
        return
        fn:concat("Type ", $et, ":
",
            fn:string-join(
               map:keys($et-report) !
                    function($report-key) {
                        "    ",
                        $report-key,
                        fn:string-join(map:get($et-report, $report-key), ", "),
                        "
"
                    }(.)
            )
        ),
        "
"
   }(.)
}
:)
    </removed-type>/text()

}

</module>/text()
    }
};

(:
 Copyright 2002-2016 MarkLogic Corporation.  All Rights Reserved.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
:)
xquery version "1.0-ml";

(: This module supports code generation for entity services.
 : This module has no public API.  See entity-services.xqy for
 : the public interface.
 :)
module namespace es-codegen = "http://marklogic.com/entity-services-codegen";

import module namespace esi = "http://marklogic.com/entity-services-impl"
    at "entity-services-impl.xqy";

import module namespace functx   = "http://www.functx.com" at "/MarkLogic/functx/functx-1.0-nodoc-2007-01.xqy";

declare namespace es = "http://marklogic.com/entity-services";
declare namespace tde = "http://marklogic.com/xdmp/tde";

declare option xdmp:mapping "false";

declare private function es-codegen:casting-function-name(
    $datatype as xs:string
) as xs:string
{
    if ($datatype eq "iri")
    then "sem:iri"
    else "xs:" || $datatype
};

declare private function es-codegen:comment(
    $comment-text as xs:string
) as xs:string
{
    concat('(: ',
        functx:pad-string-to-length(
            $comment-text,
            " ",
            max( ( (string-length($comment-text)+4), 80) )),
           ' :)')
};

declare function es-codegen:instance-converter-generate(
    $model as map:map
) as document-node()
{
    let $info := map:get($model, "info")
    let $title := map:get($info, "title")
    let $prefix := lower-case(substring($title,1,1)) || substring($title,2)
    let $version:= map:get($info, "version")
    let $base-uri := esi:resolve-base-uri($info)
    return
document {
<module>xquery version "1.0-ml";

(: This module was generated by MarkLogic Entity Services.                          :)
{es-codegen:comment("The source entity type document was " || $title || "-" || $version)}
(:                                                                                :)
(: To use this module, examine how you wish to extract data from sources,           :)
(: and modify the various extract-instance-{{X}} functions to                         :)
(: create the instances you wish.                                                   :)
(:                                                                                :)
(: You may wish to/need to alter                                                    :)
(: 1.  values.  For example, creating duration values from decimal months.          :)
(: 2.  references.  This conversion module assumes you want to denormalize          :)
(:     instances when storing them in documents.  You may choose to remove          :)
(:     code that denormalizes, and just include reference values in your instances  :)
(:     instead.                                                                     :)
(: 3.  Source XPath expressions.  The data coming into the extract-instance-{{X}}     :)
(:     functions will probably not be exactly what this module predicts.            :)
(:                                                                                :)
(: After modifying this file, put it in your project for deployment to the modules  :)
(: database of your application, and check it into your source control system.      :)
(:                                                                                :)
(: Modification History:                                                            :)
{es-codegen:comment("Generated at timestamp: " || fn:current-dateTime())}
(:   Persisted by AUTHOR                                                            :)
(:   Date: DATE                                                                     :)
module namespace {$prefix}
    = "{$base-uri}{$title}-{$version}";

import module namespace es = "http://marklogic.com/entity-services"
    at "/MarkLogic/entity-services/entity-services.xqy";

declare option xdmp:mapping "false";

(:  extract-instance-{{entity-type}} functions                                        :)
(:                                                                                :)
(:  These functions take together take a source document and create a nested        :)
(:  map structure from it.                                                          :)
(:  The resulting map is used by instance-to-canonical-xml to create documents      :)
(:  in the database.                                                                :)
(:                                                                                :)
(:  It is expected that an implementer will edit at least XPath expressions in      :)
(:  the extraction functions.  It is less likely that you will want to edit         :)
(:  the instance-to-canonical-xml or envelope functions.                            :)
{
    for $entity-type-name in map:keys(map:get($model, "definitions"))
    return
    <extract-instance>
(:~
 : Creates a map:map instance from some source document.
 : @param $source-node  A document or node that contains
 :   data for populating a {$entity-type-name}
 : @return A map:map instance with extracted data and
 :   metadata about the instance.
 :)
declare function {$prefix}:extract-instance-{$entity-type-name}(
    $source as node()?
) as map:map
{{
    let $source-node :=
        if ( ($source instance of document-node())
            or (exists($source/{$entity-type-name})))
        then $source/node()
        else $source
    let $instance := json:object()
(: Add the original source document as an attachment.                               :)
        =>map:with('$attachments',
            typeswitch($source-node)
            case object-node() return xdmp:quote($source)
            case array-node() return xdmp:quote($source)
            default return $source)
        =>map:with('$type', '{ $entity-type-name }')
    return
(: if this $source-node is a reference to another instance, then extract its key    :)
    if (empty($source-node/*))
    then $instance=>map:with('$ref', $source-node/text())
(: Otherwise, this source node contains instance data, so populate it.              :)
    else
        $instance
(: The following line identifies the type of this instance.  Do not change it.      :)
(: because this implementation uses an XML envelope.                                :)
(:                                                                                :)
(: The following code populates the properties of the                               :)
{es-codegen:comment("'" || $entity-type-name || "' entity type")}
(: Ensure that all of the property paths are correct for your source data.          :)
(: The general pattern is                                                           :)
(: =>map:with('keyName', casting-function($source-node/path/to/data))               :)
(: but you may also wish to convert values                                          :)
(: =>map:with('dateKeyName',                                                        :)
(:       xdmp:parse-dateTime("[Y0001]-[M01]-[D01]T[h01]:[m01]:[s01].[f1][Z]",       :)
(:       $source-node/path/to/data/in/the/source))                                  :)
(: You can also implement lookup functions,                                         :)
(: =>map:with('lookupKey',                                                          :)
(:       cts:search( collection('customers'),                                       :)
(:           string($source-node/path/to/lookup/key))/id                            :)
(: or populate the instance with constants.                                         :)
(: =>map:with('constantValue', 10)                                                  :)
(: The output of this function should structurally match the output of              :)
(: es:model-get-test-instances($model)                                              :)
(:                                                                                :)
 {
    (: Begin code generation block :)
    let $entity-type := $model=>map:get("definitions")=>map:get($entity-type-name)
    let $properties := map:get($entity-type, "properties")
    for $property-name in map:keys($properties)
    let $required-properties := if (empty(map:get($entity-type, "required")))
        then ()
        else json:array-values( map:get($entity-type, "required") )
    let $is-required := $property-name =
            ( map:get($entity-type, "primaryKey"), $required-properties )
    let $is-array :=
            map:get(map:get($properties, $property-name), "datatype")
            eq "array"
    let $property-datatype := esi:resolve-datatype($model, $entity-type-name, $property-name)
    let $casting-function-name := es-codegen:casting-function-name($property-datatype)
    let $wrap-if-array := function($str, $fn, $arrity-ok) {
            if ($is-array and $is-required)
            then concat("json:to-array(", $str, " ! ", $fn, "(.) )")
            else
            if ($is-array)
            then concat("es:extract-array(", $str, ", ", $fn, if ($arrity-ok) then "#1" else (), ")")
            else concat($fn, "(", $str, ")")
        }
    let $ref :=
        if ($is-array)
        then $properties=>map:get($property-name)=>map:get("items")=>map:get("$ref")
        else $properties=>map:get($property-name)=>map:get("$ref")
    let $path-to-property := concat("$source-node/", $property-name)
    let $property-comment :=
        if (empty($ref))
        then ""
        else if (contains($ref, "#/definitions"))
        then es-codegen:comment("The following property is a local reference.")=>concat("
 ")
        else concat(
        es-codegen:comment('The following property assigment comes from an external reference.'),
        "
 ",
        es-codegen:comment('Its generated value probably requires developer attention.'),
        "
 ")
    let $ref-name := functx:substring-after-last($ref, "/")
    let $extract-reference-fn :=
            concat("function($path) { json:object()",
                      "=>map:with('$type', '", $ref-name, "')",
                      "=>map:with('$ref', $path/", $ref-name, "/text() ) }")
    let $value :=
        if (empty($ref))
        then
            $wrap-if-array($path-to-property, $casting-function-name, true())
        else
            if (contains($ref, "#/definitions"))
            then
                $wrap-if-array($path-to-property, concat($prefix, ":extract-instance-", $ref-name), true())
            else
                $wrap-if-array($path-to-property, $extract-reference-fn, false())
    (: if a property is required, use map:with to force inclusion :)
    let $function-call-string :=
        if ($is-required)
        then "=>   map:with("
        else "=>es:optional("
    return
    concat($property-comment,
           $function-call-string,
           functx:pad-string-to-length("'" || $property-name || "',", " ", max((  (string-length($property-name)+4), 25) )+1 ),
           $value,
           ")
")
    (: end code generation block :)
    }
}};
</extract-instance>/text()
}

(:~
 : Turns an entity instance into an XML structure.
 : This out-of-the box implementation traverses a map structure
 : and turns it deterministically into an XML tree.
 : Using this function as-is should be sufficient for most use
 : cases, and will play well with other generated artifacts.
 : @param $entity-instance A map:map instance returned from one of the extract-instance
 :    functions.
 : @return An XML element that encodes the instance.
 :)
declare function {$prefix}:instance-to-canonical-xml(
    $entity-instance as map:map
) as element()
{{
    (: Construct an element that is named the same as the Entity Type :)
    element {{ map:get($entity-instance, "$type") }}  {{
        if ( map:contains($entity-instance, "$ref") )
        then map:get($entity-instance, "$ref")
        else
            for $key in map:keys($entity-instance)
            let $instance-property := map:get($entity-instance, $key)
            where ($key castable as xs:NCName and $key ne "$type")
            return
                typeswitch ($instance-property)
                (: This branch handles embedded objects.  You can choose to prune
                   an entity's representation of extend it with lookups here. :)
                case json:object+
                    return
                        for $prop in $instance-property
                        return element {{ $key }} {{ {$prefix}:instance-to-canonical-xml($prop) }}
                (: An array can also treated as multiple elements :)
                case json:array
                    return
                        for $val in json:array-values($instance-property)
                        return
                            if ($val instance of json:object)
                            then element {{ $key }} {{
                                attribute datatype {{ "array" }},
                                {$prefix}:instance-to-canonical-xml($val) }}
                            else element {{ $key }} {{
                                attribute datatype {{ "array" }},
                                $val }}
                (: A sequence of values should be simply treated as multiple elements :)
                case item()+
                    return
                        for $val in $instance-property
                        return element {{ $key }} {{ $val }}
                default return element {{ $key }} {{ $instance-property }}
    }}
}};

(:
 : Wraps a canonical instance (returned by instance-to-canonical-xml())
 : within an envelope patterned document, along with the source
 : document, which is stored in an attachments section.
 : @param $entity-instance an instance, as returned by an extract-instance
 : function
 : @return A document which wraps both the canonical instance and source docs.
 :)
declare function {$prefix}:instance-to-envelope(
    $entity-instance as map:map
) as document-node()
{{
    document {{
        element es:envelope {{
            element es:instance {{
                element es:info {{
                    element es:title {{ map:get($entity-instance,'$type') }},
                    element es:version {{ "{$version}" }}
                }},
                {$prefix}:instance-to-canonical-xml($entity-instance)
            }},
            element es:attachments {{
                map:get($entity-instance, "$attachments")
            }}
        }}
    }}
}};

</module>/text()
}

};

declare private function es-codegen:comment-for-conversion(
    $target-property-name as xs:string,
    $target-model as map:map,
    $source-model as map:map
) as xs:string
{
    es-codegen:comment("boo")
};

declare private function es-codegen:value-for-conversion(
    $source-model as map:map,
    $target-model as map:map,
    $target-entity-type-name as xs:string,
    $target-property-name as xs:string,
    $module-prefix
) as xs:string
{
    let $target-entity-type := $target-model
        =>map:get("definitions")
        =>map:get($target-entity-type-name)
    let $target-property := $target-entity-type
        =>map:get("properties")
        =>map:get($target-property-name)
    let $source-entity-type := $source-model
        =>map:get("definitions")
        =>map:get($target-entity-type-name)    (: this function is only called with matching types/props :)
    let $source-properties :=
        if (exists($source-entity-type))
        then $source-entity-type=>map:get("properties")
        else ()
    let $is-missing-source :=
        (empty($source-properties) or not($target-property-name = map:keys($source-properties)))
    let $source-correlate :=
        if (exists($source-properties))
        then map:get($source-properties, $target-property-name)
        else ()
    let $target-is-array := $target-property=>map:get("datatype") eq 'array'
    let $source-is-array :=
        exists($source-correlate) and $source-correlate=>map:get("datatype") eq 'array'
    let $target-ref :=
        if ($target-is-array)
        then $target-property=>map:get("items")=>map:get("$ref")
        else $target-property=>map:get("$ref")
    let $source-ref :=
        if ($source-is-array)
        then $source-correlate=>map:get("items")=>map:get("$ref")
        else
        if (exists($source-correlate))
        then $source-correlate=>map:get("$ref")
        else ()
    let $is-scalar-from-ref := empty($target-ref) and exists($source-ref)
    let $target-is-scalar-array := $target-is-array and empty($target-ref)
         and map:get($target-property, "items")=>map:contains("datatype")
    let $source-is-scalar-array := $source-is-array and empty($source-ref)
        and map:get($source-correlate, "items")=>map:contains("datatype")
    let $properties-correlate := not($target-is-scalar-array) and not($source-is-scalar-array) (: TODO and not ref :)
    let $is-array-from-scalar := $target-is-scalar-array and not($source-is-scalar-array)
    let $is-array-from-array := $target-is-scalar-array and $source-is-scalar-array
    let $truncates-array := not($target-is-array) and $source-is-scalar-array
    let $is-scalar-from-array := empty($target-ref) and $truncates-array

    let $target-datatype := esi:resolve-datatype($target-model, $target-entity-type-name, $target-property-name)
    let $casting-function-name := es-codegen:casting-function-name($target-datatype)
    let $required-properties := if (empty(map:get($target-entity-type, "required")))
        then ()
        else json:array-values( map:get($target-entity-type, "required") )
    let $is-required := $target-property-name =
            ( map:get($target-entity-type, "primaryKey"), $required-properties )
    let $path-to-property := concat("$source-node/", $target-entity-type-name, "/", $target-property-name)
    let $target-ref-name := functx:substring-after-last($target-ref, "/")
    let $source-ref-name := functx:substring-after-last($source-ref, "/")
    (: warning: property path override if source is ref and target is scalar :)
    let $path-to-property := if ($is-scalar-from-ref)
                            then $path-to-property || "/" || $source-ref-name
                            else $path-to-property
    let $wrap-if-array := function($str, $fn, $arrity-ok) {
            if ($target-is-array and $is-required)
            then concat("json:to-array(", $str, " ! ", $fn, "(.) )")
            else
            if ($target-is-array)
            then concat("es:extract-array(", $str, ", ", $fn, if ($arrity-ok) then "#1" else (), ")")
            else concat($fn, "(", $str, ")")
        }
    let $is-reference-from-scalar-or-array :=
        exists($target-ref) and empty($source-ref)

    let $extract-reference-fn :=
            concat("function($path) { json:object()",
                      "=>map:with('$type', '", $target-ref-name, "')",
                      "=>map:with('$ref', $path/", $target-ref-name, "/text() ) }")
    let $extract-scalar-fn :=
            concat("function($path) { json:object()",
                      "=>map:with('$type', '", $target-ref-name, "')",
                      "=>map:with('$ref', ", $casting-function-name, "($path)[1] ) }")

    let $comment :=
        if ($is-missing-source)
        then es-codegen:comment("The following property was missing from the source type")=>concat("
")
        else if ($truncates-array)
        then es-codegen:comment("Warning: potential data loss, truncated array.")=>concat("
")
        else ()
    let $function-call-string :=
        if ($is-required)
        then "=>   map:with("
        else "=>es:optional("
    let $property-padding :=
        functx:pad-string-to-length("'" || $target-property-name || "',", " ", max((  (string-length($target-property-name)+4), 25) )+1 )
    let $value :=
        (: case one -- missing source :)
        if ($is-missing-source)
        then concat($casting-function-name, "( () )")
        else if ($is-scalar-from-array)
        then concat($casting-function-name, "( fn:head(", $path-to-property, ") )")
        else if (empty($target-ref))
        then $wrap-if-array($path-to-property, $casting-function-name, true())
        else if ($is-reference-from-scalar-or-array)
        then $wrap-if-array($path-to-property, $extract-scalar-fn, true())
        else $wrap-if-array($path-to-property, $extract-reference-fn, false())

    return
        concat($comment,
            $function-call-string,
            $property-padding,
            $value,
            ")
"
          )
};

declare function es-codegen:version-translator-generate(
    $source-model as map:map,
    $target-model as map:map
) as document-node()
{
    let $target-info := map:get($target-model, "info")
    let $target-title := map:get($target-info, "title")
    let $target-prefix := lower-case(substring($target-title,1,1)) || substring($target-title,2)
    let $target-version:= map:get($target-info, "version")
    let $target-definitions := map:get($target-model, "definitions")
    let $target-entity-type-names := map:keys($target-definitions)
    let $target-base-uri := esi:resolve-base-uri($target-info)

    let $source-info := map:get($source-model, "info")
    let $source-title := map:get($source-info, "title")
    let $source-prefix := lower-case(substring($source-title,1,1)) || substring($source-title,2)
    let $source-version:= map:get($source-info, "version")
    let $source-definitions := map:get($source-model, "definitions")
    let $source-base-uri := esi:resolve-base-uri($source-info)

    let $module-prefix := $target-prefix || "-from-" || $source-prefix
    let $module-namespace := concat(
        $target-base-uri,
        $target-title,
        "-" ,
        $target-version,
        "-from-",
        $source-title,
        "-",
        $source-version)

    return document {
<module>xquery version "1.0-ml";
module namespace {$module-prefix}
    = "{$module-namespace}";

import module namespace es = "http://marklogic.com/entity-services"
    at "/MarkLogic/entity-services/entity-services.xqy";

(: This module was generated by MarkLogic Entity Services.                          :)
(: Its purpose is to create instances of entity types                               :)
(: defined in                                                                       :)
{es-codegen:comment($target-title || ", version " || $target-version)}
(: from documents that were persisted according to model                            :)
{es-codegen:comment($source-title || ", version " || $source-version)}
(:                                                                                :)
(: Modification History:                                                            :)
{es-codegen:comment("Generated at timestamp: " || fn:current-dateTime())}
(:   Persisted by AUTHOR                                                            :)
(:   Date: DATE                                                                     :)

{
    for $entity-type-name in $target-entity-type-names
    return
    <convert-instance>
(:~
 : Creates a map:map instance representation of the target entity type
 : from a document that contains the source entity instance.
 : @param $source-node  A document or node that contains data conforming to the
 : source entity type
 : @return A map:map instance that holds the data for this entity type.
 :)
declare function {$module-prefix}:convert-instance-{$entity-type-name}(
    $source-node as node()
) as map:map
{{
 json:object()
(: The following line identifies the type of this instance.  Do not change it.      :)
 =>map:with('$type', '{ $entity-type-name }')
{es-codegen:comment("The following lines are generated from the '"
                     || $entity-type-name ||
                     "' entity type.")=>concat("
 ")}
{
    (: Begin code generation block :)
    let $entity-type := map:get($target-definitions, $entity-type-name)
    let $properties := map:get($entity-type, "properties")
    for $property-name in map:keys($properties)
    return
        es-codegen:value-for-conversion($source-model, $target-model, $entity-type-name, $property-name, $module-prefix)
    (: end code generation block :)
    }
}};
    </convert-instance>/text(),

    (: Make comments for removed ET types :)
    for $removed-entity-type-name in map:keys($source-definitions)
    where not( $removed-entity-type-name = $target-entity-type-names)
    return
    <removed-type>
(: Entity type {$removed-entity-type-name} is in source document
 : but not in target document.
 : The following XPath expressions should get values from the source
 : instances but there is no specified target
 :
 :)
    </removed-type>/text()

}

</module>/text()
    }
};


Fix to variable name

Fix to variable name

xquery version "1.0" encoding "UTF-8";

let $xsl-fo-document as document-node() := doc("fop-readme.xml")
let $media-type as xs:string := "application/pdf"
return
    response:stream-binary(
        xslfo:render($xsl-fo-document, $media-type, ()),
        $media-type,
        "output.pdf"
    )

xquery version "1.0" encoding "UTF-8";

let $xsl-fo-document as document-node() := doc("fop-readme.xml")
let $mime-type as xs:string := "application/pdf"
return
    response:stream-binary(
        xslfo:render($xsl-fo-document, $mime-type, ()),
        $mime-type,
        "output.pdf"
    )