Addressed Ghislain's review comments

Addressed Ghislain's review comments

jsoniq version "1.0";
(:
 : Copyright 2014 28msec Inc.
 :)

(:~
 :
This module provides functionality for retrieving concepts
 : and labels.

 :
 : @author Federico Cavalieri
 :)
module namespace concepts = "http://28.io/modules/xbrl/concepts";

import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Name of the collection the concepts are stored in.
 :)
declare variable $concepts:col as xs:string := "concepts";

(:~
 : Name of the field pointing to the archive.
 :)
declare variable $concepts:ARCHIVE as xs:string := "Archive";

(:~
 : Name of the field pointing to the role.
 :)
declare variable $concepts:ROLE as xs:string := "Role";

(:~
 : Name of the field pointing to the name.
 :)
declare variable $concepts:NAME as xs:string := "Name";

(:~
 : Name of the field pointing to the labels.
 :)
declare variable $concepts:LABELS as xs:string := "Labels";

(:~
 : The default component link role.
 :)
declare variable $concepts:DEFAULT_COMPONENT_LINK_ROLE as xs:string :=
    "http://www.xbrl.org/2003/role/link";

(:~
 : The standard label role.
 :)
declare variable $concepts:STANDARD_LABEL_ROLE as xs:string :=
    "http://www.xbrl.org/2003/role/label";

(:~
 : A joker to ask for all concept names.
 :)
declare variable $concepts:ALL_CONCEPT_NAMES as xs:string := "";

(:~
 :
Retrieves all concepts.

 :
 : @return all concepts.
 :)
declare function concepts:concepts() as object*
{
  let $conn := concepts:connection()
  return concepts:find($conn, {})
};

(:~
 :
Retrieves the concepts which:
 :  - concept name matches a given one;
 :  - archive number matches that of a given component;
 :  - component role matches that of a given component or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $component-or-ids the CIDs or the components themselves.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts-for-components($concept-names as string*,
    $component-or-ids as item*) as object*
{
  let $components := components:components($component-or-ids)
  return concepts:concepts($concept-names, $components.Archive[not $$ instance of null], $components.Role)
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives the archive numbers.
 : @param $component-roles the component roles.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives as string*,
    $component-roles as string*
  ) as object*
{
  let $conn := concepts:connection()
  where exists($archives)
  return concepts:find($conn,
    {|
      {
        $concepts:ARCHIVE : { "$in" : [ $archives ] },
        $concepts:ROLE : { "$in" :
          [ $component-roles, "http://www.xbrl.org/2003/role/link" ]
        }
      },
      {
        $concepts:NAME : { "$in" : [ $concept-names ] }
      }[not $concept-names = $concepts:ALL_CONCEPT_NAMES]
    |}
  )
};

(:~
 :
Retrieves all the labels with the given label role and language in the
 : matching concepts.

 :
 :
Matching concepts are those which:
 :  - concept name matches a given one,
 :  - archive number matches that of a given component,
 :  - component role matches that of a given component or is the default
 :    component role.
 :

 :
 :
Language matching can either be exact, if no options are given,
 : or approximated, if at least one of the following options is given:

 :

 :
MatchDown: whether to match a more specific language, e.g.:
 :       "en" will match labels which language is "en" or "en-US".

 :
MatchUp: whether to match a less specific language, e.g.:
 :       "en-US" will match labels which language is "en-US" or "en".

 :

 :
 : @param $concept-names the concepts names.
 : @param $component-or-ids the CIDs or the components themselves.
 : @param $label-role the label role.
 : @param $language the label language.
 : @param $options optional parameters to control language matching.
 :
 : @return the matching labels.
 :)
 declare function concepts:labels-for-components(
     $concept-names as string*,
     $component-or-ids as item*,
     $label-role as string,
     $language as string,
     $options as object?
   ) as string*
{
  let $components := components:components($component-or-ids)
  let $concepts := concepts:concepts-for-components($concept-names, $components)
  return concepts:labels($concept-names, $components.Archive[not $$ instance of null], $components.Role,
      $label-role, $language, $concepts, $options)
};

(:~
 :
Retrieves all the labels with the given label role and language in the
 : matching concepts.

 :
 :
Matching concepts are those which:
 :  - concept name matches a given one,
 :  - archive number matches that of a given component,
 :  - component role matches that of a given component or is the default
 :    component role.
 :

 :
 :
The set of concepts to search in is specified as a parameter.

 :
 :
Language matching can either be exact, if no options are given,
 : or approximated, if at least one of the following options is given:

 :

 :
MatchDown: whether to match a more specific language, e.g.:
 :       "en" will match labels which language is "en" or "en-US".

 :
MatchUp: whether to match a less specific language, e.g.:
 :       "en-US" will match labels which language is "en-US" or "en".

 :
MatchAnyVariant: whether to match a different variant of the same
 :       language, e.g.: "en-US" will match labels which language is "en-US"
 :       or "en-UK".

 :

 :
 : @param $concept-names the concepts names.
 : @param $component-or-ids the CIDs or the components themselves.
 : @param $label-role the label role.
 : @param $language the label language.
 : @param $concepts the concepts in which the labels will be
 :                  searched.
 : @param $options optional parameters to control language matching.
 :
 : @return the matching labels.
 :)
declare function concepts:labels-for-components(
    $concept-names as string*,
    $component-or-ids as item*,
    $label-role as string,
    $language as string,
    $concepts as object*,
    $options as object?
  ) as string*
{
  let $components := components:components($component-or-ids)
    return concepts:labels($concept-names, $components.Archive[not $$ instance of null], $components.Role,
      $label-role, $language, $concepts, $options)
};

(:~
 :
Retrieves all the labels with the given label role and language in the
 : matching concepts.

 :
 :
Matching concepts are those which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 :
Language matching can either be exact, if no options are given,
 : or approximated, if at least one of the following options is given:

 :

 :
MatchDown: whether to match a more specific language, e.g.:
 :       "en" will match labels which language is "en" or "en-US".

 :
MatchUp: whether to match a less specific language, e.g.:
 :       "en-US" will match labels which language is "en-US" or "en".

 :
MatchAnyVariant: whether to match a different variant of the same
 :       language, e.g.: "en-US" will match labels which language is "en-US"
 :       or "en-UK".

 :

 :
 : @param $concept-names the concepts names.
 : @param $archives the archive numbers.
 : @param $component-roles the component roles.
 : @param $label-role the label role.
 : @param $language the label language.
 : @param $options optional parameters to control language matching.
 :
 : @return the matching labels.
 :)
declare function concepts:labels(
    $concept-names as string*,
    $archives as string*,
    $component-roles as string*,
    $label-role as string,
    $language as string,
    $options as object?
  ) as string*
{
  let $concepts := concepts:concepts($concept-names, $archives, $component-roles)
  return concepts:labels($concept-names, $archives, $component-roles, $label-role, $language,
      $concepts, $options)
};

(:~
 :
Retrieves all the labels with the given label role and language in the
 : matching concepts.

 :
 :
Matching concepts are those which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 :
The set of concepts to search in is specified as a parameter.

 :
 :
Language matching can either be exact, if no options are given,
 : or approximated, if at least one of the following options is given:

 :

 :
MatchDown: whether to match a more specific language, e.g.:
 :       "en" will match labels which language is "en" or "en-US".

 :
MatchUp: whether to match a less specific language, e.g.:
 :       "en-US" will match labels which language is "en-US" or "en".

 :
MatchAnyVariant: whether to match a different variant of the same
 :       language, e.g.: "en-US" will match labels which language is "en-US"
 :       or "en-UK".

 :

 :
 : @param $concept-names the concepts names.
 : @param $archives the archive numbers.
 : @param $component-roles the component roles.
 : @param $label-role the label role.
 : @param $language the label language.
 : @param $concepts the concepts in which the labels will be
 :                  searched.
 : @param $options Optional parameters to control language matching.
 :
 : @return the matching labels.
 :)
declare function concepts:labels(
    $concept-names as string*,
    $archives as string*,
    $component-roles as string*,
    $label-role as string,
    $language as string,
    $concepts as object*,
    $options as object?
  ) as string*
{
  let $normalized-language := concepts:normalize-language($language)
  let $concept-labels-groups-for-role :=$concepts[
      $$($concepts:NAME)    = $concept-names and
      $$($concepts:ARCHIVE) = $archives and
      $$($concepts:ROLE)    = $component-roles]($concepts:LABELS)($label-role)
  for $concept-labels-group in $concept-labels-groups-for-role
  let $perfect-match := $concept-labels-group($normalized-language)
  return
    if ($perfect-match)
    then $concept-labels-group($normalized-language)
    else concepts:approximated-labels-match($concept-labels-group, $normalized-language, $options)
};

declare %private function concepts:approximated-labels-match(
    $concept-labels-group as object,
    $normalized-language as string,
    $options as object?
  ) as string*
{
  if ($options)
  then
    let $keys := keys($concept-labels-group)
    return
    (
      if ($options.MatchDown cast as boolean?)
      then
        for $matching-key in $keys[starts-with($$, $normalized-language)]
        return $concept-labels-group($matching-key)
      else (),
      if ($options.MatchUp cast as boolean?)
      then
        for $matching-key in $keys[starts-with($normalized-language, $$)]
        return $concept-labels-group($matching-key)
      else (),
      if ($options.MatchAnyVariant cast as boolean?)
      then
        if (contains($normalized-language, "-"))
        then
          for $matching-key in $keys[starts-with($$, substring-before($normalized-language, "-") || "-")]
          return $concept-labels-group($matching-key)
        else ()
      else ()
    )
  else ()
};

declare %private function concepts:normalize-language($language as string) as string
{
  (: TODO Enable this when the DB has been reimported :)
  (:replace(lower-case($language), "_", "-"):)
  $language
};

declare function concepts:find($conn as anyURI, $query as object) as object()*
{
  mongo:find($conn, $concepts:col, concepts:hinted-query($query))
};

declare function concepts:find($conn as anyURI, $query as object, $projection as object) as object()*
{
  mongo:find($conn, $concepts:col, concepts:hinted-query($query), $projection)
};

declare %private function concepts:hinted-query($query as object) as object
{
  switch (true)
    case (exists($query("_id")))
      return { "$query": $query, "$hint": "_id_" }
    case (exists($query("Archive")) and exists($query("Role")))
      return { "$query": $query, "$hint": "Archive_1_Role_1_Name_1" }
    case (exists($query("Archive")) and exists($query("Name")))
      return { "$query": $query, "$hint": "Name_1_Archive_1" }
    case (exists($query("Archive")))
      return { "$query": $query, "$hint": "Archive_1_Role_1_Name_1" }
    default
      return $query
};

declare %private %an:strictlydeterministic function concepts:connection() as anyURI
{
  let $credentials :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return if (empty($credentials))
             then error(QName("components:CONNECTION-FAILED"), "no xbrl MongoDB configured")
             else $credentials
  return
    try {
      mongo:connect($credentials)
    } catch mongo:* {
      error(QName("components:CONNECTION-FAILED"), $err:description)
    }
};

jsoniq version "1.0";
(:
 : Copyright 2014 28msec Inc.
 :)

(:~
 :
This module provides functionality for retrieving concepts
 : and labels.

 :
 : @author Federico Cavalieri
 :)
module namespace concepts = "http://28.io/modules/xbrl/concepts";

import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Name of the collection the concepts are stored in.
 :)
declare variable $concepts:col as xs:string := "concepts";

(:~
 : Name of the field pointing to the archive.
 :)
declare variable $concepts:ARCHIVE as xs:string := "Archive";

(:~
 : Name of the field pointing to the role.
 :)
declare variable $concepts:ROLE as xs:string := "Role";

(:~
 : Name of the field pointing to the name.
 :)
declare variable $concepts:NAME as xs:string := "Name";

(:~
 : Name of the field pointing to the labels.
 :)
declare variable $concepts:LABELS as xs:string := "Labels";

(:~
 : The default component link role.
 :)
declare variable $concepts:DEFAULT_COMPONENT_LINK_ROLE as xs:string :=
    "http://www.xbrl.org/2003/role/link";

(:~
 : The standard label role.
 :)
declare variable $concepts:STANDARD_LABEL_ROLE as xs:string :=
    "http://www.xbrl.org/2003/role/label";

(:~
 : A joker to ask for all concept names.
 :)
declare variable $concepts:ALL_CONCEPT_NAMES as xs:string := "";

(:~
 :
Retrieves all concepts.

 :
 : @return all concepts.
 :)
declare function concepts:concepts() as object*
{
  let $conn := concepts:connection()
  return concepts:find($conn, {})
};

(:~
 :
Retrieves the concepts which:
 :  - concept name matches a given one;
 :  - archive number matches that of a given component;
 :  - component role matches that of a given component or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $component-or-ids the CIDs or the components themselves.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts-for-components($concept-names as string*,
    $component-or-ids as item*) as object*
{
  let $components := components:components($component-or-ids)
  return concepts:concepts($concept-names, $components.Archive[not $$ instance of null], $components.Role)
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives the archive numbers.
 : @param $component-roles the component roles.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives as string*,
    $component-roles as string*
  ) as object*
{
  let $conn := concepts:connection()
  where exists($archives)
  return concepts:find($conn,
    {|
      {
        $concepts:ARCHIVE : { "$in" : [ $archives ] },
        $concepts:ROLE : { "$in" :
          [ $component-roles, "http://www.xbrl.org/2003/role/link" ]
        }
      },
      {
        $concepts:NAME : { "$in" : [ $concept-names ] }
      }[not $concept-names = $concepts:ALL_CONCEPT_NAMES]
    |}
  )
};

(:~
 :
Retrieves all the labels with the given label role and language in the
 : matching concepts.

 :
 :
Matching concepts are those which:
 :  - concept name matches a given one,
 :  - archive number matches that of a given component,
 :  - component role matches that of a given component or is the default
 :    component role.
 :

 :
 :
Language matching can either be exact, if no options are given,
 : or approximated, if at least one of the following options is given:

 :

 :
MatchDown: whether to match a more specific language, e.g.:
 :       "en" will match labels which language is "en" or "en-US".

 :
MatchUp: whether to match a less specific language, e.g.:
 :       "en-US" will match labels which language is "en-US" or "en".

 :

 :
 : @param $concept-names the concepts names.
 : @param $component-or-ids the CIDs or the components themselves.
 : @param $label-role the label role.
 : @param $language the label language.
 : @param $options optional parameters to control language matching.
 :
 : @return the matching labels.
 :)
 declare function concepts:labels-for-components(
     $concept-names as string*,
     $component-or-ids as item*,
     $label-role as string,
     $language as string,
     $options as object?
   ) as string*
{
  let $components := components:components($component-or-ids)
  let $concepts := concepts:concepts-for-components($concept-names, $components)
  return concepts:labels($concept-names, $components.Archive[not $$ instance of null], $components.Role,
      $label-role, $language, $concepts, $options)
};

(:~
 :
Retrieves all the labels with the given label role and language in the
 : matching concepts.

 :
 :
Matching concepts are those which:
 :  - concept name matches a given one,
 :  - archive number matches that of a given component,
 :  - component role matches that of a given component or is the default
 :    component role.
 :

 :
 :
The set of concepts to search in is specified as a parameter.

 :
 :
Language matching can either be exact, if no options are given,
 : or approximated, if at least one of the following options is given:

 :

 :
MatchDown: whether to match a more specific language, e.g.:
 :       "en" will match labels which language is "en" or "en-US".

 :
MatchUp: whether to match a less specific language, e.g.:
 :       "en-US" will match labels which language is "en-US" or "en".

 :
MatchAnyVariant: whether to match a different variant of the same
 :       language, e.g.: "en-US" will match labels which language is "en-US"
 :       or "en-UK".

 :

 :
 : @param $concept-names the concepts names.
 : @param $component-or-ids the CIDs or the components themselves.
 : @param $label-role the label role.
 : @param $language the label language.
 : @param $concepts the concepts in which the labels will be
 :                  searched.
 : @param $options optional parameters to control language matching.
 :
 : @return the matching labels.
 :)
declare function concepts:labels-for-components(
    $concept-names as string*,
    $component-or-ids as item*,
    $label-role as string,
    $language as string,
    $concepts as object*,
    $options as object?
  ) as string*
{
  let $components := components:components($component-or-ids)
    return concepts:labels($concept-names, $components.Archive[not $$ instance of null], $components.Role,
      $label-role, $language, $concepts, $options)
};

(:~
 :
Retrieves all the labels with the given label role and language in the
 : matching concepts.

 :
 :
Matching concepts are those which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 :
Language matching can either be exact, if no options are given,
 : or approximated, if at least one of the following options is given:

 :

 :
MatchDown: whether to match a more specific language, e.g.:
 :       "en" will match labels which language is "en" or "en-US".

 :
MatchUp: whether to match a less specific language, e.g.:
 :       "en-US" will match labels which language is "en-US" or "en".

 :
MatchAnyVariant: whether to match a different variant of the same
 :       language, e.g.: "en-US" will match labels which language is "en-US"
 :       or "en-UK".

 :

 :
 : @param $concept-names the concepts names.
 : @param $archives the archive numbers.
 : @param $component-roles the component roles.
 : @param $label-role the label role.
 : @param $language the label language.
 : @param $options optional parameters to control language matching.
 :
 : @return the matching labels.
 :)
declare function concepts:labels(
    $concept-names as string*,
    $archives as string*,
    $component-roles as string*,
    $label-role as string,
    $language as string,
    $options as object?
  ) as string*
{
  let $concepts := concepts:concepts($concept-names, $archives, $component-roles)
  return concepts:labels($concept-names, $archives, $component-roles, $label-role, $language,
      $concepts, $options)
};

(:~
 :
Retrieves all the labels with the given label role and language in the
 : matching concepts.

 :
 :
Matching concepts are those which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 :
The set of concepts to search in is specified as a parameter.

 :
 :
Language matching can either be exact, if no options are given,
 : or approximated, if at least one of the following options is given:

 :

 :
MatchDown: whether to match a more specific language, e.g.:
 :       "en" will match labels which language is "en" or "en-US".

 :
MatchUp: whether to match a less specific language, e.g.:
 :       "en-US" will match labels which language is "en-US" or "en".

 :
MatchAnyVariant: whether to match a different variant of the same
 :       language, e.g.: "en-US" will match labels which language is "en-US"
 :       or "en-UK".

 :

 :
 : @param $concept-names the concepts names.
 : @param $archives the archive numbers.
 : @param $component-roles the component roles.
 : @param $label-role the label role.
 : @param $language the label language.
 : @param $concepts the concepts in which the labels will be
 :                  searched.
 : @param $options Optional parameters to control language matching.
 :
 : @return the matching labels.
 :)
declare function concepts:labels(
    $concept-names as string*,
    $archives as string*,
    $component-roles as string*,
    $label-role as string,
    $language as string,
    $concepts as object*,
    $options as object?
  ) as string*
{
  let $normalized-language := concepts:normalize-language($language)
  let $concept-labels-groups-for-role :=$concepts[
      $$($concepts:NAME)    = $concept-names and
      $$($concepts:ARCHIVE) = $archives and
      $$($concepts:ROLE)    = $component-roles]($concepts:LABELS)($label-role)
  for $concept-labels-group in $concept-labels-groups-for-role
  let $perfect-match := $concept-labels-group($normalized-language)
  return
    if ($perfect-match)
    then $concept-labels-group($normalized-language)
    else concepts:approximated-labels-match($concept-labels-group, $normalized-language, $options)
};

declare %private function concepts:approximated-labels-match(
    $concept-labels-group as object,
    $normalized-language as string,
    $options as object?
  ) as string*
{
  if ($options)
  then
    let $keys := keys($concept-labels-group)
    return
    (
      if ($options.MatchDown cast as boolean?)
      then
        for $matching-key in $keys[starts-with($$, $normalized-language)]
        return $concept-labels-group($matching-key)
      else (),
      if ($options.MatchUp cast as boolean?)
      then
        for $matching-key in $keys[starts-with($normalized-language, $$)]
        return $concept-labels-group($matching-key)
      else (),
      if ($options.MatchAnyVariant cast as boolean?)
      then
        if (contains($normalized-language, "-"))
        then
          for $matching-key in $keys[starts-with($$, substring-before($normalized-language, "-") || "-")]
          return $concept-labels-group($matching-key)
        else ()
      else ()
    )
  else ()
};

declare %private function concepts:normalize-language($language as string) as string
{
  (: TODO Enable this when the DB has been reimported :)
  (:replace(lower-case($language), "_", "-"):)
  $language
};

declare function concepts:find($conn as anyURI, $query as object) as object()*
{
  mongo:find($conn, $concepts:col, concepts:hinted-query($query))
};

declare function concepts:find($conn as anyURI, $query as object, $projection as object) as object()*
{
  mongo:find($conn, $concepts:col, concepts:hinted-query($query), $projection)
};

declare %private function concepts:hinted-query($query as object) as object
{
  switch (true)
    case (exists($query("_id")))
      return { "$query": $query, "$hint": "_id_" }
    case (exists($query("Archive")) and exists($query("Role")) and exists($query("Name")))
      return { "$query": $query, "$hint": "Archive_1_Role_1_Name_1" }
    case (exists($query("Archive")) and exists($query("Name")))
      return { "$query": $query, "$hint": "Name_1_Archive_1" }
    case (exists($query("Archive")))
      return { "$query": $query, "$hint": "Archive_1_Role_1_Name_1" }
    default
      return $query
};

declare %private %an:strictlydeterministic function concepts:connection() as anyURI
{
  let $credentials :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return if (empty($credentials))
             then error(QName("components:CONNECTION-FAILED"), "no xbrl MongoDB configured")
             else $credentials
  return
    try {
      mongo:connect($credentials)
    } catch mongo:* {
      error(QName("components:CONNECTION-FAILED"), $err:description)
    }
};


A fix in migration script.

A fix in migration script.

import module namespace config = "http://apps.28.io/config";

for $report in collection("reports")
return (
  for $object in descendant-objects($report)
  where exists($object.Trees)
  return replace value of json $object.Trees with [ values($object.Trees) ],

  for $object in descendant-objects($report)
  where exists($object.Domains)
  let $domains := values($object.Domains)
  return (delete json $object.Domains,
          insert json { Members : [ $domains ] } into $object),

  for $object in descendant-objects(
    $report.Hypercubes."xbrl:DefaultHypercube".Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain".Members
  )
  return for $key in keys($object)
         where not $key = ("Name", "Value", "PreferredLabelRole")
         return delete json $object.$key,

  rename json $report.Hypercubes."xbrl:DefaultHypercube" as "xbrl28:ImpliedTable",
  replace value of json $report.Hypercubes."xbrl:DefaultHypercube".Name with "xbrl28:ImpliedTable",
  replace value of json $report.Hypercubes."xbrl:DefaultHypercube".Label with "Implied Table [Table]",

  insert json {
    Concepts: [
      values($report.Hypercubes."xbrl:DefaultHypercube".Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain".Members)
    ]
  } into $report
);

import module namespace config = "http://apps.28.io/config";

for $report in collection("reports")
return (
  for $object in descendant-objects($report)
  where exists($object.Trees)
  return replace value of json $object.Trees with [ values($object.Trees) ],

  for $object in descendant-objects($report)
  where exists($object.Domains)
  let $domains := values($object.Domains)
  return (delete json $object.Domains,
          insert json { Members : [ $domains ] } into $object),

  for $object in descendant-objects(
    $report.Hypercubes."xbrl:DefaultHypercube".Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain".Members
  )
  return for $key in $object
         where not $key = ("Name", "Value", "PreferredLabelRole")
         return delete json $object.$key,

  rename json $report.Hypercubes."xbrl:DefaultHypercube" as "xbrl28:ImpliedTable",
  replace value of json $report.Hypercubes."xbrl:DefaultHypercube".Name with "xbrl28:ImpliedTable",
  replace value of json $report.Hypercubes."xbrl:DefaultHypercube".Label with "Implied Table [Table]",

  insert json {
    Concepts: [
      values($report.Hypercubes."xbrl:DefaultHypercube".Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain".Members)
    ]
  } into $report
);


Reverting.

Reverting.

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace test = "http://apps.28.io/test";
import module namespace http-client = "http://zorba.io/modules/http-client";

declare %an:nondeterministic function local:test-values($params as object, $expected-file as string) as item*
{
    let $endpoint := "spreadsheet-for-component"
    let $request := test:invoke($endpoint, $params)
    let $actual as object := $request[2]
    let $expected := parse-json(
        http-client:get("http://" || request:server-name() || ":" || request:server-port() ||
                        "/test/edinet/" || $expected-file).body.content)
    return if (deep-equal($actual, $expected))
           then true
           else { url: test:url($endpoint, $params) }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    bs-one: local:test-values(
      {
        eid: "http://info.edinet-fsa.go.jp E01225-000",
        fiscalYear: "2014",
        fiscalPeriod: "Q1",
        role: "http://disclosure.edinet-fsa.go.jp/role/jppfs/rol_QuarterlyConsolidatedBalanceSheet",
        merge: "true"
      },
      "spreadsheet-for-component-expected-bs-one.jq"),
    pl-one: local:test-values(
      {
        eid: "http://info.edinet-fsa.go.jp E01225-000",
        fiscalYear: "2014",
        fiscalPeriod: "Q1",
        role: "http://disclosure.edinet-fsa.go.jp/role/jppfs/rol_YearToQuarterEndConsolidatedStatementOfIncome",
        merge: "true"
      },
      "spreadsheet-for-component-expected-pl-one.jq"),
    cf-one: local:test-values(
      {
        eid: "http://info.edinet-fsa.go.jp E01225-000",
        fiscalYear: "2014",
        fiscalPeriod: "Q1",
        role: "http://disclosure.edinet-fsa.go.jp/role/jpcrp/rol_NotesQuarterlyConsolidatedStatementOfCashFlows",
        merge: "true"
      },
      "spreadsheet-for-component-expected-cf-one.jq"),
    bs-companies: local:test-values(
      {
        tag: "ALL",
        fiscalYear: "2014",
        fiscalPeriod: "Q1",
        role: "http://disclosure.edinet-fsa.go.jp/role/jppfs/rol_QuarterlyConsolidatedBalanceSheet",
        merge: "true"
      },
      "spreadsheet-for-component-expected-bs-companies.jq"),
    pl-companies: local:test-values(
      {
        tag: "ALL",
        fiscalYear: "2014",
        fiscalPeriod: "Q1",
        role: "http://disclosure.edinet-fsa.go.jp/role/jppfs/rol_YearToQuarterEndConsolidatedStatementOfIncome",
        merge: "true"
      },
      "spreadsheet-for-component-expected-pl-companies.jq"),
    cf-companies: local:test-values(
      {
        tag: "ALL",
        fiscalYear: "2014",
        fiscalPeriod: "Q1",
        role: "http://disclosure.edinet-fsa.go.jp/role/jpcrp/rol_NotesQuarterlyConsolidatedStatementOfCashFlows",
        merge: "true"
      },
      "spreadsheet-for-component-expected-cf-companies.jq"),
    bs-periods: local:test-values(
      {
        eid: "http://info.edinet-fsa.go.jp E01225-000",
        fiscalYear: "ALL",
        fiscalPeriod: "FY",
        role: "http://info.edinet-fsa.go.jp/jp/fr/gaap/role/ConsolidatedBalanceSheets",
        merge: "true"
      },
      "spreadsheet-for-component-expected-bs-periods.jq"),
    pl-periods: local:test-values(
      {
        eid: "http://info.edinet-fsa.go.jp E01225-000",
        fiscalYear: "ALL",
        fiscalPeriod: "FY",
        role: "http://info.edinet-fsa.go.jp/jp/fr/gaap/role/ConsolidatedStatementsOfIncome",
        merge: "true"
      },
      "spreadsheet-for-component-expected-pl-periods.jq"),
    cf-periods: local:test-values(
      {
        eid: "http://info.edinet-fsa.go.jp E01225-000",
        fiscalYear: "ALL",
        fiscalPeriod: "FY",
        role: "http://info.edinet-fsa.go.jp/jp/fr/gaap/role/ConsolidatedStatementsOfCashFlowsIndirect",
        merge: "true"
      },
      "spreadsheet-for-component-expected-cf-periods.jq")
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace test = "http://apps.28.io/test";
import module namespace http-client = "http://zorba.io/modules/http-client";

declare %an:nondeterministic function local:test-values($params as object, $expected-file as string) as item*
{
    let $endpoint := "spreadsheet-for-component"
    let $request := test:invoke($endpoint, $params)
    let $actual as object := $request[2]
    let $expected := parse-json(
        http-client:get("http://" || request:server-name() || ":" || request:server-port() ||
                        "/test/edinet/" || $expected-file).body.content)
    return if (deep-equal($actual, $expected))
           then true
           else { url: test:url($endpoint, $params), actual: $actual, expected: $expected }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    bs-one: local:test-values(
      {
        eid: "http://info.edinet-fsa.go.jp E01225-000",
        fiscalYear: "2014",
        fiscalPeriod: "Q1",
        role: "http://disclosure.edinet-fsa.go.jp/role/jppfs/rol_QuarterlyConsolidatedBalanceSheet",
        merge: "true"
      },
      "spreadsheet-for-component-expected-bs-one.jq"),
    pl-one: local:test-values(
      {
        eid: "http://info.edinet-fsa.go.jp E01225-000",
        fiscalYear: "2014",
        fiscalPeriod: "Q1",
        role: "http://disclosure.edinet-fsa.go.jp/role/jppfs/rol_YearToQuarterEndConsolidatedStatementOfIncome",
        merge: "true"
      },
      "spreadsheet-for-component-expected-pl-one.jq"),
    cf-one: local:test-values(
      {
        eid: "http://info.edinet-fsa.go.jp E01225-000",
        fiscalYear: "2014",
        fiscalPeriod: "Q1",
        role: "http://disclosure.edinet-fsa.go.jp/role/jpcrp/rol_NotesQuarterlyConsolidatedStatementOfCashFlows",
        merge: "true"
      },
      "spreadsheet-for-component-expected-cf-one.jq"),
    bs-companies: local:test-values(
      {
        tag: "ALL",
        fiscalYear: "2014",
        fiscalPeriod: "Q1",
        role: "http://disclosure.edinet-fsa.go.jp/role/jppfs/rol_QuarterlyConsolidatedBalanceSheet",
        merge: "true"
      },
      "spreadsheet-for-component-expected-bs-companies.jq"),
    pl-companies: local:test-values(
      {
        tag: "ALL",
        fiscalYear: "2014",
        fiscalPeriod: "Q1",
        role: "http://disclosure.edinet-fsa.go.jp/role/jppfs/rol_YearToQuarterEndConsolidatedStatementOfIncome",
        merge: "true"
      },
      "spreadsheet-for-component-expected-pl-companies.jq"),
    cf-companies: local:test-values(
      {
        tag: "ALL",
        fiscalYear: "2014",
        fiscalPeriod: "Q1",
        role: "http://disclosure.edinet-fsa.go.jp/role/jpcrp/rol_NotesQuarterlyConsolidatedStatementOfCashFlows",
        merge: "true"
      },
      "spreadsheet-for-component-expected-cf-companies.jq"),
    bs-periods: local:test-values(
      {
        eid: "http://info.edinet-fsa.go.jp E01225-000",
        fiscalYear: "ALL",
        fiscalPeriod: "FY",
        role: "http://info.edinet-fsa.go.jp/jp/fr/gaap/role/ConsolidatedBalanceSheets",
        merge: "true"
      },
      "spreadsheet-for-component-expected-bs-periods.jq"),
    pl-periods: local:test-values(
      {
        eid: "http://info.edinet-fsa.go.jp E01225-000",
        fiscalYear: "ALL",
        fiscalPeriod: "FY",
        role: "http://info.edinet-fsa.go.jp/jp/fr/gaap/role/ConsolidatedStatementsOfIncome",
        merge: "true"
      },
      "spreadsheet-for-component-expected-pl-periods.jq"),
    cf-periods: local:test-values(
      {
        eid: "http://info.edinet-fsa.go.jp E01225-000",
        fiscalYear: "ALL",
        fiscalPeriod: "FY",
        role: "http://info.edinet-fsa.go.jp/jp/fr/gaap/role/ConsolidatedStatementsOfCashFlowsIndirect",
        merge: "true"
      },
      "spreadsheet-for-component-expected-cf-periods.jq")
})


Fixed link.

Fixed link.

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace http-request = "http://www.28msec.com/modules/http/request";
import module namespace csv = "http://zorba.io/modules/json-csv";

import module namespace entities = "http://28.io/modules/xbrl/entities";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token        as string? external;
declare  %rest:env                              variable $request-uri  as string  external;
declare  %rest:case-insensitive                 variable $format       as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $eid          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cik          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker       as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic          as string* external;
declare  %rest:case-insensitive                 variable $profile-name as string  external := $config:profile-name;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)
let $tag := if (exists(($cik, $tag, $ticker, $sic)))
             then $tag
             else "ALL"

(: Object resolution :)
let $entities :=
    switch($profile-name)
    case "sec" return
        for $entity in companies:companies(
            $cik,
            $tag,
            $ticker,
            $sic)
        order by $entity.Profiles.SEC.CompanyName
        return $entity
    default return
        for $entity in
            if(exists($eid)) then entities:entities($eid)
                                else entities:entities()
        return {
            EID: $entity._id
        }
let $comment :=
{
    NumEntities: count($entities),
    TotalNumEntities: session:num-entities()
}
let $entities :=
    for $entity in $entities
    return copy $e := $entity
    modify
      switch($profile-name)
      case "sec"
      return
        insert json {
          Archives: "http://" || http-request:server-name() || ":" || http-request:server-port() ||
          "/v1/_queries/public/api/filings.jq?_method=POST&cik="|| tokenize($e._id, " ")[2] ||
          "&fiscalYear=ALL&fiscalPeriod=ALL&format=html" ||
          "&profile-name=" || $profile-name ||
          "&token=" || http-request:parameter-values("token")
        } into $e
      default return
        insert json {
          Archives: "http://" || http-request:server-name() || ":" || http-request:server-port() ||
          "/v1/_queries/public/api/filings.jq?_method=POST&eid="|| encode-for-uri($e._id) ||
          "&format=html" ||
          "&profile-name=" || $profile-name ||
          "&token=" || http-request:parameter-values("token")
        } into $e
    return $e
let $result := { "Entities" : [ $entities ] }
let $serializers := {
    to-xml : function($res as object) as node() {
        switch($profile-name)
        case "sec" return
        <Entities>{
            companies:to-xml($res.Entities[])
        }</Entities>
        default return <Entities>
            {
                for $id in $res.Entities[].EID
                return <EID>{$id}</EID>
            }
        </Entities>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec"
        return string-join(companies:to-csv($res.Entities[]))
        default return
            string-join(
                csv:serialize($res.Entities[], { serialize-null-as : "" }),
        "")
    }
}
return api:serialize($result, $comment, $serializers, $format, "entities")

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace http-request = "http://www.28msec.com/modules/http/request";
import module namespace csv = "http://zorba.io/modules/json-csv";

import module namespace entities = "http://28.io/modules/xbrl/entities";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token        as string? external;
declare  %rest:env                              variable $request-uri  as string  external;
declare  %rest:case-insensitive                 variable $format       as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $eid          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cik          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker       as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic          as string* external;
declare  %rest:case-insensitive                 variable $profile-name as string  external := $config:profile-name;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)
let $tag := if (exists(($cik, $tag, $ticker, $sic)))
             then $tag
             else "ALL"

(: Object resolution :)
let $entities :=
    switch($profile-name)
    case "sec" return
        for $entity in companies:companies(
            $cik,
            $tag,
            $ticker,
            $sic)
        order by $entity.Profiles.SEC.CompanyName
        return $entity
    default return
        for $entity in
            if(exists($eid)) then entities:entities($eid)
                                else entities:entities()
        return {
            EID: $entity._id
        }
let $comment :=
{
    NumEntities: count($entities),
    TotalNumEntities: session:num-entities()
}
let $entities :=
    for $entity in $entities
    return copy $e := $entity
    modify
      switch($profile-name)
      case "sec"
      return
        insert json {
          Archives: "http://" || http-request:server-name() || ":" || http-request:server-port() ||
          "/v1/_queries/public/api/filings.jq?_method=POST&cik="|| tokenize($e._id, " ")[2] ||
          "&fiscalYear=ALL&fiscalPeriod=ALL&format=html" ||
          "&profile-name=" || $profile-name ||
          "&token=" || http-request:parameter-values("token")
        } into $e
      default return
        insert json {
          Archives: "http://" || http-request:server-name() || ":" || http-request:server-port() ||
          "/v1/_queries/public/api/filings.jq?_method=POST&entity="|| encode-for-uri($e._id) ||
          "&format=html" ||
          "&profile-name=" || $profile-name ||
          "&token=" || http-request:parameter-values("token")
        } into $e
    return $e
let $result := { "Entities" : [ $entities ] }
let $serializers := {
    to-xml : function($res as object) as node() {
        switch($profile-name)
        case "sec" return
        <Entities>{
            companies:to-xml($res.Entities[])
        }</Entities>
        default return <Entities>
            {
                for $id in $res.Entities[].EID
                return <EID>{$id}</EID>
            }
        </Entities>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec"
        return string-join(companies:to-csv($res.Entities[]))
        default return
            string-join(
                csv:serialize($res.Entities[], { serialize-null-as : "" }),
        "")
    }
}
return api:serialize($result, $comment, $serializers, $format, "entities")


Update spreadsheet-for-report.jq

Update spreadsheet-for-report.jq

import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace api = "http://apps.28.io/api";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token         as string? external;
declare  %rest:env                              variable $request-uri   as string  external;
declare  %rest:case-insensitive                 variable $format        as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker        as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear    as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid           as string* external;
declare  %rest:case-insensitive                 variable $validate      as boolean external := false;
declare  %rest:case-insensitive                 variable $eliminate     as boolean external := false;
declare  %rest:case-insensitive                 variable $report        as string? external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := "sec";

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities :=
    for $entity in
        companies:companies(
            $cik,
            $tag,
            $ticker,
            $sic)
    order by $entity.Profiles.SEC.CompanyName
    return $entity
let $report as object? := reports:reports($report)

(: Fact resolution :)
let $filter-override as object? := fiscal-core:filter-override(
    $entities,
    $fiscalYear,
    $fiscalPeriod,
    $aid)[$profile-name eq "sec"]

(: Fact resolution :)
let $hypercube := hypercubes:hypercubes-for-components($report, "xbrl:DefaultHypercube")
let $filtered-aspects := values($hypercube.Aspects)[exists(($$.Domains, $$.DomainRestriction))]
let $spreadsheet as object? :=
    if(count($filtered-aspects) lt 2 and not exists(($filter-override)))
    then {
          response:status-code(403);
          session:error("The report filters are too weak, which leads to too big an output.", $format)
    } else
        components:spreadsheet(
            $report,
            {|
                { FilterOverride: $filter-override}[exists($filter-override)],
                {
                    FlattenRows: true,
                    Eliminate: $eliminate,
                    Validate: $validate
                }
            |}
        )
let $results :=
    {
        response:content-type("application/json");
        response:serialization-parameters({"indent" : true});
        $spreadsheet
    }
return api:check-and-return-results($token, $results, $format)

import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace api = "http://apps.28.io/api";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token         as string? external;
declare  %rest:env                              variable $request-uri   as string  external;
declare  %rest:case-insensitive                 variable $format        as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker        as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear    as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid           as string* external;
declare  %rest:case-insensitive                 variable $validate      as boolean external := false;
declare  %rest:case-insensitive                 variable $eliminate     as boolean external := false;
declare  %rest:case-insensitive                 variable $report        as string? external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := "generic";

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities :=
    for $entity in
        companies:companies(
            $cik,
            $tag,
            $ticker,
            $sic)
    order by $entity.Profiles.SEC.CompanyName
    return $entity
let $report as object? := reports:reports($report)

(: Fact resolution :)
let $filter-override as object? := fiscal-core:filter-override(
    $entities,
    $fiscalYear,
    $fiscalPeriod,
    $aid)[$profile-name eq "sec"]

(: Fact resolution :)
let $hypercube := hypercubes:hypercubes-for-components($report, "xbrl:DefaultHypercube")
let $filtered-aspects := values($hypercube.Aspects)[exists(($$.Domains, $$.DomainRestriction))]
let $spreadsheet as object? :=
    if(count($filtered-aspects) lt 2 and not exists(($filter-override)))
    then {
          response:status-code(403);
          session:error("The report filters are too weak, which leads to too big an output.", $format)
    } else
        components:spreadsheet(
            $report,
            {|
                { FilterOverride: $filter-override}[exists($filter-override)],
                {
                    FlattenRows: true,
                    Eliminate: $eliminate,
                    Validate: $validate
                }
            |}
        )
let $results :=
    {
        response:content-type("application/json");
        response:serialization-parameters({"indent" : true});
        $spreadsheet
    }
return api:check-and-return-results($token, $results, $format)


Update test.jq

Update test.jq

jsoniq version "1.0";
module namespace test = "http://apps.28.io/test";
import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace config = "http://apps.28.io/config";
import module namespace functx = "http://www.functx.com";
import module namespace backend = "http://apps.28.io/backend";

declare function test:url($endpoint as string, $parameters as object) as string
{
    test:url($endpoint, $parameters, false)
};

declare function test:url($endpoint as string, $parameters as object, $includeToken as boolean) as string
{
  backend:url(
    $endpoint,
    {|
      $config:test-token,
      $parameters
    |}
  )
};

declare %an:nondeterministic function test:invoke($endpoint as string, $parameters as object) as item*
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:get($url)
  return ($response.status, parse-json($response.body.content))
};

declare %an:nondeterministic function test:invoke-xml($endpoint as string, $parameters as object) as item*
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:get($url)
  return ($response.status, parse-xml($response.body.content))
};

declare %an:sequential function test:invoke-body($endpoint as string, $parameters as object, $body as string) as item*
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:post($url, $body, "application/x-www-form-urlencoded")
  return ($response.status, parse-json($response.body.content))
};

declare %an:nondeterministic function test:invoke-raw($endpoint as string, $parameters as object) as object
{
  let $url as string:= test:url($endpoint, $parameters, true)
  return http-client:get($url)
};

declare %an:nondeterministic function test:invoke-public($endpoint as string, $parameters as object) as item*
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:get($url)
  return ($response.status, parse-json($response.body.content))
};

declare %an:nondeterministic function test:get-expected-result(
  $expected-file as string
) as item*
{
  parse-json(
    http-client:get("http://" || request:server-name() || ":" || request:server-port() ||
                    "/test/" || $expected-file).body.content)
};

declare %an:nondeterministic function test:get-expected-result-xml(
  $expected-file as string
) as item*
{
  parse-xml(
    http-client:get("http://" || request:server-name() || ":" || request:server-port() ||
                    "/test/" || $expected-file).body.content)
};

declare %an:nondeterministic function test:invoke-and-assert-deep-equal(
  $endpoint as string,
  $parameters as object,
  $transform as function(object) as item*,
  $expected as item*
) as item
{
  test:invoke-and-assert-deep-equal($endpoint, $parameters, $transform, $expected, {})
};

declare %an:sequential function test:invoke-and-assert-status(
  $endpoint as string,
  $parameters as object,
  $expected-status as integer
) as item
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:send-request({href: $url, method: "GET", "options" : { "status-only": true, "follow-redirect" : false } })
  let $actual-status := $response.status
  return if($actual-status eq $expected-status)
         then true
         else {
           url: test:url($endpoint, $parameters),
           expected: $expected-status,
           actual: $actual-status
         }
};

(:
  Note on NoArrayOrder option: it allows for false positives in case arrays have duplicate elements. The number
  of occurrences of duplicate elements is not checked.
:)
declare %an:nondeterministic function test:invoke-and-assert-deep-equal(
  $endpoint as string,
  $parameters as object,
  $transform as function(object) as item*,
  $expected as item*,
  $options as object?
) as item
{
  let $request := switch($options.Format)
                  case "xml" return test:invoke-xml($endpoint, $parameters)
                  default return test:invoke($endpoint, $parameters)
  let $status as integer := $request[1]
  let $actual as item* := $transform($request[2])
  let $expected as item* := if($options.TrimIdField) then test:trim-ids($expected) else $expected
  let $actual as item* := if($options.TrimIdField) then test:trim-ids($actual) else $actual
  return if($options.NoArrayOrder)
          then test:assert-deep-equal-no-array-order($expected, $actual, $status, test:url($endpoint, $parameters))
          else test:assert-deep-equal($expected, $actual, $status, test:url($endpoint, $parameters))
};

declare %an:sequential function test:check-all-success($o as object) as object
{
  if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
  then {
    response:status-code(500);
    $o
  } else $o
};

declare %private function test:trim-ids($input as item*) as item*
{
  for $i in $input
  return typeswitch($i)
          case object return {|
                                 for $key in keys($i)
                                 where $key ne "_id"
                                 return {$key: test:trim-ids($i.$key) }
                               |}
          case array return [ test:trim-ids($i[]) ]
          default return $i
};

(:    return
        if ($actual eq $expected)
        then true
        else "false [Actual="||$actual||", Expected="||$expected ||"]":)

declare function test:assert-eq(
    $expected as atomic,
    $actual as atomic?,
    $status as integer,
    $url as string) as item
{
    switch(true)
    case $status ne 200 return { "url": $url, status: $status }
    case try {$expected eq $actual} catch * { false } return true
    default return
    {
        "url": $url,
        "expected": $expected,
        "actual": $actual
    }
};

declare function test:assert-deep-equal(
    $expected as item*,
    $actual as item*,
    $status as integer,
    $url as string) as item
{
    switch(true)
    case $status ne 200 return { "url": $url, status: $status }
    case functx:sequence-deep-equal($expected, $actual) return true
    default return
    {
        "url": $url,
        "expected": typeswitch($expected) case json-item return $expected default return serialize($expected),
        "actual": typeswitch($actual) case json-item return $actual default return serialize($actual)
    }
};

declare function test:assert-deep-equal-no-array-order(
    $expected as item*,
    $actual as item*,
    $status as integer,
    $url as string) as item
{
    switch(true)
    case $status ne 200 return { "url": $url, status: $status }
    case test:deep-equal-no-array-order($expected, $actual) return true
    default return
    {
        "url": $url,
        "expected": typeswitch($expected) case json-item return $expected default return serialize($expected),
        "actual": typeswitch($actual) case json-item return $actual default return serialize($actual)
    }
};

declare function test:deep-equal-no-array-order(
    $expected as item*,
    $actual as item*) as boolean
{
    if(count($expected) ne count($actual))
    then false
    else
      every $i in 1 to count($expected)
      satisfies
        let $expected as item := $expected[$i]
        let $actual as item := $actual[$i]
        return
        typeswitch(($expected, $actual))
        case object* return
          switch(true)
          case not (every $key in keys($expected) satisfies keys($actual) = $key)
            return false
          case not (every $key in keys($actual) satisfies keys($expected) = $key)
            return false
          default return
            let $recursive-calls :=
                for $key in keys($expected)
                return test:deep-equal-no-array-order($expected.$key, $actual.$key)
            return if (every $item in $recursive-calls satisfies $item)
                    then true
                    else false
        case array* return
          if(not size($expected) eq size($actual))
          then false
          else
            let $expected-items as item* :=
              for $e as item in $expected[]
              let $s as string := serialize($e)
              order by $s
              return $e
            let $actual-items as item* :=
              for $a as item in $actual[]
              let $s as string := serialize($a)
              order by $s
              return $a
            let $size as integer := size($expected)
            return every $i in 1 to $size
                   satisfies test:deep-equal-no-array-order(
                       $expected-items[$i],
                       $actual-items[$i])
        default return deep-equal($expected, $actual)
};

declare function test:assert-eq-array(
    $expected as array,
    $actual as array,
    $status as integer,
    $url as string) as item
{
    let $diff := try {
            {
                actual: [
                    for $a in flatten($actual)
                    where not($a = flatten($expected))
                    return $a ],
                actualDuplicates: [
                    for $a in flatten($actual)
                    where $a = flatten($expected) and count(flatten($actual)[$$ eq $a]) ne count(flatten($expected)[$$ eq $a])
                    return $a ],
                expected: [
                    for $e in flatten($expected)
                    where not($e = flatten($actual))
                    return $e ]
            }
        } catch * { "error: " || $err:description }

    return
        switch(true)
        case $status ne 200 return { "url": $url, status: $status }
        case empty(($diff.actual[],$diff.actualDuplicates[],$diff.expected[])) return true
        default return
        {
            "url": $url,
            "diff": $diff,
            "expected": $expected,
            "actual": $actual
        }
};

jsoniq version "1.0";
module namespace test = "http://apps.28.io/test";
import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace config = "http://apps.28.io/config";
import module namespace functx = "http://www.functx.com";
import module namespace backend = "http://apps.28.io/backend";

declare function test:url($endpoint as string, $parameters as object) as string
{
    test:url($endpoint, $parameters, false)
};

declare function test:url($endpoint as string, $parameters as object, $includeToken as boolean) as string
{
  backend:url(
    $endpoint,
    {|
      { token: if($includeToken) then $config:test-token else "{{token}}" },
      $parameters
    |}
  )
};

declare %an:nondeterministic function test:invoke($endpoint as string, $parameters as object) as item*
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:get($url)
  return ($response.status, parse-json($response.body.content))
};

declare %an:nondeterministic function test:invoke-xml($endpoint as string, $parameters as object) as item*
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:get($url)
  return ($response.status, parse-xml($response.body.content))
};

declare %an:sequential function test:invoke-body($endpoint as string, $parameters as object, $body as string) as item*
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:post($url, $body, "application/x-www-form-urlencoded")
  return ($response.status, parse-json($response.body.content))
};

declare %an:nondeterministic function test:invoke-raw($endpoint as string, $parameters as object) as object
{
  let $url as string:= test:url($endpoint, $parameters, true)
  return http-client:get($url)
};

declare %an:nondeterministic function test:invoke-public($endpoint as string, $parameters as object) as item*
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:get($url)
  return ($response.status, parse-json($response.body.content))
};

declare %an:nondeterministic function test:get-expected-result(
  $expected-file as string
) as item*
{
  parse-json(
    http-client:get("http://" || request:server-name() || ":" || request:server-port() ||
                    "/test/" || $expected-file).body.content)
};

declare %an:nondeterministic function test:get-expected-result-xml(
  $expected-file as string
) as item*
{
  parse-xml(
    http-client:get("http://" || request:server-name() || ":" || request:server-port() ||
                    "/test/" || $expected-file).body.content)
};

declare %an:nondeterministic function test:invoke-and-assert-deep-equal(
  $endpoint as string,
  $parameters as object,
  $transform as function(object) as item*,
  $expected as item*
) as item
{
  test:invoke-and-assert-deep-equal($endpoint, $parameters, $transform, $expected, {})
};

declare %an:sequential function test:invoke-and-assert-status(
  $endpoint as string,
  $parameters as object,
  $expected-status as integer
) as item
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:send-request({href: $url, method: "GET", "options" : { "status-only": true, "follow-redirect" : false } })
  let $actual-status := $response.status
  return if($actual-status eq $expected-status)
         then true
         else {
           url: test:url($endpoint, $parameters),
           expected: $expected-status,
           actual: $actual-status
         }
};

(:
  Note on NoArrayOrder option: it allows for false positives in case arrays have duplicate elements. The number
  of occurrences of duplicate elements is not checked.
:)
declare %an:nondeterministic function test:invoke-and-assert-deep-equal(
  $endpoint as string,
  $parameters as object,
  $transform as function(object) as item*,
  $expected as item*,
  $options as object?
) as item
{
  let $request := switch($options.Format)
                  case "xml" return test:invoke-xml($endpoint, $parameters)
                  default return test:invoke($endpoint, $parameters)
  let $status as integer := $request[1]
  let $actual as item* := $transform($request[2])
  let $expected as item* := if($options.TrimIdField) then test:trim-ids($expected) else $expected
  let $actual as item* := if($options.TrimIdField) then test:trim-ids($actual) else $actual
  return if($options.NoArrayOrder)
          then test:assert-deep-equal-no-array-order($expected, $actual, $status, test:url($endpoint, $parameters))
          else test:assert-deep-equal($expected, $actual, $status, test:url($endpoint, $parameters))
};

declare %an:sequential function test:check-all-success($o as object) as object
{
  if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
  then {
    response:status-code(500);
    $o
  } else $o
};

declare %private function test:trim-ids($input as item*) as item*
{
  for $i in $input
  return typeswitch($i)
          case object return {|
                                 for $key in keys($i)
                                 where $key ne "_id"
                                 return {$key: test:trim-ids($i.$key) }
                               |}
          case array return [ test:trim-ids($i[]) ]
          default return $i
};

(:    return
        if ($actual eq $expected)
        then true
        else "false [Actual="||$actual||", Expected="||$expected ||"]":)

declare function test:assert-eq(
    $expected as atomic,
    $actual as atomic?,
    $status as integer,
    $url as string) as item
{
    switch(true)
    case $status ne 200 return { "url": $url, status: $status }
    case try {$expected eq $actual} catch * { false } return true
    default return
    {
        "url": $url,
        "expected": $expected,
        "actual": $actual
    }
};

declare function test:assert-deep-equal(
    $expected as item*,
    $actual as item*,
    $status as integer,
    $url as string) as item
{
    switch(true)
    case $status ne 200 return { "url": $url, status: $status }
    case functx:sequence-deep-equal($expected, $actual) return true
    default return
    {
        "url": $url,
        "expected": typeswitch($expected) case json-item return $expected default return serialize($expected),
        "actual": typeswitch($actual) case json-item return $actual default return serialize($actual)
    }
};

declare function test:assert-deep-equal-no-array-order(
    $expected as item*,
    $actual as item*,
    $status as integer,
    $url as string) as item
{
    switch(true)
    case $status ne 200 return { "url": $url, status: $status }
    case test:deep-equal-no-array-order($expected, $actual) return true
    default return
    {
        "url": $url,
        "expected": typeswitch($expected) case json-item return $expected default return serialize($expected),
        "actual": typeswitch($actual) case json-item return $actual default return serialize($actual)
    }
};

declare function test:deep-equal-no-array-order(
    $expected as item*,
    $actual as item*) as boolean
{
    if(count($expected) ne count($actual))
    then false
    else
      every $i in 1 to count($expected)
      satisfies
        let $expected as item := $expected[$i]
        let $actual as item := $actual[$i]
        return
        typeswitch(($expected, $actual))
        case object* return
          switch(true)
          case not (every $key in keys($expected) satisfies keys($actual) = $key)
            return false
          case not (every $key in keys($actual) satisfies keys($expected) = $key)
            return false
          default return
            let $recursive-calls :=
                for $key in keys($expected)
                return test:deep-equal-no-array-order($expected.$key, $actual.$key)
            return if (every $item in $recursive-calls satisfies $item)
                    then true
                    else false
        case array* return
          if(not size($expected) eq size($actual))
          then false
          else
            let $expected-items as item* :=
              for $e as item in $expected[]
              let $s as string := serialize($e)
              order by $s
              return $e
            let $actual-items as item* :=
              for $a as item in $actual[]
              let $s as string := serialize($a)
              order by $s
              return $a
            let $size as integer := size($expected)
            return every $i in 1 to $size
                   satisfies test:deep-equal-no-array-order(
                       $expected-items[$i],
                       $actual-items[$i])
        default return deep-equal($expected, $actual)
};

declare function test:assert-eq-array(
    $expected as array,
    $actual as array,
    $status as integer,
    $url as string) as item
{
    let $diff := try {
            {
                actual: [
                    for $a in flatten($actual)
                    where not($a = flatten($expected))
                    return $a ],
                actualDuplicates: [
                    for $a in flatten($actual)
                    where $a = flatten($expected) and count(flatten($actual)[$$ eq $a]) ne count(flatten($expected)[$$ eq $a])
                    return $a ],
                expected: [
                    for $e in flatten($expected)
                    where not($e = flatten($actual))
                    return $e ]
            }
        } catch * { "error: " || $err:description }

    return
        switch(true)
        case $status ne 200 return { "url": $url, status: $status }
        case empty(($diff.actual[],$diff.actualDuplicates[],$diff.expected[])) return true
        default return
        {
            "url": $url,
            "diff": $diff,
            "expected": $expected,
            "actual": $actual
        }
};


only watch use-compiled-releases.yml

only watch use-compiled-releases.yml

include "./concourse/pipeline-helpers";

{
  "jobs": [
    import_repo_release("github.com/dpb587/openvpn-bosh-release"),

    compile_boshio_release("cloudfoundry/bosh"; "ubuntu-trusty"),
    compile_boshio_release("concourse/concourse"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/garden-runc-release"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/uaa-release"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry-community/docker-registry-boshrelease"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/syslog-release"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/dns-release"; "ubuntu-trusty"),

    {
      "name": "import-cf-deployment",
      "plan": [
        {
          "get": "cf-deployment",
          "trigger": true
        },
        {
          "get": "repo"
        },
        {
          "task": "import-cf-deployment",
          "file": "repo/concourse/tasks/import-cf-deployment/config.yml",
          "params": {
            "git_user_email": "((maintainer_email))",
            "git_user_name": "((maintainer_name))"
          }
        },
        {
          "put": "repo",
          "params": {
            "rebase": true,
            "repository": "repo"
          }
        }
      ]
    }
  ],
  "resources": [
    {
      "name": "repo",
      "type": "git",
      "source": {
        "branch": "master",
        "private_key": "((git_private_key))",
        "uri": "git@github.com:dpb587/bosh-compiled-releases.git"
      }
    },
    {
      "name": "release-compiler",
      "type": "git",
      "source": {
        "uri": "https://github.com/dpb587/bosh-release-compiler.git"
      }
    },
    {
      "name": "ubuntu-trusty",
      "type": "metalink-repository",
      "source": {
        "uri": "https://dpb587.github.io/upstream-blob-mirror/repository/bosh.io/stemcell/bosh-warden-boshlite-ubuntu-trusty-go_agent/index.xml"
      }
    },

    repo_release("github.com/dpb587/openvpn-bosh-release"; {"uri": "https://github.com/dpb587/openvpn-bosh-release.git"}),

    boshio_release("cloudfoundry/bosh"),
    boshio_release("concourse/concourse"),
    boshio_release("cloudfoundry/garden-runc-release"),
    boshio_release("cloudfoundry/uaa-release"),
    boshio_release("cloudfoundry-community/docker-registry-boshrelease"),
    boshio_release("cloudfoundry/syslog-release"),
    boshio_release("cloudfoundry/dns-release"),

    {
      "name": "cf-deployment",
      "type": "git",
      "source": {
        "uri": "https://github.com/cloudfoundry/cf-deployment.git",
        "paths": [
          "operations/use-compiled-releases.yml"
        ]
      }
    }
  ],
  "resource_types": [
    {
      "name": "metalink-repository",
      "type": "docker-image",
      "source": {
        "repository": "dpb587/metalink-repository-resource"
      }
    }
  ]
}

include "./concourse/pipeline-helpers";

{
  "jobs": [
    import_repo_release("github.com/dpb587/openvpn-bosh-release"),

    compile_boshio_release("cloudfoundry/bosh"; "ubuntu-trusty"),
    compile_boshio_release("concourse/concourse"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/garden-runc-release"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/uaa-release"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry-community/docker-registry-boshrelease"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/syslog-release"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/dns-release"; "ubuntu-trusty"),

    {
      "name": "import-cf-deployment",
      "plan": [
        {
          "get": "cf-deployment",
          "trigger": true
        },
        {
          "get": "repo"
        },
        {
          "task": "import-cf-deployment",
          "file": "repo/concourse/tasks/import-cf-deployment/config.yml",
          "params": {
            "git_user_email": "((maintainer_email))",
            "git_user_name": "((maintainer_name))"
          }
        },
        {
          "put": "repo",
          "params": {
            "rebase": true,
            "repository": "repo"
          }
        }
      ]
    }
  ],
  "resources": [
    {
      "name": "repo",
      "type": "git",
      "source": {
        "branch": "master",
        "private_key": "((git_private_key))",
        "uri": "git@github.com:dpb587/bosh-compiled-releases.git"
      }
    },
    {
      "name": "release-compiler",
      "type": "git",
      "source": {
        "uri": "https://github.com/dpb587/bosh-release-compiler.git"
      }
    },
    {
      "name": "ubuntu-trusty",
      "type": "metalink-repository",
      "source": {
        "uri": "https://dpb587.github.io/upstream-blob-mirror/repository/bosh.io/stemcell/bosh-warden-boshlite-ubuntu-trusty-go_agent/index.xml"
      }
    },

    repo_release("github.com/dpb587/openvpn-bosh-release"; {"uri": "https://github.com/dpb587/openvpn-bosh-release.git"}),

    boshio_release("cloudfoundry/bosh"),
    boshio_release("concourse/concourse"),
    boshio_release("cloudfoundry/garden-runc-release"),
    boshio_release("cloudfoundry/uaa-release"),
    boshio_release("cloudfoundry-community/docker-registry-boshrelease"),
    boshio_release("cloudfoundry/syslog-release"),
    boshio_release("cloudfoundry/dns-release"),

    {
      "name": "cf-deployment",
      "type": "git",
      "source": {
        "uri": "https://github.com/cloudfoundry/cf-deployment.git"
      }
    }
  ],
  "resource_types": [
    {
      "name": "metalink-repository",
      "type": "docker-image",
      "source": {
        "repository": "dpb587/metalink-repository-resource"
      }
    }
  ]
}


Fix HTML layout

Fix HTML layout

jsoniq version "1.0";

module namespace api = "http://apps.28.io/api";

import module namespace session    = "http://apps.28.io/session";
import module namespace resp       = "http://www.28msec.com/modules/http-response";
import module namespace sec-fiscal = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare function api:json-to-csv($objects as object*) as string
{
  let $flattened as object* := api:flatten-json-object($objects, ())
  return string-join(
    csv:serialize($flattened, { field-names: [ keys($flattened) ], serialize-null-as : "" }),
    "")
  };

declare function api:json-to-xml($objects as object*, $root-name as string) as element()*
{
  for $flattened-object as object in api:flatten-json-object($objects, { KeepArray: true })
  return element { $root-name } {
    for $key in keys($flattened-object)
    let $value := $flattened-object.$key
    return typeswitch($value)
           case array return
             let $key-with-s := if(substring($key, string-length($key), 1) eq "s")
                                then $key
                                else $key || "s"
             let $key-no-s := if(substring($key, string-length($key), 1) eq "s")
                                then substring($key, 1, string-length($key) - 1)
                                else $key
             return element {$key-with-s} { for $v in $value[] return element { $key-no-s } { $v } }
           default return element { $key } { $value }
  }
};

declare %private function api:flatten-json-object(
  $items as item*,
  $options as object?) as item*
{
  for $item in $items
  return typeswitch($item)
         case atomic return $item
         case array return let $atomic-items := flatten($item)[$$ instance of atomic]
                           return if ($options.KeepArray)
                                 then [ $atomic-items ]
                                  else string-join($atomic-items, ", ")
         case object return {|
             for $key in keys($item)
             return typeswitch($item.$key)
                    case object return api:flatten-json-object($item.$key, $options)
                    default return { $key: api:flatten-json-object($item.$key, $options) }
         |}
         default return ()
};

declare function api:validate-regexp($name as string, $value as string?, $regexp as string)
as ()
{
  if (exists($value))
  then
    if (matches($value, "^" || $regexp || "$"))
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " does not match reg. expression " || $regexp || ".");
  else ();
};

declare function api:validate-enum($name as string, $value as string?, $enum as string*)
as ()
{
  if (exists($value))
  then
    if ($value = $enum)
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " is not one of these: " || string-join($enum,", ") || ".");
  else ();
};

declare %private function api:sgpl($count as integer, $singular as string, $plural as string)
as string
{
    if ($count eq 1)
    then $count || " " || $singular
    else $count || " " || $plural
};

declare function api:passed($timestamp as xs:dateTime?)
as string
{
    if (empty($timestamp) or $timestamp eq null)
    then "Never"
    else
        let $now := current-dateTime()
        let $difference := $now - $timestamp
        let $years := years-from-duration($difference)
        let $month := months-from-duration($difference)
        let $days := days-from-duration($difference)
        let $hours := hours-from-duration($difference)
        let $minutes := minutes-from-duration($difference)
        return
            if ($years gt 0) then api:sgpl($years,"year ago","years ago")
            else if ($month gt 0) then api:sgpl($month,"month ago","month ago")
            else if ($days gt 0) then api:sgpl($days, "day ago", "days ago")
            else if ($hours gt 0) then api:sgpl($hours, "hour ago", "hours ago")
            else if ($minutes gt 0) then api:sgpl($minutes, "minute ago", "minutes ago")
            else if ($difference div xs:dayTimeDuration("PT1S") lt 0) then "future"
            else "Just Now"
};

declare function api:success() as object
{
  { "success" : true }
};

declare function api:success($data as object()) as object
{
  {|
     {"success" : true },
     $data
  |}
};

declare %an:sequential function api:check-and-return-results(
    $token as string?,
    $results as item*,
    $format as string?
) as item*
{
    switch(session:has-access($token, "data_sec"))
    case $session:ACCESS-ALLOWED return
        $results
    case $session:ACCESS-DENIED return {
          resp:status-code(403);
          session:error("access denied", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          resp:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()
};

declare function api:normalize-facts(
    $facts as object*) as object*
{
    for $fact in $facts
    return {|
        {
            "Aspects" : {|
                trim($fact.Aspects, ("xbrl:Unit")),
                { "sec:Archive" : $fact.Aspects."xbrl28:Archive" }[exists($fact.Aspects."xbrl28:Archive")]
            |}
        },
        trim($fact, ("Aspects", "_id")),
        { Unit: $fact.Aspects."xbrl:Unit" }[exists($fact.Aspects."xbrl:Unit") and not exists($fact.Unit)]
    |}
};

declare %an:sequential function api:serialize(
    $result as json-item,
    $comment as object,
    $serializers as object,
    $format as string?,
    $file-name as string) as item*
{
    switch ($format)
    case "xml" return {
        resp:serialization-parameters({"omit-xml-declaration" : false, indent : true });
        session:comment("xml", $comment),
        $serializers.to-xml($result)
    }
    case "text" case "csv" return {
        resp:content-type("text/csv");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    case "html" return {
        resp:content-type("text/html");
        let $csv as string := $serializers.to-csv($result)
        return api:csv-to-html($csv)
    }
    case "excel" return {
        resp:content-type("application/vnd.ms-excel");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    default return {
        resp:content-type("application/json");
        resp:serialization-parameters({"indent" : true});
        {|
            session:comment("json", $comment),
            $result
        |}
    }
};

declare %an:sequential function api:csv-to-html(
    $csv as string) as item*
{
    let $csv as string* := $csv
    let $rows := tokenize($csv, "\n")
    let $header := $rows[1]
    let $body := subsequence($rows, 2)
    let $display-cell := function($row as string, $cell-tag-name as string) {
      for tumbling window $cells as string* in tokenize($row, ",")
      start $start at $i when true
      only end $end at $j when not contains(replace($start, "\"\"", ""), "\"") or ($j gt $i and contains(replace($end, "\"\"", ""), "\""))
      let $cell := replace(string-join($cells, ","), "\"", "")
      return if(contains($cell, "http://") and (contains($cell, "28.io") or contains($cell, "rendering.secxbrl.info")))
      then element $cell-tag-name { Link }
      else element $cell-tag-name {$cell}
    }
    return

        <title>Cell Store REST API</title>

{$display-cell($header)}

          {
            for $row as string in $body
            return
{$display-cell($row)}

          }

};

declare function api:preprocess-fiscal-years($fiscal-years as string*) as integer*
{
  distinct-values(
    for $fy in $fiscal-years ! upper-case($$)
    return switch($fy)
           case "LATEST" return $sec-fiscal:LATEST_FISCAL_YEAR
           case "ALL" return $sec-fiscal:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
  )
};

declare function api:preprocess-fiscal-periods($fiscal-periods as string*) as string*
{
  distinct-values(
    for $fp in $fiscal-periods ! upper-case($$)
    return switch($fp)
           case "Q1"
           case "Q2"
           case "Q3"
           case "FY"
             return $fp
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIODS
           default
             return error(xs:QName("local:INVALID-PERIOD"),
               $fp || ": fiscalPeriod values must be one or more of Q1, Q2, Q3, FY, ALL")
  )
};

declare function api:preprocess-fiscal-period-types($fiscal-period-types as string*) as string*
{
  distinct-values(
    for $fpt in $fiscal-period-types
    return switch($fpt)
           case "instant"
           case "QTD"
           case "YTD"
             return $fpt
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIOD_TYPES
           default
             return error(xs:QName("local:INVALID-PERIOD-TYPE"),
               $fpt || ": fiscalPeriodType values must be one or more of instant, YTD, QTD")
  )
};

declare function api:preprocess-format($format as string?, $request-uri as string) as string?
{
  let $request-path := tokenize($request-uri, "\\?")[1]
  return lower-case(($format, substring-after($request-path, ".jq."))[1])
};

declare function api:preprocess-tags($tags as string*) as string*
{
  distinct-values($tags ! upper-case($$))
};

declare function api:preprocess-boolean($name as string, $value as string)
as boolean
{
  if ($value eq "")
  then true
  else
  {
      api:validate-enum($name, lower-case($value), ("true", "false"));
      boolean($value)
  }
};

jsoniq version "1.0";

module namespace api = "http://apps.28.io/api";

import module namespace session    = "http://apps.28.io/session";
import module namespace resp       = "http://www.28msec.com/modules/http-response";
import module namespace sec-fiscal = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare function api:json-to-csv($objects as object*) as string
{
  let $flattened as object* := api:flatten-json-object($objects, ())
  return string-join(
    csv:serialize($flattened, { field-names: [ keys($flattened) ], serialize-null-as : "" }),
    "")
  };

declare function api:json-to-xml($objects as object*, $root-name as string) as element()*
{
  for $flattened-object as object in api:flatten-json-object($objects, { KeepArray: true })
  return element { $root-name } {
    for $key in keys($flattened-object)
    let $value := $flattened-object.$key
    return typeswitch($value)
           case array return
             let $key-with-s := if(substring($key, string-length($key), 1) eq "s")
                                then $key
                                else $key || "s"
             let $key-no-s := if(substring($key, string-length($key), 1) eq "s")
                                then substring($key, 1, string-length($key) - 1)
                                else $key
             return element {$key-with-s} { for $v in $value[] return element { $key-no-s } { $v } }
           default return element { $key } { $value }
  }
};

declare %private function api:flatten-json-object(
  $items as item*,
  $options as object?) as item*
{
  for $item in $items
  return typeswitch($item)
         case atomic return $item
         case array return let $atomic-items := flatten($item)[$$ instance of atomic]
                           return if ($options.KeepArray)
                                 then [ $atomic-items ]
                                  else string-join($atomic-items, ", ")
         case object return {|
             for $key in keys($item)
             return typeswitch($item.$key)
                    case object return api:flatten-json-object($item.$key, $options)
                    default return { $key: api:flatten-json-object($item.$key, $options) }
         |}
         default return ()
};

declare function api:validate-regexp($name as string, $value as string?, $regexp as string)
as ()
{
  if (exists($value))
  then
    if (matches($value, "^" || $regexp || "$"))
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " does not match reg. expression " || $regexp || ".");
  else ();
};

declare function api:validate-enum($name as string, $value as string?, $enum as string*)
as ()
{
  if (exists($value))
  then
    if ($value = $enum)
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " is not one of these: " || string-join($enum,", ") || ".");
  else ();
};

declare %private function api:sgpl($count as integer, $singular as string, $plural as string)
as string
{
    if ($count eq 1)
    then $count || " " || $singular
    else $count || " " || $plural
};

declare function api:passed($timestamp as xs:dateTime?)
as string
{
    if (empty($timestamp) or $timestamp eq null)
    then "Never"
    else
        let $now := current-dateTime()
        let $difference := $now - $timestamp
        let $years := years-from-duration($difference)
        let $month := months-from-duration($difference)
        let $days := days-from-duration($difference)
        let $hours := hours-from-duration($difference)
        let $minutes := minutes-from-duration($difference)
        return
            if ($years gt 0) then api:sgpl($years,"year ago","years ago")
            else if ($month gt 0) then api:sgpl($month,"month ago","month ago")
            else if ($days gt 0) then api:sgpl($days, "day ago", "days ago")
            else if ($hours gt 0) then api:sgpl($hours, "hour ago", "hours ago")
            else if ($minutes gt 0) then api:sgpl($minutes, "minute ago", "minutes ago")
            else if ($difference div xs:dayTimeDuration("PT1S") lt 0) then "future"
            else "Just Now"
};

declare function api:success() as object
{
  { "success" : true }
};

declare function api:success($data as object()) as object
{
  {|
     {"success" : true },
     $data
  |}
};

declare %an:sequential function api:check-and-return-results(
    $token as string?,
    $results as item*,
    $format as string?
) as item*
{
    switch(session:has-access($token, "data_sec"))
    case $session:ACCESS-ALLOWED return
        $results
    case $session:ACCESS-DENIED return {
          resp:status-code(403);
          session:error("access denied", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          resp:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()
};

declare function api:normalize-facts(
    $facts as object*) as object*
{
    for $fact in $facts
    return {|
        {
            "Aspects" : {|
                trim($fact.Aspects, ("xbrl:Unit")),
                { "sec:Archive" : $fact.Aspects."xbrl28:Archive" }[exists($fact.Aspects."xbrl28:Archive")]
            |}
        },
        trim($fact, ("Aspects", "_id")),
        { Unit: $fact.Aspects."xbrl:Unit" }[exists($fact.Aspects."xbrl:Unit") and not exists($fact.Unit)]
    |}
};

declare %an:sequential function api:serialize(
    $result as json-item,
    $comment as object,
    $serializers as object,
    $format as string?,
    $file-name as string) as item*
{
    switch ($format)
    case "xml" return {
        resp:serialization-parameters({"omit-xml-declaration" : false, indent : true });
        session:comment("xml", $comment),
        $serializers.to-xml($result)
    }
    case "text" case "csv" return {
        resp:content-type("text/csv");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    case "html" return {
        resp:content-type("text/html");
        let $csv as string := $serializers.to-csv($result)
        return api:csv-to-html($csv)
    }
    case "excel" return {
        resp:content-type("application/vnd.ms-excel");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    default return {
        resp:content-type("application/json");
        resp:serialization-parameters({"indent" : true});
        {|
            session:comment("json", $comment),
            $result
        |}
    }
};

declare %an:sequential function api:csv-to-html(
    $csv as string) as item*
{
    let $csv as string* := $csv
    let $rows := tokenize($csv, "\n")
    let $header := $rows[1]
    let $body := subsequence($rows, 2)
    let $display-cell := function($row as string) {
      for tumbling window $cells as string* in tokenize($row, ",")
      start $start at $i when true
      only end $end at $j when not contains(replace($start, "\"\"", ""), "\"") or ($j gt $i and contains(replace($end, "\"\"", ""), "\""))
      let $cell := replace(string-join($cells, ","), "\"", "")
      return if(contains($cell, "http://") and (contains($cell, "28.io") or contains($cell, "rendering.secxbrl.info")))
      then Link
      else {$cell}
    }
    return

        <title>Cell Store REST API</title>

	          {$display-cell($header)}

          {
            for $row as string in $body
            return
{$display-cell($row)}

          }

};

declare function api:preprocess-fiscal-years($fiscal-years as string*) as integer*
{
  distinct-values(
    for $fy in $fiscal-years ! upper-case($$)
    return switch($fy)
           case "LATEST" return $sec-fiscal:LATEST_FISCAL_YEAR
           case "ALL" return $sec-fiscal:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
  )
};

declare function api:preprocess-fiscal-periods($fiscal-periods as string*) as string*
{
  distinct-values(
    for $fp in $fiscal-periods ! upper-case($$)
    return switch($fp)
           case "Q1"
           case "Q2"
           case "Q3"
           case "FY"
             return $fp
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIODS
           default
             return error(xs:QName("local:INVALID-PERIOD"),
               $fp || ": fiscalPeriod values must be one or more of Q1, Q2, Q3, FY, ALL")
  )
};

declare function api:preprocess-fiscal-period-types($fiscal-period-types as string*) as string*
{
  distinct-values(
    for $fpt in $fiscal-period-types
    return switch($fpt)
           case "instant"
           case "QTD"
           case "YTD"
             return $fpt
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIOD_TYPES
           default
             return error(xs:QName("local:INVALID-PERIOD-TYPE"),
               $fpt || ": fiscalPeriodType values must be one or more of instant, YTD, QTD")
  )
};

declare function api:preprocess-format($format as string?, $request-uri as string) as string?
{
  let $request-path := tokenize($request-uri, "\\?")[1]
  return lower-case(($format, substring-after($request-path, ".jq."))[1])
};

declare function api:preprocess-tags($tags as string*) as string*
{
  distinct-values($tags ! upper-case($$))
};

declare function api:preprocess-boolean($name as string, $value as string)
as boolean
{
  if ($value eq "")
  then true
  else
  {
      api:validate-enum($name, lower-case($value), ("true", "false"));
      boolean($value)
  }
};