Use of constants in mongo-wrapper. Relaxed selection criterions to choose an index in more cases.

module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

(: Indexes declarations :)
declare variable $mw:ARCHIVES-ID as string := "_id_";
declare variable $mw:ARCHIVES-ENTITY as string := "Entity_hashed";

declare variable $mw:COMPONENTS-ID as string := "_id_";
declare variable $mw:COMPONENTS-ARCHIVE as string := "Archive_hashed";
declare variable $mw:COMPONENTS-PROFILE_SEC_DISCLOSURE as string := "Profiles.SEC.Disclosure_hashed";
declare variable $mw:COMPONENTS-ROLE as string := "Role_hashed";

declare variable $mw:CONCEPTS-ID as string := "_id_";
declare variable $mw:CONCEPTS-ARCHIVE-ROLE-NAME as string := "Archive_1_Role_1_Name_1";
declare variable $mw:CONCEPTS-NAME-ARCHIVE as string := "Name_1_Archive_1";

declare variable $mw:ENTITIES-ID as string := "_id_";
declare variable $mw:ENTITIES-PROFILES_SEC_COMPANYTYPE as string := "Profiles.SEC.CompanyType_hashed";
declare variable $mw:ENTITIES-PROFILES_SEC_SIC as string := "Profiles.SEC.SIC_hashed";
declare variable $mw:ENTITIES-PROFILES_SEC_SECTOR as string := "Profiles.SEC.Sector_hashed";
declare variable $mw:ENTITIES-PROFILES_SEC_TAGS as string := "Profiles.SEC.Tags_1";
declare variable $mw:ENTITIES-PROFILES_SEC_TICKERS as string := "Profiles.SEC.Tickers_1";

declare variable $mw:FACTS-ID as string := "_id_";
declare variable $mw:FACTS-AXCONCEPT_AXENTITY_ASFY_ASFP as string := "Aspects.xbrl:Concept_1_Aspects.xbrl:Entity_1_Aspects.sec:FiscalYear_1_Aspects.sec:FiscalPeriod_1";
declare variable $mw:FACTS-ASARCHIVE_AXCONCEPT as string := "Aspects.sec:Archive_1_Aspects.xbrl:Concept_1";

declare variable $mw:FOOTNOTES-ID as string := "_id_";
declare variable $mw:FOOTNOTES-ARCHIVE as string := "Archive_hashed";
declare variable $mw:FOOTNOTES-FACTS_ID-LANG as string := "Facts._id_1_Lang_1";

declare variable $mw:ALL-INDEXES as object :=
{
  "archives" : [ $mw:ARCHIVES-ID, $mw:ARCHIVES-ENTITY ],
  "components": [ $mw:COMPONENTS-ID, $mw:COMPONENTS-ARCHIVE, $mw:COMPONENTS-PROFILE_SEC_DISCLOSURE, $mw:COMPONENTS-ROLE ],
  "concepts": [ $mw:CONCEPTS-ID, $mw:CONCEPTS-ARCHIVE-ROLE-NAME, $mw:CONCEPTS-NAME-ARCHIVE ],
  "entities": [ $mw:ENTITIES-ID, $mw:ENTITIES-PROFILES_SEC_COMPANYTYPE, $mw:ENTITIES-PROFILES_SEC_SIC,
               $mw:ENTITIES-PROFILES_SEC_SECTOR, $mw:ENTITIES-PROFILES_SEC_TAGS, $mw:ENTITIES-PROFILES_SEC_TICKERS ],
  "facts": [ $mw:FACTS-ID, $mw:FACTS-AXCONCEPT_AXENTITY_ASFY_ASFP, $mw:FACTS-ASARCHIVE_AXCONCEPT ],
  "footnotes": [ $mw:FOOTNOTES-ID, $mw:FOOTNOTES-ARCHIVE, $mw:FOOTNOTES-FACTS-LANG ]
};

declare %an:strictlydeterministic function mw:connection() as anyURI
{
  let $credentials :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return if (empty($credentials))
             then error(QName("mw:CONNECTION-FAILED"), "no xbrl MongoDB configured")
             else $credentials
  return
    try {
      mongo:connect($credentials)
    } catch mongo:* {
      error(QName("mw:CONNECTION-FAILED"), $err:description)
    }
};

declare function mw:find($collection as string, $query as object) as object()*
{
  let $conn := mw:connection()
  return mongo:find($conn, $collection, mw:hint($collection, $query))
};

declare function mw:find($collection as string, $query as object, $projection as object) as object()*
{
  let $conn := mw:connection()
  return mongo:find($conn, $collection, mw:hint($collection, $query), $projection)
};

declare function mw:run-cmd-deterministic($command as object) as object*
{
  let $conn := mw:connection()
  return mongo:run-cmd-deterministic($conn, $command)
};

declare %private function mw:hint($collection as string, $query as object) as object
{
  switch($collection)
    case "archives"
      return mw:archives-hint($query)
    case "components"
      return mw:components-hint($query)
    case "concepts"
      return mw:concepts-hint($query)
    case "entities"
      return mw:entities-hint($query)
    case "facts"
      return mw:facts-hint($query)
    case "footnotes"
      return mw:footnotes-hint($query)
    default
      return $query
};

declare %private function mw:archives-hint($query as object) as object
{
  switch (true)
    case (exists($query("_id")))
      return { "$query": $query, "$hint": $mw:ARCHIVES-ID }
    case (exists($query("Entity")))
      return { "$query": $query, "$hint": $mw:ARCHIVES-ENTITY }
    default
      return $query
};

declare %private function mw:components-hint($query as object) as object
{
  switch (true)
    case (exists($query("_id")))
      return { "$query": $query, "$hint": $mw:COMPONENTS-ID }
    case (exists($query("Archive")))
      return { "$query": $query, "$hint": $mw:COMPONENTS-ARCHIVE }
    case (exists($query("Profiles.SEC.Disclosure")))
      return { "$query": $query, "$hint": $mw:COMPONENTS-PROFILE_SEC_DISCLOSURE }
    case (exists($query("Role")))
      return { "$query": $query, "$hint": $mw:COMPONENTS-ROLE }
    default
      return $query
};

declare %private function mw:concepts-hint($query as object) as object
{
  (:
     Q      I
     N   -> NA
     R   -> DEF
     RN  -> DEF
     A   -> ARN
     AN  -> NA
     AR  -> ARN
     ARN -> ARN
  :)
  switch (true)
    case (exists($query("_id")))
      return { "$query": $query, "$hint": $mw:CONCEPTS-ID }
    case (exists($query("Archive")) and (not(exists($query("Name"))) or exists($query("Role"))))
      return { "$query": $query, "$hint": $mw:CONCEPTS-ARCHIVE-ROLE-NAME }
    case (exists($query("Name")))
      return { "$query": $query, "$hint": $mw:CONCEPTS-NAME-ARCHIVE }
    default
      return $query
};

declare %private function mw:entities-hint($query as object) as object
{
  switch (true)
    case (exists($query("_id")))
      return { "$query": $query, "$hint": $mw:ENTITIES-ID }
    case (exists($query("Profiles.SEC.CompanyType")))
      return { "$query": $query, "$hint": $mw:ENTITIES-PROFILES_SEC_COMPANYTYPE }
    case (exists($query("Profiles.SEC.SIC")))
      return { "$query": $query, "$hint": $mw:ENTITIES-PROFILES_SEC_SIC }
    case (exists($query("Profiles.SEC.Sector")))
      return { "$query": $query, "$hint": $mw:ENTITIES-PROFILES_SEC_SECTOR }
    case (exists($query("Profiles.SEC.Tags")))
      return { "$query": $query, "$hint": $mw:ENTITIES-PROFILES_SEC_TAGS }
    case (exists($query("Profiles.SEC.Tickers")))
      return { "$query": $query, "$hint": $mw:ENTITIES-PROFILES_SEC_TICKERS }
    default
      return $query
};

declare %private function mw:facts-hint($query as object) as object
{
    switch (true)
    case (exists($query("_id")))
      return { "$query": $query, "$hint": $mw:FACTS-ID }
    case (mw:is-selective($query, "Aspects.xbrl:Concept") and
          mw:is-selective($query, "Aspects.xbrl:Entity"))
      return { "$query": $query, "$hint": $mw:FACTS-AXCONCEPT_AXENTITY_ASFY_ASFP }
    case (mw:is-selective($query, "Aspects.sec:Archive"))
      return { "$query": $query, "$hint": $mw:FACTS-ASARCHIVE_AXCONCEPT }
    default
      return $query
};

declare %private function mw:footnotes-hint($query as object) as object
{
    switch (true)
    case (exists($query("_id")))
      return { "$query": $query, "$hint": $mw:FOOTNOTES-ID }
    case (exists($query("Archive")))
      return { "$query": $query, "$hint": $mw:FOOTNOTES-ARCHIVE }
    case (exists($query("Facts._id")))
      return { "$query": $query, "$hint": $mw:FOOTNOTES-FACTS_ID-LANG }
    default
      return $query
};

(:
  Heuristics detection of whether a condition in the query is selective and should be
  used to decide the index to use
:)
declare %private function mw:is-selective($query as object, $field as string) as boolean
{
    exists($query($field)) and
      (
        not($query($field) instance of object) or (: Equality :)
        exists($query($field)("$in")) (: $in operator :)
      )
};

module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

(: Indexes declarations :)
declare variable $mw:ARCHIVES-ID as string := "_id_";
declare variable $mw:ARCHIVES-ENTITY as string := "Entity_hashed";

declare variable $mw:COMPONENTS-ID as string := "_id_";
declare variable $mw:COMPONENTS-ARCHIVE as string := "Archive_hashed";
declare variable $mw:COMPONENTS-PROFILE_SEC_DISCLOSURE as string := "Profiles.SEC.Disclosure_hashed";
declare variable $mw:COMPONENTS-ROLE as string := "Role_hashed";

declare variable $mw:CONCEPTS-ID as string := "_id_";
declare variable $mw:CONCEPTS-ARCHIVE-ROLE-NAME as string := "Archive_1_Role_1_Name_1";
declare variable $mw:CONCEPTS-NAME-ARCHIVE as string := "Name_1_Archive_1";

declare variable $mw:ENTITIES-ID as string := "_id_";
declare variable $mw:ENTITIES-PROFILES_SEC_COMPANYTYPE as string := "Profiles.SEC.CompanyType_hashed";
declare variable $mw:ENTITIES-PROFILES_SEC_SIC as string := "Profiles.SEC.SIC_hashed";
declare variable $mw:ENTITIES-PROFILES_SEC_SECTOR as string := "Profiles.SEC.Sector_hashed";
declare variable $mw:ENTITIES-PROFILES_SEC_TAGS as string := "Profiles.SEC.Tags_1";
declare variable $mw:ENTITIES-PROFILES_SEC_TICKERS as string := "Profiles.SEC.Tickers_1";

declare variable $mw:FACTS-ID as string := "_id_";
declare variable $mw:FACTS-AXCONCEPT_AXENTITY_ASFY_ASFP as string := "Aspects.xbrl:Concept_1_Aspects.xbrl:Entity_1_Aspects.sec:FiscalYear_1_Aspects.sec:FiscalPeriod_1";
declare variable $mw:FACTS-ASARCHIVE_AXCONCEPT as string := "Aspects.sec:Archive_1_Aspects.xbrl:Concept_1";

declare variable $mw:FOOTNOTES-ID as string := "_id_";
declare variable $mw:FOOTNOTES-ARCHIVE as string := "Archive_hashed";
declare variable $mw:FOOTNOTES-FACTS-LANG as string := "Facts._id_1_Lang_1";

declare variable $mw:ALL-INDEXES as object :=
{
  "archives" : [ $mw:ARCHIVES-ID, $mw:ARCHIVES-ENTITY ],
  "components": [ $mw:COMPONENTS-ID, $mw:COMPONENTS-ARCHIVE, $mw:COMPONENTS-PROFILE_SEC_DISCLOSURE, $mw:COMPONENTS-ROLE ],
  "concepts": [ $mw:CONCEPTS-ID, $mw:CONCEPTS-ARCHIVE-ROLE-NAME, $mw:CONCEPTS-NAME-ARCHIVE ],
  "entities": [ $mw:ENTITIES-ID, $mw:ENTITIES-PROFILES_SEC_COMPANYTYPE, $mw:ENTITIES-PROFILES_SEC_SIC,
               $mw:ENTITIES-PROFILES_SEC_SECTOR, $mw:ENTITIES-PROFILES_SEC_TAGS, $mw:ENTITIES-PROFILES_SEC_TICKERS ],
  "facts": [ $mw:FACTS-ID, $mw:FACTS-AXCONCEPT_AXENTITY_ASFY_ASFP, $mw:FACTS-ASARCHIVE_AXCONCEPT ],
  "footnotes": [ $mw:FOOTNOTES-ID, $mw:FOOTNOTES-ARCHIVE, $mw:FOOTNOTES-FACTS-LANG ]
};

declare %an:strictlydeterministic function mw:connection() as anyURI
{
  let $credentials :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return if (empty($credentials))
             then error(QName("mw:CONNECTION-FAILED"), "no xbrl MongoDB configured")
             else $credentials
  return
    try {
      mongo:connect($credentials)
    } catch mongo:* {
      error(QName("mw:CONNECTION-FAILED"), $err:description)
    }
};

declare function mw:find($collection as string, $query as object) as object()*
{
  let $conn := mw:connection()
  return mongo:find($conn, $collection, mw:hint($collection, $query))
};

declare function mw:find($collection as string, $query as object, $projection as object) as object()*
{
  let $conn := mw:connection()
  return mongo:find($conn, $collection, mw:hint($collection, $query), $projection)
};

declare function mw:run-cmd-deterministic($command as object) as object*
{
  let $conn := mw:connection()
  return mongo:run-cmd-deterministic($conn, $command)
};

declare %private function mw:hint($collection as string, $query as object) as object
{
  switch($collection)
    case "archives"
      return mw:archives-hint($query)
    case "components"
      return mw:components-hint($query)
    case "concepts"
      return mw:concepts-hint($query)
    case "entities"
      return mw:entities-hint($query)
    case "facts"
      return mw:facts-hint($query)
    case "footnotes"
      return mw:footnotes-hint($query)
    default
      return $query
};

declare %private function mw:archives-hint($query as object) as object
{
  switch (true)
    case (exists($query("_id")))
      return { "$query": $query, "$hint": "_id_" }
    case (exists($query("Entity")))
      return { "$query": $query, "$hint": "Entity_hashed" }
    default
      return $query
};

declare %private function mw:components-hint($query as object) as object
{
  switch (true)
    case (exists($query("_id")))
      return { "$query": $query, "$hint": "_id_" }
    case (exists($query("Archive")))
      return { "$query": $query, "$hint": "Archive_hashed" }
    case (exists($query("Profiles.SEC.Disclosure")))
      return { "$query": $query, "$hint": "Profiles.SEC.Disclosure_hashed" }
    case (exists($query("Role")))
      return { "$query": $query, "$hint": "Role_hashed" }
    default
      return $query
};

declare %private function mw:concepts-hint($query as object) as object
{
  switch (true)
    case (exists($query("_id")))
      return { "$query": $query, "$hint": "_id_" }
    case (exists($query("Archive")) and exists($query("Role")))
      return { "$query": $query, "$hint": "Archive_1_Role_1_Name_1" }
    case (exists($query("Archive")) and exists($query("Name")))
      return { "$query": $query, "$hint": "Name_1_Archive_1" }
    case (exists($query("Archive")))
      return { "$query": $query, "$hint": "Archive_1_Role_1_Name_1" }
    default
      return $query
};

declare %private function mw:entities-hint($query as object) as object
{
  switch (true)
    case (exists($query("_id")))
      return { "$query": $query, "$hint": "_id_" }
    case (exists($query("Profiles.SEC.CompanyType")))
      return { "$query": $query, "$hint": "Profiles.SEC.CompanyType_hashed" }
    case (exists($query("Profiles.SEC.SIC")))
      return { "$query": $query, "$hint": "Profiles.SEC.SIC_hashed" }
    case (exists($query("Profiles.SEC.Sector")))
      return { "$query": $query, "$hint": "Profiles.SEC.Sector_hashed" }
    case (exists($query("Profiles.SEC.Tags")))
      return { "$query": $query, "$hint": "Profiles.SEC.Tags_1" }
    case (exists($query("Profiles.SEC.Tickers")))
      return { "$query": $query, "$hint": "Profiles.SEC.Tickers_1" }
    default
      return $query
};

declare %private function mw:facts-hint($query as object) as object
{
    switch (true)
    case (exists($query("_id")))
      return { "$query": $query, "$hint": "_id_" }
    case (exists($query("Aspects.xbrl:Concept")) and
          exists($query("Aspects.xbrl:Entity")) and
          exists($query("Aspects.sec:FiscalYear")) and
          exists($query("Aspects.sec:FiscalPeriod")))
      return { "$query": $query, "$hint": "Aspects.xbrl:Concept_1_Aspects.xbrl:Entity_1_Aspects.sec:FiscalYear_1_Aspects.sec:FiscalPeriod_1" }
    case (exists($query("Aspects.sec:Archive")) and exists($query("Aspects.xbrl:Concept")))
      return { "$query": $query, "$hint": "Aspects.sec:Archive_1_Aspects.xbrl:Concept_1" }
    default
      return $query
};

declare %private function mw:footnotes-hint($query as object) as object
{
    switch (true)
    case (exists($query("_id")))
      return { "$query": $query, "$hint": "_id_" }
    case (exists($query("Archive")))
      return { "$query": $query, "$hint": "Archive_hashed" }
    case (exists($query("Facts._id")) and exists($query("Lang")))
      return { "$query": $query, "$hint": "Facts._id_1_Lang_1" }
    default
      return $query
};


:lock: remove exposing query details and reconsuming streamable strings

jsoniq version "1.0";
module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace mw-sec = "http://28.io/modules/xbrl/profiles/sec/mongo-wrapper";
import module namespace mw-japan = "http://28.io/modules/xbrl/profiles/japan/mongo-wrapper";
import module namespace config = "http://apps.28.io/config";
import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

declare %an:strictlydeterministic function mw:connection() as anyURI
{
  let $credentials :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return if (empty($credentials))
             then error(QName("mw:CONNECTION-FAILED"), "no xbrl MongoDB configured")
             else $credentials
  return
    try {
      mongo:connect($credentials)
    } catch mongo:* {
      error(QName("mw:CONNECTION-FAILED"), $err:description)
    }
};

declare %private function mw:check-result(
    $results as object()*,
    $collection as string,
    $hint as object) as object()*
{
    if(empty($results."$err"))
    then $results
    else error(QName("mw:INVALID-PARAMETER"),
               string-join((
                 $results."$err",
                 "COLLECTION: " || $collection
               ), " "))
};

declare function mw:find($collection as string, $query as object) as object()*
{
  let $conn := mw:connection()
  let $hint as object := mw:hint($collection, $query)
  return
    mw:check-result(
        mongo:find($conn, $collection, $hint),
        $collection,
        $hint)
};

declare function mw:find($collection as string, $query as object, $projection as object) as object()*
{
  let $conn := mw:connection()
  let $hint as object := mw:hint($collection, $query)
  return
    mw:check-result(
      mongo:find($conn, $collection, $hint, $projection, {}),
      $collection,
      $hint)
};

declare function mw:run-cmd-deterministic($command as object) as object*
{
  let $conn := mw:connection()
  return mongo:run-cmd-deterministic($conn, $command)
};

declare %private function mw:hint($collection as string, $query as object) as object
{
  switch(lower-case($config:profile-name))
    case "sec"
      return mw-sec:hint($collection, $query)
    case "japan"
      return mw-japan:hint($collection, $query)
    default
      return $query
};

(:
  Heuristics detection of whether a condition in the query is selective and should be
  used to decide the index to use
:)
declare %private function mw:is-selective($query as object, $field as string) as boolean
{
    exists($query($field)) and
      (
        not($query($field) instance of object) or (: Equality checks are of type atomic - not object - other kinds are disallowed :)
        exists($query($field)("$in")) (: $in operator :)
      )
};

jsoniq version "1.0";
module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace mw-sec = "http://28.io/modules/xbrl/profiles/sec/mongo-wrapper";
import module namespace mw-japan = "http://28.io/modules/xbrl/profiles/japan/mongo-wrapper";
import module namespace config = "http://apps.28.io/config";
import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

declare %an:strictlydeterministic function mw:connection() as anyURI
{
  let $credentials :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return if (empty($credentials))
             then error(QName("mw:CONNECTION-FAILED"), "no xbrl MongoDB configured")
             else $credentials
  return
    try {
      mongo:connect($credentials)
    } catch mongo:* {
      error(QName("mw:CONNECTION-FAILED"), $err:description)
    }
};

declare %private function mw:check-result(
    $results as object()*,
    $collection as string,
    $hint as object) as object()*
{
    if(empty($results."$err"))
    then $results
    else error(QName("mw:INVALID-PARAMETER"),
               string-join((
                 $results."$err",
                 "COLLECTION: " || $collection,
                 "HINT: " || serialize($hint)
               ), " "))
};

declare function mw:find($collection as string, $query as object) as object()*
{
  let $conn := mw:connection()
  let $hint as object := mw:hint($collection, $query)
  return
    mw:check-result(
        mongo:find($conn, $collection, $hint),
        $collection,
        $hint)
};

declare function mw:find($collection as string, $query as object, $projection as object) as object()*
{
  let $conn := mw:connection()
  let $hint as object := mw:hint($collection, $query)
  return
    mw:check-result(
      mongo:find($conn, $collection, $hint, $projection, {}),
      $collection,
      $hint)
};

declare function mw:run-cmd-deterministic($command as object) as object*
{
  let $conn := mw:connection()
  return mongo:run-cmd-deterministic($conn, $command)
};

declare %private function mw:hint($collection as string, $query as object) as object
{
  switch(lower-case($config:profile-name))
    case "sec"
      return mw-sec:hint($collection, $query)
    case "japan"
      return mw-japan:hint($collection, $query)
    default
      return $query
};

(:
  Heuristics detection of whether a condition in the query is selective and should be
  used to decide the index to use
:)
declare %private function mw:is-selective($query as object, $field as string) as boolean
{
    exists($query($field)) and
      (
        not($query($field) instance of object) or (: Equality checks are of type atomic - not object - other kinds are disallowed :)
        exists($query($field)("$in")) (: $in operator :)
      )
};


Update facttable-for-report.jq

Update facttable-for-report.jq

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace api = "http://apps.28.io/api";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token         as string? external;
declare  %rest:env                              variable $request-uri   as string  external;
declare  %rest:case-insensitive                 variable $format        as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker        as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear    as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid           as string* external;
declare  %rest:case-insensitive                 variable $validate      as boolean external := false;
declare  %rest:case-insensitive                 variable $labels        as boolean external := false;
declare  %rest:case-insensitive                 variable $report        as string? external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := "generic";

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities :=
    for $entity in
        companies:companies(
            $cik,
            $tag,
            $ticker,
            $sic,
            $aid)
    order by $entity.Profiles.SEC.CompanyName
    return $entity
let $report-id as string? := $report
let $report as object? := reports:reports($report-id)

return
if(empty($report))
then
{
      response:status-code(404);
      session:error("report with id '" || $report-id || "' does not exist.", $format)
} else

    (: Fact resolution :)
    let $filter-override as object? := fiscal-core:filter-override(
        $entities,
        $fiscalYear,
        $fiscalPeriod,
        $aid)[$profile-name eq "sec"]
    let $facts as object* :=
        let $hypercube := hypercubes:hypercubes-for-components($report, "xbrl:DefaultHypercube")
        let $filtered-aspects := values($hypercube.Aspects)[exists(($$.Domains, $$.DomainRestriction))]
        return if(count($filtered-aspects) lt 2 and not exists(($filter-override)))
        then {
              response:status-code(403);
              session:error("The report filters are too weak, which leads to too big an output.", $format)
        } else
            components:facts(
                    $report,
                    {|
                        { FilterOverride: $filter-override }[exists($filter-override)],
                        { Validate: $validate }
                    |}
                )

    let $concepts as object* :=
        reports:concepts($report)
    let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $concepts:AMERICAN_ENGLISH )[1]
    let $role as string := ( $report.Role, $concepts:ANY_COMPONENT_LINK_ROLE )[1]
    let $facts :=
      if($profile-name eq "sec")
      then
        for $fact in $facts
        group by $archive := $fact.Aspects."sec:Archive"
        let $archive := archives:archives($archive)
        let $entity := entities:entities($archive.Entity)
        for $fact in $fact
        return
        {|
            trim($fact, ("Labels", "EntityRegistrantName")),
            { "EntityRegistrantName" : $entity.Profiles.SEC.CompanyName },
            if($labels)
            then
                let $labels as object? := facts:labels($fact, $role, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
                return
                    { Labels : $labels }
            else ()
        |}
      else
        for $fact in $facts
        let $labels-object as object? := facts:labels($fact, $role, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
        return
        {|
            trim($fact, "Labels"),
            { Labels : $labels-object }[exists($labels)]
        |}

    let $facts := api:normalize-facts($facts)

    let $results :=
        switch ($format)
        case "xml" return {
            response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
            session:comment("xml",
            {
                NumFacts : count($facts),
                TotalNumFacts: session:num-facts(),
                TotalNumArchives: session:num-archives(),
                TotalNumEntities: session:num-entities()
            }),
            <FactTable NetworkIdentifier="http://bizql.io/facttable-for-report"
                    TableName="xbrl:FactTableForReport">{
                conversion:facts-to-xml($facts, { Caller: "Report" })
            }</FactTable>
        }
        case "text" case "csv" return {
            response:content-type("text/csv");
            response:header("Content-Disposition", "attachment; filename=facts.csv");
            conversion:facts-to-csv($facts, { Caller: "Report"})
        }
        case "excel" return {
            response:content-type("application/vnd.ms-excel");
            response:header("Content-Disposition", "attachment; filename=fact.csv");
            conversion:facts-to-csv($facts, { Caller: "Report"})
        }
        default return {
            response:content-type("application/json");
            response:serialization-parameters({"indent" : true});
            {|
                { NetworkIdentifier : "http://secxbrl.info/facts" },
                { TableName : "xbrl:Facts" },
                { FactTable : [ $facts ] },
                session:comment("json", {
                        NumFacts : count($facts),
                        TotalNumFacts: session:num-facts(),
                        TotalNumArchives: session:num-archives(),
                        TotalNumEntities: session:num-entities()
                    })
            |}
        }
    return api:check-and-return-results($token, $results, $format)

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace api = "http://apps.28.io/api";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token         as string? external;
declare  %rest:env                              variable $request-uri   as string  external;
declare  %rest:case-insensitive                 variable $format        as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker        as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear    as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid           as string* external;
declare  %rest:case-insensitive                 variable $validate      as boolean external := false;
declare  %rest:case-insensitive                 variable $labels        as boolean external := false;
declare  %rest:case-insensitive                 variable $report        as string? external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := "generic";

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities :=
    for $entity in
        companies:companies(
            $cik,
            $tag,
            $ticker,
            $sic,
            $aid)
    order by $entity.Profiles.SEC.CompanyName
    return $entity
let $report-id as string? := $report
let $report as object? := reports:reports($report-id)

return
if(empty($report))
then
{
      response:status-code(404);
      session:error("report with id '" || $report-id || "' does not exist.", $format)
} else

    (: Fact resolution :)
    let $filter-override as object? := fiscal-core:filter-override(
        $entities,
        $fiscalYear,
        $fiscalPeriod,
        $aid)[$profile-name eq "sec"]
    let $facts as object* :=
        let $hypercube := hypercubes:hypercubes-for-components($report, "xbrl:DefaultHypercube")
        let $filtered-aspects := values($hypercube.Aspects)[exists(($$.Domains, $$.DomainRestriction))]
        return if(false)
        then {
              response:status-code(403);
              session:error("The report filters are too weak, which leads to too big an output.", $format)
        } else
            components:facts(
                    $report,
                    {|
                        { FilterOverride: $filter-override }[exists($filter-override)],
                        { Validate: $validate }
                    |}
                )

    let $concepts as object* :=
        reports:concepts($report)
    let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $concepts:AMERICAN_ENGLISH )[1]
    let $role as string := ( $report.Role, $concepts:ANY_COMPONENT_LINK_ROLE )[1]
    let $facts :=
      if($profile-name eq "sec")
      then
        for $fact in $facts
        group by $archive := $fact.Aspects."sec:Archive"
        let $archive := archives:archives($archive)
        let $entity := entities:entities($archive.Entity)
        for $fact in $fact
        return
        {|
            trim($fact, ("Labels", "EntityRegistrantName")),
            { "EntityRegistrantName" : $entity.Profiles.SEC.CompanyName },
            if($labels)
            then
                let $labels as object? := facts:labels($fact, $role, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
                return
                    { Labels : $labels }
            else ()
        |}
      else
        for $fact in $facts
        let $labels-object as object? := facts:labels($fact, $role, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
        return
        {|
            trim($fact, "Labels"),
            { Labels : $labels-object }[exists($labels)]
        |}

    let $facts := api:normalize-facts($facts)

    let $results :=
        switch ($format)
        case "xml" return {
            response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
            session:comment("xml",
            {
                NumFacts : count($facts),
                TotalNumFacts: session:num-facts(),
                TotalNumArchives: session:num-archives(),
                TotalNumEntities: session:num-entities()
            }),
            <FactTable NetworkIdentifier="http://bizql.io/facttable-for-report"
                    TableName="xbrl:FactTableForReport">{
                conversion:facts-to-xml($facts, { Caller: "Report" })
            }</FactTable>
        }
        case "text" case "csv" return {
            response:content-type("text/csv");
            response:header("Content-Disposition", "attachment; filename=facts.csv");
            conversion:facts-to-csv($facts, { Caller: "Report"})
        }
        case "excel" return {
            response:content-type("application/vnd.ms-excel");
            response:header("Content-Disposition", "attachment; filename=fact.csv");
            conversion:facts-to-csv($facts, { Caller: "Report"})
        }
        default return {
            response:content-type("application/json");
            response:serialization-parameters({"indent" : true});
            {|
                { NetworkIdentifier : "http://secxbrl.info/facts" },
                { TableName : "xbrl:Facts" },
                { FactTable : [ $facts ] },
                session:comment("json", {
                        NumFacts : count($facts),
                        TotalNumFacts: session:num-facts(),
                        TotalNumArchives: session:num-archives(),
                        TotalNumEntities: session:num-entities()
                    })
            |}
        }
    return api:check-and-return-results($token, $results, $format)


Began refactoring facts.jq

Began refactoring facts.jq

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace sec-fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace core = "http://xbrl.io/modules/bizql/profiles/sec/core";

import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";
import module namespace conversion = "http://xbrl.io/modules/bizql/conversion";
import module namespace hypercubes2 = "http://xbrl.io/modules/bizql/hypercubes2";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";

import module namespace util = "http://secxbrl.info/modules/util";

declare function local:to-xml($o as object*) as node()*
{
    (session:comment("xml", {
        NumFacts: count($o),
        TotalNumFacts: session:num-facts(),
        TotalNumArchives: session:num-archives(),
        TotalNumEntities: session:num-entities()
    }),
    <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
        conversion:facts-to-xml($o, { Caller: "Report"})
    }</FactTable>)
};

declare function local:contains-aspect($name as string) as boolean
{
  some $x in (request:param-names() ! starts-with($$, $name))
  satisfies $x
};

declare function local:cast($value as string, $type as string) as atomic
{
  switch ($type)
  case "integer" return $value cast as integer
  default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube() as object
{
    hypercubes:user-defined-hypercube({|
        let $concepts := (request:param-values("concept"), request:param-values("xbrl:Concept"))
        return
            if (exists($concepts))
            then { "xbrl:Concept" : { Domain : [ $concepts ] } }
            else (),

        if (not(request:param-names() = "dei:LegalEntityAxis"))
        then {
                "dei:LegalEntityAxis" : {
                    "Domain" : [ "sec:DefaultLegalEntity" ],
                    "Default" : "sec:DefaultLegalEntity"
                }
        } else (),

        if (not(local:contains-aspect("sec:Accepted")))
        then { "sec:Accepted" : {  } } else (),

        if (not(local:contains-aspect("sec:FiscalYear")))
        then { "sec:FiscalYear" : {  } } else (),

        if (not(local:contains-aspect("sec:FiscalPeriod")))
        then { "sec:FiscalPeriod" : {  } } else (),

        for $p in request:param-names()
        where contains($p, ":") and not(starts-with($p, "xbrl:Concept"))
        group by $dimension-name := if (ends-with(lower-case($p), "::default"))
                                    then substring-before($p, "::default")
                                    else if (ends-with(lower-case($p), ":default"))
                                    then substring-before($p, ":default")
                                    else if (ends-with(lower-case($p), "::type"))
                                    then substring-before($p, "::type")
                                    else $p
        let $all := (request:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $type := (request:param-values($dimension-name || "::type"), request:param-values($dimension-name || ":type"))[1]
        return
        {
            $dimension-name : {|
                let $v := request:param-values($dimension-name)
                return
                    if (exists($v) and not($all))
                    then { "Domain" : [ if (exists($type)) then local:cast($v, $type) else $v ] }
                    else (),

            let $is-default := ($p = $dimension-name || "::default") or ($p = $dimension-name || ":default")
            let $default := (request:param-values($dimension-name || "::default"), request:param-values($dimension-name || ":default"))[1]
            return
                (if ($is-default)
                then {
                    "Default" : if (exists($type)) then local:cast($default, $type) else $default
                  } else (),
                if (exists($type))
                then { "Type" : $type }
                else ())
            |}
        }
    |})
};

declare function local:facts(
    $archives as object*,
    $map as string?,
    $rules as string?) as object*
{
    let $hypercube := hypercubes2:modify-hypercube(
        local:hypercube(),
        {
            "sec:Archive" : {
                Type: "string",
                Domain : [archives:aid($archives)]
            }
        }
    )
    for $fact in core:facts-for(
        {|
            { Hypercube : $hypercube },
            if (exists($map)) then { "ConceptMaps" : $map } else (),
            if (exists($rules)) then { "Rules" : $rules } else ()
        |}
    )
    return {|
        { Aspects : {|
            for $a in keys($fact.Aspects)
            where $a ne "xbrl:Unit"
            return { $a : $fact.Aspects.$a }
        |} },
        { Type: $fact.Type },
        if (exists($fact.Aspects."xbrl:Unit"))
        then { Unit: $fact.Aspects."xbrl:Unit" }
        else  (),
        if (exists($fact.Decimals))
        then { Decimals: $fact.Decimals }
        else (),
        { Value: $fact.Value },
        { "EntityRegistrantName" : companies:companies($fact.Aspects."xbrl:Entity").Profiles.SEC.CompanyName},
        if (exists(flatten($fact.AuditTrails[])))
        then { "AuditTrails" : $fact.AuditTrails }
        else ()
    |}
};

declare function local:filings(
    $ciks as string*,
    $tags as string*,
    $tickers as string*,
    $sics as string*,
    $fp as string*,
    $fy as string*) as object*
{
    let $entities := if ($tags = "ALL") then companies:companies()
                                        else (
                                            companies:companies($ciks),
                                            companies:companies-for-tags($tags),
                                            companies:companies-for-tickers($tickers),
                                            companies:companies-for-SIC($sics)
                                        )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then sec-fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else sec-fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                        case "ALL" return  $sec-fiscal:ALL_FISCAL_YEARS
                    default return $fy
                )
    for $fp in $fp
    return sec-fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy cast as integer)
};

session:audit-call();

let $format      := lower-case(request:param-values("format")[1])

(: choose
    1. entities (using ciks, tags, tickers, sics), FY, and FP ) or
    2. accession numbers
:)
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $sec-fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids     := request:param-values("aid")
let $map      := request:param-values("map")[1]
let $rules    := request:param-values("rules")[1]
let $archives := (
                    local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                    archives:archives($aids)
                 )
let $entities := companies:companies($archives.Entity)

let $results :=
    let $facts := local:facts($archives, $map, $rules)
    return
        switch ($format)
        case "xml" return {
            response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
            local:to-xml($facts)
        }
        case "text" case "csv" return {
            response:content-type("text/csv");
            response:header("Content-Disposition", "attachment; filename=facts.csv");
            conversion:facts-to-csv($facts, { Caller: "Report" })
        }
        case "excel" return {
            response:content-type("application/vnd.ms-excel");
            response:header("Content-Disposition", "attachment; filename=fact.csv");
            conversion:facts-to-csv($facts, { Caller: "Report" })
        }
        default return {
            response:content-type("application/json");
            response:serialization-parameters({"indent" : true});
            {|
                { NetworkIdentifier : "http://bizql.io/facts" },
                { TableName : "xbrl:Facts" },
                { FactTable : [ $facts ] },
                session:comment("json", {
                        NumFacts: count($facts),
                        TotalNumFacts: session:num-facts(),
                        TotalNumArchives: session:num-archives(),
                        TotalNumEntities: session:num-entities()
                    })
            |}
        }
return
    switch(true)
    case empty($archives) return {
        response:status-code(400);
        session:error("entities or archives not found (valid parameters: cik, ticker, tag, sic, aid)", $format)
    }
    default return util:check-and-return-results($entities, $results, $format)

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace sec-fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace core = "http://xbrl.io/modules/bizql/profiles/sec/core";

import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:modify-hypercube(
    $hypercube as object,
    $options as object?) as object
{
    let $options :=
    {|
        for $dimension in keys(($hypercube.Aspects, $options))
        let $hypercube-metadata := $hypercube.Aspects.$dimension
        let $new-metadata := $options.$dimension
        return {
            $dimension : if(exists($new-metadata))
            then
                $new-metadata
            else {|
                { Type : $hypercube.Aspects.$dimension.Type }[$hypercube-metadata.Kind eq "TypedDimension"],
                let $hypercube-domain := descendant-objects(values($hypercube-metadata.Domains)).Name
                where exists($hypercube-domain)
                return { Domain : [ $hypercube-domain ] },
                let $hypercube-default := $hypercube-metadata.Default
                where exists($hypercube-default)
                return { Default : $hypercube-default }
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($options)
};

declare function local:to-csv($o as object*) as string?
{
    if (exists($o)) (: bug in csv:serialize :)
    then
    string-join(
        csv:serialize(
            for $o in $o
            let $a := $o.Aspects
            return {|
                for $k in keys($a)
                return { $k : $a.$k },
                { "Value" : $o.Value },
                if (exists($o."Unit"))
                then { "Unit" :  $o."Unit" }
                else (),
                if (exists($o.Decimals))
                then { "Decimals" : $o.Decimals }
                else(),
                { "EntityRegistrantName" : $o.EntityRegistrantName },
                if (exists($o.ReportedConcept))
                then { "ReportedConcept" : $o.ReportedConcept }
                else ()
            |},
            { serialize-null-as : "" }
        )
    )
    else ()
};

declare function local:audittrail-to-xml($audit as item) as element()
{
    <AuditTrails>{
        for $a in flatten($audit)
        return (
            <Type>{$a.Type}</Type>,
            <Label>{$a.Label}</Label>,
            <Message>{$a.Message}</Message>,
            <Data>{
                if (exists($a.Data.OriginalConcept))
                then <OriginalConcept>{$a.Data.OriginalConcept}</OriginalConcept>
                else (),
                if (exists($a.Data.OutputConcept))
                then <OutputConcept>{$a.Data.OutputConcept}</OutputConcept>
                else (),
                if (exists($a.Data.AuditTrails))
                then local:audittrail-to-xml($a.Data.AuditTrails)
                else ()
            }</Data>
        )
    }</AuditTrails>
};

declare function local:to-xml($o as object*) as node()*
{
    (session:comment("xml", {
        NumFacts: count($o),
        TotalNumFacts: session:num-facts(),
        TotalNumArchives: session:num-archives(),
        TotalNumEntities: session:num-entities()
    }),
    <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
        for $o in $o
        let $a := $o.Aspects
        return
            <Fact>{
                <Aspects>{
                    for $k in keys($a)
                    return
                        <Aspect>
                            <Name>{$k}</Name>
                            <Value>{$a.$k}</Value>
                        </Aspect>
                }</Aspects>,
                <Value>{$o.Value}</Value>,
                <Type>{$o.Type}</Type>,
                if (exists($o.Unit))
                then <Unit>{$o.Unit}</Unit>
                else(),
                if (exists($o.Decimals))
                then <Decimals>{$o.Decimals}</Decimals>
                else (),
                <EntityRegistrantName>{$o.EntityRegistrantName}</EntityRegistrantName>,
                if (exists($o.AuditTrails))
                then local:audittrail-to-xml($o."AuditTrails")
                else ()
            }</Fact>
    }</FactTable>)
};

declare function local:contains-aspect($name as string) as boolean
{
  some $x in (request:param-names() ! starts-with($$, $name))
  satisfies $x
};

declare function local:cast($value as string, $type as string) as atomic
{
  switch ($type)
  case "integer" return $value cast as integer
  default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube() as object
{
    hypercubes:user-defined-hypercube({|
        let $concepts := (request:param-values("concept"), request:param-values("xbrl:Concept"))
        return
            if (exists($concepts))
            then { "xbrl:Concept" : { Domain : [ $concepts ] } }
            else (),

        if (not(request:param-names() = "dei:LegalEntityAxis"))
        then {
                "dei:LegalEntityAxis" : {
                    "Domain" : [ "sec:DefaultLegalEntity" ],
                    "Default" : "sec:DefaultLegalEntity"
                }
        } else (),

        if (not(local:contains-aspect("sec:Accepted")))
        then { "sec:Accepted" : {  } } else (),

        if (not(local:contains-aspect("sec:FiscalYear")))
        then { "sec:FiscalYear" : {  } } else (),

        if (not(local:contains-aspect("sec:FiscalPeriod")))
        then { "sec:FiscalPeriod" : {  } } else (),

        for $p in request:param-names()
        where contains($p, ":") and not(starts-with($p, "xbrl:Concept"))
        group by $dimension-name := if (ends-with(lower-case($p), "::default"))
                                    then substring-before($p, "::default")
                                    else if (ends-with(lower-case($p), ":default"))
                                    then substring-before($p, ":default")
                                    else if (ends-with(lower-case($p), "::type"))
                                    then substring-before($p, "::type")
                                    else $p
        let $all := (request:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $type := (request:param-values($dimension-name || "::type"), request:param-values($dimension-name || ":type"))[1]
        return
        {
            $dimension-name : {|
                let $v := request:param-values($dimension-name)
                return
                    if (exists($v) and not($all))
                    then { "Domain" : [ if (exists($type)) then local:cast($v, $type) else $v ] }
                    else (),

            let $is-default := ($p = $dimension-name || "::default") or ($p = $dimension-name || ":default")
            let $default := (request:param-values($dimension-name || "::default"), request:param-values($dimension-name || ":default"))[1]
            return
                (if ($is-default)
                then {
                    "Default" : if (exists($type)) then local:cast($default, $type) else $default
                  } else (),
                if (exists($type))
                then { "Type" : $type }
                else ())
            |}
        }
    |})
};

declare function local:facts(
    $archives as object*,
    $map as string?,
    $rules as string?) as object*
{
    let $hypercube := local:modify-hypercube(
        local:hypercube(),
        {
            "sec:Archive" : {
                Type: "string",
                Domain : [archives:aid($archives)]
            }
        }
    )
    for $fact in core:facts-for(
        {|
            { Hypercube : $hypercube },
            if (exists($map)) then { "ConceptMaps" : $map } else (),
            if (exists($rules)) then { "Rules" : $rules } else ()
        |}
    )
    return {|
        { Aspects : {|
            for $a in keys($fact.Aspects)
            where $a ne "xbrl:Unit"
            return { $a : $fact.Aspects.$a }
        |} },
        { Type: $fact.Type },
        if (exists($fact.Aspects."xbrl:Unit"))
        then { Unit: $fact.Aspects."xbrl:Unit" }
        else  (),
        if (exists($fact.Decimals))
        then { Decimals: $fact.Decimals }
        else (),
        { Value: $fact.Value },
        { "EntityRegistrantName" : companies:companies($fact.Aspects."xbrl:Entity").Profiles.SEC.CompanyName},
        if (exists(flatten($fact.AuditTrails[])))
        then { "AuditTrails" : $fact.AuditTrails }
        else ()
    |}
};

declare function local:filings(
    $ciks as string*,
    $tags as string*,
    $tickers as string*,
    $sics as string*,
    $fp as string*,
    $fy as string*) as object*
{
    let $entities := if ($tags = "ALL") then companies:companies()
                                        else (
                                            companies:companies($ciks),
                                            companies:companies-for-tags($tags),
                                            companies:companies-for-tickers($tickers),
                                            companies:companies-for-SIC($sics)
                                        )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then sec-fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else sec-fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                        case "ALL" return  $sec-fiscal:ALL_FISCAL_YEARS
                    default return $fy
                )
    for $fp in $fp
    return sec-fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy cast as integer)
};

session:audit-call();

let $format      := lower-case(request:param-values("format")[1])

(: choose
    1. entities (using ciks, tags, tickers, sics), FY, and FP ) or
    2. accession numbers
:)
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $sec-fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids     := request:param-values("aid")
let $map      := request:param-values("map")[1]
let $rules    := request:param-values("rules")[1]
let $archives := (
                    local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                    archives:archives($aids)
                 )
let $entities := companies:companies($archives.Entity)
return
    switch(true)
    case empty($archives) return {
        response:status-code(400);
        session:error("entities or archives not found (valid parameters: cik, ticker, tag, sic, aid)", $format)
    }
    default return
      switch(session:check-access($entities, "data_sec"))
      case $session:ACCESS-ALLOWED return {
        let $facts := local:facts($archives, $map, $rules)
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($facts)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=facts.csv");
                local:to-csv($facts)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=fact.csv");
                local:to-csv($facts)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { NetworkIdentifier : "http://bizql.io/facts" },
                    { TableName : "xbrl:Facts" },
                    { FactTable : [ $facts ] },
                    session:comment("json", {
                            NumFacts: count($facts),
                            TotalNumFacts: session:num-facts(),
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })
                |}
            }
       }
    case $session:ACCESS-DENIED return {
          response:status-code(403);
          session:error("accessing filings of an entity that is not in the DOW30", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          response:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()


:bug: Fix prefix.

:bug: Fix prefix.

jsoniq version "1.0";
module namespace backend = "http://apps.28.io/backend";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace config = "http://apps.28.io/config";

(:~
 :
Builds the URL for a backend call on the specified endpoint and parameters.

 :
 : @param $endpoint the name of the endpoint (without the .jq extension).
 : @param $parameters the parameters, passed as an object.
 :
 : @return the URL.
 :)
declare function backend:url($endpoint as string, $parameters as object) as string
{
    backend:url($endpoint, $parameters, false)
};

(:~
 :
Builds the URL for a backend call on the specified endpoint and parameters.

 :
 : @param $endpoint the name of the endpoint (without the .jq extension).
 : @param $parameters the parameters, passed as an object.
 : @param whether to show the token.
 :
 : @return the URL.
 :)
declare function backend:url($endpoint as string, $parameters as object, $includeToken as boolean) as string
{
    "http://" || request:server-name() || ":" || request:server-port() ||
    "/v1/_queries/public/api/"||$endpoint||".jq?_method=POST&token=" || (if($includeToken) then $config:test-token else "{{token}}") || "&"||
    string-join(
        for $key in keys($parameters)
        for $value as string in (flatten($parameters.$key) ! string($$))
        return ($key||"="||encode-for-uri($value)),
        "&")
};

jsoniq version "1.0";
module namespace backend = "http://apps.28.io/backend";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace config = "http://apps.28.io/config";

(:~
 :
Builds the URL for a backend call on the specified endpoint and parameters.

 :
 : @param $endpoint the name of the endpoint (without the .jq extension).
 : @param $parameters the parameters, passed as an object.
 :
 : @return the URL.
 :)
declare function backend:url($endpoint as string, $parameters as object) as string
{
    test:url($endpoint, $parameters, false)
};

(:~
 :
Builds the URL for a backend call on the specified endpoint and parameters.

 :
 : @param $endpoint the name of the endpoint (without the .jq extension).
 : @param $parameters the parameters, passed as an object.
 : @param whether to show the token.
 :
 : @return the URL.
 :)
declare function backend:url($endpoint as string, $parameters as object, $includeToken as boolean) as string
{
    "http://" || request:server-name() || ":" || request:server-port() ||
    "/v1/_queries/public/api/"||$endpoint||".jq?_method=POST&token=" || (if($includeToken) then $config:test-token else "{{token}}") || "&"||
    string-join(
        for $key in keys($parameters)
        for $value as string in (flatten($parameters.$key) ! string($$))
        return ($key||"="||encode-for-uri($value)),
        "&")
};


:green_heart: Fix report element tests.

:green_heart: Fix report element tests.

import module namespace test = "http://apps.28.io/test";

test:check-all-success({
    taxonomy-label: test:invoke-and-assert-deep-equal(
      "report-elements",
      {aid:"STANDARD-TAXONOMY-2014", label:"Accounting policy for deferred assets"},
      function($b as item*) as item* { $b.ReportElements },
      test:get-expected-result("edinet/report-elements-expected1.jq"),
      { NoArrayOrder: true }
    ),
    taxonomy-name: test:invoke-and-assert-deep-equal(
      "report-elements",
      {aid:"STANDARD-TAXONOMY-2014", name:"jppfs-cor:CurrentAssets"},
      function($b as item*) as item* { $b.ReportElements },
      test:get-expected-result("edinet/report-elements-expected2.jq"),
      { NoArrayOrder: true }
    ),
    company-label: test:invoke-and-assert-deep-equal(
      "report-elements",
      {edinetcode:"E01225", label:"Loss of inactive facilities"},
      function($b as item*) as item* { $b.ReportElements },
      test:get-expected-result("edinet/report-elements-expected3.jq"),
      { NoArrayOrder: true }
    ),
    company-name: test:invoke-and-assert-deep-equal(
      "report-elements",
      {edinetcode:"E01225", name:"jpcrp-cor:NotesConsolidatedStatementOfCashFlowsHeading"},
      function($b as item*) as item* { $b.ReportElements },
      test:get-expected-result("edinet/report-elements-expected4.jq"),
      { NoArrayOrder: true }
    ),
    company-name-xml : test:invoke-and-assert-deep-equal(
      "report-elements",
      {edinetcode:"E01225", name:"jpcrp-cor:NotesConsolidatedStatementOfCashFlowsHeading", format: "xml"},
      function($b as item*) as item* { document { $b/ReportElements } },
      test:get-expected-result-xml("edinet/report-elements-expected4-xml.jq"),
      { Format : "xml" }
    ),
    abstract-true : test:invoke-and-assert-deep-equal(
      "report-elements",
      {aid:"STANDARD-TAXONOMY-2014", abstract: "true", onlyNames: "true" },
      function($b as item*) as item* { count($b.ReportElements[]) },
      1029,
      ()
    ),
    abstract-false : test:invoke-and-assert-deep-equal(
      "report-elements",
      {aid:"STANDARD-TAXONOMY-2014", abstract: "false", onlyNames: "true" },
      function($b as item*) as item* { count($b.ReportElements[]) },
      4771,
      ()
    )
})

import module namespace test = "http://apps.28.io/test";

test:check-all-success({
    taxonomy-label: test:invoke-and-assert-deep-equal(
      "report-elements",
      {aid:"STANDARD-TAXONOMY-2014", label:"Accounting policy for deferred assets"},
      function($b as item*) as item* { $b.ReportElements },
      test:get-expected-result("edinet/report-elements-expected1.jq"),
      { NoArrayOrder: true }
    ),
    taxonomy-name: test:invoke-and-assert-deep-equal(
      "report-elements",
      {aid:"STANDARD-TAXONOMY-2014", name:"jppfs-cor:CurrentAssets"},
      function($b as item*) as item* { $b.ReportElements },
      test:get-expected-result("edinet/report-elements-expected2.jq"),
      { NoArrayOrder: true }
    ),
    company-label: test:invoke-and-assert-deep-equal(
      "report-elements",
      {edinetcode:"E01225", label:"Loss of inactive facilities"},
      function($b as item*) as item* { $b.ReportElements },
      test:get-expected-result("edinet/report-elements-expected3.jq"),
      { NoArrayOrder: true }
    ),
    company-name: test:invoke-and-assert-deep-equal(
      "report-elements",
      {edinetcode:"E01225", name:"jpcrp-cor:NotesConsolidatedStatementOfCashFlowsHeading"},
      function($b as item*) as item* { $b.ReportElements },
      test:get-expected-result("edinet/report-elements-expected4.jq"),
      { NoArrayOrder: true }
    ),
    company-name-xml : test:invoke-and-assert-deep-equal(
      "report-elements",
      {edinetcode:"E01225", name:"jpcrp-cor:NotesConsolidatedStatementOfCashFlowsHeading", format: "xml"},
      function($b as item*) as item* { document { $b/ReportElements } },
      test:get-expected-result-xml("edinet/report-elements-expected4-xml.jq"),
      { Format : "xml" }
    ),
    abstract-true : test:invoke-and-assert-deep-equal(
      "report-elements",
      {aid:"STANDARD-TAXONOMY-2014", abstract: "true", onlyNames: "true" },
      function($b as item*) as item* { count($b.ReportElements[]) },
      1027,
      ()
    ),
    abstract-false : test:invoke-and-assert-deep-equal(
      "report-elements",
      {aid:"STANDARD-TAXONOMY-2014", abstract: "false", onlyNames: "true" },
      function($b as item*) as item* { count($b.ReportElements[]) },
      4771,
      ()
    )
})


:bug: allow elastic search usage without authentication.

jsoniq version "1.0";
(:
 : Copyright 2012 28msec Inc.
 :)

(:~
 :
This module provides a driver to access
 : ElasticSearch.

 :
 : @author Dennis Knochenwefel
 :)
module namespace es = "http://28.io/modules/elasticsearch";

import module namespace credentials =
    "http://www.28msec.com/modules/credentials";
import module namespace http = "http://zorba.io/modules/http-client";

declare namespace ver = "http://zorba.io/options/versioning";

declare option ver:module-version "1.0";

declare %private variable $es:CREDENTIALS-CATEGORY as xs:string :=
    "ElasticSearch";
declare %private variable $es:RETRY-OPTIONS :=
  {
    retry :
    {
      "delay" : [1000, 2000, 4000],
      "on-connection-error" : true(),
      "on-statuses" : [ 500, 503 ]
    }
  };

declare function es:connection($credentials-name as xs:string?)
as object?
{
  let $credentials as object? := credentials:credentials(
      $es:CREDENTIALS-CATEGORY,
      $credentials-name)
  where exists($credentials)
  return
      let $urls as string+ :=
        for $url in $credentials.urls[]
        return $url || ("/"[not ends-with($url, "/")])
      let $user as string? := $credentials.user
      let $password as string? := $credentials.pass
      let $authentication as string? :=
        (
          encode-for-uri($user) || ":" || encode-for-uri($password) || "@"
        )[exists($user) and exists($password)]
      return
        {
          "urls": [
            for $url in $urls
            return "https://" || $authentication || $url
          ]
        }
};

declare %private function es:url(
  $connection as object,
  $paths as string*
) as string
{
  es:url($connection, $paths, ())
};

declare %private function es:url(
  $connection as object,
  $paths as string*,
  $parameters as string*
) as string
{
  let $server as string := $connection.urls[][2]
  let $parameters as string* :=
    for $parameter in $parameters
    return encode-for-uri($parameter)
  let $path as string := string-join($paths, "/")
  let $query as string := string-join($parameters, "&")
  return
    $server || string-join(($path, $query), "?")
};

declare %private function es:request(
  $method as string,
  $url as string,
  $body as item?
) as object
{
  {|
    {
      "method": $method,
      "href": $url,
      "options": $es:RETRY-OPTIONS
    },
    switch(true)

    case empty($body) return ()

    case $body instance of string
    return
      {
        "body":
        {
          "media-type": "text/plain",
          "content": $body
        }
      }

    case $body instance of object
    case $body instance of array
    return
      {
        "body":
        {
          "media-type": "application/json",
          "content": serialize($body)
        }
      }

    default return
      error(xs:QName("es:UNKNOWN_BODY_TYPE"),
            "cannot create request from body content",
            $body)
  |}
};

declare %an:nondeterministic function es:exists(
  $connection as object,
  $index as string
) as boolean
{
  es:exists($connection, $index, (), ())
};

declare %an:nondeterministic function es:exists(
  $connection as object,
  $index as string,
  $type as string?
) as boolean
{
  es:exists($connection, $index, $type, ())
};

declare %an:nondeterministic function es:exists(
  $connection as object,
  $index as string,
  $type as string?,
  $id as string?
) as boolean
{
  let $url as string := es:url($connection, ($index, $type, $id))
  let $request as object := es:request("HEAD", $url, ())
  let $response as object := http:send-nondeterministic-request($request)
  return
    switch ($response.status)
    case 200
    case 204 return true
    case 404 return false
    default return error(xs:QName("es:SERVER_ERROR"),
                         $response.body.content,
                         $response)
};

declare %an:sequential function es:create(
  $connection as object,
  $index as string
) as object
{
  es:create($connection, $index, ())
};

declare %an:sequential function es:create(
  $connection as object,
  $index as string,
  $mapping as object?
) as object
{
  let $method as string :=
    if(empty($mapping))
    then "POST"
    else "PUT"
  let $url as string := es:url($connection, ($index))
  let $request as object := es:request($method, $url, $mapping)
  let $response as object := http:send-request($request)
  return
    if($response.status eq 200)
    then parse-json($response.body.content)
    else error(xs:QName("es:SERVER_ERROR"), $response.body.content, $response)
};

declare %an:sequential function es:create(
  $connection as object,
  $index as string,
  $type as string,
  $mapping as object
) as object?
{
  let $url as string := es:url($connection, ($index, "_mapping", $type))
  let $request as object := es:request("PUT", $url, $mapping)
  let $response as object := http:send-request($request)
  return
    if($response.status eq 200)
    then parse-json($response.body.content)
    else error(xs:QName("es:SERVER_ERROR"), $response.body.content, $response)
};

declare %an:sequential function es:delete(
  $connection as object,
  $index as string
) as object?
{
  es:delete($connection, $index, (),())
};

declare %an:sequential function es:delete(
  $connection as object,
  $index as string,
  $type as string?
) as object?
{
  es:delete($connection, $index, $type, ())
};

declare %an:sequential function es:delete(
  $connection as object,
  $index as string,
  $type as string?,
  $id as string?
) as object?
{
  let $url as string := es:url($connection, ($index, $type, $id))
  let $request as object := es:request("DELETE", $url, ())
  let $response as object := http:send-request($request)
  return
    if($response.status eq 200)
    then parse-json($response.body.content)
    else error(xs:QName("es:SERVER_ERROR"), $response.body.content, $response)
};

declare %an:nondeterministic function es:get(
  $connection as object,
  $index as string,
  $type as string,
  $id as string
) as object?
{
  let $url as string := es:url($connection, ($index, $type, $id))
  let $response as object := http:get($url)
  return
    switch ($response.status)
    case 200 return parse-json($response.body.content)
    case 404 return ()
    default return error(xs:QName("es:CONNECTION_ERROR"),
                         $response.body.content,
                         $response)
};

declare %an:sequential function es:insert(
  $connection as object,
  $index as string,
  $type as string,
  $doc as object
) as object?
{
  let $id as string? := $doc._id
  return es:insert($connection, $index, $type, $id, $doc)
};

declare %an:sequential function es:insert(
  $connection as object,
  $index as string,
  $type as string,
  $id as string?,
  $doc as object
) as object?
{
  let $method as string := if(empty($id)) then "POST" else "PUT"
  let $url as string := es:url($connection, ($index, $type, $id))
  let $request as object := es:request($method, $url, $doc)
  let $response as object := http:send-request($request)
  return
    switch ($response.status)
    case 200
    case 201 return parse-json($response.body.content)
    default return error(xs:QName("es:CONNECTION_ERROR"),
                         $response.body.content,
                         $response)
};

declare %an:nondeterministic function es:list(
  $connection as object,
  $index as string,
  $type as string
) as object?
{
  let $url as string := es:url($connection, ($index, $type, "_search"))
  let $request as object := es:request("GET", $url, ())
  let $response as object := http:send-nondeterministic-request($request)
  return
    if($response.status eq 200)
    then parse-json($response.body.content)
    else error(xs:QName("es:CONNECTION_ERROR"),
                        $response.body.content,
                        $response)
};

declare %an:nondeterministic function es:search(
  $connection as object,
  $index as string,
  $type as string,
  $query as object
) as object?
{
  let $url as string := es:url($connection, ($index, $type, "_search"))
  let $request as object := es:request("GET", $url, $query)
  let $response as object := http:send-nondeterministic-request($request)
  return
    if($response.status eq 200)
    then parse-json($response.body.content)
    else error(xs:QName("es:CONNECTION_ERROR"),
                        $response.body.content,
                        $response)
};

jsoniq version "1.0";
(:
 : Copyright 2012 28msec Inc.
 :)

(:~
 :
This module provides a driver to access
 : ElasticSearch.

 :
 : @author Dennis Knochenwefel
 :)
module namespace es = "http://28.io/modules/elasticsearch";

import module namespace credentials =
    "http://www.28msec.com/modules/credentials";
import module namespace http = "http://zorba.io/modules/http-client";

declare namespace ver = "http://zorba.io/options/versioning";

declare option ver:module-version "1.0";

declare %private variable $es:CREDENTIALS-CATEGORY as xs:string :=
    "ElasticSearch";
declare %private variable $es:RETRY-OPTIONS :=
  {
    retry :
    {
      "delay" : [1000, 2000, 4000],
      "on-connection-error" : true(),
      "on-statuses" : [ 500, 503 ]
    }
  };

declare function es:connection($credentials-name as xs:string?)
as object?
{
  let $credentials as object? := credentials:credentials(
      $es:CREDENTIALS-CATEGORY,
      $credentials-name)
  where exists($credentials)
  return
      let $urls as string+ :=
        for $url in $credentials.urls[]
        return $url || ("/"[not ends-with($url, "/")])
      let $user as string := $credentials.user
      let $password as string := $credentials.pass
      return
        {
          "urls": [
            for $url in $urls
            return
              "https://" || encode-for-uri($user) || ":" ||
                            encode-for-uri($password) ||"@" || $url
          ]
        }
};

declare %private function es:url(
  $connection as object,
  $paths as string*
) as string
{
  es:url($connection, $paths, ())
};

declare %private function es:url(
  $connection as object,
  $paths as string*,
  $parameters as string*
) as string
{
  let $server as string := $connection.urls[][2]
  let $parameters as string* :=
    for $parameter in $parameters
    return encode-for-uri($parameter)
  let $path as string := string-join($paths, "/")
  let $query as string := string-join($parameters, "&")
  return
    $server || string-join(($path, $query), "?")
};

declare %private function es:request(
  $method as string,
  $url as string,
  $body as item?
) as object
{
  {|
    {
      "method": $method,
      "href": $url,
      "options": $es:RETRY-OPTIONS
    },
    switch(true)

    case empty($body) return ()

    case $body instance of string
    return
      {
        "body":
        {
          "media-type": "text/plain",
          "content": $body
        }
      }

    case $body instance of object
    case $body instance of array
    return
      {
        "body":
        {
          "media-type": "application/json",
          "content": serialize($body)
        }
      }

    default return
      error(xs:QName("es:UNKNOWN_BODY_TYPE"),
            "cannot create request from body content",
            $body)
  |}
};

declare %an:nondeterministic function es:exists(
  $connection as object,
  $index as string
) as boolean
{
  es:exists($connection, $index, (), ())
};

declare %an:nondeterministic function es:exists(
  $connection as object,
  $index as string,
  $type as string?
) as boolean
{
  es:exists($connection, $index, $type, ())
};

declare %an:nondeterministic function es:exists(
  $connection as object,
  $index as string,
  $type as string?,
  $id as string?
) as boolean
{
  let $url as string := es:url($connection, ($index, $type, $id))
  let $request as object := es:request("HEAD", $url, ())
  let $response as object := http:send-nondeterministic-request($request)
  return
    switch ($response.status)
    case 200
    case 204 return true
    case 404 return false
    default return error(xs:QName("es:SERVER_ERROR"),
                         $response.body.content,
                         $response)
};

declare %an:sequential function es:create(
  $connection as object,
  $index as string
) as object
{
  es:create($connection, $index, ())
};

declare %an:sequential function es:create(
  $connection as object,
  $index as string,
  $mapping as object?
) as object
{
  let $method as string :=
    if(empty($mapping))
    then "POST"
    else "PUT"
  let $url as string := es:url($connection, ($index))
  let $request as object := es:request($method, $url, $mapping)
  let $response as object := http:send-request($request)
  return
    if($response.status eq 200)
    then parse-json($response.body.content)
    else error(xs:QName("es:SERVER_ERROR"), $response.body.content, $response)
};

declare %an:sequential function es:create(
  $connection as object,
  $index as string,
  $type as string,
  $mapping as object
) as object?
{
  let $url as string := es:url($connection, ($index, "_mapping", $type))
  let $request as object := es:request("PUT", $url, $mapping)
  let $response as object := http:send-request($request)
  return
    if($response.status eq 200)
    then parse-json($response.body.content)
    else error(xs:QName("es:SERVER_ERROR"), $response.body.content, $response)
};

declare %an:sequential function es:delete(
  $connection as object,
  $index as string
) as object?
{
  es:delete($connection, $index, (),())
};

declare %an:sequential function es:delete(
  $connection as object,
  $index as string,
  $type as string?
) as object?
{
  es:delete($connection, $index, $type, ())
};

declare %an:sequential function es:delete(
  $connection as object,
  $index as string,
  $type as string?,
  $id as string?
) as object?
{
  let $url as string := es:url($connection, ($index, $type, $id))
  let $request as object := es:request("DELETE", $url, ())
  let $response as object := http:send-request($request)
  return
    if($response.status eq 200)
    then parse-json($response.body.content)
    else error(xs:QName("es:SERVER_ERROR"), $response.body.content, $response)
};

declare %an:nondeterministic function es:get(
  $connection as object,
  $index as string,
  $type as string,
  $id as string
) as object?
{
  let $url as string := es:url($connection, ($index, $type, $id))
  let $response as object := http:get($url)
  return
    switch ($response.status)
    case 200 return parse-json($response.body.content)
    case 404 return ()
    default return error(xs:QName("es:CONNECTION_ERROR"),
                         $response.body.content,
                         $response)
};

declare %an:sequential function es:insert(
  $connection as object,
  $index as string,
  $type as string,
  $doc as object
) as object?
{
  let $id as string? := $doc._id
  return es:insert($connection, $index, $type, $id, $doc)
};

declare %an:sequential function es:insert(
  $connection as object,
  $index as string,
  $type as string,
  $id as string?,
  $doc as object
) as object?
{
  let $method as string := if(empty($id)) then "POST" else "PUT"
  let $url as string := es:url($connection, ($index, $type, $id))
  let $request as object := es:request($method, $url, $doc)
  let $response as object := http:send-request($request)
  return
    switch ($response.status)
    case 200
    case 201 return parse-json($response.body.content)
    default return error(xs:QName("es:CONNECTION_ERROR"),
                         $response.body.content,
                         $response)
};

declare %an:nondeterministic function es:list(
  $connection as object,
  $index as string,
  $type as string
) as object?
{
  let $url as string := es:url($connection, ($index, $type, "_search"))
  let $request as object := es:request("GET", $url, ())
  let $response as object := http:send-nondeterministic-request($request)
  return
    if($response.status eq 200)
    then parse-json($response.body.content)
    else error(xs:QName("es:CONNECTION_ERROR"),
                        $response.body.content,
                        $response)
};

declare %an:nondeterministic function es:search(
  $connection as object,
  $index as string,
  $type as string,
  $query as object
) as object?
{
  let $url as string := es:url($connection, ($index, $type, "_search"))
  let $request as object := es:request("GET", $url, $query)
  let $response as object := http:send-nondeterministic-request($request)
  return
    if($response.status eq 200)
    then parse-json($response.body.content)
    else error(xs:QName("es:CONNECTION_ERROR"),
                        $response.body.content,
                        $response)
};