:racehorse: Made report element queries faster.

import module namespace api = "http://apps.28.io/api";

import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/backend";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace config = "http://apps.28.io/config";

import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

declare option rest:response "first-item";

declare function local:concepts-for-archives-and-labels($aids as string*, $labels as string) as object*
{
   mw:run-cmd-deterministic(
      {
        "text" : "concepts",
        "filter" : { "Archive" : { "$in" : [ $aids ] } },
        "search" : $labels,
        "limit" : 100,
        "score" : { "$meta" : "textScore" },
        "sort" : { score: { "$meta" : "textScore" } }
      }).results[].obj
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token          as string?  external;
declare  %rest:case-insensitive                 variable $profile-name   as string  external := $config:profile-name;
declare  %rest:env                              variable $request-uri    as string   external;
declare  %rest:case-insensitive                 variable $format         as string?  external;
declare  %rest:case-insensitive %rest:distinct  variable $cik            as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode     as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag            as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker         as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $sic            as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear     as string*  external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod   as string*  external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $filingKind     as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $aid            as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure     as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $section        as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $role           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $eid            as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $label          as string*  external;
declare  %rest:case-insensitive                 variable $report         as string?  external;
declare  %rest:case-insensitive %rest:distinct  variable $name           as string*  external;
declare  %rest:case-insensitive                 variable $onlyNames      as boolean? external := false;
declare  %rest:case-insensitive                 variable $onlyTextBlocks as boolean? external := ();
declare  %rest:case-insensitive                 variable $abstract       as boolean? external := ();
declare  %rest:case-insensitive                 variable $kind           as string*  external := ();
declare  %rest:case-insensitive                 variable $count          as boolean external := false;

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $kind as string* := api:preprocess-concept-kind($kind)
let $section := distinct-values(($section, $role))

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic,
  (),
  (),
  (),
  ())

let $entities-not-found as boolean :=
  exists(($eid, $cik, $tag, $ticker, $sic)) and empty($entities)

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $filingKind,
  $aid)

let $archives-not-found as boolean :=
  exists(($entities, $fiscalPeriod, $fiscalYear, $filingKind, $aid)) and empty($archives)

let $entities as object* :=
    ($entities[entities:eid($$) = $archives.Entity],
    let $not-found := $archives.Entity[not entities:eid($entities) = $$]
    where exists($not-found)
    return entities:entities($not-found))

let $report as object? := reports:reports($report)

let $concepts as object* :=
    multiplexer:concepts(
      $profile-name,
      $archives,
      $name,
      $disclosure,
      $section,
      $label[$profile-name ne "sec"],
      $label[$profile-name eq "sec"],
      $report,
      $onlyNames)[
    (empty($onlyTextBlocks) or $$.IsTextBlock eq $onlyTextBlocks) and
    (empty($abstract) or $$.IsAbstract eq $abstract) and
    (empty($kind) or $$.Kind = $kind)
  ]

let $result :=
  let $all-aids as string* := distinct-values($concepts.Archive)
  let $roles as string* := distinct-values($concepts.Role)
  let $components := components:components-for-archives-and-roles($all-aids, $roles)
  return {
    ReportElements : [
      switch(true)
      case $count return ()
      case $onlyNames return distinct-values($concepts.Name)
      default return
        for $concept in $concepts
        group by $archive := $concept.Archive,  $role := $concept.Role
        let $component as object := $components[$$.Archive eq $archive and $$.Role eq $role]
        return
          if($profile-name eq "sec")
          then
                let $archive-object as object := $archives[$$._id eq $archive]
                let $entity as object := $entities[entities:eid($$) = $archive-object.Entity]
                let $metadata := {
                    Archive : $archive,
                    Section : $component.Label,
                    SectionURI : $component.Role,
                    CIK : entities:eid($entity),
                    EntityRegistrantName : $entity.Profiles.SEC.CompanyName,
                    FiscalYear : $archive-object.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                    FiscalPeriod : $archive-object.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
                    (: Deprecated :)
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label,
                    AccessionNumber : $archive
                }
                for $concept in $concept
                let $original-name := ($concept.Origin, $concept.Name)[1]
                return {|
                    project($concept, ("Name", "Origin")),
                    {
                      Labels: backend:url("labels", {|
                        {
                          token: $token,
                          concept: $original-name,
                          aid: $archive,
                          section: $role,
                          format: $format,
                          profile-name: $profile-name
                        }
                      |})
                    },
                    trim($concept, ("Origin", "_id", "Archive", "Role", "Name", "Labels")),
                    $metadata
                |}
         else
                let $metadata := {
                    Archive : $archive,
                    Section : $component.Label,
                    SectionURI : $component.Role,
                    (: Deprecated :)
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label
                }
                for $concept in $concept
                let $original-name := ($concept.Origin, $concept.Name)[1]
                return {|
                    project($concept, ("Name", "Origin")),
                    {
                      Labels: backend:url("labels",
                        {
                          token: $token,
                          concept: $original-name,
                          aid: $archive,
                          section: $role,
                          format: $format,
                          profile-name: $profile-name
                        }
                      )
                    },
                    {
                      Facts:
                        if($concept.Kind eq "Concept")
                        then backend:url("facts", {|
                        {
                          token: $token,
                          "xbrl:Concept": $original-name,
                          aid: $archive,
                          format: $format,
                          profile-name: $profile-name
                        },
                        {
                          fiscalYear: "ALL",
                          fiscalPeriod: "ALL",
                          fiscalPeriodType: "ALL"
                        }[$profile-name eq "japan"]
                      |})
                      else "None"
                    },
                    trim($concept, ("_id", "Archive", "Role", "Name", "Labels", "Label", "IsAbstract", "SubstitutionGroup")),
                    $metadata
                |}
        ]
    }
let $comment := {
    NumConcepts: count($result.ReportElements[]),
    TotalNumConcepts: session:num-concepts()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <ReportElements>{
          if ($onlyNames)
          then
            <ReportElement>{
              for $c in $res.ReportElements[]
              return <Name>{$c}</Name>
            }</ReportElement>
          else api:json-to-xml($res.ReportElements[], "ReportElement")
        }</ReportElements>
    },
    to-csv : function($res as object) as string {
      if ($onlyNames)
      then
          string-join(("Name", $res.ReportElements[]), "
")
      else
          api:json-to-csv($res.ReportElements[])
    }
}
let $results := api:serialize($result, $comment, $serializers, $format, "report-elements")
return switch(true)
       case $entities-not-found return api:not-found("entity")
       case $archives-not-found return api:not-found("archive")
       default return api:check-and-return-results($token, $results, $format)

import module namespace api = "http://apps.28.io/api";

import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/backend";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace config = "http://apps.28.io/config";

import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

declare option rest:response "first-item";

declare function local:concepts-for-archives-and-labels($aids as string*, $labels as string) as object*
{
   mw:run-cmd-deterministic(
      {
        "text" : "concepts",
        "filter" : { "Archive" : { "$in" : [ $aids ] } },
        "search" : $labels,
        "limit" : 100,
        "score" : { "$meta" : "textScore" },
        "sort" : { score: { "$meta" : "textScore" } }
      }).results[].obj
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token          as string?  external;
declare  %rest:case-insensitive                 variable $profile-name   as string  external := $config:profile-name;
declare  %rest:env                              variable $request-uri    as string   external;
declare  %rest:case-insensitive                 variable $format         as string?  external;
declare  %rest:case-insensitive %rest:distinct  variable $cik            as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode     as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag            as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker         as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $sic            as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear     as string*  external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod   as string*  external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $filingKind     as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $aid            as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure     as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $section        as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $role           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $eid            as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $label          as string*  external;
declare  %rest:case-insensitive                 variable $report         as string?  external;
declare  %rest:case-insensitive %rest:distinct  variable $name           as string*  external;
declare  %rest:case-insensitive                 variable $onlyNames      as boolean? external := false;
declare  %rest:case-insensitive                 variable $onlyTextBlocks as boolean? external := ();
declare  %rest:case-insensitive                 variable $abstract       as boolean? external := ();
declare  %rest:case-insensitive                 variable $kind           as string*  external := ();
declare  %rest:case-insensitive                 variable $count          as boolean external := false;

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $kind as string* := api:preprocess-concept-kind($kind)
let $section := distinct-values(($section, $role))

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic,
  (),
  (),
  (),
  ())

let $entities-not-found as boolean :=
  exists(($eid, $cik, $tag, $ticker, $sic)) and empty($entities)

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $filingKind,
  $aid)

let $archives-not-found as boolean :=
  exists(($entities, $fiscalPeriod, $fiscalYear, $filingKind, $aid)) and empty($archives)

let $entities as object* :=
    ($entities[entities:eid($$) = $archives.Entity],
    let $not-found := $archives.Entity[not entities:eid($entities) = $$]
    where exists($not-found)
    return entities:entities($not-found))

let $report as object? := reports:reports($report)

let $concepts as object* :=
    multiplexer:concepts(
      $profile-name,
      $archives,
      $name,
      $disclosure,
      $section,
      $label[$profile-name ne "sec"],
      $label[$profile-name eq "sec"],
      $report,
      $onlyNames)[
    (empty($onlyTextBlocks) or $$.IsTextBlock eq $onlyTextBlocks) and
    (empty($abstract) or $$.IsAbstract eq $abstract) and
    (empty($kind) or $$.Kind = $kind)
  ]

let $result :=
  let $all-aids := $concepts.Archive
  let $roles := $concepts.Role
  let $components := components:components-for-archives-and-roles($all-aids, $roles)
  return {
    ReportElements : [
      switch(true)
      case $count return ()
      case $onlyNames return distinct-values($concepts.Name)
      default return
        for $concept in $concepts
        group by $archive := $concept.Archive,  $role := $concept.Role
        let $component as object := $components[$$.Archive eq $archive and $$.Role eq $role]
        return
          if($profile-name eq "sec")
          then
                let $archive-object as object := $archives[$$._id eq $archive]
                let $entity as object := $entities[entities:eid($$) = $archive-object.Entity]
                let $metadata := {
                    Archive : $archive,
                    Section : $component.Label,
                    SectionURI : $component.Role,
                    CIK : entities:eid($entity),
                    EntityRegistrantName : $entity.Profiles.SEC.CompanyName,
                    FiscalYear : $archive-object.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                    FiscalPeriod : $archive-object.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
                    (: Deprecated :)
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label,
                    AccessionNumber : $archive
                }
                for $concept in $concept
                let $original-name := ($concept.Origin, $concept.Name)[1]
                return {|
                    project($concept, ("Name", "Origin")),
                    {
                      Labels: backend:url("labels", {|
                        {
                          token: $token,
                          concept: $original-name,
                          aid: $archive,
                          section: $role,
                          format: $format,
                          profile-name: $profile-name
                        }
                      |})
                    },
                    trim($concept, ("Origin", "_id", "Archive", "Role", "Name", "Labels")),
                    $metadata
                |}
         else
                let $metadata := {
                    Archive : $archive,
                    Section : $component.Label,
                    SectionURI : $component.Role,
                    (: Deprecated :)
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label
                }
                for $concept in $concept
                let $original-name := ($concept.Origin, $concept.Name)[1]
                return {|
                    project($concept, ("Name", "Origin")),
                    {
                      Labels: backend:url("labels",
                        {
                          token: $token,
                          concept: $original-name,
                          aid: $archive,
                          section: $role,
                          format: $format,
                          profile-name: $profile-name
                        }
                      )
                    },
                    {
                      Facts:
                        if($concept.Kind eq "Concept")
                        then backend:url("facts", {|
                        {
                          token: $token,
                          "xbrl:Concept": $original-name,
                          aid: $archive,
                          format: $format,
                          profile-name: $profile-name
                        },
                        {
                          fiscalYear: "ALL",
                          fiscalPeriod: "ALL",
                          fiscalPeriodType: "ALL"
                        }[$profile-name eq "japan"]
                      |})
                      else "None"
                    },
                    trim($concept, ("_id", "Archive", "Role", "Name", "Labels", "Label", "IsAbstract", "SubstitutionGroup")),
                    $metadata
                |}
        ]
    }
let $comment := {
    NumConcepts: count($result.ReportElements[]),
    TotalNumConcepts: session:num-concepts()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <ReportElements>{
          if ($onlyNames)
          then
            <ReportElement>{
              for $c in $res.ReportElements[]
              return <Name>{$c}</Name>
            }</ReportElement>
          else api:json-to-xml($res.ReportElements[], "ReportElement")
        }</ReportElements>
    },
    to-csv : function($res as object) as string {
      if ($onlyNames)
      then
          string-join(("Name", $res.ReportElements[]), "
")
      else
          api:json-to-csv($res.ReportElements[])
    }
}
let $results := api:serialize($result, $comment, $serializers, $format, "report-elements")
return switch(true)
       case $entities-not-found return api:not-found("entity")
       case $archives-not-found return api:not-found("archive")
       default return api:check-and-return-results($token, $results, $format)


Update filings.jq

Update filings.jq

import module namespace test = "http://apps.28.io/test";
import module namespace config = "http://apps.28.io/config";
import module namespace mw = "http://28.io/modules/xbrl/data-source-wrapper";

if($config:content[] = "risk-data")
then {
  mw:insert("archives", { _id: "foobar" });
  test:check-all-success({
    remove: test:delete-and-assert-status(
      "delete-filings",
      {aid:"foobar"},
      403,
      ()
    ),
    remove-admin: test:delete-and-assert-status(
      "delete-filings",
      {aid:"foobar"},
      204,
      { IsAdmin: true }
    )
  })
}
else ()

import module namespace test = "http://apps.28.io/test";
import module namespace config = "http://apps.28.io/config";
import module namespace mw = "http://28.io/modules/xbrl/data-source-wrapper";

if($config:content[] = "risk-data")
then {
  mw:insert("archives", { _id: "foobar" });
  test:check-all-success({
    remove: test:delete-and-assert-status(
      "delete-filings",
      {aid:"foobar"},
      401,
      ()
    ),
    remove-admin: test:delete-and-assert-status(
      "delete-filings",
      {aid:"foobar"},
      204,
      { IsAdmin: true }
    )
  })
}
else ()


Update concept-map.jq

Update concept-map.jq

import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace concept-maps = "http://28.io/modules/xbrl/concept-maps";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare option rest:response "first-item";

declare function local:to-xml($c as object) as node()*
{
    (
        session:comment("xml"),
        <ConceptMap name="{$c.ShortName}" linkrole="{$c.LinkRole}">{
            for $o in $c.Trees[]
            return
                <Mapping>{
                     (
                         <Concept>{$o.Name}</Concept>,
                         for $t in $o.To[].Name
                         return
                             <MappedTo>{$t}</MappedTo>
                     )
                }
                </Mapping>
        }</ConceptMap>
    )
};

declare function local:to-csv($c as object) as string
{
    string-join(
        csv:serialize(
            for $o as object in $c.Trees[]
            for $v as object at $y in $o.To[]
            return {
                "Concept" : $o.Name,
                "MappedTo" : $v.Name,
                "TryOrder" : $y
            }
        )
    )
};

(: Query parameters :)
declare  %rest:case-insensitive  variable $token        as string? external;
declare  %rest:env               variable $request-uri  as string  external;
declare  %rest:case-insensitive  variable $format       as string? external;
declare  %rest:case-insensitive  variable $map          as string? external;
declare  %rest:case-insensitive  variable $name         as string? external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)

(: Object resolution :)
let $map := ($map, $name)
let $map := concept-maps:concept-maps($map)
let $comment := {
    NumMaps: count($map)
}
let $serializers := {
    to-xml : local:to-xml#1,
    to-csv : local:to-csv#1
}

return
    if (exists($map))
    then api:serialize($map, $comment, $serializers, $format, "components")
    else {
        { status: 404 },
        session:error("concept map not found", "json")
    }

import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace concept-maps = "http://28.io/modules/xbrl/concept-maps";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare option rest:response "first-item";

declare function local:to-xml($c as object) as node()*
{
    (
        session:comment("xml"),
        <ConceptMap name="{$c.ShortName}" linkrole="{$c.LinkRole}">{
            for $o in $c.Trees[]
            return
                <Mapping>{
                     (
                         <Concept>{$o.Name}</Concept>,
                         for $t in $o.To[].Name
                         return
                             <MappedTo>{$t}</MappedTo>
                     )
                }
                </Mapping>
        }</ConceptMap>
    )
};

declare function local:to-csv($c as object) as string
{
    string-join(
        csv:serialize(
            for $o as object in $c.Trees[]
            for $v as object at $y in $o.To[]
            return {
                "Concept" : $o.Name,
                "MappedTo" : $v.Name,
                "TryOrder" : $y
            }
        )
    )
};

(: Query parameters :)
declare  %rest:case-insensitive  variable $token        as string? external;
declare  %rest:env               variable $request-uri  as string  external;
declare  %rest:case-insensitive  variable $format       as string? external;
declare  %rest:case-insensitive  variable $map          as string? external;
declare  %rest:case-insensitive  variable $name         as string? external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)

(: Object resolution :)
let $map := ($map, $name)
let $map := concept-maps:concept-maps($map)
let $comment := {
    NumMaps: count($map)
}
let $serializers := {
    to-xml : local:to-xml#1,
    to-csv : local:to-csv#1
}

return
    if (exists($map))
    then api:serialize($map, $comment, $serializers, $format, "components")
    else {
        { status: 403 },
        session:error("concept map not found", "json")
    }


Process filetree and meta files in one step.

Instead of converting with `to_entries` and processing the resulting
array, we can just reduce all input files at once. This reduces the
length and groups this singular conceptual step together.

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Select the descendant described by the array of names in `$path`.
def descend($path):
    if ($path | length) > 0
    then
        .contents[]? | select(.name == $path[0]) | descend($path[1:])
    else
        .
    end
;

# Insert a child element at a particular path.
# Like `setpath/2`, but instead of making objects like `{"a":{"b":{…}}}`, this
# makes objects like `{"contents":[{"name":"a", "contents":[{"name":"b",…}]}]}`
# I don't really like this function, because it's so verbose. Making a path
# expression for updating with the |= operator (see `descend/1`) doesn't seem
# to work when it is recursive?
def insert($path; $child):
    if ($path | length) > 0
    then
        if ([.contents[]? | select(.name == $path[0])] | length) > 0
        then
            (.contents[] | select(.name == $path[0])) |= (. | insert($path[1:]; $child))
        else
            (.contents = [.contents[]?] + [{"name":$path[0]} | insert($path[1:];$child)])
        end
    else
        . * $child
    end
;

# Merge together `filetree.json` and `*.meta.json` files, with behaviour
# depending on the filename of the object. Use with the `--null-input` switch.
def process_files:
    reduce inputs as $input
        (   {}
        ;   ($input | input_filename) as $f |
            if ($f | endswith(".meta.json")) then
                insert
                ( $f | ltrimstr($prefix) | rtrimstr(".meta.json") | split("/")
                ; {"meta": $input} )
            else
                reduce $input[] as $entry
                ( .
                ;   insert
                    ( $entry.path | split("/")
                    ; $entry )
                )
            end * .
        )
;

# Adds links to each page object. The link should be the same as the path,
# except in the case of directories, which may either link to its index.md or
# have no link at all. If there is only one child to a page, perhaps it should
# collapse onto its parent.
# Also not great.
def add_links:
    if (.path and (.path | endswith(".md")))
    then
        .link = (.path | rtrimstr(".md") | . + ".html")
    else
        [.contents[]? | add_links] as $newcontents
        |   [ $newcontents[] | select(.name=="index.md") ] as $indexfiles
        |   [ $newcontents[] | select(.name!="index.md") ] as $otherfiles
        |   if ($newcontents | length) == 0 then
                .
            elif ($indexfiles | length) == 0 then
                .contents = $newcontents
            else
                $indexfiles[0] as $index
                | .contents = $otherfiles + $indexfiles[1:]
                | .link = $index.link
            end
    end
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    process_files
    | add_links
;

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Select the descendant described by the array of names in `$path`.
def descend($path):
    if ($path | length) > 0
    then
        .contents[]? | select(.name == $path[0]) | descend($path[1:])
    else
        .
    end
;

# Insert a child element at a particular path.
# Like `setpath/2`, but instead of making objects like `{"a":{"b":{…}}}`, this
# makes objects like `{"contents":[{"name":"a", "contents":[{"name":"b",…}]}]}`
# I don't really like this function, because it's so verbose. Making a path
# expression for updating with the |= operator (see `descend/1`) doesn't seem
# to work when it is recursive?
def insert($path; $child):
    if ($path | length) > 0
    then
        if ([.contents[]? | select(.name == $path[0])] | length) > 0
        then
            (.contents[] | select(.name == $path[0])) |= (. | insert($path[1:]; $child))
        else
            (.contents = [.contents[]?] + [{"name":$path[0]} | insert($path[1:];$child)])
        end
    else
        . * $child
    end
;

# Merge together `filetree.json` and `*.meta.json` files, as given in the
# format produced by `file_entries`.
def combine_files:
    reduce (to_entries | .[]) as $entry
        (   {}
        ;   . * (
            if ($entry.key | endswith(".meta.json"))
            then
                ( $entry.key
                    | ltrimstr($prefix)
                    | rtrimstr(".meta.json")
                    | split("/")
                ) as $path
                | insert($path; {"meta": $entry.value})
            else
                reduce $entry.value[] as $f
                ( .
                ; insert($f.path | split("/"); $f)
                )
            end
            )
        )
;

# Make an object out of a stream of files, associating the each filename with
# its contents. Use with the `--null-input` switch.
def file_entries:
    reduce inputs as $input
        ( {}
        ; . + { ($input | input_filename) : $input }
        )
;

# Adds links to each page object. The link should be the same as the path,
# except in the case of directories, which may either link to its index.md or
# have no link at all. If there is only one child to a page, perhaps it should
# collapse onto its parent.
# Also not great.
def add_links:
    if (.path and (.path | endswith(".md")))
    then
        .link = (.path | rtrimstr(".md") | . + ".html")
    else
        [.contents[]? | add_links] as $newcontents
        |   [ $newcontents[] | select(.name=="index.md") ] as $indexfiles
        |   [ $newcontents[] | select(.name!="index.md") ] as $otherfiles
        |   if ($newcontents | length) == 0 then
                .
            elif ($indexfiles | length) == 0 then
                .contents = $newcontents
            else
                $indexfiles[0] as $index
                | .contents = $otherfiles + $indexfiles[1:]
                | .link = $index.link
            end
    end
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    file_entries
    | combine_files
    | add_links
;


Update core.jq

Update core.jq

jsoniq version "1.0";

(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for querying fiscal reports (10-K, 10-Q)
 : submitted to the SEC.

 :
 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace facts = "http://28.io/modules/xbrl/facts";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";
import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Joker for all fiscal periods.
 :)
declare variable $fiscal-core:ALL_FISCAL_PERIODS as string := "ALL";

(:~
 : Joker for all fiscal years.
 :)
declare variable $fiscal-core:ALL_FISCAL_YEARS as integer := 0;

(:~
 : Joker for the latest fiscal years.
 :)
declare variable $fiscal-core:LATEST_FISCAL_YEAR as integer := 1;

(:~
 :
Return a distinct set of companies identified by either
 :   CIKs, tags, tickers, or sics.

 :
 : @param $ciks a set of CIKs.
 : @param $tags a set of tags (ALL retrieves all companies).
 : @param $tickers a set of tickers.
 : @param $sics a set of SIC codes.
 :
 : @return the companies with the given identifiers, tags, tickers, or sic codes.
 :)
declare function fiscal-core:filings(
    $entities-or-eids as item*,
    $fiscal-periods as string*,
    $fiscal-years as integer*,
    $archives-or-aids as item*) as object*
{
    let $entities := entities:entities($entities-or-eids)
    return
        if(exists(index-of($fiscal-years, $fiscal-core:LATEST_FISCAL_YEAR)))
        then
            for $entity in $entities
            return
            fiscal-core:latest-filings($entity, $fiscal-periods)
        else
            let $fiscal-years as integer* :=
                $fiscal-years[$$ ne $fiscal-core:LATEST_FISCAL_YEAR]
            return
                fiscal-core:filings-for-entities-and-fiscal-periods-and-years($entities, $fiscal-periods, $fiscal-years),
    archives:archives($archives-or-aids)
};

declare function fiscal-core:latest-filings(
    $entities as object*,
    $fiscal-periods as string*) as object*
{
    for $entity in $entities
    for $fiscal-period in $fiscal-periods
    let $latest-fiscal-year :=
        fiscal-core:latest-reported-fiscal-period($entity, $fiscal-period).year
    return fiscal-core:filings-for-entities-and-fiscal-periods-and-years(
        $entity,
        $fiscal-period,
        $latest-fiscal-year cast as integer?
    )
};

declare function fiscal-core:filter-override(
    $entities-or-eids as item*,
    $fiscal-years as integer*,
    $fiscal-periods as string*,
    $archives-or-aids as item*
) as object?
{
    let $entities := entities:entities($entities-or-eids)
    let $aids := archives:aid($archives-or-aids)
    let $latest-filings := fiscal-core:latest-filings($entities, $fiscal-periods)
    return
    switch(true)
    case count($aids) gt 0 return {
        "xbrl:Entity" : {
        },
        "sec:Archive" : {
            Type: "string",
            Domain : [ $aids ]
        },
        "sec:FiscalYear" : {
            Type: "integer"
        },
        "sec:FiscalPeriod" : {|
            {
                Type: "string",
                Domain: [ $fiscal-periods ]
            }[exists($fiscal-periods) and empty(index-of($fiscal-periods, $fiscal-core:ALL_FISCAL_PERIODS))]
        |}
    }
    case exists(index-of($fiscal-years, $fiscal-core:LATEST_FISCAL_YEAR))
    return {
        "sec:Archive" : {
            Type: "string",
            Domain : [archives:aid($latest-filings)]
        },
        "sec:FiscalPeriod" : {|
            {
                Type: "string",
                Domain: [ $fiscal-periods ]
            }[exists($fiscal-periods) and empty(index-of($fiscal-periods, $fiscal-core:ALL_FISCAL_PERIODS))]
        |}
    }
    case exists(($entities, $fiscal-years, $fiscal-periods))
    return {
        "xbrl:Entity" : {|
            {
                Type: "string",
                Domain: [ $entities._id ]
            }[exists($entities)]
        |},
        "sec:FiscalYear" :
            let $fiscal-years as integer* :=
                $fiscal-years[$$ ne $fiscal-core:LATEST_FISCAL_YEAR]
            return {|
                {
                    Type: "integer",
                    Domain: [ $fiscal-years ]
                }[exists($fiscal-years) and empty(index-of($fiscal-years, $fiscal-core:ALL_FISCAL_YEARS))]
            |},
        "sec:FiscalPeriod" : {|
            {
                Type: "string",
                Domain: [ $fiscal-periods ]
            }[exists($fiscal-periods) and empty(index-of($fiscal-periods, $fiscal-core:ALL_FISCAL_PERIODS))]
        |},
        "sec:Archive" : {}
    }
    default return ()
};

(:~
 :
Retrieves the fiscal period of a filing.

 :
 : @param $filing-or-id a filing or its id, or the empty sequence.
 :
 : @return the fiscal period (FY, YTD2, YTD3, Q1, Q2 or Q3) if any.
 :)
declare function fiscal-core:fiscal-period-for-filing(
    $filing-or-id as item?) as string?
{
  archives:archives($filing-or-id).Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
};

(:~
 :
Retrieves the fiscal period of a fact.

 :
 : @param $fact a fact object or the empty sequence.
 :
 : @return the fiscal period (FY, YTD2, YTD3, Q1, Q2 or Q3) if any.
 :)
declare function fiscal-core:fiscal-period-for-fact(
    $fact as object?) as string?
{
  $fact.Aspects."sec:FiscalPeriod"
};

(:~
 :
Retrieves the fiscal period of a filing or a fact.

 :
 : @deprecated use fiscal-period-for-fact#1 and fiscal-period-for-filing#1
 :
 : @param $filing-fact-or-id a filing, a fact, or its id.
 :
 : @return the fiscal period (FY, YTD2, YTD3, Q1, Q2 or Q3).
 :)
declare function fiscal-core:fiscal-period(
    $filing-fact-or-id as item?) as string?
{
  if(empty($filing-fact-or-id))
  then ()
  else
    typeswitch ($filing-fact-or-id)
    case object return
      switch (true)
      (: archive :)
      case exists($filing-fact-or-id.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus)
        return $filing-fact-or-id.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
      (: fact :)
      case exists($filing-fact-or-id.Aspects."sec:FiscalPeriod")
        return $filing-fact-or-id.Aspects."sec:FiscalPeriod"
      default return ()
    default return
      let $id as xs:string := string($filing-fact-or-id)
      return
        switch(true)
        (:
          "The first ten digits are the CIK of the entity that logged
           in to submit the filing. The next two digits define the
           current year. The final digits are assigned sequentially by
           CIK starting with one on the first business day of each year."

        http://www.sec.gov/info/edgar/edgarfm-vol2-v25.pdf :)
        case matches($id, "\\d{10}-\\d{2}-\\d{6}")
          return archives:archives($id).Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
        default return facts:facts($id).Aspects."sec:FiscalPeriod"
};

(:~
 :
Retrieves the fiscal year of a filing.

 :
 : @param $filing-or-id a filing or its id, or the empty sequence.
 :
 : @return the fiscal year (e.g., 2012) if any.
 :)
declare function fiscal-core:fiscal-year-for-filing(
    $filing-or-id as item?) as integer?
{
  archives:archives($filing-or-id).Profiles.SEC.Fiscal.DocumentFiscalYearFocus
};

(:~
 :
Retrieves the fiscal year of a fact.

 :
 : @param $fact a fact object or the empty sequence.
 :
 : @return the fiscal year (e.g., 2012) if any.
 :)
declare function fiscal-core:fiscal-year-for-fact(
    $fact as object?) as integer?
{
  $fact.Aspects."sec:FiscalYear"
};

(:~
 :
Retrieves the fiscal year of a filing or a fact.

 :
 : @deprecated use fiscal-core:fiscal-year-for-archive#1 or fiscal-core:fiscal-year-for-fact#1.
 :
 : @param $filing-fact-or-id a filing, a fact, or its id.
 :
 : @return the fiscal year (e.g., 2012).
 :)
declare function fiscal-core:fiscal-year(
    $filing-fact-or-id as item?) as integer?
{
  if(empty($filing-fact-or-id))
  then ()
  else
    typeswitch ($filing-fact-or-id)
    case object return
      switch (true)
      (: archive :)
      case exists($filing-fact-or-id.Profiles.SEC.Fiscal.DocumentFiscalYearFocus)
        return $filing-fact-or-id.Profiles.SEC.Fiscal.DocumentFiscalYearFocus
      (: fact :)
      case exists($filing-fact-or-id.Aspects."sec:FiscalYear")
        return $filing-fact-or-id.Aspects."sec:FiscalYear"
      default return ()
    default return
      let $id as xs:string := string($filing-fact-or-id)
      return
        switch(true)
        (:
          "The first ten digits are the CIK of the entity that logged
           in to submit the filing. The next two digits define the
           current year. The final digits are assigned sequentially by
           CIK starting with one on the first business day of each year."

          http://www.sec.gov/info/edgar/edgarfm-vol2-v25.pdf :)
        case matches($id, "\\d{10}-\\d{2}-\\d{6}")
          return archives:archives($id).Profiles.SEC.Fiscal.DocumentFiscalYearFocus
        default return facts:facts($id).Aspects."sec:FiscalYear"
};

(:~
 :
Retrieves all filings by entities and for given fiscal periods and years.

 :
 : @param $entities-or-ids entities or their ids.
 : @param $fiscal-period-focus fiscal periods (FY, YTD1, YTD2, YTD3, Q1, Q2, Q3, Q4) or $fiscal-core:ALL_FISCAL_PERIODS to accept all..
 : @param $fiscal-year-focus fiscal years or $fiscal-core:ALL_FISCAL_YEARS to accept all..
 :
 : @return all filings that match the criteria.
 :)
declare function
fiscal-core:filings-for-entities-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  let $fiscal-period-focus :=
    switch(true)
    case $fiscal-period-focus = ( "10-K", "FY") return "FY"
    case $fiscal-period-focus eq "10-Q" return ( "Q3", "Q2", "Q1" )
    case $fiscal-period-focus eq "YTD1" return "Q1"
    case $fiscal-period-focus eq "YTD2" return "Q2"
    case $fiscal-period-focus eq "YTD3" return "Q3"
    default return $fiscal-period-focus
  let $conn := fiscal-core:connection()
  for $cik-or-entity in $entities-or-ids
  let $entity-id as xs:string := companies:eid($cik-or-entity)
  let $query := {|
      { "Entity" : $entity-id },
      switch (true)
      case deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS)
        return ()
      case count($fiscal-period-focus) eq 1
        return
        {
          "Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus": $fiscal-period-focus
        }
      default return
        {
          "Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus":
            { "$in" : [ $fiscal-period-focus ] }
        },
      switch (true)
      case deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS)
        return ()
      case count($fiscal-year-focus) eq 1
        return
        {
          "Profiles.SEC.Fiscal.DocumentFiscalYearFocus": $fiscal-year-focus
        }
      default
        return
        {
          "Profiles.SEC.Fiscal.DocumentFiscalYearFocus":
            { "$in" : [ $fiscal-year-focus ] }
        }
    |}
  return mongo:find($conn, $archives:col, $query)
};

(:~
 :
Retrieves the balance sheet for the given entities and fiscal periods and years.

 :
 : @param $entity-or-ids a list of entities or their EIDs.
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 :
 : @error entities:INVALID_PARAMETER if the CIK or entity is not valid.
 :
 : @return the balance sheet at that date and for that formType.
 :)
declare function fiscal-core:balance-sheets-for-fiscal-periods-and-years(
    $entity-or-ids as item*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  for $archive in
    fiscal-core:filings-for-entities-and-fiscal-periods-and-years($entity-or-ids,
                                                                  $fiscal-period-focus,
                                                                  $fiscal-year-focus)
  return sec-networks:networks-for-filings-and-disclosures(
      $archive,
      $sec-networks:BALANCE_SHEET)
};

(:~
 :
Retrieves facts for the given aspects, fiscal periods and years.

 :
 : @param $aspects an object containing aspects to filter, among which
 :                 xbrl:Concept, xbrl:Entity and xbrl:Period (all optional).
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY).
 : @param $fiscal-year-focus the years (2012)
 :
 : @return all facts matching the given aspects, fiscal period and year.
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects as object?,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects,
    $fiscal-period-focus,
    $fiscal-year-focus,
    ())
};

(:~
 :
Retrieves facts for the given aspects, fiscal periods and years.

 :
 : @param $aspects an object containing aspects to filter, among which
 :                 xbrl:Concept, xbrl:Entity and xbrl:Period (all optional).
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 : @param $options standard SEC BizQL options.
 :
 : @return all facts matching the given aspects, fiscal period and year.
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects as object?,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*,
    $options as object?) as object*
{
  let $filter as object := {|
    {
      "Filter":
        {|
          if(exists($aspects))
          then { $facts:ASPECTS : $aspects }
          else (),
          if( deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS) and deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS))
          then { "Profiles.SEC.Fiscal" : { "$exists" : true } }
          else { "Profiles": { "SEC": { "Fiscal":
                 {|
                   if(deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS) )
                   then ()
                   else { "Period" : [ $fiscal-period-focus ] },
                   if( deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS) )
                   then ()
                   else { "Year": [ $fiscal-year-focus ] }
                 |}
               }}}
        |}
    },
    if (empty($options.Hypercube))
    then { Hypercube: sec:dimensionless-hypercube(()) }
    else ()
  |}
  let $facts :=
    facts:facts-for(
      facts:merge-objects($filter, $options, true (: giving parameters higher priority :))
    )
  return sec:hide-amended-facts($facts, $options)
};

(:~
 :
Retrieves facts for the given entities, concepts, fiscal periods and years.

 :
 : @param $entities-or-ids a list of entities or their EIDs.
 : @param $concepts a list of concept names (us-gaap:Assets)
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 :
 : @error entities:INVALID_PARAMETER if the CIK or entity is not valid.
 : @return all facts matching entities, concepts, fiscal periods and years
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $concepts as string*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids,
    $concepts,
    $fiscal-period-focus,
    $fiscal-year-focus,
    ())
};

(:~
 :
Retrieves facts for the given entities, concepts, fiscal periods and years.

 :
 : @param $entities-or-ids a list of entities or their EIDs.
 : @param $concepts a list of concept names (us-gaap:Assets)
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 : @param $options standard SEC BizQL options.
 :
 : @error entities:INVALID_PARAMETER if the CIK or entity is not valid.
 : @return all facts matching entities, concepts, fiscal periods and years
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $concepts as string*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*,
    $options as object?) as object*
{
  let $entity-ids as xs:string* := $entities-or-ids ! companies:eid($$)
  let $aspects := {|
    if(exists($entity-ids))
    then { "xbrl:Entity": [ $entity-ids ] }
    else (),
    if(exists($concepts))
    then { "xbrl:Concept": [ $concepts ] }
    else ()
  |}
  return
    fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
      $aspects,
      $fiscal-period-focus,
      $fiscal-year-focus,
      $options)
};

(:~
 :
Retrieves the latest reported fiscal period and year for the
 : supplied entity.

 :
 : @param $entity-or-id an entity or its EID.
 :
 : @return an object with two fields: period and year.
 :)
declare function fiscal-core:latest-reported-fiscal-period(
    $entity-or-id as item) as object?
{
  (
  let $entity-or-id := companies:eid($entity-or-id)
  for $archive in archives:archives-for-entities($entity-or-id)
  let $archive-fiscal-period := fiscal-core:fiscal-period-for-filing($archive)
  let $archive-fiscal-year := fiscal-core:fiscal-year-for-filing($archive)
  group by $archive-fiscal-year
  order by $archive-fiscal-year descending empty least
  return {
    year: $archive-fiscal-year,
    period: switch(true)
            case $archive-fiscal-period = "FY" return "FY"
            case $archive-fiscal-period = ("Q3", "YTD3") return "Q3"
            case $archive-fiscal-period = ("Q2", "YTD2") return "Q2"
            case $archive-fiscal-period = ("Q1", "YTD1") return "Q1"
            default return () }
  )[1]
};

(:~
 :
Retrieves the latest fiscal period and year for the
 : supplied entity.

 :
 : @param $entity-or-id an entity or its EID.
 : @param $type 10-K or 10-Q or FY, YTD1, YTD2, YTD3, Q1, Q2, or Q3, or $fiscal-core:ALL_FISCAL_PERIODS
 : to accept all.
 :
 : @return an object with two fields: period and year.
 :)
declare function fiscal-core:latest-reported-fiscal-period(
    $entity-or-id as item,
    $type as string) as object?
{
  if(deep-equal($type, $fiscal-core:ALL_FISCAL_PERIODS))
  then fiscal-core:latest-reported-fiscal-period($entity-or-id)
  else
  (
  let $type-filter :=
    switch(true)
    case $type = ( "10-K", "FY") return "FY"
    case $type eq "10-Q" return ( "Q3", "Q2", "Q1" )
    case $type eq "YTD1" return "Q1"
    case $type eq "YTD2" return "Q2"
    case $type eq "YTD3" return "Q3"
    default return $type
  let $entity-or-id := companies:eid($entity-or-id)
  for $archive in archives:archives-for-entities($entity-or-id)
  let $archive-fiscal-period := fiscal-core:fiscal-period-for-filing($archive)
  let $archive-fiscal-year := fiscal-core:fiscal-year-for-filing($archive)
  where $archive-fiscal-period = $type-filter
  group by $archive-fiscal-year
  order by $archive-fiscal-year descending empty least
  return {
    year: $archive-fiscal-year,
    period:
      switch(true)
      case $archive-fiscal-period = "FY" and ($type eq "10-K" or $type eq "FY") return "FY"
      case $archive-fiscal-period = "Q3" and ($type eq "10-Q" or $type eq "Q3") return "Q3"
      case $archive-fiscal-period = "Q2" and ($type eq "10-Q" or $type eq "Q2") return "Q2"
      case $archive-fiscal-period = "Q1" and ($type eq "10-Q" or $type eq "Q1") return "Q1"
      default return () }
  )[1]
};

(:~
 :)
declare %private %an:strictlydeterministic function fiscal-core:connection() as anyURI
{
  let $credentials :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return if (empty($credentials))
             then error(QName("fiscal-core:CONNECTION-FAILED"), "no xbrl MongoDB configured")
             else $credentials
  return
    try {
      mongo:connect($credentials)
    } catch mongo:* {
      error(QName("fiscal-core:CONNECTION-FAILED"), $err:description)
    }
};

jsoniq version "1.0";

(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for querying fiscal reports (10-K, 10-Q)
 : submitted to the SEC.

 :
 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace facts = "http://28.io/modules/xbrl/facts";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";
import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Joker for all fiscal periods.
 :)
declare variable $fiscal-core:ALL_FISCAL_PERIODS as string := "ALL";

(:~
 : Joker for all fiscal years.
 :)
declare variable $fiscal-core:ALL_FISCAL_YEARS as integer := 0;

(:~
 : Joker for the latest fiscal years.
 :)
declare variable $fiscal-core:LATEST_FISCAL_YEAR as integer := 1;

(:~
 :
Return a distinct set of companies identified by either
 :   CIKs, tags, tickers, or sics.

 :
 : @param $ciks a set of CIKs.
 : @param $tags a set of tags (ALL retrieves all companies).
 : @param $tickers a set of tickers.
 : @param $sics a set of SIC codes.
 :
 : @return the companies with the given identifiers, tags, tickers, or sic codes.
 :)
declare function fiscal-core:filings(
    $entities-or-eids as item*,
    $fiscal-periods as string*,
    $fiscal-years as integer*,
    $archives-or-aids as item*) as object*
{
    let $entities := entities:entities($entities-or-eids)
    return
        if(exists(index-of($fiscal-years, $fiscal-core:LATEST_FISCAL_YEAR)))
        then
            for $entity in $entities
            return
            fiscal-core:latest-filings($entity, $fiscal-periods)
        else
            let $fiscal-years as integer* :=
                $fiscal-years[$$ ne $fiscal-core:LATEST_FISCAL_YEAR]
            return
                fiscal-core:filings-for-entities-and-fiscal-periods-and-years($entities, $fiscal-periods, $fiscal-years),
    archives:archives($archives-or-aids)
};

declare function fiscal-core:latest-filings(
    $entities as object*,
    $fiscal-periods as string*) as object*
{
    for $entity in $entities
    for $fiscal-period in $fiscal-periods
    let $latest-fiscal-year :=
        fiscal-core:latest-reported-fiscal-period($entity, $fiscal-period).year
    return fiscal-core:filings-for-entities-and-fiscal-periods-and-years(
        $entity,
        $fiscal-period,
        $latest-fiscal-year cast as integer?
    )
};

declare function fiscal-core:filter-override(
    $entities-or-eids as item*,
    $fiscal-years as integer*,
    $fiscal-periods as string*,
    $archives-or-aids as item*
) as object?
{
    let $entities := entities:entities($entities-or-eids)
    let $aids := archives:aid($archives-or-aids)
    let $latest-filings := fiscal-core:latest-filings($entities, $fiscal-periods)
    return
    switch(true)
    case count($aids) gt 0 return {
        "sec:Archive" : {
            Type: "string",
            Domain : [ $aids ]
        },
        "sec:FiscalYear" : {
            Type: "integer"
        },
        "sec:FiscalPeriod" : {|
            {
                Type: "string",
                Domain: [ $fiscal-periods ]
            }[exists($fiscal-periods) and empty(index-of($fiscal-periods, $fiscal-core:ALL_FISCAL_PERIODS))]
        |}
    }
    case exists(index-of($fiscal-years, $fiscal-core:LATEST_FISCAL_YEAR))
    return {
        "sec:Archive" : {
            Type: "string",
            Domain : [archives:aid($latest-filings)]
        },
        "sec:FiscalPeriod" : {|
            {
                Type: "string",
                Domain: [ $fiscal-periods ]
            }[exists($fiscal-periods) and empty(index-of($fiscal-periods, $fiscal-core:ALL_FISCAL_PERIODS))]
        |}
    }
    case exists(($entities, $fiscal-years, $fiscal-periods))
    return {
        "xbrl:Entity" : {|
            {
                Type: "string",
                Domain: [ $entities._id ]
            }[exists($entities)]
        |},
        "sec:FiscalYear" :
            let $fiscal-years as integer* :=
                $fiscal-years[$$ ne $fiscal-core:LATEST_FISCAL_YEAR]
            return {|
                {
                    Type: "integer",
                    Domain: [ $fiscal-years ]
                }[exists($fiscal-years) and empty(index-of($fiscal-years, $fiscal-core:ALL_FISCAL_YEARS))]
            |},
        "sec:FiscalPeriod" : {|
            {
                Type: "string",
                Domain: [ $fiscal-periods ]
            }[exists($fiscal-periods) and empty(index-of($fiscal-periods, $fiscal-core:ALL_FISCAL_PERIODS))]
        |},
        "sec:Archive" : {}
    }
    default return ()
};

(:~
 :
Retrieves the fiscal period of a filing.

 :
 : @param $filing-or-id a filing or its id, or the empty sequence.
 :
 : @return the fiscal period (FY, YTD2, YTD3, Q1, Q2 or Q3) if any.
 :)
declare function fiscal-core:fiscal-period-for-filing(
    $filing-or-id as item?) as string?
{
  archives:archives($filing-or-id).Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
};

(:~
 :
Retrieves the fiscal period of a fact.

 :
 : @param $fact a fact object or the empty sequence.
 :
 : @return the fiscal period (FY, YTD2, YTD3, Q1, Q2 or Q3) if any.
 :)
declare function fiscal-core:fiscal-period-for-fact(
    $fact as object?) as string?
{
  $fact.Aspects."sec:FiscalPeriod"
};

(:~
 :
Retrieves the fiscal period of a filing or a fact.

 :
 : @deprecated use fiscal-period-for-fact#1 and fiscal-period-for-filing#1
 :
 : @param $filing-fact-or-id a filing, a fact, or its id.
 :
 : @return the fiscal period (FY, YTD2, YTD3, Q1, Q2 or Q3).
 :)
declare function fiscal-core:fiscal-period(
    $filing-fact-or-id as item?) as string?
{
  if(empty($filing-fact-or-id))
  then ()
  else
    typeswitch ($filing-fact-or-id)
    case object return
      switch (true)
      (: archive :)
      case exists($filing-fact-or-id.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus)
        return $filing-fact-or-id.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
      (: fact :)
      case exists($filing-fact-or-id.Aspects."sec:FiscalPeriod")
        return $filing-fact-or-id.Aspects."sec:FiscalPeriod"
      default return ()
    default return
      let $id as xs:string := string($filing-fact-or-id)
      return
        switch(true)
        (:
          "The first ten digits are the CIK of the entity that logged
           in to submit the filing. The next two digits define the
           current year. The final digits are assigned sequentially by
           CIK starting with one on the first business day of each year."

        http://www.sec.gov/info/edgar/edgarfm-vol2-v25.pdf :)
        case matches($id, "\\d{10}-\\d{2}-\\d{6}")
          return archives:archives($id).Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
        default return facts:facts($id).Aspects."sec:FiscalPeriod"
};

(:~
 :
Retrieves the fiscal year of a filing.

 :
 : @param $filing-or-id a filing or its id, or the empty sequence.
 :
 : @return the fiscal year (e.g., 2012) if any.
 :)
declare function fiscal-core:fiscal-year-for-filing(
    $filing-or-id as item?) as integer?
{
  archives:archives($filing-or-id).Profiles.SEC.Fiscal.DocumentFiscalYearFocus
};

(:~
 :
Retrieves the fiscal year of a fact.

 :
 : @param $fact a fact object or the empty sequence.
 :
 : @return the fiscal year (e.g., 2012) if any.
 :)
declare function fiscal-core:fiscal-year-for-fact(
    $fact as object?) as integer?
{
  $fact.Aspects."sec:FiscalYear"
};

(:~
 :
Retrieves the fiscal year of a filing or a fact.

 :
 : @deprecated use fiscal-core:fiscal-year-for-archive#1 or fiscal-core:fiscal-year-for-fact#1.
 :
 : @param $filing-fact-or-id a filing, a fact, or its id.
 :
 : @return the fiscal year (e.g., 2012).
 :)
declare function fiscal-core:fiscal-year(
    $filing-fact-or-id as item?) as integer?
{
  if(empty($filing-fact-or-id))
  then ()
  else
    typeswitch ($filing-fact-or-id)
    case object return
      switch (true)
      (: archive :)
      case exists($filing-fact-or-id.Profiles.SEC.Fiscal.DocumentFiscalYearFocus)
        return $filing-fact-or-id.Profiles.SEC.Fiscal.DocumentFiscalYearFocus
      (: fact :)
      case exists($filing-fact-or-id.Aspects."sec:FiscalYear")
        return $filing-fact-or-id.Aspects."sec:FiscalYear"
      default return ()
    default return
      let $id as xs:string := string($filing-fact-or-id)
      return
        switch(true)
        (:
          "The first ten digits are the CIK of the entity that logged
           in to submit the filing. The next two digits define the
           current year. The final digits are assigned sequentially by
           CIK starting with one on the first business day of each year."

          http://www.sec.gov/info/edgar/edgarfm-vol2-v25.pdf :)
        case matches($id, "\\d{10}-\\d{2}-\\d{6}")
          return archives:archives($id).Profiles.SEC.Fiscal.DocumentFiscalYearFocus
        default return facts:facts($id).Aspects."sec:FiscalYear"
};

(:~
 :
Retrieves all filings by entities and for given fiscal periods and years.

 :
 : @param $entities-or-ids entities or their ids.
 : @param $fiscal-period-focus fiscal periods (FY, YTD1, YTD2, YTD3, Q1, Q2, Q3, Q4) or $fiscal-core:ALL_FISCAL_PERIODS to accept all..
 : @param $fiscal-year-focus fiscal years or $fiscal-core:ALL_FISCAL_YEARS to accept all..
 :
 : @return all filings that match the criteria.
 :)
declare function
fiscal-core:filings-for-entities-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  let $fiscal-period-focus :=
    switch(true)
    case $fiscal-period-focus = ( "10-K", "FY") return "FY"
    case $fiscal-period-focus eq "10-Q" return ( "Q3", "Q2", "Q1" )
    case $fiscal-period-focus eq "YTD1" return "Q1"
    case $fiscal-period-focus eq "YTD2" return "Q2"
    case $fiscal-period-focus eq "YTD3" return "Q3"
    default return $fiscal-period-focus
  let $conn := fiscal-core:connection()
  for $cik-or-entity in $entities-or-ids
  let $entity-id as xs:string := companies:eid($cik-or-entity)
  let $query := {|
      { "Entity" : $entity-id },
      switch (true)
      case deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS)
        return ()
      case count($fiscal-period-focus) eq 1
        return
        {
          "Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus": $fiscal-period-focus
        }
      default return
        {
          "Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus":
            { "$in" : [ $fiscal-period-focus ] }
        },
      switch (true)
      case deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS)
        return ()
      case count($fiscal-year-focus) eq 1
        return
        {
          "Profiles.SEC.Fiscal.DocumentFiscalYearFocus": $fiscal-year-focus
        }
      default
        return
        {
          "Profiles.SEC.Fiscal.DocumentFiscalYearFocus":
            { "$in" : [ $fiscal-year-focus ] }
        }
    |}
  return mongo:find($conn, $archives:col, $query)
};

(:~
 :
Retrieves the balance sheet for the given entities and fiscal periods and years.

 :
 : @param $entity-or-ids a list of entities or their EIDs.
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 :
 : @error entities:INVALID_PARAMETER if the CIK or entity is not valid.
 :
 : @return the balance sheet at that date and for that formType.
 :)
declare function fiscal-core:balance-sheets-for-fiscal-periods-and-years(
    $entity-or-ids as item*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  for $archive in
    fiscal-core:filings-for-entities-and-fiscal-periods-and-years($entity-or-ids,
                                                                  $fiscal-period-focus,
                                                                  $fiscal-year-focus)
  return sec-networks:networks-for-filings-and-disclosures(
      $archive,
      $sec-networks:BALANCE_SHEET)
};

(:~
 :
Retrieves facts for the given aspects, fiscal periods and years.

 :
 : @param $aspects an object containing aspects to filter, among which
 :                 xbrl:Concept, xbrl:Entity and xbrl:Period (all optional).
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY).
 : @param $fiscal-year-focus the years (2012)
 :
 : @return all facts matching the given aspects, fiscal period and year.
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects as object?,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects,
    $fiscal-period-focus,
    $fiscal-year-focus,
    ())
};

(:~
 :
Retrieves facts for the given aspects, fiscal periods and years.

 :
 : @param $aspects an object containing aspects to filter, among which
 :                 xbrl:Concept, xbrl:Entity and xbrl:Period (all optional).
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 : @param $options standard SEC BizQL options.
 :
 : @return all facts matching the given aspects, fiscal period and year.
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects as object?,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*,
    $options as object?) as object*
{
  let $filter as object := {|
    {
      "Filter":
        {|
          if(exists($aspects))
          then { $facts:ASPECTS : $aspects }
          else (),
          if( deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS) and deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS))
          then { "Profiles.SEC.Fiscal" : { "$exists" : true } }
          else { "Profiles": { "SEC": { "Fiscal":
                 {|
                   if(deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS) )
                   then ()
                   else { "Period" : [ $fiscal-period-focus ] },
                   if( deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS) )
                   then ()
                   else { "Year": [ $fiscal-year-focus ] }
                 |}
               }}}
        |}
    },
    if (empty($options.Hypercube))
    then { Hypercube: sec:dimensionless-hypercube(()) }
    else ()
  |}
  let $facts :=
    facts:facts-for(
      facts:merge-objects($filter, $options, true (: giving parameters higher priority :))
    )
  return sec:hide-amended-facts($facts, $options)
};

(:~
 :
Retrieves facts for the given entities, concepts, fiscal periods and years.

 :
 : @param $entities-or-ids a list of entities or their EIDs.
 : @param $concepts a list of concept names (us-gaap:Assets)
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 :
 : @error entities:INVALID_PARAMETER if the CIK or entity is not valid.
 : @return all facts matching entities, concepts, fiscal periods and years
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $concepts as string*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids,
    $concepts,
    $fiscal-period-focus,
    $fiscal-year-focus,
    ())
};

(:~
 :
Retrieves facts for the given entities, concepts, fiscal periods and years.

 :
 : @param $entities-or-ids a list of entities or their EIDs.
 : @param $concepts a list of concept names (us-gaap:Assets)
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 : @param $options standard SEC BizQL options.
 :
 : @error entities:INVALID_PARAMETER if the CIK or entity is not valid.
 : @return all facts matching entities, concepts, fiscal periods and years
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $concepts as string*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*,
    $options as object?) as object*
{
  let $entity-ids as xs:string* := $entities-or-ids ! companies:eid($$)
  let $aspects := {|
    if(exists($entity-ids))
    then { "xbrl:Entity": [ $entity-ids ] }
    else (),
    if(exists($concepts))
    then { "xbrl:Concept": [ $concepts ] }
    else ()
  |}
  return
    fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
      $aspects,
      $fiscal-period-focus,
      $fiscal-year-focus,
      $options)
};

(:~
 :
Retrieves the latest reported fiscal period and year for the
 : supplied entity.

 :
 : @param $entity-or-id an entity or its EID.
 :
 : @return an object with two fields: period and year.
 :)
declare function fiscal-core:latest-reported-fiscal-period(
    $entity-or-id as item) as object?
{
  (
  let $entity-or-id := companies:eid($entity-or-id)
  for $archive in archives:archives-for-entities($entity-or-id)
  let $archive-fiscal-period := fiscal-core:fiscal-period-for-filing($archive)
  let $archive-fiscal-year := fiscal-core:fiscal-year-for-filing($archive)
  group by $archive-fiscal-year
  order by $archive-fiscal-year descending empty least
  return {
    year: $archive-fiscal-year,
    period: switch(true)
            case $archive-fiscal-period = "FY" return "FY"
            case $archive-fiscal-period = ("Q3", "YTD3") return "Q3"
            case $archive-fiscal-period = ("Q2", "YTD2") return "Q2"
            case $archive-fiscal-period = ("Q1", "YTD1") return "Q1"
            default return () }
  )[1]
};

(:~
 :
Retrieves the latest fiscal period and year for the
 : supplied entity.

 :
 : @param $entity-or-id an entity or its EID.
 : @param $type 10-K or 10-Q or FY, YTD1, YTD2, YTD3, Q1, Q2, or Q3, or $fiscal-core:ALL_FISCAL_PERIODS
 : to accept all.
 :
 : @return an object with two fields: period and year.
 :)
declare function fiscal-core:latest-reported-fiscal-period(
    $entity-or-id as item,
    $type as string) as object?
{
  if(deep-equal($type, $fiscal-core:ALL_FISCAL_PERIODS))
  then fiscal-core:latest-reported-fiscal-period($entity-or-id)
  else
  (
  let $type-filter :=
    switch(true)
    case $type = ( "10-K", "FY") return "FY"
    case $type eq "10-Q" return ( "Q3", "Q2", "Q1" )
    case $type eq "YTD1" return "Q1"
    case $type eq "YTD2" return "Q2"
    case $type eq "YTD3" return "Q3"
    default return $type
  let $entity-or-id := companies:eid($entity-or-id)
  for $archive in archives:archives-for-entities($entity-or-id)
  let $archive-fiscal-period := fiscal-core:fiscal-period-for-filing($archive)
  let $archive-fiscal-year := fiscal-core:fiscal-year-for-filing($archive)
  where $archive-fiscal-period = $type-filter
  group by $archive-fiscal-year
  order by $archive-fiscal-year descending empty least
  return {
    year: $archive-fiscal-year,
    period:
      switch(true)
      case $archive-fiscal-period = "FY" and ($type eq "10-K" or $type eq "FY") return "FY"
      case $archive-fiscal-period = "Q3" and ($type eq "10-Q" or $type eq "Q3") return "Q3"
      case $archive-fiscal-period = "Q2" and ($type eq "10-Q" or $type eq "Q2") return "Q2"
      case $archive-fiscal-period = "Q1" and ($type eq "10-Q" or $type eq "Q1") return "Q1"
      default return () }
  )[1]
};

(:~
 :)
declare %private %an:strictlydeterministic function fiscal-core:connection() as anyURI
{
  let $credentials :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return if (empty($credentials))
             then error(QName("fiscal-core:CONNECTION-FAILED"), "no xbrl MongoDB configured")
             else $credentials
  return
    try {
      mongo:connect($credentials)
    } catch mongo:* {
      error(QName("fiscal-core:CONNECTION-FAILED"), $err:description)
    }
};


Simplify content merging.

Simplify content merging.

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    (. == {} or . == [] or . == false or . == null or . == 0) | not
;

# Group an array of objects into an object of arrays, such that the key-value
# pairs of the new object are an accumulation of those of the old object. For
# example, `[{"a":1, "b":2}, {"a":3}]` turns into `{"a":[1, 3], "b":[3]}`. This
# can be used to partition an array.
# Try: `[1,2,3] | map({(.%2|tostring):.}) | group`
def group:
    reduce ([.[] | to_entries[]] | group_by(.key))[] as $group
    ( {}
    ; .[ $group[0].key ] |= (. // []) + [ $group[].value ]
    )
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`.
def tree($path):
    def tree_aux($future):
        { "name": $future[0] } +
        if ($future | length) > 1 then
            { "contents": [ tree_aux($future[1:]) ] }
        else
            .
        end
    ;
    tree_aux($path)
;

# Add paths to each object, that is, the names of the ancestors.
def add_paths:
    def add_path_aux($history):
        ($history + [.name]) as $present |
        .path = ($present | join("/")) |
        (.contents[]? |= add_path_aux($present))
    ;
    add_path_aux([])
;

# Merge two values. If the values are both objects, the values at equal indices
# will be recursively merged; if the values are both arrays, the arrays are
# concatenated; if the values are some other type, we only check if the values
# do not conflict.
def merge(other):
    if (. | type) == "object" and (other | type) == "object" then
        (. | to_entries) + (other | to_entries)
        | group_by(.key)
        | map({ "key": (.[0].key), "value" : (reduce .[1:][].value as $x (.[0].value; merge($x)))})
        | from_entries
    elif (. | type) == "array" and (other | type) == "array" then
        . + other
    elif . == other then
        .
    else
        error("trying to merge incompatible objects " + (.|type) + " and " + (other|type))
    end
;

# Determine whether a page should be uploaded and included in the table of
# contents: it should either be marked as `publish`, or be *not* marked as
# `draft` and have children that *are* marked as `publish`.
def is_publication:
    ( .publish or .meta.publish | bool)
    or
    ( (.meta.draft | bool | not) and ((.contents // []) | any(is_publication)) )
;

# Merge an array of content by grouping it by name and then merging all the
# pages with the same name. This is necessary because simply merging content
# will only concatenate pages, even if they are the same page.
def merge_content:
    reduce .[] as $x ({}; merge($x)) |
    if has("contents") then
        .contents |= ( group_by(.name) | map(merge_content) )
    else
        .
    end
;

# Any page gets a link to its HTML.
def add_links:
    if (has("path") and (.path | endswith(".md"))) then
        .link = (.path | rtrimstr(".md") | . + ".html")
    else
        .contents[]? |= add_links
    end
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def add_frontmatter:
    if has("contents") then
        ( .contents | map({(.name=="index.md" or .meta.frontmatter|tostring):.}) | group) as {$true, $false}
        | .frontmatter = $true[0]
        | .contents = ($false + $true[1:])
        | (.contents[] |= add_frontmatter)
    else
        .
    end
;

# If marked as such, drafts can be excluded from being uploaded and included
# in the table of contents.
def add_drafts:
    if has("contents") then
        ( .contents | map({(is_publication|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .drafts = ($false // [])
        | (.contents[] |= add_drafts)
    else
        .
    end
;

# A resource is anything that is neither a directory nor a page with metadata
# nor a forced link.
def add_resources:
    if has("contents") then
        ( .contents | map({(has("contents") or has("meta") or has("publish")|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .resources = ($false // [])
        | (.contents[]? |= add_resources)
    else
        .
    end;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    merge_content
    | add_paths
    | add_links
    | add_drafts
    | add_frontmatter
    | add_resources
;

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    (. == {} or . == [] or . == false or . == null or . == 0) | not
;

# Group an array of objects into an object of arrays, such that the key-value
# pairs of the new object are an accumulation of those of the old object. For
# example, `[{"a":1, "b":2}, {"a":3}]` turns into `{"a":[1, 3], "b":[3]}`. This
# can be used to partition an array.
# Try: `[1,2,3] | map({(.%2|tostring):.}) | group`
def group:
    reduce ([.[] | to_entries[]] | group_by(.key))[] as $group
    ( {}
    ; .[ $group[0].key ] |= (. // []) + [ $group[].value ]
    )
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`.
def tree($path):
    def tree_aux($future):
        { "name": $future[0] } +
        if ($future | length) > 1 then
            { "contents": [ tree_aux($future[1:]) ] }
        else
            .
        end
    ;
    tree_aux($path)
;

# Add paths to each object, that is, the names of the ancestors.
def add_paths:
    def add_path_aux($history):
        ($history + [.name]) as $present |
        .path = ($present | join("/")) |
        (.contents[]? |= add_path_aux($present))
    ;
    add_path_aux([])
;

# Merge two values. If the values are both objects, the values at equal indices
# will be recursively merged; if the values are both arrays, the arrays are
# concatenated; if the values are some other type, we only check if the values
# do not conflict.
def merge(other):
    if (. | type) == "object" and (other | type) == "object" then
        (. | to_entries) + (other | to_entries)
        | group_by(.key)
        | map({ "key": (.[0].key), "value" : (reduce .[1:][].value as $x (.[0].value; merge($x)))})
        | from_entries
    elif (. | type) == "array" and (other | type) == "array" then
        . + other
    elif . == other then
        .
    else
        error("trying to merge incompatible objects " + (.|type) + " and " + (other|type))
    end
;

# Determine whether a page should be uploaded and included in the table of
# contents: it should either be marked as `publish`, or be *not* marked as
# `draft` and have children that *are* marked as `publish`.
def is_publication:
    ( .publish or .meta.publish | bool)
    or
    ( (.meta.draft | bool | not) and ((.contents // []) | any(is_publication)) )
;

# Merge filetrees by grouping them by name and then merging all the pages with
# the same name. This is necessary because simply merging content will
# concatenate pages.
def merge_content:
    if has("contents") then
        .contents |= (
            group_by(.name)
            | map(reduce .[] as $x ({}; merge($x)) | merge_content)
        )
    else
        .
    end
;

# Any page gets a link to its HTML.
def add_links:
    if (has("path") and (.path | endswith(".md"))) then
        .link = (.path | rtrimstr(".md") | . + ".html")
    else
        .contents[]? |= add_links
    end
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def add_frontmatter:
    if has("contents") then
        ( .contents | map({(.name=="index.md" or .meta.frontmatter|tostring):.}) | group) as {$true, $false}
        | .frontmatter = $true[0]
        | .contents = ($false + $true[1:])
        | (.contents[] |= add_frontmatter)
    else
        .
    end
;

# If marked as such, drafts can be excluded from being uploaded and included
# in the table of contents.
def add_drafts:
    if has("contents") then
        ( .contents | map({(is_publication|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .drafts = ($false // [])
        | (.contents[] |= add_drafts)
    else
        .
    end
;

# A resource is anything that is neither a directory nor a page with metadata
# nor a forced link.
def add_resources:
    if has("contents") then
        ( .contents | map({(has("contents") or has("meta") or has("publish")|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .resources = ($false // [])
        | (.contents[]? |= add_resources)
    else
        .
    end;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    reduce .[] as $x ({}; merge($x))
    | merge_content
    | add_paths
    | add_links
    | add_drafts
    | add_frontmatter
    | add_resources
;


Fix issue with Thema conversion

Fix issue with Thema conversion

"http://uri.gbv.de/terminology/thema" as $BASE |

# date of publication as found at EDItEUR website
{
  "1.0": "2013",
  "1.1": "2015",
  "1.2": "2016-05",
  "1.3": "2018-04",
  "1.4": "2020-04"
} as $VERSIONS |

# IssueNumber and Modified may contain dates or version numbers
def dateValue:
  if match("^\\d\\.\\d") then
    $VERSIONS[.[0:3]]
  elif match("^2\\d\\d\\d\\d\\d$") then
    .[0:4]+"-"+.[4:6]
  else
    null
  end
;

.CodeList.ThemaCodes.Code[] |
{
  uri: ($BASE + "/" + .CodeValue),
  notation: [.CodeValue],
  prefLabel: { LANGUAGE: .CodeDescription },
  inScheme: [ { uri: $BASE } ],
  issued: .IssueNumber|dateValue
}
+
if .CodeNotes | length > 0 then
  { scopeNote: { LANGUAGE: [.CodeNotes] } }
else
  { }
end
+
if .CodeParent | length > 0 then
  { broader: [ { uri: ($BASE + "/" + .CodeParent) } ] }
else
  { topConceptOf: [ { uri: $BASE } ] }
end
+
if .Modified | length > 0 then
  { modified: .Modified|dateValue }
else
  {}
end

"http://uri.gbv.de/terminology/thema" as $BASE |

# date of publication as found at EDItEUR website
{
  "1.0": "2013",
  "1.1": "2015",
  "1.2": "2016-05",
  "1.3": "2018-04",
  "1.4": "2020-04"
} as $VERSIONS |

# IssueNumber and Modified may contain dates or version numbers
def dateValue:
  if match("^\\d\\.\\d") then
    $VERSIONS[.[0:3]]
  elif match("^2\\d\\d\\d\\d\\d$") then
    .[0:4]+"-"+.[4:6]
  else
    null
  end
;

.CodeList.ThemaCodes.Code[] |
{
  uri: ($BASE + "/" + .CodeValue),
  notation: [.CodeValue],
  prefLabel: { LANGUAGE: .CodeDescription },
  inScheme: [ { uri: $BASE } ],
  issued: .IssueNumber|dateValue
}
+
if .CodeNotes | length > 0 then
  { scopeNote: { LANGUAGE: [.CodeNotes] } }
else
  { }
end
+
if .CodeParent then
  { broader: [ { uri: ($BASE + "/" + .CodeParent) } ] }
else
  { topConceptOf: [ { uri: $BASE } ] }
end
+
if .Modified | length > 0 then
  { modified: .Modified|dateValue }
else
  {}
end


:pencil2: Removed taxonomy entity from expected results.

import module namespace test = "http://apps.28.io/test";

declare variable $local:expected as object :=
{
  "all" : [
    "http://info.edinet-fsa.go.jp E01225-000",
    "http://info.edinet-fsa.go.jp E01264-000",
    "http://info.edinet-fsa.go.jp E02166-000",
    "http://info.edinet-fsa.go.jp E02274-000",
    "http://info.edinet-fsa.go.jp E02513-000",
    "http://info.edinet-fsa.go.jp E02529-000",
    "http://info.edinet-fsa.go.jp E04147-000",
    "http://info.edinet-fsa.go.jp E04425-000",
    "http://info.edinet-fsa.go.jp E04430-000",
    "http://info.edinet-fsa.go.jp EDINET-000"
  ],
  "NIKKEI" : [
    "http://info.edinet-fsa.go.jp E01225-000",
    "http://info.edinet-fsa.go.jp E01264-000",
    "http://info.edinet-fsa.go.jp E02166-000",
    "http://info.edinet-fsa.go.jp E02274-000",
    "http://info.edinet-fsa.go.jp E02513-000",
    "http://info.edinet-fsa.go.jp E02529-000",
    "http://info.edinet-fsa.go.jp E04147-000",
    "http://info.edinet-fsa.go.jp E04425-000",
    "http://info.edinet-fsa.go.jp E04430-000"
  ]
};

test:check-all-success({
    all: test:invoke-and-assert-deep-equal(
      "entities",
      {},
      function($b as item*) as item* { [ $b.Entities[]._id ] },
      $local:expected.all
    ),
    nikkei: test:invoke-and-assert-deep-equal(
      "entities",
      {tag:"NIKKEI"},
      function($b as item*) as item* { [ $b.Entities[]._id ] },
      $local:expected.NIKKEI
    ),
    example-json: test:invoke-and-assert-deep-equal(
      "entities",
      {cik:"E01225"},
      function($b as item*) as item* { $b.Entities[] },
      test:get-expected-result("edinet/entities-expected1.jq")
    ),
    example-xml: test:invoke-and-assert-deep-equal(
      "entities",
      {cik:"E01225", format: "xml"},
      function($b as item*) as item* { document { $b/Entities } },
      test:get-expected-result-xml("edinet/entities-expected1-xml.jq"),
      { Format: "xml" }
    )
})

import module namespace test = "http://apps.28.io/test";

declare variable $local:expected as object :=
{
  "all" : [
    "http://info.edinet-fsa.go.jp E01225-000",
    "http://info.edinet-fsa.go.jp E01264-000",
    "http://info.edinet-fsa.go.jp E02166-000",
    "http://info.edinet-fsa.go.jp E02274-000",
    "http://info.edinet-fsa.go.jp E02513-000",
    "http://info.edinet-fsa.go.jp E02529-000",
    "http://info.edinet-fsa.go.jp E04147-000",
    "http://info.edinet-fsa.go.jp E04425-000",
    "http://info.edinet-fsa.go.jp E04430-000",
    "http://info.edinet-fsa.go.jp EDINET-000"
  ],
  "NIKKEI" : [
    "http://info.edinet-fsa.go.jp E01225-000",
    "http://info.edinet-fsa.go.jp E01264-000",
    "http://info.edinet-fsa.go.jp E02166-000",
    "http://info.edinet-fsa.go.jp E02274-000",
    "http://info.edinet-fsa.go.jp E02513-000",
    "http://info.edinet-fsa.go.jp E02529-000",
    "http://info.edinet-fsa.go.jp E04147-000",
    "http://info.edinet-fsa.go.jp E04425-000",
    "http://info.edinet-fsa.go.jp E04430-000",
    "http://info.edinet-fsa.go.jp EDINET-000"
  ]
};

test:check-all-success({
    all: test:invoke-and-assert-deep-equal(
      "entities",
      {},
      function($b as item*) as item* { [ $b.Entities[]._id ] },
      $local:expected.all
    ),
    nikkei: test:invoke-and-assert-deep-equal(
      "entities",
      {tag:"NIKKEI"},
      function($b as item*) as item* { [ $b.Entities[]._id ] },
      $local:expected.NIKKEI
    ),
    example-json: test:invoke-and-assert-deep-equal(
      "entities",
      {cik:"E01225"},
      function($b as item*) as item* { $b.Entities[] },
      test:get-expected-result("edinet/entities-expected1.jq")
    ),
    example-xml: test:invoke-and-assert-deep-equal(
      "entities",
      {cik:"E01225", format: "xml"},
      function($b as item*) as item* { document { $b/Entities } },
      test:get-expected-result-xml("edinet/entities-expected1-xml.jq"),
      { Format: "xml" }
    )
})


test failure

test failure

import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

declare %an:nondeterministic function local:test-concepts($expected as integer, $params as string) as atomic
{
    let $elements as item* := parse-json(http-client:get("http://" || request:server-name() || ":" || request:server-port() || "/v1/_queries/public/api/report-elements.jq?_method=POST" || $params).body.content).ReportElements[]
    let $actual as integer := count($elements)
    let $duplicates := for $element in $elements where $element.CIK instance of array return $element
        if (exists($duplicates)) then
            "false [CIK " || serialize($duplicates[1].CIK, ()) || " is not unique]"
        else if ($actual eq $expected) then
            true
        else
            "false [Actual="||$actual||", Expected="||$expected ||"]"
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    all: local:test-concepts(3, "&cik=4962&name=us-gaap:Assets"),
    onlyNames: local:test-concepts(16143, "&tag=DOW30&fiscalYear=2012&fiscalPeriod=FY&onlyNames=true"),
    disclosures: local:test-concepts(1855, "&ticker=ko&fiscalYear=2012&fiscalPeriod=FY&report=Disclosure&concept=disc:ConsolidationVariableInterestEntityPolicy")
})

import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

declare %an:nondeterministic function local:test-concepts($expected as integer, $params as string) as atomic
{
    let $elements as item* := parse-json(http-client:get("http://" || request:server-name() || ":" || request:server-port() || "/v1/_queries/public/api/report-elements.jq?_method=POST" || $params).body.content).ReportElements[]
    let $actual as integer := count($elements)
    let $duplicates := for $element in $elements where $element.CIK instance of array return $element
    return
        if (exists($duplicates)) then
            "false [CIK " || serialize($duplicates[1].CIK, ()) || " is not unique]"
        else if ($actual eq $expected) then
            true
        else
            "false [Actual="||$actual||", Expected="||$expected ||"]"
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    all: local:test-concepts(3, "&cik=4962&name=us-gaap:Assets"),
    onlyNames: local:test-concepts(16143, "&tag=DOW30&fiscalYear=2012&fiscalPeriod=FY&onlyNames=true"),
    disclosures: local:test-concepts(1855, "&ticker=ko&fiscalYear=2012&fiscalPeriod=FY&report=Disclosure&concept=disc:ConsolidationVariableInterestEntityPolicy")
})


:bug: Some fixes.

:bug: Some fixes.