Update facttable-for-report.jq

Update facttable-for-report.jq

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace api = "http://apps.28.io/api";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token         as string? external;
declare  %rest:env                              variable $request-uri   as string  external;
declare  %rest:case-insensitive                 variable $format        as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker        as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear    as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid           as string* external;
declare  %rest:case-insensitive                 variable $validate      as boolean external := false;
declare  %rest:case-insensitive                 variable $labels        as boolean external := false;
declare  %rest:case-insensitive                 variable $report        as string? external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := "sec";

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities :=
    for $entity in
        companies:companies(
            $cik,
            $tag,
            $ticker,
            $sic)
    order by $entity.Profiles.SEC.CompanyName
    return $entity
let $report as object? := reports:reports($report)

(: Fact resolution :)
let $filter-override as object? := fiscal-core:filter-override(
    $entities,
    $fiscalYear,
    $fiscalPeriod,
    $aid)[$profile-name eq "sec"]
let $facts as object* :=
    let $hypercube := hypercubes:hypercubes-for-components($report, "xbrl:DefaultHypercube")
    let $filtered-aspects := values($hypercube.Aspects)[exists(($$.Domains, $$.DomainRestriction))]
    return if(count($filtered-aspects) lt 2 and not exists(($filter-override)))
    then {
          response:status-code(403);
          session:error("The report filters are too weak, which leads to too big an output.", $format)
    } else
        components:facts(
                $report,
                {|
                    { FilterOverride: $filter-override }[exists($filter-override)],
                    { Validate: $validate }
                |}
            )

let $concepts as object* :=
    reports:concepts($report)
let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $concepts:AMERICAN_ENGLISH )[1]
let $role as string := ( $report.Role, $concepts:ANY_COMPONENT_LINK_ROLE )[1]
let $facts :=
    if($profile-name eq "sec")
    then
        for $fact in $facts
        group by $archive := $fact.Aspects."sec:Archive"
        let $archive := archives:archives($archive)
        let $entity := entities:entities($archive.Entity)
        for $fact in $fact
        let $labels-object as object? := facts:labels($fact, $role, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
        return
        {|
            trim($fact, ("Labels", "EntityRegistrantName")),
            { "EntityRegistrantName" : $entity.Profiles.SEC.CompanyName },
            { Labels : $labels-object }[$labels]
        |}
    else
        for $fact in $facts
        let $labels as object? := facts:labels($fact, $role, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
        return
        {|
            trim($fact, "Labels"),
            { Labels : $labels }[exists($labels)]
        |}

let $facts := if($profile-name eq "sec")
              then api:normalize-facts($facts)
              else $facts

let $results :=
    switch ($format)
    case "xml" return {
        response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
        session:comment("xml",
        {
            NumFacts : count($facts),
            TotalNumFacts: session:num-facts(),
            TotalNumArchives: session:num-archives(),
            TotalNumEntities: session:num-entities()
        }),
        <FactTable NetworkIdentifier="http://bizql.io/facttable-for-report"
                TableName="xbrl:FactTableForReport">{
            conversion:facts-to-xml($facts, { Caller: "Report" })
        }</FactTable>
    }
    case "text" case "csv" return {
        response:content-type("text/csv");
        response:header("Content-Disposition", "attachment; filename=facts.csv");
        conversion:facts-to-csv($facts, { Caller: "Report"})
    }
    case "excel" return {
        response:content-type("application/vnd.ms-excel");
        response:header("Content-Disposition", "attachment; filename=fact.csv");
        conversion:facts-to-csv($facts, { Caller: "Report"})
    }
    default return {
        response:content-type("application/json");
        response:serialization-parameters({"indent" : true});
        {|
            { NetworkIdentifier : "http://secxbrl.info/facts" },
            { TableName : "xbrl:Facts" },
            { FactTable : [ $facts ] },
            session:comment("json", {
                    NumFacts : count($facts),
                    TotalNumFacts: session:num-facts(),
                    TotalNumArchives: session:num-archives(),
                    TotalNumEntities: session:num-entities()
                })
        |}
    }
return api:check-and-return-results($token, $results, $format)

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace api = "http://apps.28.io/api";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token         as string? external;
declare  %rest:env                              variable $request-uri   as string  external;
declare  %rest:case-insensitive                 variable $format        as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker        as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear    as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid           as string* external;
declare  %rest:case-insensitive                 variable $validate      as boolean external := false;
declare  %rest:case-insensitive                 variable $labels        as boolean external := false;
declare  %rest:case-insensitive                 variable $report        as string? external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := "sec";

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities :=
    for $entity in
        companies:companies(
            $cik,
            $tag,
            $ticker,
            $sic)
    order by $entity.Profiles.SEC.CompanyName
    return $entity
let $report as object? := reports:reports($report)

(: Fact resolution :)
let $filter-override as object? := fiscal-core:filter-override(
    $entities,
    $fiscalYear,
    $fiscalPeriod,
    $aid)[$profile-name eq "sec"]
let $facts as object* :=
    let $hypercube := hypercubes:hypercubes-for-components($report, "xbrl:DefaultHypercube")
    let $filtered-aspects := values($hypercube.Aspects)[exists(($$.Domains, $$.DomainRestriction))]
    return if(count($filtered-aspects) lt 2 and not exists(($filter-override)))
    then {
          response:status-code(403);
          session:error("The report filters are too weak, which leads to too big an output.", $format)
    } else
        components:facts(
                $report,
                {|
                    { FilterOverride: $filter-override }[exists($filter-override)],
                    { Validate: $validate }
                |}
            )

let $concepts as object* :=
    reports:concepts($report)
let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $concepts:AMERICAN_ENGLISH )[1]
let $role as string := ( $report.Role, $concepts:ANY_COMPONENT_LINK_ROLE )[1]
let $facts :=
    if($profile-name eq "sec")
    then
        for $fact in $facts
        group by $archive := $fact.Aspects."sec:Archive"
        let $archive := archives:archives($archive)
        let $entity := entities:entities($archive.Entity)
        for $fact in $fact
        let $labels as object? := facts:labels($fact, $role, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
        return
        {|
            trim($fact, ("Labels", "EntityRegistrantName")),
            { "EntityRegistrantName" : $entity.Profiles.SEC.CompanyName },
            { Labels : $labels }[exists($labels)]
        |}
    else
        for $fact in $facts
        let $labels as object? := facts:labels($fact, $role, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
        return
        {|
            trim($fact, "Labels"),
            { Labels : $labels }[exists($labels)]
        |}

let $facts := if($profile-name eq "sec")
              then api:normalize-facts($facts)
              else $facts

let $results :=
    switch ($format)
    case "xml" return {
        response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
        session:comment("xml",
        {
            NumFacts : count($facts),
            TotalNumFacts: session:num-facts(),
            TotalNumArchives: session:num-archives(),
            TotalNumEntities: session:num-entities()
        }),
        <FactTable NetworkIdentifier="http://bizql.io/facttable-for-report"
                TableName="xbrl:FactTableForReport">{
            conversion:facts-to-xml($facts, { Caller: "Report" })
        }</FactTable>
    }
    case "text" case "csv" return {
        response:content-type("text/csv");
        response:header("Content-Disposition", "attachment; filename=facts.csv");
        conversion:facts-to-csv($facts, { Caller: "Report"})
    }
    case "excel" return {
        response:content-type("application/vnd.ms-excel");
        response:header("Content-Disposition", "attachment; filename=fact.csv");
        conversion:facts-to-csv($facts, { Caller: "Report"})
    }
    default return {
        response:content-type("application/json");
        response:serialization-parameters({"indent" : true});
        {|
            { NetworkIdentifier : "http://secxbrl.info/facts" },
            { TableName : "xbrl:Facts" },
            { FactTable : [ $facts ] },
            session:comment("json", {
                    NumFacts : count($facts),
                    TotalNumFacts: session:num-facts(),
                    TotalNumArchives: session:num-archives(),
                    TotalNumEntities: session:num-entities()
                })
        |}
    }
return api:check-and-return-results($token, $results, $format)


:pencil: Unbox.

:pencil: Unbox.

jsoniq version "1.0";
module namespace japan = "http://28.io/modules/xbrl/profiles/japan/core";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";
import module namespace es = "http://28.io/modules/elasticsearch";

import module namespace config = "http://apps.28.io/config";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace entities = "http://28.io/modules/xbrl/entities";

(:
 : Copyright 2012-2015 28msec Inc.
 :)
(:~
 :
This module provides functionality for querying against the Japan profile.

 :
 : @author Ghislain Fourny
 :
 :)

(:~
 :
Retrieves the entities with the given identifiers.

 :
 : @param $edinetcode a sequence of edinet codes like E01225.
 : @param $securitiescode a sequence of securities codes, like 9432.
 : @param $eid a sequence of raw EIDs.
 : @param $tag a sequence of tags (e.g., indices).
 : @param $full-text a full text search phrase
 : @param $language a language identifier for the full text search (e.g. ja/en)
 : @param $options optional settings such as full-text-offset or full-text-limit
 :
 : @return the entities that match those identifiers.
 :)
declare %an:nondeterministic function japan:entities(
    $edinetcode as string*,
    $securitiescode as string*,
    $eid as string*,
    $tag as string*,
    $full-text as string?,
    $language as string,
    $aid as string*,
    $options as object?
) as object*
{
    switch(true)
    case $tag = "ALL" return entities:entities()
    case exists(($eid, $edinetcode, $securitiescode, $tag, $full-text)) return
       for $entity in (
         mw:find($entities:col, { "Profiles" : { "FSA" : { "Tags" : [ $tag ] } } }),

         mw:find($entities:col, { "Profiles" : { "FSA" : { "EDINETCode" : [ $edinetcode ] } } }),

         mw:find($entities:col, { "Profiles" : { "FSA" : { "SecuritiesCode" : [ $securitiescode ] } } }),

         let $conn := es:connection("xbrl")
         where exists($full-text)
         return es:search(
           $conn,
           $config:db-name,
           "entities",
           {
             "from" : ($options.full-text-offset, 0)[1],
             "size" : ($options.full-text-limit, 10)[1],
             "query" : {
               "multi_match" : {
                 "query": $full-text,
                 "type": "most_fields",
                 "fields": [
                   switch($language)
                   case "ja" return
                     ("SubmitterName", "FundName", "IssuerName", "Province", "SecuritiesType")
                   default return ("SubmitterNameAlphabetic","Industry")
                 ]
               }
             }
           }
         ).hits.hits[]._source,

         entities:entities(($eid, archives:archives($aid).Entity))

       )
       group by $entity._id
       return $entity[1]
    default return ()
};

(:~
 :
Retrieves the filings with the given characteristics.

 :
 : @param $entities a sequence of entities or EIDs.
 : @param $fiscalYear a sequence of fiscal years.
 : @param $fiscalPeriod a sequence of fiscal periods.
 : @param $filingKind a sequence of filing kinds to filter for.
 : @param $aid a sequence of AIDs.
 :
 : @return the filings that match those criteria.
 :)
declare function japan:filings(
    $entities as item*,
    $fiscalYear as integer*,
    $fiscalPeriod as string*,
    $filingKind as string*,
    $aid as string*) as object*
{
    archives:archives($aid),
    if($fiscalYear = 1)
    then
        for $a as object in archives:archives-for-entities($entities)
        where empty($filingKind) or $a.Profiles.FSA.FilingKinds[] = $filingKind
        where (empty($fiscalPeriod) or ($fiscalPeriod = "ALL") or
               $a.Profiles.FSA.DocumentFiscalPeriodFocus = $fiscalPeriod)
        group by $a.Entity
        return
            for $filing in $a
            group by $fy := $filing.Profiles.FSA.DocumentFiscalYearFocus
            order by $fy descending
            count $i where $i eq 1
            return $filing
    else
        for $a as object in archives:archives-for-entities($entities)
        where empty($filingKind) or $a.Profiles.FSA.FilingKinds[] = $filingKind
        where (empty($fiscalYear) or
               $fiscalYear = 0 or
               $fiscalYear = $a.Profiles.FSA.DocumentFiscalYearFocus)
               and (empty($fiscalPeriod) or ($fiscalPeriod = "ALL") or
               $a.Profiles.FSA.DocumentFiscalPeriodFocus = $fiscalPeriod)
        return $a
};

(:~
 :
Return latest filings of entities and fiscal periods.

 :
 : @param $profile-name the name of the profile (e.g., SEC, Japan, Generic).
 : @param $entities a sequence of entity objects.
 : @param $fiscal-periods a set of fiscal periods such as "Q1", "Q2", "Q3", "Q4", "FY"
 :
 : @return the latest archives for given entities and fiscal periods.
 :)
declare function japan:latest-filings(
    $entities as object*,
    $fiscal-periods as string*) as object*
{
    (
        error(xs:QName("japan:NOT_IMPLEMENTED"),
            "japan profile: Getting latest-filings for entities not implemented."),

        $entities,
        $fiscal-periods
    )
};

jsoniq version "1.0";
module namespace japan = "http://28.io/modules/xbrl/profiles/japan/core";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";
import module namespace es = "http://28.io/modules/elasticsearch";

import module namespace config = "http://apps.28.io/config";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace entities = "http://28.io/modules/xbrl/entities";

(:
 : Copyright 2012-2015 28msec Inc.
 :)
(:~
 :
This module provides functionality for querying against the Japan profile.

 :
 : @author Ghislain Fourny
 :
 :)

(:~
 :
Retrieves the entities with the given identifiers.

 :
 : @param $edinetcode a sequence of edinet codes like E01225.
 : @param $securitiescode a sequence of securities codes, like 9432.
 : @param $eid a sequence of raw EIDs.
 : @param $tag a sequence of tags (e.g., indices).
 : @param $full-text a full text search phrase
 : @param $language a language identifier for the full text search (e.g. ja/en)
 : @param $options optional settings such as full-text-offset or full-text-limit
 :
 : @return the entities that match those identifiers.
 :)
declare %an:nondeterministic function japan:entities(
    $edinetcode as string*,
    $securitiescode as string*,
    $eid as string*,
    $tag as string*,
    $full-text as string?,
    $language as string,
    $aid as string*,
    $options as object?
) as object*
{
    switch(true)
    case $tag = "ALL" return entities:entities()
    case exists(($eid, $edinetcode, $securitiescode, $tag, $full-text)) return
       for $entity in (
         mw:find($entities:col, { "Profiles.FSA.Tags" : [ $tag ] }),

         mw:find($entities:col, { "Profiles.FSA.EDINETCode" : [ $edinetcode ] }),

         mw:find($entities:col, { "Profiles.FSA.SecuritiesCode" : [ $securitiescode ] }),

         let $conn := es:connection("xbrl")
         where exists($full-text)
         return es:search(
           $conn,
           $config:db-name,
           "entities",
           {
             "from" : ($options.full-text-offset, 0)[1],
             "size" : ($options.full-text-limit, 10)[1],
             "query" : {
               "multi_match" : {
                 "query": $full-text,
                 "type": "most_fields",
                 "fields": [
                   switch($language)
                   case "ja" return
                     ("SubmitterName", "FundName", "IssuerName", "Province", "SecuritiesType")
                   default return ("SubmitterNameAlphabetic","Industry")
                 ]
               }
             }
           }
         ).hits.hits[]._source,

         entities:entities(($eid, archives:archives($aid).Entity))

       )
       group by $entity._id
       return $entity[1]
    default return ()
};

(:~
 :
Retrieves the filings with the given characteristics.

 :
 : @param $entities a sequence of entities or EIDs.
 : @param $fiscalYear a sequence of fiscal years.
 : @param $fiscalPeriod a sequence of fiscal periods.
 : @param $filingKind a sequence of filing kinds to filter for.
 : @param $aid a sequence of AIDs.
 :
 : @return the filings that match those criteria.
 :)
declare function japan:filings(
    $entities as item*,
    $fiscalYear as integer*,
    $fiscalPeriod as string*,
    $filingKind as string*,
    $aid as string*) as object*
{
    archives:archives($aid),
    if($fiscalYear = 1)
    then
        for $a as object in archives:archives-for-entities($entities)
        where empty($filingKind) or $a.Profiles.FSA.FilingKinds[] = $filingKind
        where (empty($fiscalPeriod) or ($fiscalPeriod = "ALL") or
               $a.Profiles.FSA.DocumentFiscalPeriodFocus = $fiscalPeriod)
        group by $a.Entity
        return
            for $filing in $a
            group by $fy := $filing.Profiles.FSA.DocumentFiscalYearFocus
            order by $fy descending
            count $i where $i eq 1
            return $filing
    else
        for $a as object in archives:archives-for-entities($entities)
        where empty($filingKind) or $a.Profiles.FSA.FilingKinds[] = $filingKind
        where (empty($fiscalYear) or
               $fiscalYear = 0 or
               $fiscalYear = $a.Profiles.FSA.DocumentFiscalYearFocus)
               and (empty($fiscalPeriod) or ($fiscalPeriod = "ALL") or
               $a.Profiles.FSA.DocumentFiscalPeriodFocus = $fiscalPeriod)
        return $a
};

(:~
 :
Return latest filings of entities and fiscal periods.

 :
 : @param $profile-name the name of the profile (e.g., SEC, Japan, Generic).
 : @param $entities a sequence of entity objects.
 : @param $fiscal-periods a set of fiscal periods such as "Q1", "Q2", "Q3", "Q4", "FY"
 :
 : @return the latest archives for given entities and fiscal periods.
 :)
declare function japan:latest-filings(
    $entities as object*,
    $fiscal-periods as string*) as object*
{
    (
        error(xs:QName("japan:NOT_IMPLEMENTED"),
            "japan profile: Getting latest-filings for entities not implemented."),

        $entities,
        $fiscal-periods
    )
};


Give clearer example for `jq` function `group`.

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    (. == {} or . == [] or . == false or . == null or . == 0) | not
;

# Format a date into its wordy equivalent.
def format_date:
    try
    (
        (   try (. | strptime("%Y-%m-%d"))
            catch (
                try (. | strptime("%Y/%m/%d"))
                catch (
                    try (. | strptime("%Y-%m"))
                )
            )
        ) | mktime | strftime("%B %-d, %Y")
    )
;

# Group an array of objects into an object of arrays, such that the key-value
# pairs of the new object are an accumulation of those of the old object. For
# example, `[{"a":1, "b":2}, {"a":3}]` turns into `{"a":[1, 3], "b":[3]}`. This
# can be used to partition an array.
# Try: `[1,"2",3] | map ({(.|type):.}) | group`
def group:
    reduce ([.[] | to_entries[]] | group_by(.key))[] as $group
    ( {}
    ; .[ $group[0].key ] |= (. // []) + [ $group[].value ]
    )
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`.
def tree($path):
    def tree_aux($future):
        { "name": $future[0] } +
        if ($future | length) > 1 then
            { "contents": [ tree_aux($future[1:]) ] }
        else
            .
        end
    ;
    tree_aux($path)
;

# Merge two values. If the values are both objects, the values at equal indices
# will be recursively merged; if the values are both arrays, the arrays are
# concatenated; if the values are some other type, we only check if the values
# do not conflict.
def merge(other):
    if (. | type) == "object" and (other | type) == "object" then
        (. | to_entries) + (other | to_entries)
        | group_by(.key)
        | map({ "key": (.[0].key), "value" : (reduce .[1:][].value as $x (.[0].value; merge($x)))})
        | from_entries
    elif (. | type) == "array" and (other | type) == "array" then
        . + other
    elif . == other then
        .
    else
        error("trying to merge incompatible objects " + (.|type) + " and " + (other|type))
    end
;

# Merge an array of content by grouping it by name and then merging all the
# pages with the same name. This is necessary because simply merging content
# will only concatenate pages, even if they are the same page.
def merge_content:
    reduce .[] as $x ({}; merge($x)) |
    if has("contents") then
        .contents |= ( group_by(.name) | map(merge_content) )
    else
        .
    end
;

# Add paths to each object, that is, the names of the ancestors.
def add_directories:
    def add_directories_aux($history):
        ($history + [.name]) as $present |
        .directory = $history |
        (.contents[]? |= add_directories_aux($present))
    ;
    add_directories_aux([])
;

# Any page gets a link to its target.
def add_links(target_extension):
    if (has("directory") and (.name | endswith(".md"))) then
        .link = ((.directory + [.name]) | join("/") | rtrimstr(".md") | . + "." + target_extension)
    else
        .contents[]? |= add_links(target_extension)
    end
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def add_frontmatter:
    if has("contents") then
        ( .contents | map({(.name=="index.md" or .meta.frontmatter|tostring):.}) | group) as {$true, $false}
        | .frontmatter = $true[0]
        | .contents = (($false // []) + $true[1:])
        | (.contents[] |= add_frontmatter)
    else
        .
    end
;

# If marked as such, drafts can be excluded from being uploaded and included in
# the table of contents. To not count as a draft, a document should either be
# marked as `publish`, or be *not* marked as `draft` and have children that
# *are* marked as `publish`.
def add_drafts:
    def is_publication:
        ( .publish or .meta.publish | bool)
        or
        ( .external | bool )
        or
        ( (.meta.draft | bool | not) and ((.contents // []) | any(is_publication)) )
    ;
    if has("contents") then
        ( .contents | map({(is_publication|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .drafts = ($false // [])
        | (.contents[] |= add_drafts)
    else
        .
    end
;

# A resource is anything that is neither a directory nor a page with metadata
# nor an external link.
def add_resources:
    if has("contents") then
        ( .contents | map({(has("contents") or has("meta") or (has("external") and (.external | bool))|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .resources = ($false // [])
        | (.contents[]? |= add_resources)
    else
        .
    end
;

def add_formatted_dates:
    if has("contents") then
        .contents[]? |= add_formatted_dates
    elif has("meta") then
        .meta.formatted_date = (.meta.date | format_date)
    else
        .
    end
;

# Sort content first according to sort order in metadata.
def sort_content:
    if has("contents") then
        .contents |= (sort_by(.meta.sort // .meta.title // .name) | map(sort_content))
    else
        .
    end
;

# Add a note that tells us whether this has only children who have no more
# subchildren.
def annotate_leaves:
    if has("contents") then
        .only_leaves = (.contents | all(has("contents") | not))
        | (.contents |= map(annotate_leaves))
    else
        .
    end
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index(target_extension):
    merge_content
    | add_directories
    | add_links(target_extension)
    | add_drafts
    | add_frontmatter
    | add_resources
    | sort_content
    | annotate_leaves
;

# Get publishable targets from an index
def targets:
    recurse(.contents[]?)
    | ., (.frontmatter // empty)
    | select(has("link") and (has("external") | not))
    | .directory as $dir
    | [.link] + [.targets?[] | $dir + [.] | join("/") ]
    | .[]
;

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    (. == {} or . == [] or . == false or . == null or . == 0) | not
;

# Format a date into its wordy equivalent.
def format_date:
    try
    (
        (   try (. | strptime("%Y-%m-%d"))
            catch (
                try (. | strptime("%Y/%m/%d"))
                catch (
                    try (. | strptime("%Y-%m"))
                )
            )
        ) | mktime | strftime("%B %-d, %Y")
    )
;

# Group an array of objects into an object of arrays, such that the key-value
# pairs of the new object are an accumulation of those of the old object. For
# example, `[{"a":1, "b":2}, {"a":3}]` turns into `{"a":[1, 3], "b":[3]}`. This
# can be used to partition an array.
# Try: `[1,2,3] | map({(.%2|tostring):.}) | group`
def group:
    reduce ([.[] | to_entries[]] | group_by(.key))[] as $group
    ( {}
    ; .[ $group[0].key ] |= (. // []) + [ $group[].value ]
    )
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`.
def tree($path):
    def tree_aux($future):
        { "name": $future[0] } +
        if ($future | length) > 1 then
            { "contents": [ tree_aux($future[1:]) ] }
        else
            .
        end
    ;
    tree_aux($path)
;

# Merge two values. If the values are both objects, the values at equal indices
# will be recursively merged; if the values are both arrays, the arrays are
# concatenated; if the values are some other type, we only check if the values
# do not conflict.
def merge(other):
    if (. | type) == "object" and (other | type) == "object" then
        (. | to_entries) + (other | to_entries)
        | group_by(.key)
        | map({ "key": (.[0].key), "value" : (reduce .[1:][].value as $x (.[0].value; merge($x)))})
        | from_entries
    elif (. | type) == "array" and (other | type) == "array" then
        . + other
    elif . == other then
        .
    else
        error("trying to merge incompatible objects " + (.|type) + " and " + (other|type))
    end
;

# Merge an array of content by grouping it by name and then merging all the
# pages with the same name. This is necessary because simply merging content
# will only concatenate pages, even if they are the same page.
def merge_content:
    reduce .[] as $x ({}; merge($x)) |
    if has("contents") then
        .contents |= ( group_by(.name) | map(merge_content) )
    else
        .
    end
;

# Add paths to each object, that is, the names of the ancestors.
def add_directories:
    def add_directories_aux($history):
        ($history + [.name]) as $present |
        .directory = $history |
        (.contents[]? |= add_directories_aux($present))
    ;
    add_directories_aux([])
;

# Any page gets a link to its target.
def add_links(target_extension):
    if (has("directory") and (.name | endswith(".md"))) then
        .link = ((.directory + [.name]) | join("/") | rtrimstr(".md") | . + "." + target_extension)
    else
        .contents[]? |= add_links(target_extension)
    end
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def add_frontmatter:
    if has("contents") then
        ( .contents | map({(.name=="index.md" or .meta.frontmatter|tostring):.}) | group) as {$true, $false}
        | .frontmatter = $true[0]
        | .contents = (($false // []) + $true[1:])
        | (.contents[] |= add_frontmatter)
    else
        .
    end
;

# If marked as such, drafts can be excluded from being uploaded and included in
# the table of contents. To not count as a draft, a document should either be
# marked as `publish`, or be *not* marked as `draft` and have children that
# *are* marked as `publish`.
def add_drafts:
    def is_publication:
        ( .publish or .meta.publish | bool)
        or
        ( .external | bool )
        or
        ( (.meta.draft | bool | not) and ((.contents // []) | any(is_publication)) )
    ;
    if has("contents") then
        ( .contents | map({(is_publication|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .drafts = ($false // [])
        | (.contents[] |= add_drafts)
    else
        .
    end
;

# A resource is anything that is neither a directory nor a page with metadata
# nor an external link.
def add_resources:
    if has("contents") then
        ( .contents | map({(has("contents") or has("meta") or (has("external") and (.external | bool))|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .resources = ($false // [])
        | (.contents[]? |= add_resources)
    else
        .
    end
;

def add_formatted_dates:
    if has("contents") then
        .contents[]? |= add_formatted_dates
    elif has("meta") then
        .meta.formatted_date = (.meta.date | format_date)
    else
        .
    end
;

# Sort content first according to sort order in metadata.
def sort_content:
    if has("contents") then
        .contents |= (sort_by(.meta.sort // .meta.title // .name) | map(sort_content))
    else
        .
    end
;

# Add a note that tells us whether this has only children who have no more
# subchildren.
def annotate_leaves:
    if has("contents") then
        .only_leaves = (.contents | all(has("contents") | not))
        | (.contents |= map(annotate_leaves))
    else
        .
    end
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index(target_extension):
    merge_content
    | add_directories
    | add_links(target_extension)
    | add_drafts
    | add_frontmatter
    | add_resources
    | sort_content
    | annotate_leaves
;

# Get publishable targets from an index
def targets:
    recurse(.contents[]?)
    | ., (.frontmatter // empty)
    | select(has("link") and (has("external") | not))
    | .directory as $dir
    | [.link] + [.targets?[] | $dir + [.] | join("/") ]
    | .[]
;


A simplification.

A simplification.

jsoniq version "1.0";

module namespace accountant-converter = "http://28.io/modules/xbrl/profiles/accountant/converter";

(:~
 :
Flattens the row headers of the layout, leading to a rendering more familiar to
 : accountants.

 :
 : @param the original layout model.
 : @return the layout model with row headers flattened.
 :)
declare function accountant-converter:flatten-row-headers($layout-model as object) as object
{
  copy $l := $layout-model
  modify
    for $table as object in $l.TableSet[]
    let $breakdown as object := $table.TableHeaders.y[]
    let $headers as array := $breakdown.GroupCells
    let $first-header as array := $headers[[1]]
    let $other-headers as array := [ $headers[][position() gt 1] ]
    let $cells as array := $table.TableCells.Facts
    let $result as object := accountant-converter:flatten-headers({ Headers: $other-headers, Cells: $cells}, 1)
    return (replace value of json $breakdown.GroupCells with [
                copy $f := $first-header
                modify replace value of json $f[[1]].CellSpan with size($result.Headers[])
                return $f,
                $result.Headers[]
            ],
            replace value of json $table.TableCells.Facts with $result.Cells)
  return $l
};

declare %private function accountant-converter:flatten-headers(
    $headers-and-cells as object,
    $depth as integer) as object?
{
    switch(true)

    case size($headers-and-cells.Headers) le 1
    return copy $result := $headers-and-cells
           modify for $header in $result.Headers[][]
                  return (insert json { Depth: $depth } into $header,
                          delete json $header.CellSpan,
                          delete json $header.RollUp)
           return $result

    case size($headers-and-cells.Headers[[1]]) le 0
    return {
        Headers: null,
        Cells: $headers-and-cells.Cells
    }

    default return
        let $headers as array := $headers-and-cells.Headers
        let $cells as array := $headers-and-cells.Cells
        let $first-header as object := $headers[[1]][[1]]
        let $span as integer* := $first-header.CellSpan
        let $is-rightmost as boolean := size($headers) eq 1

        let $cells-to-the-right as array := accountant-converter:take-span([$headers[][position() gt 1]], $span)
        let $first-column-of-cells-to-the-right as object* := ($cells-to-the-right[[1]])[]
        let $has-rollup := exists($first-column-of-cells-to-the-right.RollUp[$$])
        let $roll-up-position as integer? :=
            switch(true)
            case not $has-rollup return 0
            case $first-column-of-cells-to-the-right[1].RollUp return 1
            default return $span
        let $roll-up-span as integer? :=
            for $cell in $first-column-of-cells-to-the-right
            where $cell.RollUp
            return $cell.CellSpan
        let $cells-to-the-right-no-rollup:= switch($roll-up-position)
                                            case 1 return accountant-converter:remove-span($cells-to-the-right, $roll-up-span)
                                            case $span return accountant-converter:take-span($cells-to-the-right, $span - $roll-up-span)
                                            case 0 return $cells-to-the-right
                                            default return []

        let $children-cells := [$cells[][position() le $span]]
        let $converted-children := accountant-converter:flatten-headers(
            {
                Headers: $cells-to-the-right-no-rollup,
                Cells: $children-cells
            },
            $depth + 1
        )

        let $bottom := accountant-converter:remove-span($headers, $span)
        let $bottom-cells := [$cells[][position() gt $span]]
        let $converted-bottom := accountant-converter:flatten-headers(
            {
                Headers: $bottom,
                Cells: $bottom-cells
            },
            $depth
        )

        let $new-first-header as object := copy $n := $first-header
                                           modify (insert json {
                                                        Depth: $depth,
                                                        IsAbstract: size($cells-to-the-right[[1]]) gt 1 or not $cells-to-the-right[[1]][[1]].RollUp
                                                   } into $n,
                                                   delete json $n.CellSpan,
                                                   delete json $n.RollUp)
                                           return $n

        return {
            (:OriginalCells: $cells,
            CellsToTheRight: $cells-to-the-right,
            RollUpPosition: $roll-up-position,
            RollUpSpan: $roll-up-span,
            CellsToTheRightNoRollUp: $cells-to-the-right-no-rollup,
            ChildrenCells: $children-cells,:)
            Headers: [
                [
                    $new-first-header[$roll-up-position = (0, 1)],
                    $converted-children.Headers[][],
                    $new-first-header[not $roll-up-position = (0, 1)],
                    $converted-bottom.Headers[][]
                ]
            ],
            Cells: [
                if(not $is-rightmost and $roll-up-position eq 0)
                then [ (1 to size($cells[[1]])) ! null ]
                else (),
                $converted-children.Cells[],
                $converted-bottom.Cells[]
            ]
        }
};

declare %private function accountant-converter:take-span($headers as array, $span as integer) as array
{
    [
        for $column as array in $headers[]
        return [ for $n in 1 to size($column)
                 let $first-n-headers := $column[][position() le $n]
                 let $sum := sum($first-n-headers.CellSpan)
                 where $sum eq $span
                 return $first-n-headers
        ]
    ]
};

declare %private function accountant-converter:remove-span($headers as array, $span as integer) as array
{
    [
        for $column as array in $headers[]
        return [ for $n in 1 to size($column)
                 let $first-n-headers := $column[][position() le $n]
                 let $sum := sum($first-n-headers ! ($$.CellSpan, 1)[1])
                 where $sum eq $span
                 return $column[][position() gt $n]
        ]
    ]
};

jsoniq version "1.0";

module namespace accountant-converter = "http://28.io/modules/xbrl/profiles/accountant/converter";

(:~
 :
Flattens the row headers of the layout, leading to a rendering more familiar to
 : accountants.

 :
 : @param the original layout model.
 : @return the layout model with row headers flattened.
 :)
declare function accountant-converter:flatten-row-headers($layout-model as object) as object
{
  copy $l := $layout-model
  modify
    for $table as object in $l.TableSet[]
    let $breakdown as object := $table.TableHeaders.y[]
    let $headers as array := $breakdown.GroupCells
    let $first-header as array := $headers[[1]]
    let $other-headers as array := [ $headers[][position() gt 1] ]
    let $cells as array := $table.TableCells.Facts
    let $result as object := accountant-converter:flatten-headers({ Headers: $other-headers, Cells: $cells}, 1)
    return (replace value of json $breakdown.GroupCells with [
                copy $f := $first-header
                modify replace value of json $f[[1]].CellSpan with size($result.Headers[])
                return $f,
                $result.Headers[]
            ],
            replace value of json $table.TableCells.Facts with $result.Cells)
  return $l
};

declare %private function accountant-converter:flatten-headers(
    $headers-and-cells as object,
    $depth as integer) as object?
{
    switch(true)

    case size($headers-and-cells.Headers) le 1
    return copy $result := $headers-and-cells
           modify for $header in $result.Headers[][]
                  return (insert json { Depth: $depth } into $header,
                          delete json $header.CellSpan,
                          delete json $header.RollUp)
           return $result

    case size($headers-and-cells.Headers[[1]]) le 0
    return {
        Headers: null,
        Cells: $headers-and-cells.Cells
    }

    default return
        let $headers as array := $headers-and-cells.Headers
        let $cells as array := $headers-and-cells.Cells
        let $first-header as object := $headers[[1]][[1]]
        let $span as integer* := $first-header.CellSpan
        let $is-rightmost as boolean := size($headers) eq 1

        let $cells-to-the-right as array := accountant-converter:take-span([$headers[][position() gt 1]], $span)
        let $first-column-of-cells-to-the-right as object* := ($cells-to-the-right[[1]])[]
        let $has-rollup := exists($first-column-of-cells-to-the-right.RollUp[$$])
        let $roll-up-position as integer? :=
            for tumbling window $cells in $first-column-of-cells-to-the-right
            start at $i when $i eq 1
            end $e when $e.Rollup
            return sum($cells.CellSpan)
        let $roll-up-position as integer := if($has-rollup) then $roll-up-position else 0
        let $roll-up-span as integer? :=
            for $cell in $first-column-of-cells-to-the-right
            where $cell.RollUp
            return $cell.CellSpan
        let $cells-to-the-right-no-rollup:= switch($roll-up-position)
                                            case 1 return accountant-converter:remove-span($cells-to-the-right, $roll-up-span)
                                            case $span return accountant-converter:take-span($cells-to-the-right, $span - $roll-up-span)
                                            case 0 return $cells-to-the-right
                                            default return []

        let $children-cells := [$cells[][position() le $span]]
        let $converted-children := accountant-converter:flatten-headers(
            {
                Headers: $cells-to-the-right-no-rollup,
                Cells: $children-cells
            },
            $depth + 1
        )

        let $bottom := accountant-converter:remove-span($headers, $span)
        let $bottom-cells := [$cells[][position() gt $span]]
        let $converted-bottom := accountant-converter:flatten-headers(
            {
                Headers: $bottom,
                Cells: $bottom-cells
            },
            $depth
        )

        let $new-first-header as object := copy $n := $first-header
                                           modify (insert json {
                                                        Depth: $depth,
                                                        IsAbstract: size($cells-to-the-right[[1]]) gt 1 or not $cells-to-the-right[[1]][[1]].RollUp
                                                   } into $n,
                                                   delete json $n.CellSpan,
                                                   delete json $n.RollUp)
                                           return $n

        return {
            (:OriginalCells: $cells,
            CellsToTheRight: $cells-to-the-right,
            RollUpPosition: $roll-up-position,
            RollUpSpan: $roll-up-span,
            CellsToTheRightNoRollUp: $cells-to-the-right-no-rollup,
            ChildrenCells: $children-cells,:)
            Headers: [
                [
                    $new-first-header[$roll-up-position = (0, 1)],
                    $converted-children.Headers[][],
                    $new-first-header[not $roll-up-position = (0, 1)],
                    $converted-bottom.Headers[][]
                ]
            ],
            Cells: [
                if(not $is-rightmost and $roll-up-position eq 0)
                then [ (1 to size($cells[[1]])) ! null ]
                else (),
                $converted-children.Cells[],
                $converted-bottom.Cells[]
            ]
        }
};

declare %private function accountant-converter:take-span($headers as array, $span as integer) as array
{
    [
        for $column as array in $headers[]
        return [ for $n in 1 to size($column)
                 let $first-n-headers := $column[][position() le $n]
                 let $sum := sum($first-n-headers.CellSpan)
                 where $sum eq $span
                 return $first-n-headers
        ]
    ]
};

declare %private function accountant-converter:remove-span($headers as array, $span as integer) as array
{
    [
        for $column as array in $headers[]
        return [ for $n in 1 to size($column)
                 let $first-n-headers := $column[][position() le $n]
                 let $sum := sum($first-n-headers ! ($$.CellSpan, 1)[1])
                 where $sum eq $span
                 return $column[][position() gt $n]
        ]
    ]
};


:pencil: Output periods as dateTimes on midnight UTC.

jsoniq version "1.0";

module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace backend = "http://apps.28.io/backend";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare %private function conversion:aspect-label(
    $aspect as string,
    $fact as object) as string
{
    ($fact.Labels.$aspect, $aspect)[1]
};

declare %private function conversion:aspect-value-or-label(
    $aspect as string,
    $fact as object) as atomic
{
    let $aspect-value as atomic := $fact.Aspects.($aspect)
    let $labels as object? := $fact.Labels
    return
        switch(true)
        case ($aspect-value instance of string and exists($labels.($aspect-value)))
          return $labels.($aspect-value)
        default return $aspect-value
};

declare function conversion:facts-to-csv(
    $facts as object*,
    $options as object?) as string?
{
    let $projection :=
        if($options.Caller eq "Report")
        then ("Value",
              "Decimals",
              "EntityRegistrantName",
              "ReportedConcept")
        else ("Value",
              "Type",
              "Decimals")
    let $use-labels as boolean := exists($facts.Labels)
    return if (exists($facts)) (: bug in csv:serialize :)
           then string-join(
                csv:serialize(
                    for $fact in $facts
                    return
                        {|
                            if($use-labels)
                            then
                                for $aspect as string in keys($fact.Aspects)[$$ ne "sec:Archive"]
                                let $aspect-label as string := conversion:aspect-label($aspect, $fact)
                                let $aspect-value as atomic := conversion:aspect-value-or-label($aspect, $fact)
                                return
                                    {
                                        $aspect-label : $aspect-value
                                    }
                            else
                                $fact.Aspects,
                            project($fact, $projection)
                        |},
                    { serialize-null-as : "" }
                )
            )
            else ()
};

declare function conversion:facts-to-xml(
    $facts as object*,
    $options as object?) as element()*
{
    for $fact in $facts
    let $aspects := $fact.Aspects
    let $use-labels as boolean := exists($facts.Labels)
    return
        <Fact>{
            <Aspects>{
                for $aspect in keys($aspects)
                return
                    <Aspect>
                        <Name>{
                            if($use-labels)
                            then attribute { "label" } { conversion:aspect-label($aspect, $fact) }
                            else (),
                            $aspect
                        }</Name>
                        <Value>{
                            if($use-labels)
                            then attribute { "label" } { conversion:aspect-value-or-label($aspect, $fact) }
                            else (),
                            $aspects.$aspect
                        }</Value>
                    </Aspect>
            }</Aspects>,
            if($options.Caller eq "Report")
            then (
                <Value>{$fact.Value}</Value>,
                <Type>{$fact.Type}</Type>,
                <Decimals>{$fact.Decimals}</Decimals>[exists($fact.Decimals)],
                <EntityRegistrantName>{$fact.EntityRegistrantName}</EntityRegistrantName>,
                conversion:audittrail-to-xml($fact.AuditTrails)[exists($fact.AuditTrails)]
            ) else (
                <Type>{$fact.Type}</Type>,
                <Value>{$fact.Value}</Value>,
                <Decimals>{$fact.Decimals}</Decimals>[exists($fact.Decimals)]
            )
        }</Fact>
};

declare function conversion:facts-to-odata(
  $facts as object*,
  $hypercube as object,
  $parameters as object?) as node()*
{
    for $fact in $facts
    let $entity-registrant-name := $fact.Labels.($fact.Aspects."xbrl:Entity")
    let $id := backend:url("facts", {|
      {
        format: "atom-xml"
      },
      $parameters,
      for $key in $fact.KeyAspects[]
      return {
        $key: string($fact.Aspects.$key)
      }
    |})
    return
       <entry xmlns='http://www.w3.org/2005/Atom'
        xmlns:m='http://docs.oasis-open.org/odata/ns/metadata'
        xmlns:d='http://docs.oasis-open.org/odata/ns/data'
        xml:base='http://www.28.io/'>
        <id>{$id}</id>
        <link rel="self">{$id}</link>
        <category term='#Model.Fact' scheme='http://docs.oasis-open.org/odata/ns/scheme'/>
        <title/>
        <updated>{current-dateTime()}</updated>
        <author><name>28msec, Inc.</name></author>
        <content type='application/xml'>
            <m:properties>{
            <d:EntityRegistrantName>{$entity-registrant-name}</d:EntityRegistrantName>,
            for $key in keys($fact.Aspects)
            let $value := $fact.Aspects.$key
            let $type as string :=
              switch(true)
              case $hypercube.Aspects.$key.Kind eq "TypedDimension" and
                   $hypercube.Aspects.$key.Type eq "integer" and
                        $fact.Aspects.$key castable as int
                return "Edm.Int32"
              case $hypercube.Aspects.$key.Kind eq "TypedDimension" and
                   $hypercube.Aspects.$key.Type eq "integer"
                return "Edm.Integer"
              case $hypercube.Aspects.$key.Kind eq "TypedDimension" and
                   $hypercube.Aspects.$key.Type eq "boolean"
                return "Edm.Boolean"
              case $key eq "xbrl:Period" and
                   $fact.Aspects.$key castable as date
                return "Edm.DateTime"
              default
                return "Edm.String"
            return
                element { "d:" || substring-after($key, ":") }
                {
                  attribute m:type { $type },
                  switch($type)
                  case "Edm.String" return ($fact.Labels.$value, $value)[1]
                  case "Edm.DateTime" return $value || "T00:00:00Z"
                  default return $value
                },
            element {"d:Value"} { attribute { "m:type" } {
                    switch ($fact.Concept.DataType)
                    case "xbrli:booleanItemType" return "Edm.Boolean"
                    case "xbrli:base64BinaryItemType" return "Edm.Binary"
                    case "xbrli:dateTimeItemType" return "Edm.DateTime"
                    case "xbrli:timeItemType" return "Edm.Time"
                    case "xbrli:stringItemType"
                    case "xbrli:anyURIItemType" return "Edm.String"
                    case "xbrli:sharesItemType"
                    case "xbrli:pureItemType"
                    case "xbrli:decimalItemType"
                    case "xbrli:monetaryItemType" return "Edm.Decimal"
                    case "xbrli:floatItemType" return "Edm.Single"
                    case "xbrli:doubleItemType" return "Edm.Double"
                    case "xbrli:integerItemType"
                    case "xbrli:nonPositiveIntegerItemType"
                    case "xbrli:negativeIntegerItemType"
                    case "xbrli:nonNegativeIntegerItemType"
                    case "xbrli:unsignedLongItemType"
                    case "xbrli:unsignedIntItemType"
                    case "xbrli:unsignedShortItemType"
                    case "xbrli:positiveIntegerItemType" return "Edm.Integer"
                    case "xbrli:unsignedByteItemType" return "Edm.Byte"
                    case "xbrli:longItemType" return "Edm.Int64"
                    case "xbrli:intItemType" return "Edm.Int32"
                    case "xbrli:shortItemType" return "Edm.Int16"
                    case "xbrli:byteItemType" return "Edm.SByte"
                    default return "Edm.String"
                }, ($fact.Labels.($fact.Value), $fact.Value)[1] }
            }</m:properties>
        </content>
        </entry>
};

declare function conversion:audittrail-to-xml($audit as item) as element()
{
    <AuditTrails>{
        for $a in flatten($audit)
        return (
            <Type>{$a.Type}</Type>,
            <Label>{$a.Label}</Label>,
            <Message>{$a.Message}</Message>,
            <Data>{
                if (exists($a.Data.OriginalConcept))
                then <OriginalConcept>{$a.Data.OriginalConcept}</OriginalConcept>
                else (),
                if (exists($a.Data.OutputConcept))
                then <OutputConcept>{$a.Data.OutputConcept}</OutputConcept>
                else (),
                if (exists($a.Data.AuditTrails))
                then conversion:audittrail-to-xml($a.Data.AuditTrails)
                else ()
            }</Data>
        )
    }</AuditTrails>
};

jsoniq version "1.0";

module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace backend = "http://apps.28.io/backend";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare %private function conversion:aspect-label(
    $aspect as string,
    $fact as object) as string
{
    ($fact.Labels.$aspect, $aspect)[1]
};

declare %private function conversion:aspect-value-or-label(
    $aspect as string,
    $fact as object) as atomic
{
    let $aspect-value as atomic := $fact.Aspects.($aspect)
    let $labels as object? := $fact.Labels
    return
        switch(true)
        case ($aspect-value instance of string and exists($labels.($aspect-value)))
          return $labels.($aspect-value)
        default return $aspect-value
};

declare function conversion:facts-to-csv(
    $facts as object*,
    $options as object?) as string?
{
    let $projection :=
        if($options.Caller eq "Report")
        then ("Value",
              "Decimals",
              "EntityRegistrantName",
              "ReportedConcept")
        else ("Value",
              "Type",
              "Decimals")
    let $use-labels as boolean := exists($facts.Labels)
    return if (exists($facts)) (: bug in csv:serialize :)
           then string-join(
                csv:serialize(
                    for $fact in $facts
                    return
                        {|
                            if($use-labels)
                            then
                                for $aspect as string in keys($fact.Aspects)[$$ ne "sec:Archive"]
                                let $aspect-label as string := conversion:aspect-label($aspect, $fact)
                                let $aspect-value as atomic := conversion:aspect-value-or-label($aspect, $fact)
                                return
                                    {
                                        $aspect-label : $aspect-value
                                    }
                            else
                                $fact.Aspects,
                            project($fact, $projection)
                        |},
                    { serialize-null-as : "" }
                )
            )
            else ()
};

declare function conversion:facts-to-xml(
    $facts as object*,
    $options as object?) as element()*
{
    for $fact in $facts
    let $aspects := $fact.Aspects
    let $use-labels as boolean := exists($facts.Labels)
    return
        <Fact>{
            <Aspects>{
                for $aspect in keys($aspects)
                return
                    <Aspect>
                        <Name>{
                            if($use-labels)
                            then attribute { "label" } { conversion:aspect-label($aspect, $fact) }
                            else (),
                            $aspect
                        }</Name>
                        <Value>{
                            if($use-labels)
                            then attribute { "label" } { conversion:aspect-value-or-label($aspect, $fact) }
                            else (),
                            $aspects.$aspect
                        }</Value>
                    </Aspect>
            }</Aspects>,
            if($options.Caller eq "Report")
            then (
                <Value>{$fact.Value}</Value>,
                <Type>{$fact.Type}</Type>,
                <Decimals>{$fact.Decimals}</Decimals>[exists($fact.Decimals)],
                <EntityRegistrantName>{$fact.EntityRegistrantName}</EntityRegistrantName>,
                conversion:audittrail-to-xml($fact.AuditTrails)[exists($fact.AuditTrails)]
            ) else (
                <Type>{$fact.Type}</Type>,
                <Value>{$fact.Value}</Value>,
                <Decimals>{$fact.Decimals}</Decimals>[exists($fact.Decimals)]
            )
        }</Fact>
};

declare function conversion:facts-to-odata(
  $facts as object*,
  $hypercube as object,
  $parameters as object?) as node()*
{
    for $fact in $facts
    let $entity-registrant-name := $fact.Labels.($fact.Aspects."xbrl:Entity")
    let $id := backend:url("facts", {|
      {
        format: "atom-xml"
      },
      $parameters,
      for $key in $fact.KeyAspects[]
      return {
        $key: string($fact.Aspects.$key)
      }
    |})
    return
       <entry xmlns='http://www.w3.org/2005/Atom'
        xmlns:m='http://docs.oasis-open.org/odata/ns/metadata'
        xmlns:d='http://docs.oasis-open.org/odata/ns/data'
        xml:base='http://www.28.io/'>
        <id>{$id}</id>
        <link rel="self">{$id}</link>
        <category term='#Model.Fact' scheme='http://docs.oasis-open.org/odata/ns/scheme'/>
        <title/>
        <updated>{current-dateTime()}</updated>
        <author><name>28msec, Inc.</name></author>
        <content type='application/xml'>
            <m:properties>{
            <d:EntityRegistrantName>{$entity-registrant-name}</d:EntityRegistrantName>,
            for $key in keys($fact.Aspects)
            let $value := $fact.Aspects.$key
            return
                element { "d:" || substring-after($key, ":") }
                {
                  attribute m:type {
                    switch(true)
                    case $hypercube.Aspects.$key.Kind eq "TypedDimension" and
                         $hypercube.Aspects.$key.Type eq "integer" and
                              $fact.Aspects.$key castable as int
                      return "Edm.Int32"
                    case $hypercube.Aspects.$key.Kind eq "TypedDimension" and
                         $hypercube.Aspects.$key.Type eq "integer"
                      return "Edm.Integer"
                    case $hypercube.Aspects.$key.Kind eq "TypedDimension" and
                         $hypercube.Aspects.$key.Type eq "boolean"
                      return "Edm.Boolean"
                    case $key eq "xbrl:Period" and
                         $fact.Aspects.$key castable as date
                      return "Edm.DateTime"
                    default
                      return "Edm.String"
                  },
                  ($fact.Labels.$value, $value)[1]
                },
            element {"d:Value"} { attribute { "m:type" } {
                    switch ($fact.Concept.DataType)
                    case "xbrli:booleanItemType" return "Edm.Boolean"
                    case "xbrli:base64BinaryItemType" return "Edm.Binary"
                    case "xbrli:dateTimeItemType" return "Edm.DateTime"
                    case "xbrli:timeItemType" return "Edm.Time"
                    case "xbrli:stringItemType"
                    case "xbrli:anyURIItemType" return "Edm.String"
                    case "xbrli:sharesItemType"
                    case "xbrli:pureItemType"
                    case "xbrli:decimalItemType"
                    case "xbrli:monetaryItemType" return "Edm.Decimal"
                    case "xbrli:floatItemType" return "Edm.Single"
                    case "xbrli:doubleItemType" return "Edm.Double"
                    case "xbrli:integerItemType"
                    case "xbrli:nonPositiveIntegerItemType"
                    case "xbrli:negativeIntegerItemType"
                    case "xbrli:nonNegativeIntegerItemType"
                    case "xbrli:unsignedLongItemType"
                    case "xbrli:unsignedIntItemType"
                    case "xbrli:unsignedShortItemType"
                    case "xbrli:positiveIntegerItemType" return "Edm.Integer"
                    case "xbrli:unsignedByteItemType" return "Edm.Byte"
                    case "xbrli:longItemType" return "Edm.Int64"
                    case "xbrli:intItemType" return "Edm.Int32"
                    case "xbrli:shortItemType" return "Edm.Int16"
                    case "xbrli:byteItemType" return "Edm.SByte"
                    default return "Edm.String"
                }, ($fact.Labels.($fact.Value), $fact.Value)[1] }
            }</m:properties>
        </content>
        </entry>
};

declare function conversion:audittrail-to-xml($audit as item) as element()
{
    <AuditTrails>{
        for $a in flatten($audit)
        return (
            <Type>{$a.Type}</Type>,
            <Label>{$a.Label}</Label>,
            <Message>{$a.Message}</Message>,
            <Data>{
                if (exists($a.Data.OriginalConcept))
                then <OriginalConcept>{$a.Data.OriginalConcept}</OriginalConcept>
                else (),
                if (exists($a.Data.OutputConcept))
                then <OutputConcept>{$a.Data.OutputConcept}</OutputConcept>
                else (),
                if (exists($a.Data.AuditTrails))
                then conversion:audittrail-to-xml($a.Data.AuditTrails)
                else ()
            }</Data>
        )
    }</AuditTrails>
};


Update facts.jq

Update facts.jq

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facts"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-empty($params as object) as item
{
    let $endpoint := "facts"
    let $res as object := test:invoke-raw($endpoint, $params)
    return if($res.status eq 200 and $res.headers."Content-Length" eq "0") then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res
    }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw($endpoint, $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "Accession Number,Concept,Entity,Period,Fiscal Period,Fiscal Year,Accepted,Legal Entity,Unit,Value,Decimals",
        "0000021344-13-000050,\"Cash and Cash Equivalents, at Carrying Value\",COCA COLA CO,2013-09-27,Q3,2013,20131024121047,Default Legal Entity,USD,11118000000,-6",
        "0000021344-13-000050,Assets,COCA COLA CO,2013-09-27,Q3,2013,20131024121047,Default Legal Entity,USD,89432000000,-6"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res
    }
};

declare %an:nondeterministic function local:test-labels-aids() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: "disc:AdvertisingCostsPolicyTextBlock",
        map: "Disclosures",
        format: "json",
        fiscalYear: 2014,
        fiscalPeriod: "FY",
        labels: true,
        aid: "0000858877-14-000029"
    }
    let $res := test:invoke($endpoint, $params)
    let $actual := [
            for $labels in $res[2].FactTable[].Labels
            return (keys($labels) ! $labels.$$)
        ]
    let $expected := [ "Advertising Costs, Policy", "Default Legal Entity", "Accession Number", "Concept", "Entity", "CISCO SYSTEMS, INC.", "Period", "Fiscal Period", "Fiscal Year", "Accepted", "Unit", "Legal Entity" ]
    let $status as integer := $res[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

local:check({
    cocacola: local:test-facttable(468, {
        ticker:"ko"
    }),
    cocacola-all: local:test-facttable(468, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-instant: local:test-facttable(163, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant" ]
    }),
    cocacola-ytd: local:test-facttable(305, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-qtd: local:test-facttable(0, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "Q2" ],
        fiscalPeriodType: [ "QTD" ]
    }),
    cocacolaCSVLabels: local:test-labels(),
    ciscoLabelsByAid: local:test-labels-aids(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(30, {
        tag:"DOW30",
        concept:"us-gaap:Assets"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-empty({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    }),
    quartersfact: local:test-facttable(0, { (: this fact is not reported in the raw XML filing :)
        aid: "0000051143-13-000007",
        concept: "disc:AssetRetirementObligationsPolicy",
        fiscalPeriod: "Q3",
        fiscalYear: "2013",
        map: "Disclosures"
    }),
    quartersfact2: local:test-facttable(1, {
        aid: "0001193125-12-207154",
        concept: "disc:CashFlowOperatingCapitalTableTextBlock",
        fiscalPeriod: "Q1",
        fiscalYear: "2012",
        map: "Disclosures"
    }),
    generic: local:test-facttable(2, {
        "xbrl:Entity":"http://www.sec.gov/CIK%200000021344",
        "sec:Accepted":"20140227132423",
        concept:"us-gaap:Assets",
        profile-name: "generic"
    })
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facts"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-empty($params as object) as item
{
    let $endpoint := "facts"
    let $res as object := test:invoke-raw($endpoint, $params)
    return if($res.status eq 200 and $res.headers."Content-Length" eq "0") then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res
    }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw($endpoint, $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "Accession Number,Concept,Entity,Period,Fiscal Period,Fiscal Year,Accepted,Legal Entity,Unit,Value,Decimals",
        "0000021344-13-000050,\"Cash and Cash Equivalents, at Carrying Value\",COCA COLA CO,2013-09-27,Q3,2013,20131024121047,Default Legal Entity,USD,11118000000,-6",
        "0000021344-13-000050,Assets,COCA COLA CO,2013-09-27,Q3,2013,20131024121047,Default Legal Entity,USD,89432000000,-6"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res
    }
};

declare %an:nondeterministic function local:test-labels-aids() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: "disc:AdvertisingCostsPolicyTextBlock",
        map: "Disclosures",
        format: "json",
        fiscalYear: 2014,
        fiscalPeriod: "FY",
        labels: true,
        aid: "0000858877-14-000029"
    }
    let $res := test:invoke($endpoint, $params)
    let $actual := [
            for $labels in $res[2].FactTable[].Labels
            return (keys($labels) ! $labels.$$)
        ]
    let $expected := [ "Advertising Costs, Policy", "Default Legal Entity", "Accession Number", "Concept", "Entity", "CISCO SYSTEMS, INC.", "Period", "Fiscal Period", "Fiscal Year", "Accepted", "Unit", "Legal Entity" ]
    let $status as integer := $res[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

local:check({
    cocacola: local:test-facttable(468, {
        ticker:"ko"
    }),
    cocacola-all: local:test-facttable(468, {
        ticker:"ko",
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-instant: local:test-facttable(468, {
        ticker:"ko",
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant" ]
    }),
    cocacola-ytd: local:test-facttable(468, {
        ticker:"ko",
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-qtd: local:test-facttable(468, {
        ticker:"ko",
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "QTD" ]
    }),
    cocacolaCSVLabels: local:test-labels(),
    ciscoLabelsByAid: local:test-labels-aids(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(30, {
        tag:"DOW30",
        concept:"us-gaap:Assets"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-empty({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    }),
    quartersfact: local:test-facttable(0, { (: this fact is not reported in the raw XML filing :)
        aid: "0000051143-13-000007",
        concept: "disc:AssetRetirementObligationsPolicy",
        fiscalPeriod: "Q3",
        fiscalYear: "2013",
        map: "Disclosures"
    }),
    quartersfact2: local:test-facttable(1, {
        aid: "0001193125-12-207154",
        concept: "disc:CashFlowOperatingCapitalTableTextBlock",
        fiscalPeriod: "Q1",
        fiscalYear: "2012",
        map: "Disclosures"
    }),
    generic: local:test-facttable(2, {
        "xbrl:Entity":"http://www.sec.gov/CIK%200000021344",
        "sec:Accepted":"20140227132423",
        concept:"us-gaap:Assets",
        profile-name: "generic"
    })
})