fix CIK not returned in XML result

fix CIK not returned in XML result

jsoniq version "1.0";

import module namespace components = "http://xbrl.io/modules/bizql/components";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace filings = "http://xbrl.io/modules/bizql/profiles/sec/filings";
import module namespace fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace sec = "http://xbrl.io/modules/bizql/profiles/sec/core";
import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";

import module namespace sec-networks = "http://xbrl.io/modules/bizql/profiles/sec/networks";
import module namespace networks = "http://xbrl.io/modules/bizql/networks";
import module namespace concept-maps = "http://xbrl.io/modules/bizql/concept-maps";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

declare function local:to-csv($o as object*) as string
{
    string-join(
        csv:serialize(
            for $o in $o
            let $a := $o.Aspects
            return {|
                (for $k in keys($a) return { $k : $a.$k }),
                { "Unit" :  $o.Unit },
                { "Value" : $o.Value },
                { "Type" : $o.Type },
                { "Decimals" : $o.Decimals }
            |},
            { serialize-null-as : "" }
        )
    )
};

declare function local:to-xml($o as object*) as element()*
{
    for $o in $o
    let $a := $o.Aspects
    return
        <Fact>{
            <Aspects>{
                for $k in keys($a)
                return
                    <Aspect>
                        <Name>{$k}</Name>
                        <Value>{$a.$k}</Value>
                    </Aspect>
            }</Aspects>,
            <Type>{$o.Type}</Type>,
            <Value>{$o.Value}</Value>,
            if (exists($o.Unit))
            then <Unit>{$o.Unit}</Unit>
            else(),
            if (exists($o.Decimals))
            then <Decimals>{$o.Decimals}</Decimals>
            else ()
        }</Fact>
};

declare function local:components-by-disclosures($disclosures as string*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    for $aid in $aids
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: $aid,
            "Profiles.SEC.Disclosure": { "$in" : [ $disclosures ] }
        })
};

declare function local:components-by-roles($roles as string*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: { "$in" : [ $aids ] },
            "Role": { "$in" : [ $roles ] }
        })
};

declare function local:components-by-concepts($concepts as string*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    let $ids := mongo:find($conn, "concepts",
        {|
            (
                { "Name" : { "$in" : [ $concepts ] } },
                { "Archive" : { "$in" : [ $aids ] } }
            )
        |}).Component
    return components:components($ids)
};

declare function local:filings(
    $ciks as string*,
    $tags as string*,
    $tickers as string*,
    $sics as string*,
    $fp as string*,
    $fy as string*) as object*
{
    let $entities := if ($tags = "ALL") then companies:companies()
                                        else (
                                            companies:companies($ciks),
                                            companies:companies-for-tags($tags),
                                            companies:companies-for-tickers($tickers),
                                            companies:companies-for-SIC($sics)
                                        )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                        case "ALL" return  $fiscal:ALL_FISCAL_YEARS
                    default return $fy cast as integer
                )
    for $fp in $fp
    return fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy)
};

(:
 : This function needs to go into the core. It returns all facts for a network
 : - not only the ones for the specific FY/FP
 :)
declare function local:facts($networks-or-ids as item*, $options as object?)
as object*
{
  for $component as object in components:components($networks-or-ids)
  for $table as string? allowing empty in sec-networks:tables($component).Name
  let $hypercube as object? := hypercubes:hypercubes-for-components($component, $table)
  let $hypercube as object := if (exists($hypercube))
                              then
                                if(exists($hypercube.Aspects."dei:LegalEntityAxis"))
                                then $hypercube
                                else
                                  copy $h := $hypercube
                                  modify (
                                    insert json { "dei:LegalEntityAxis" :
                                      {
                                        "Name": "dei:LegalEntityAxis",
                                        "Label": "Legal Entity Dimension",
                                        "Default" : "sec:DefaultLegalEntity"
                                      }} into $h.Aspects
                                    )
                                  return $h
                              else sec:dimensionless-hypercube({
                                  Concepts: [ sec-networks:line-items($component).Name ]
                              })
  let $facts := hypercubes:facts-for-hypercube(
    $hypercube,
    $component.Archive,
    $options
  )
  let $hide-amended-facts as boolean :=
    if (exists($options.HideAmendedFacts))
    then $options.HideAmendedFacts
    else true (: default :)
  return
    if ($hide-amended-facts)
    then sec:hide-amended-facts($facts)
    else $facts
};

session:audit-call();

let $format      := lower-case((request:param-values("format"), substring-after(request:path(), ".jq."))[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids        := archives:aid(request:param-values("aid"))
let $roles       := request:param-values("networkIdentifier")
let $archives    := (
                        local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                        archives:archives($aids)
                    )
let $cid         := request:param-values("cid")
let $concepts    := distinct-values(request:param-values("concept"))
let $rollup      := distinct-values(request:param-values("rollup"))
let $map         := request:param-values("map")
let $disclosures := request:param-values("disclosure")
let $components  := (if (exists($cid))
                    then components:components($cid)
                    else (),
                    if (exists($concepts) or exists($disclosures) or exists($roles))
                    then (
                            local:components-by-concepts($concepts, $archives._id),
                            local:components-by-disclosures($disclosures, $archives._id),
                            local:components-by-roles($roles, $archives._id)
                        )
                    else components:components-for-archives($archives._id))
let $component := $components[1] (: only one for know :)
let $archive   := archives:archives($component.Archive)
let $entity    := entities:entities($archive.Entity)

return
     if (session:only-dow30($entity) or session:valid())
     then {
        let $facts := if (exists($rollup))
                     then
                         let $calc-network := networks:networks-for-components-and-short-names($component, $networks:CALCULATION_NETWORK)
                         let $hc := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube") (: sec-networks:tables($component).Name) :)
                         let $p := hypercubes:populate-networks-with-facts($calc-network, $hc, $archive)
                         let $map := concept-maps:concept-maps($map)
                         let $concepts :=
                            if (exists($map))
                            then
                                for $d in $rollup
                                return
                                    keys(descendant-objects($p)[$$.Name = keys($map.Trees($d).To)][1].To)
                            else
                                for $d in $rollup
                                return ($d, keys(descendant-objects($p)[$$.Name eq $d].To))
                         return sec:facts-for-archives-and-concepts($archive, $concepts, { Hypercube: $hc })
                     else local:facts($component, {| |})
        let $fact-table :=  for $f in $facts
                            let $a := $f.Aspects
                            return {|
                                { "Aspects" : {|
                                    { "xbrl:Entity": $a."xbrl:Entity" },
                                    if (exists($a."dei:LegalEntityAxis")) then { "dei:LegalEntityAxis": $a."dei:LegalEntityAxis" } else (),
                                    { "xbrl:Period" : $a."xbrl:Period" },
                                    { "xbrl:Concept" : $a."xbrl:Concept" },
                                    for $k in keys($f.Aspects)
                                    where $k ne "xbrl:Unit" and $k ne "xbrl:Entity" and $k ne "dei:LegalEntityAxis" and
                                          $k ne "xbrl:Period" and $k ne "xbrl:Concept"
                                    return { $k : $f.Aspects.$k }
                                |} },
                                { "Type" : $f.Type },
                                { "Value" : $f.Value },
                                if (exists($f.Decimals))
                                then { "Decimals" : $f.Decimals }
                                else (),
                                if (exists($f."Aspects"."xbrl:Unit"))
                                then { "Unit" : $f."Aspects"."xbrl:Unit" }
                                else ()
                            |}
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                (session:comment("xml", {
                            NumFacts : count($fact-table),
                            TotalNumFacts: session:num-facts(),
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        }),
                <FactTable entityRegistrantName="{$entity.Profiles.SEC.CompanyName}"
                    cik="{$entity._id}"
                    tableName="{sec-networks:tables($component, {IncludeImpliedTable: true}).Name}"
                    label="{$component.Label}"
                    accessionNumber="{$component.Archive}"
                    networkIdentifier="{$component.Role}"
                    formType="{$archive.Profiles.SEC.FormType}"
                    fiscalPeriod="{$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus}"
                    fiscalYear="{$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus}"
                    acceptanceDatetime="{filings:acceptance-dateTimes($archive)}"
                    disclosure="{$component.Profiles.SEC.Disclosure}"
                    >{
                    local:to-xml($fact-table)
                }</FactTable>)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=facttable-" || $cid || ".csv");
                local:to-csv($fact-table)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=facttable-" || $cid || ".csv");
                local:to-csv($fact-table)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { CIK : $entity._id },
                    { EntityRegistrantName : $entity.Profiles.SEC.CompanyName },
                    { TableName : sec-networks:tables($component, {IncludeImpliedTable: true}).Name },
                    { Label : $component.Label },
                    { AccessionNumber : $component.Archive },
                    { FormType : $archive.Profiles.SEC.FormType },
                    { FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus },
                    { FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus },
                    { AcceptanceDatetime : filings:acceptance-dateTimes($archive) },
                    { NetworkIdentifier: $component.Role },
                    { Disclosure : $component.Profiles.SEC.Disclosure },
                    { FactTable : [ $fact-table ] },
                    session:comment("json", {
                            NumFacts : count($fact-table),
                            TotalNumFacts: session:num-facts(),
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })
                |}
            }
     } else {
        response:status-code(401);
        session:error("accessing fact table for an entity that is not in the DOW30", $format)
     }

jsoniq version "1.0";

import module namespace components = "http://xbrl.io/modules/bizql/components";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace filings = "http://xbrl.io/modules/bizql/profiles/sec/filings";
import module namespace fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace sec = "http://xbrl.io/modules/bizql/profiles/sec/core";
import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";

import module namespace sec-networks = "http://xbrl.io/modules/bizql/profiles/sec/networks";
import module namespace networks = "http://xbrl.io/modules/bizql/networks";
import module namespace concept-maps = "http://xbrl.io/modules/bizql/concept-maps";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

declare function local:to-csv($o as object*) as string
{
    string-join(
        csv:serialize(
            for $o in $o
            let $a := $o.Aspects
            return {|
                (for $k in keys($a) return { $k : $a.$k }),
                { "Unit" :  $o.Unit },
                { "Value" : $o.Value },
                { "Type" : $o.Type },
                { "Decimals" : $o.Decimals }
            |},
            { serialize-null-as : "" }
        )
    )
};

declare function local:to-xml($o as object*) as element()*
{
    for $o in $o
    let $a := $o.Aspects
    return
        <Fact>{
            <Aspects>{
                for $k in keys($a)
                return
                    <Aspect>
                        <Name>{$k}</Name>
                        <Value>{$a.$k}</Value>
                    </Aspect>
            }</Aspects>,
            <Type>{$o.Type}</Type>,
            <Value>{$o.Value}</Value>,
            if (exists($o.Unit))
            then <Unit>{$o.Unit}</Unit>
            else(),
            if (exists($o.Decimals))
            then <Decimals>{$o.Decimals}</Decimals>
            else ()
        }</Fact>
};

declare function local:components-by-disclosures($disclosures as string*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    for $aid in $aids
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: $aid,
            "Profiles.SEC.Disclosure": { "$in" : [ $disclosures ] }
        })
};

declare function local:components-by-roles($roles as string*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: { "$in" : [ $aids ] },
            "Role": { "$in" : [ $roles ] }
        })
};

declare function local:components-by-concepts($concepts as string*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    let $ids := mongo:find($conn, "concepts",
        {|
            (
                { "Name" : { "$in" : [ $concepts ] } },
                { "Archive" : { "$in" : [ $aids ] } }
            )
        |}).Component
    return components:components($ids)
};

declare function local:filings(
    $ciks as string*,
    $tags as string*,
    $tickers as string*,
    $sics as string*,
    $fp as string*,
    $fy as string*) as object*
{
    let $entities := if ($tags = "ALL") then companies:companies()
                                        else (
                                            companies:companies($ciks),
                                            companies:companies-for-tags($tags),
                                            companies:companies-for-tickers($tickers),
                                            companies:companies-for-SIC($sics)
                                        )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                        case "ALL" return  $fiscal:ALL_FISCAL_YEARS
                    default return $fy cast as integer
                )
    for $fp in $fp
    return fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy)
};

(:
 : This function needs to go into the core. It returns all facts for a network
 : - not only the ones for the specific FY/FP
 :)
declare function local:facts($networks-or-ids as item*, $options as object?)
as object*
{
  for $component as object in components:components($networks-or-ids)
  for $table as string? allowing empty in sec-networks:tables($component).Name
  let $hypercube as object? := hypercubes:hypercubes-for-components($component, $table)
  let $hypercube as object := if (exists($hypercube))
                              then
                                if(exists($hypercube.Aspects."dei:LegalEntityAxis"))
                                then $hypercube
                                else
                                  copy $h := $hypercube
                                  modify (
                                    insert json { "dei:LegalEntityAxis" :
                                      {
                                        "Name": "dei:LegalEntityAxis",
                                        "Label": "Legal Entity Dimension",
                                        "Default" : "sec:DefaultLegalEntity"
                                      }} into $h.Aspects
                                    )
                                  return $h
                              else sec:dimensionless-hypercube({
                                  Concepts: [ sec-networks:line-items($component).Name ]
                              })
  let $facts := hypercubes:facts-for-hypercube(
    $hypercube,
    $component.Archive,
    $options
  )
  let $hide-amended-facts as boolean :=
    if (exists($options.HideAmendedFacts))
    then $options.HideAmendedFacts
    else true (: default :)
  return
    if ($hide-amended-facts)
    then sec:hide-amended-facts($facts)
    else $facts
};

session:audit-call();

let $format      := lower-case((request:param-values("format"), substring-after(request:path(), ".jq."))[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids        := archives:aid(request:param-values("aid"))
let $roles       := request:param-values("networkIdentifier")
let $archives    := (
                        local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                        archives:archives($aids)
                    )
let $cid         := request:param-values("cid")
let $concepts    := distinct-values(request:param-values("concept"))
let $rollup      := distinct-values(request:param-values("rollup"))
let $map         := request:param-values("map")
let $disclosures := request:param-values("disclosure")
let $components  := (if (exists($cid))
                    then components:components($cid)
                    else (),
                    if (exists($concepts) or exists($disclosures) or exists($roles))
                    then (
                            local:components-by-concepts($concepts, $archives._id),
                            local:components-by-disclosures($disclosures, $archives._id),
                            local:components-by-roles($roles, $archives._id)
                        )
                    else components:components-for-archives($archives._id))
let $component := $components[1] (: only one for know :)
let $archive   := archives:archives($component.Archive)
let $entity    := entities:entities($archive.Entity)

return
     if (session:only-dow30($entity) or session:valid())
     then {
        let $facts := if (exists($rollup))
                     then
                         let $calc-network := networks:networks-for-components-and-short-names($component, $networks:CALCULATION_NETWORK)
                         let $hc := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube") (: sec-networks:tables($component).Name) :)
                         let $p := hypercubes:populate-networks-with-facts($calc-network, $hc, $archive)
                         let $map := concept-maps:concept-maps($map)
                         let $concepts :=
                            if (exists($map))
                            then
                                for $d in $rollup
                                return
                                    keys(descendant-objects($p)[$$.Name = keys($map.Trees($d).To)][1].To)
                            else
                                for $d in $rollup
                                return ($d, keys(descendant-objects($p)[$$.Name eq $d].To))
                         return sec:facts-for-archives-and-concepts($archive, $concepts, { Hypercube: $hc })
                     else local:facts($component, {| |})
        let $fact-table :=  for $f in $facts
                            let $a := $f.Aspects
                            return {|
                                { "Aspects" : {|
                                    { "xbrl:Entity": $a."xbrl:Entity" },
                                    if (exists($a."dei:LegalEntityAxis")) then { "dei:LegalEntityAxis": $a."dei:LegalEntityAxis" } else (),
                                    { "xbrl:Period" : $a."xbrl:Period" },
                                    { "xbrl:Concept" : $a."xbrl:Concept" },
                                    for $k in keys($f.Aspects)
                                    where $k ne "xbrl:Unit" and $k ne "xbrl:Entity" and $k ne "dei:LegalEntityAxis" and
                                          $k ne "xbrl:Period" and $k ne "xbrl:Concept"
                                    return { $k : $f.Aspects.$k }
                                |} },
                                { "Type" : $f.Type },
                                { "Value" : $f.Value },
                                if (exists($f.Decimals))
                                then { "Decimals" : $f.Decimals }
                                else (),
                                if (exists($f."Aspects"."xbrl:Unit"))
                                then { "Unit" : $f."Aspects"."xbrl:Unit" }
                                else ()
                            |}
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                (session:comment("xml", {
                            NumFacts : count($fact-table),
                            TotalNumFacts: session:num-facts(),
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        }),
                <FactTable entityRegistrantName="{$entity.Profiles.SEC.CompanyName}"
                    cik="{$entity.CIK}"
                    tableName="{sec-networks:tables($component, {IncludeImpliedTable: true}).Name}"
                    label="{$component.Label}"
                    accessionNumber="{$component.Archive}"
                    networkIdentifier="{$component.Role}"
                    formType="{$archive.Profiles.SEC.FormType}"
                    fiscalPeriod="{$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus}"
                    fiscalYear="{$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus}"
                    acceptanceDatetime="{filings:acceptance-dateTimes($archive)}"
                    disclosure="{$component.Profiles.SEC.Disclosure}"
                    >{
                    local:to-xml($fact-table)
                }</FactTable>)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=facttable-" || $cid || ".csv");
                local:to-csv($fact-table)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=facttable-" || $cid || ".csv");
                local:to-csv($fact-table)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { CIK : $entity._id },
                    { EntityRegistrantName : $entity.Profiles.SEC.CompanyName },
                    { TableName : sec-networks:tables($component, {IncludeImpliedTable: true}).Name },
                    { Label : $component.Label },
                    { AccessionNumber : $component.Archive },
                    { FormType : $archive.Profiles.SEC.FormType },
                    { FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus },
                    { FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus },
                    { AcceptanceDatetime : filings:acceptance-dateTimes($archive) },
                    { NetworkIdentifier: $component.Role },
                    { Disclosure : $component.Profiles.SEC.Disclosure },
                    { FactTable : [ $fact-table ] },
                    session:comment("json", {
                            NumFacts : count($fact-table),
                            TotalNumFacts: session:num-facts(),
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })
                |}
            }
     } else {
        response:status-code(401);
        session:error("accessing fact table for an entity that is not in the DOW30", $format)
     }


Remove useless code.

Remove useless code.

jsoniq version "1.0";

module namespace accountant-converter = "http://28.io/modules/xbrl/profiles/accountant/converter";

(:~
 :
Flattens the row headers of the layout, leading to a rendering more familiar to
 : accountants.

 :
 : @param the original layout model.
 : @return the layout model with row headers flattened.
 :)
declare function accountant-converter:flatten-row-headers($layout-model as object) as object
{
  copy $l := $layout-model
  modify
    for $table in $l.TableSet[]
    let $breakdown := $table.TableHeaders.y[]
    let $headers as array := $breakdown.GroupCells
    let $first-header as array := $headers[[1]]
    let $other-headers as array := [ $headers[][position() gt 1] ]
    let $cells as array := $table.TableCells.Facts
    let $result := accountant-converter:flatten-headers({ Headers: $other-headers, Cells: $cells}, 1)
    return (replace value of json $breakdown.GroupCells with [
                copy $f := $first-header
                modify replace value of json $f[[1]].CellSpan with size($result.Headers[])
                return $f,
                $result.Headers[]
            ],
            replace value of json $table.TableCells.Facts with $result.Cells)
  return $l
};

declare %private function accountant-converter:flatten-headers(
    $headers-and-cells as object,
    $depth as integer) as object?
{
    switch(true)

    case size($headers-and-cells.Headers) le 1
    return copy $result := $headers-and-cells
           modify for $header in $result.Headers[][]
                  return (insert json { Depth: $depth } into $header,
                          delete json $header.CellSpan,
                          delete json $header.RollUp)
           return $result

    case size($headers-and-cells.Headers[[1]]) le 0
    return {
        Headers: null,
        Cells: $headers-and-cells.Cells
    }

    default return
        let $headers as array := $headers-and-cells.Headers
        let $cells as array := $headers-and-cells.Cells
        let $first-header as object := $headers[[1]][[1]]
        let $span as integer* := $first-header.CellSpan
        let $is-rightmost as boolean := size($headers) eq 1

        let $cells-to-the-right := accountant-converter:take-span([$headers[][position() gt 1]], $span)
        let $roll-up-position := (for $cell at $i in ($cells-to-the-right[[1]])[]
                                  where $cell.RollUp
                                  return $i, 0)[1]
        let $roll-up-span := for $cell in ($cells-to-the-right[[1]])[]
                             where $cell.RollUp
                             return $cell.CellSpan
        let $cells-to-the-right-no-rollup:= switch($roll-up-position)
                                            case 1 return accountant-converter:remove-span($cells-to-the-right, $roll-up-span)
                                            case $span return accountant-converter:take-span($cells-to-the-right, $span - $roll-up-span)
                                            case 0 return $cells-to-the-right
                                            default return []

        let $children-cells := [$cells[][position() le $span]]
        let $converted-children := accountant-converter:flatten-headers(
            {
                Headers: $cells-to-the-right-no-rollup,
                Cells: $children-cells
            },
            $depth + 1
        )

        let $bottom := accountant-converter:remove-span($headers, $span)
        let $bottom-cells := [$cells[][position() gt $span]]
        let $converted-bottom := accountant-converter:flatten-headers(
            {
                Headers: $bottom,
                Cells: $bottom-cells
            },
            $depth
        )

        let $new-first-header as object := copy $n := $first-header
                                           modify (insert json {
                                                        Depth: $depth,
                                                        IsAbstract: size($cells-to-the-right[[1]]) gt 1 or not $cells-to-the-right[[1]][[1]].RollUp
                                                   } into $n,
                                                   delete json $n.CellSpan,
                                                   delete json $n.RollUp)
                                           return $n

        return {
            (:OriginalCells: $cells,
            CellsToTheRight: $cells-to-the-right,
            RollUpPosition: $roll-up-position,
            RollUpSpan: $roll-up-span,
            CellsToTheRightNoRollUp: $cells-to-the-right-no-rollup,
            ChildrenCells: $children-cells,:)
            Headers: [
                [
                    if($roll-up-position = (0, 1))
                    then $new-first-header
                    else (),
                    $converted-children.Headers[][],
                    if($roll-up-position eq $span and $roll-up-position ne 1)
                    then $new-first-header
                    else (),
                    $converted-bottom.Headers[][]
                ]
            ],
            Cells: [
                if(not $is-rightmost and $roll-up-position eq 0)
                then [ (1 to size($cells[[1]])) ! null ]
                else (),
                $converted-children.Cells[],
                $converted-bottom.Cells[]
            ]
        }
};

declare %private function accountant-converter:take-span($headers as array, $span as integer) as array
{
    [
        for $column as array in $headers[]
        return [ for $n in 1 to size($column)
                 let $first-n-headers := $column[][position() le $n]
                 let $sum := sum($first-n-headers.CellSpan)
                 where $sum eq $span
                 return $first-n-headers
        ]
    ]
};

declare %private function accountant-converter:remove-span($headers as array, $span as integer) as array
{
    [
        for $column as array in $headers[]
        return [ for $n in 1 to size($column)
                 let $first-n-headers := $column[][position() le $n]
                 let $sum := sum($first-n-headers ! ($$.CellSpan, 1)[1])
                 where $sum eq $span
                 return $column[][position() gt $n]
        ]
    ]
};

jsoniq version "1.0";

module namespace accountant-converter = "http://28.io/modules/xbrl/profiles/accountant/converter";

(:~
 :
Flattens the row headers of the layout, leading to a rendering more familiar to
 : accountants.

 :
 : @param the original layout model.
 : @return the layout model with row headers flattened.
 :)
declare function accountant-converter:flatten-row-headers($layout-model as object) as object
{
  copy $l := $layout-model
  modify
    for $table in $l.TableSet[]
    let $breakdown := $table.TableHeaders.y[]
    let $headers as array := $breakdown.GroupCells
    let $first-header as array := $headers[[1]]
    let $other-headers as array := [ $headers[][position() gt 1] ]
    let $cells as array := $table.TableCells.Facts
    let $result := accountant-converter:flatten-headers({ Headers: $other-headers, Cells: $cells}, 1)
(:    return insert json { Debug: $result } into $breakdown:)
    return (replace value of json $breakdown.GroupCells with [
                copy $f := $first-header
                modify replace value of json $f[[1]].CellSpan with size($result.Headers[])
                return $f,
                $result.Headers[]
            ],
            replace value of json $table.TableCells.Facts with $result.Cells)
  return $l
};

declare %private function accountant-converter:flatten-headers(
    $headers-and-cells as object,
    $depth as integer) as object?
{
    switch(true)

    case size($headers-and-cells.Headers) le 1
    return copy $result := $headers-and-cells
           modify for $header in $result.Headers[][]
                  return (insert json { Depth: $depth } into $header,
                          delete json $header.CellSpan,
                          delete json $header.RollUp)
           return $result

    case size($headers-and-cells.Headers[[1]]) le 0
    return {
        Headers: null,
        Cells: $headers-and-cells.Cells
    }

    default return
        let $headers as array := $headers-and-cells.Headers
        let $cells as array := $headers-and-cells.Cells
        let $first-header as object := $headers[[1]][[1]]
        let $span as integer* := $first-header.CellSpan
        let $is-rightmost as boolean := size($headers) eq 1

        let $cells-to-the-right := accountant-converter:take-span([$headers[][position() gt 1]], $span)
        let $roll-up-position := (for $cell at $i in ($cells-to-the-right[[1]])[]
                                  where $cell.RollUp
                                  return $i, 0)[1]
        let $roll-up-span := for $cell in ($cells-to-the-right[[1]])[]
                             where $cell.RollUp
                             return $cell.CellSpan
        let $cells-to-the-right-no-rollup:= switch($roll-up-position)
                                            case 1 return accountant-converter:remove-span($cells-to-the-right, $roll-up-span)
                                            case $span return accountant-converter:take-span($cells-to-the-right, $span - $roll-up-span)
                                            case 0 return $cells-to-the-right
                                            default return []

        let $children-cells := [$cells[][position() le $span]]
        let $converted-children := accountant-converter:flatten-headers(
            {
                Headers: $cells-to-the-right-no-rollup,
                Cells: $children-cells
            },
            $depth + 1
        )

        let $bottom := accountant-converter:remove-span($headers, $span)
        let $bottom-cells := [$cells[][position() gt $span]]
        let $converted-bottom := accountant-converter:flatten-headers(
            {
                Headers: $bottom,
                Cells: $bottom-cells
            },
            $depth
        )

        let $new-first-header as object := copy $n := $first-header
                                           modify (insert json {
                                                        Depth: $depth,
                                                        IsAbstract: size($cells-to-the-right[[1]]) gt 1 or not $cells-to-the-right[[1]][[1]].RollUp
                                                   } into $n,
                                                   delete json $n.CellSpan,
                                                   delete json $n.RollUp)
                                           return $n

        return {
            (:OriginalCells: $cells,
            CellsToTheRight: $cells-to-the-right,
            RollUpPosition: $roll-up-position,
            RollUpSpan: $roll-up-span,
            CellsToTheRightNoRollUp: $cells-to-the-right-no-rollup,
            ChildrenCells: $children-cells,:)
            Headers: [
                [
                    if($roll-up-position = (0, 1))
                    then $new-first-header
                    else (),
                    $converted-children.Headers[][],
                    if($roll-up-position eq $span and $roll-up-position ne 1)
                    then $new-first-header
                    else (),
                    $converted-bottom.Headers[][]
                ]
            ],
            Cells: [
                if(not $is-rightmost and $roll-up-position eq 0)
                then [ (1 to size($cells[[1]])) ! null ]
                else (),
                $converted-children.Cells[],
                $converted-bottom.Cells[]
            ]
        }
};

declare %private function accountant-converter:take-span($headers as array, $span as integer) as array
{
    [
        for $column as array in $headers[]
        return [ for $n in 1 to size($column)
                 let $first-n-headers := $column[][position() le $n]
                 let $sum := sum($first-n-headers.CellSpan)
                 where $sum eq $span
                 return $first-n-headers
        ]
    ]
};

declare %private function accountant-converter:remove-span($headers as array, $span as integer) as array
{
    [
        for $column as array in $headers[]
        return [ for $n in 1 to size($column)
                 let $first-n-headers := $column[][position() le $n]
                 let $sum := sum($first-n-headers ! ($$.CellSpan, 1)[1])
                 where $sum eq $span
                 return $column[][position() gt $n]
        ]
    ]
};


Add `tree/1`.

Add `tree/1`.

Same rationale as previous update to `index.jq`: this will make things
easier in the future, since we can just merge whole tables of contents
without thinking about it more than necessary.

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    (. == {} or . == [] or . == false or . == null or . == 0) | not
;

# Determine whether an index entry was manually entered into the table of
# contents "base".
def is_manual:
    .type == "hyperlink"
;

# Determine whether a page should be uploaded and included in the table of
# contents: it should either be marked as `publish`, or be *not* marked as
# `draft` and have children that *are* marked as `publish`.
def is_publication:
    (.meta.publish|bool)
    or
    ((.meta.draft|bool|not) and ((.contents // [])|any(is_publication or is_manual))
    )
;

# Group the values of properties of an array of objects. For example, [{"a":1,
# "b":2}, {"a":3}] turns into {"a":[1, 3], "b":[3]}. This can be used to
# partition an array; example: `[1,2,3] | map({(.%2|tostring):.}) | group`
def group:
    reduce ([.[] | to_entries[]] | group_by(.key))[] as $group
    ( {}
    ; .[ $group[0].key ] |= (. // []) + [ $group[].value ]
    )
;

# Insert a child element at a particular path.
# Like `setpath/2`, but instead of making objects like `{"a":{"b":{…}}}`, this
# makes objects like `{"contents":[{"name":"a", "contents":[{"name":"b",…}]}]}`
def insert($path; $child):
    if ($path | length) > 0 then
        if ([.contents[]? | select(.name == $path[0])] | length) > 0 then
            (.contents[] | select(.name == $path[0])) |= (. | insert($path[1:]; $child))
        else
            (.contents |= (. // []) + [ {} | .name = $path[0] | insert($path[1:]; $child)])
        end
    else
        . * $child
    end
;

# Put parent objects so that the current object exists at a particular path.
# Like `setpath/2`, but instead of making objects like `{"a":{"b":{…}}}`, this
# makes objects like `{"contents":[{"name":"a", "contents":[{"name":"b",…}]}]}`
def tree($path):
    def tree_aux($history; $future):
        { "name": $future[0]
        , "path": ($history + [$future[0]]) | join("/") } *
        if ($future | length) > 1 then
            { "contents": [ tree_aux($history + [$future[0]]; $future[1:]) ] }
        else
            { "contents": [] } * .
        end
    ;
    tree_aux([]; $path)
;

# Merge two values. If the values are both objects, the values at equal indices
# will be merged; if the values are both arrays, the arrays are concatenated;
# if the values are some other type, we only check if the values do not
# conflict.
def merge(other):
    if (. | type) == "object" and (other | type) == "object" then
        (. | to_entries) + (other | to_entries)
        | group_by(.key)
        | map({ "key": (.[0].key), "value" : (reduce .[1:][].value as $x (.[0].value; merge($x)))})
        | from_entries
    elif (. | type) == "array" and (other | type) == "array" then
        . + other
    elif . == other then
        .
    else
        error("trying to merge incompatible objects")
    end
;

# Merge an array of pages by grouping them by path and then merging all the
# pages with the same path.
def merge_pages:
    group_by(.path) | map(reduce .[] as $x ({}; merge($x)))
;

# Merge together `filetree.json` and `*.meta.json` files, with behaviour
# depending on the filename of the object. Use with the `--null-input` switch.
def process_files:
    reduce inputs as $input
        (   {}
        ;   ($input | input_filename) as $f |
            if ($f | endswith(".meta.json")) then
                insert
                ( $f | ltrimstr($prefix) | rtrimstr(".meta.json") | split("/")
                ; {"meta": $input} )
            else
                reduce $input[] as $entry
                ( .
                ;   insert
                    ( $entry.path | split("/")
                    ; $entry )
                )
            end
        )
;

# Any page gets a link to its HTML.
def add_links:
    if (has("path") and (.path | endswith(".md"))) then
        .link = (.path | rtrimstr(".md") | . + ".html")
    else
        .contents[]? |= add_links
    end
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def add_frontmatter:
    if has("contents") then
        ( .contents | map({(.name=="index.md" or .meta.frontmatter|tostring):.}) | group) as {$true, $false}
        | .frontmatter = $true[0]
        | .contents = ($false + $true[1:])
        | (.contents[]? |= add_frontmatter)
    else
        .
    end
;

# If marked as such, drafts can be excluded from being uploaded and included
# in the table of contents.
def add_drafts:
    if has("contents") then
        ( .contents | map({(is_publication or is_manual|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .drafts = ($false // [])
        | (.contents[]? |= add_drafts)
    else
        .
    end
;

# A resource is anything that is neither a directory nor a page with metadata.
def add_resources:
    if has("contents") then
        ( .contents | map({(has("contents") or has("meta") or is_manual|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .resources = ($false // [])
        | (.contents[]? |= add_resources)
    else
        .
    end;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    process_files
    | add_links
    | add_frontmatter
    | add_drafts
    | add_resources
;

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    (. == {} or . == [] or . == false or . == null or . == 0) | not
;

# Determine whether an index entry was manually entered into the table of
# contents "base".
def is_manual:
    .type == "hyperlink"
;

# Determine whether a page should be uploaded and included in the table of
# contents: it should either be marked as `publish`, or be *not* marked as
# `draft` and have children that *are* marked as `publish`.
def is_publication:
    (.meta.publish|bool)
    or
    ((.meta.draft|bool|not) and ((.contents // [])|any(is_publication or is_manual))
    )
;

# Group the values of properties of an array of objects. For example, [{"a":1,
# "b":2}, {"a":3}] turns into {"a":[1, 3], "b":[3]}. This can be used to
# partition an array; example: [1,2,3] | map({(.%2|tostring):.}) | group
def group:
    reduce ([.[] | to_entries[]] | group_by(.key))[] as $group
    ( {}
    ; .[ $group[0].key ] |= (. // []) + [ $group[].value ]
    )
;

# Insert a child element at a particular path.
# Like `setpath/2`, but instead of making objects like `{"a":{"b":{…}}}`, this
# makes objects like `{"contents":[{"name":"a", "contents":[{"name":"b",…}]}]}`
def insert($path; $child):
    if ($path | length) > 0 then
        if ([.contents[]? | select(.name == $path[0])] | length) > 0 then
            (.contents[] | select(.name == $path[0])) |= (. | insert($path[1:]; $child))
        else
            (.contents |= (. // []) + [ {} | .name = $path[0] | insert($path[1:]; $child)])
        end
    else
        . * $child
    end
;

# Merge two values. If the values are both objects, the values at equal indices
# will be merged; if the values are both arrays, the arrays are concatenated;
# if the values are some other type, we only check if the values do not
# conflict.
def merge(other):
    if (. | type) == "object" and (other | type) == "object" then
        (. | to_entries) + (other | to_entries)
        | group_by(.key)
        | map({ "key": (.[0].key), "value" : (reduce .[1:][].value as $x (.[0].value; merge($x)))})
        | from_entries
    elif (. | type) == "array" and (other | type) == "array" then
        . + other
    elif . == other then
        .
    else
        error("trying to merge incompatible objects")
    end
;

# Merge an array of pages by grouping them by path and then merging all the
# pages with the same path.
def merge_pages:
    group_by(.path) | map(reduce .[] as $x ({}; merge($x)))
;

# Merge together `filetree.json` and `*.meta.json` files, with behaviour
# depending on the filename of the object. Use with the `--null-input` switch.
def process_files:
    reduce inputs as $input
        (   {}
        ;   ($input | input_filename) as $f |
            if ($f | endswith(".meta.json")) then
                insert
                ( $f | ltrimstr($prefix) | rtrimstr(".meta.json") | split("/")
                ; {"meta": $input} )
            else
                reduce $input[] as $entry
                ( .
                ;   insert
                    ( $entry.path | split("/")
                    ; $entry )
                )
            end
        )
;

# Any page gets a link to its HTML.
def add_links:
    if (has("path") and (.path | endswith(".md"))) then
        .link = (.path | rtrimstr(".md") | . + ".html")
    else
        .contents[]? |= add_links
    end
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def add_frontmatter:
    if has("contents") then
        ( .contents | map({(.name=="index.md" or .meta.frontmatter|tostring):.}) | group) as {$true, $false}
        | .frontmatter = $true[0]
        | .contents = ($false + $true[1:])
        | (.contents[]? |= add_frontmatter)
    else
        .
    end
;

# If marked as such, drafts can be excluded from being uploaded and included
# in the table of contents.
def add_drafts:
    if has("contents") then
        ( .contents | map({(is_publication or is_manual|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .drafts = ($false // [])
        | (.contents[]? |= add_drafts)
    else
        .
    end
;

# A resource is anything that is neither a directory nor a page with metadata.
def add_resources:
    if has("contents") then
        ( .contents | map({(has("contents") or has("meta") or is_manual|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .resources = ($false // [])
        | (.contents[]? |= add_resources)
    else
        .
    end;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    process_files
    | add_links
    | add_frontmatter
    | add_drafts
    | add_resources
;


:fire: remove unused parameter

:fire: remove unused parameter

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace config = "http://apps.28.io/config";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace random = "http://zorba.io/modules/random";

declare variable $newId as string := random:uuid();

declare %private function local:url($parameters as object?, $endpoint as string, $includeToken as boolean) as string
{
    "http://" || request:server-name() || ":" || request:server-port() ||
    "/v1/_queries/public/reports/" || $endpoint || ".jq?token=" || (if($includeToken) then $config:test-token else "{{token}}") || "&"||
    string-join(
        for $key in keys($parameters)
        for $value as string in (flatten($parameters.$key) ! string($$))
        return ($key||"="||$value),
        "&")
};

declare %an:sequential function local:export-facs-xbrlb() as item*
{
    let $parameters := {
        "_id": "supportFundamentalAccountingConcepts",
        "export": "true"
    }
    let $endpoint as string := "reports"
    let $url := local:url($parameters, $endpoint, true())
    let $request := http-client:get($url)
    return (local:url($parameters, $endpoint, false()),$request)
};

declare %an:sequential function local:import-facs-xbrlb($archive as xs:base64Binary) as item*
{
    let $parameters := {
        "import": "true",
        "_id": $newId,
        "label": encode-for-uri("Import test")
    }
    let $endpoint as string := "add-report"
    let $url := local:url($parameters, $endpoint, true())
    let $request := http-client:post($url, $archive)
    return (local:url($parameters, $endpoint, false()),$request)
};

declare %an:sequential function local:test-export-import() as item
{
  let $export := local:export-facs-xbrlb()
  let $export-url := $export[1]
  let $export-response := $export[2]
  let $status as integer := $export-response.status
  (:let $body as string := string:materialize($export-response.body.content):)
  return
      if($status ne 200)
      then
        {
            "url": $export-url,
            "status": $status,
            "reponse": $export-response
        }
      else
          let $fac-xbrlb as xs:base64Binary := xs:base64Binary($export-response.body.content)
          let $import := local:import-facs-xbrlb($fac-xbrlb)
          let $import-url := $import[1]
          let $import-response := $import[2]
          let $status as integer := $import-response.status
          let $body as object := parse-json($import-response.body.content)
          return
              if($status eq 200 and exists($body."Label") and $body.Label eq "Import test" and $body."_id" eq $newId)
              then true()
              else
                    {
                        "url": $import-url,
                        "status": $status,
                        "reponse": $body
                    }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    "export-import": local:test-export-import()
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace config = "http://apps.28.io/config";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace random = "http://zorba.io/modules/random";

declare variable $newId as string := random:uuid();

declare %private function local:url($parameters as object?, $endpoint as string, $includeToken as boolean) as string
{
    "http://" || request:server-name() || ":" || request:server-port() ||
    "/v1/_queries/public/reports/" || $endpoint || ".jq?_method=POST&token=" || (if($includeToken) then $config:test-token else "{{token}}") || "&"||
    string-join(
        for $key in keys($parameters)
        for $value as string in (flatten($parameters.$key) ! string($$))
        return ($key||"="||$value),
        "&")
};

declare %an:sequential function local:export-facs-xbrlb() as item*
{
    let $parameters := {
        "_id": "supportFundamentalAccountingConcepts",
        "export": "true"
    }
    let $endpoint as string := "reports"
    let $url := local:url($parameters, $endpoint, true())
    let $request := http-client:get($url)
    return (local:url($parameters, $endpoint, false()),$request)
};

declare %an:sequential function local:import-facs-xbrlb($archive as xs:base64Binary) as item*
{
    let $parameters := {
        "import": "true",
        "_id": $newId,
        "label": encode-for-uri("Import test")
    }
    let $endpoint as string := "add-report"
    let $url := local:url($parameters, $endpoint, true())
    let $request := http-client:post($url, $archive)
    return (local:url($parameters, $endpoint, false()),$request)
};

declare %an:sequential function local:test-export-import() as item
{
  let $export := local:export-facs-xbrlb()
  let $export-url := $export[1]
  let $export-response := $export[2]
  let $status as integer := $export-response.status
  (:let $body as string := string:materialize($export-response.body.content):)
  return
      if($status ne 200)
      then
        {
            "url": $export-url,
            "status": $status,
            "reponse": $export-response
        }
      else
          let $fac-xbrlb as xs:base64Binary := xs:base64Binary($export-response.body.content)
          let $import := local:import-facs-xbrlb($fac-xbrlb)
          let $import-url := $import[1]
          let $import-response := $import[2]
          let $status as integer := $import-response.status
          let $body as object := parse-json($import-response.body.content)
          return
              if($status eq 200 and exists($body."Label") and $body.Label eq "Import test" and $body."_id" eq $newId)
              then true()
              else
                    {
                        "url": $import-url,
                        "status": $status,
                        "reponse": $body
                    }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    "export-import": local:test-export-import()
})


Fix: 6242 Missing elements from the model structure

Fixed query to make sure the root of the model structure is returned.

jsoniq version "1.0";

import module namespace components = "http://xbrl.io/modules/bizql/components";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace filings = "http://xbrl.io/modules/bizql/profiles/sec/filings";
import module namespace fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

import module namespace sec-networks = "http://xbrl.io/modules/bizql/profiles/sec/networks";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-xml-rec($o, $level as integer)
{
    for $o in $o
    return
        element { if ($o.Kind eq "Domain") then "Member" else $o.Kind } {
            attribute { "name" } { $o.Name },
            attribute { "label" } { $o.Label },
            if (exists($o.Balance)) then attribute { "balance" } { $o.Balance } else (),
            if (exists($o.DataType)) then attribute { "dataType" } { $o.DataType } else (),
            if (exists($o.BaseType)) then attribute { "baseDataType" } { $o.BaseType } else (),
            if (exists($o.IsAbstract)) then attribute { "abstract" } { $o.IsAbstract } else (),
            if (exists($o.PeriodType)) then attribute { "periodType" } { $o.PeriodType } else (),
            attribute { "level" } { $level },
            local:to-xml-rec($o.Children[], $level + 1)
        }
};

declare function local:to-xml($model)
{
    ((session:comment("xml")),
    <Component>
        <Network entityRegistrantName="{$model.EntityRegistrantName}"
                 accessionNumber="{$model.AccessionNumber}"
                 tableName="{$model.TableName}"
                 cik="{$model.CIK}"
                 label="{$model.Label}"
                 networkIdentifier="{$model.NetworkIdentifier}"
                 formType="{$model.FormType}"
                 fiscalPeriod="{$model.FiscalPeriod}"
                 fiscalYear="{$model.FiscalYear}"
                 acceptanceDatetime="{$model.AcceptanceDatetime}"
                 disclosure="{$model.Disclosure}"
                 >{
            local:to-xml-rec($model.ModelStructure.Children[], 0)
        }</Network>
    </Component>)

};

declare function local:to-csv-rec($objects, $level as integer)
{
    for $o in $objects
    let $object := {
        "Label" : $o.Label,
        "Name" : $o.Name,
        "ObjectClass" : if ($o.Kind eq "Domain") then "Member" else $o.Kind,
        "DataType" : $o.DataType,
        "BaseDataType" : $o.BaseType,
        "Balance" : $o.Balance,
        "Abstract" : $o.IsAbstract,
        "PeriodType" : $o.PeriodType,
        "Level" : $level
    }
    return (
        $object,
        local:to-csv-rec($o.Children[], $level + 1)
    )
};

declare function local:to-csv($model)
{
    let $lines := local:to-csv-rec($model.ModelStructure.Children[], 0)
    return
        if (exists($lines))
        then string-join(csv:serialize($lines, { serialize-null-as : "" }))
        else ""
};

declare function local:enrich-json-rec($objects, $level as integer)
{
    for $object in $objects
    return
        copy $o := $object
        modify (
            if (exists($o.Children)) then delete json $o("Children") else (),
            insert json { Level : $level } into $o,
            if ($o.Kind eq "Domain") then replace value of json $o("Kind") with "Member" else ()
        )
        return {|
            $o,
            let $children := local:enrich-json-rec($object.Children[], $level + 1)
            return if (exists($children)) then { Children : [ $children ] } else ()
        |}
};

declare function local:enrich-json($component)
{
    {
        ModelStructure : [ local:enrich-json-rec($component.ModelStructure, 0) ] ,
        CIK : $component.CIK,
        EntityRegistrantName : $component.EntityRegistrantName,
        Label : $component.Label,
        AccessionNumber : $component.AccessionNumber,
        TableName : $component.TableName,
        FormType : $component.FormType,
        FiscalPeriod : $component.FiscalPeriod,
        FiscalYear : $component.FiscalYear,
        AcceptanceDatetime : $component.AcceptanceDatetime,
        NetworkIdentifier: $component.NetworkIdentifier,
        Disclosure : $component.Disclosure
    }
};
declare function local:components-by-disclosures($disclosures, $aids)
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    for $aid in $aids
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: $aid,
            "Profiles.SEC.Disclosure": { "$in" : [ $disclosures ] }
        })
};

declare function local:components-by-concepts($concepts, $aids)
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    let $ids := mongo:find($conn, "concepts",
        {|
            (
                { "Name" : { "$in" : [ $concepts ] } },
                { "Archive" : { "$in" : [ $aids ] } }
            )
        |}).Component
    return components:components($ids)
};

declare function local:filings(
    $ciks,
    $tags,
    $tickers,
    $sics,
    $fp,
    $fy)
{
    let $entities := (
        companies:companies($ciks),
        companies:companies-for-tags($tags),
        companies:companies-for-tickers($tickers),
        companies:companies-for-SIC($sics)
    )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                        case "ALL" return ()
                    default return $fy
                )
    for $fp in $fp
    return fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy cast as integer)
};

let $format      := lower-case((request:param-values("format"), substring-after(request:path(), ".jq."))[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y cast as integer
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then ("Q1", "Q2", "Q3", "FY")
                        else $fp)
let $aids        := archives:aid(request:param-values("aid"))
let $archives    := (
                        local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                        archives:archives($aids)
                    )
let $cid         := request:param-values("cid")
let $concepts    := distinct-values(request:param-values("concept"))
let $disclosures := request:param-values("disclosure")
let $components  := if (exists($cid))
                    then components:components($cid)
                    else if (exists($concepts) or exists($disclosures))
                    then (local:components-by-concepts($concepts, $archives._id), local:components-by-disclosures($disclosures, $archives._id))
                    else components:components-for-archives($archives)
let $component := $components[1] (: only one for know :)
let $archive   := archives:archives($component.Archive)
let $entity    := entities:entities($archives.Entity)
return
     if (session:only-dow30($entity) or session:valid())
     then {
        let $model := {|
                    { CIK : $entity._id },
                    { EntityRegistrantName : $entity.Profiles.SEC.CompanyName },
                    { ModelStructure : sec-networks:model-structures($component) },
                    { TableName : sec-networks:tables($component, {IncludeImpliedTable: true}).Name },
                    { Label : $component.Label },
                    { AccessionNumber : $component.Archive },
                    { FormType : $archive.Profiles.SEC.FormType },
                    { FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus },
                    { FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus },
                    { AcceptanceDatetime : filings:acceptance-dateTimes($archive) },
                    { NetworkIdentifier: $component.Role },
                    { Disclosure : $component.Profiles.SEC.Disclosure }
        |}
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($model)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=modelstructure-" || $cid || ".csv");
                local:to-csv($model)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=modelstructure-" || $cid || ".csv");
                local:to-csv($model)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    local:enrich-json($model),
                    session:comment("json")
                |}
            }
     } else {
        response:status-code(401);
        session:error("accessing fact table for an entity that is not in the DOW30", $format)
     }

jsoniq version "1.0";

import module namespace components = "http://xbrl.io/modules/bizql/components";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace filings = "http://xbrl.io/modules/bizql/profiles/sec/filings";
import module namespace fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

import module namespace sec-networks = "http://xbrl.io/modules/bizql/profiles/sec/networks";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-xml-rec($o, $level as integer)
{
    for $o in $o
    return
        element { if ($o.Kind eq "Domain") then "Member" else $o.Kind } {
            attribute { "name" } { $o.Name },
            attribute { "label" } { $o.Label },
            if (exists($o.Balance)) then attribute { "balance" } { $o.Balance } else (),
            if (exists($o.DataType)) then attribute { "dataType" } { $o.DataType } else (),
            if (exists($o.BaseType)) then attribute { "baseDataType" } { $o.BaseType } else (),
            if (exists($o.IsAbstract)) then attribute { "abstract" } { $o.IsAbstract } else (),
            if (exists($o.PeriodType)) then attribute { "periodType" } { $o.PeriodType } else (),
            attribute { "level" } { $level },
            local:to-xml-rec($o.Children[], $level + 1)
        }
};

declare function local:to-xml($model)
{
    ((session:comment("xml")),
    <Component>
        <Network entityRegistrantName="{$model.EntityRegistrantName}"
                 accessionNumber="{$model.AccessionNumber}"
                 tableName="{$model.TableName}"
                 cik="{$model.CIK}"
                 label="{$model.Label}"
                 networkIdentifier="{$model.NetworkIdentifier}"
                 formType="{$model.FormType}"
                 fiscalPeriod="{$model.FiscalPeriod}"
                 fiscalYear="{$model.FiscalYear}"
                 acceptanceDatetime="{$model.AcceptanceDatetime}"
                 disclosure="{$model.Disclosure}"
                 >{
            local:to-xml-rec($model.ModelStructure.Children[], 0)
        }</Network>
    </Component>)

};

declare function local:to-csv-rec($objects, $level as integer)
{
    for $o in $objects
    let $object := {
        "Label" : $o.Label,
        "Name" : $o.Name,
        "ObjectClass" : if ($o.Kind eq "Domain") then "Member" else $o.Kind,
        "DataType" : $o.DataType,
        "BaseDataType" : $o.BaseType,
        "Balance" : $o.Balance,
        "Abstract" : $o.IsAbstract,
        "PeriodType" : $o.PeriodType,
        "Level" : $level
    }
    return (
        $object,
        local:to-csv-rec($o.Children[], $level + 1)
    )
};

declare function local:to-csv($model)
{
    let $lines := local:to-csv-rec($model.ModelStructure.Children[], 0)
    return
        if (exists($lines))
        then string-join(csv:serialize($lines, { serialize-null-as : "" }))
        else ""
};

declare function local:enrich-json-rec($objects, $level as integer)
{
    for $object in $objects
    return
        copy $o := $object
        modify (
            if (exists($o.Children)) then delete json $o("Children") else (),
            insert json { Level : $level } into $o,
            if ($o.Kind eq "Domain") then replace value of json $o("Kind") with "Member" else ()
        )
        return {|
            $o,
            let $children := local:enrich-json-rec($object.Children[], $level + 1)
            return if (exists($children)) then { Children : [ $children ] } else ()
        |}
};

declare function local:enrich-json($component)
{
    {
        ModelStructure : [ local:enrich-json-rec($component.ModelStructure.Children[], 0) ] ,
        CIK : $component.CIK,
        EntityRegistrantName : $component.EntityRegistrantName,
        Label : $component.Label,
        AccessionNumber : $component.AccessionNumber,
        TableName : $component.TableName,
        FormType : $component.FormType,
        FiscalPeriod : $component.FiscalPeriod,
        FiscalYear : $component.FiscalYear,
        AcceptanceDatetime : $component.AcceptanceDatetime,
        NetworkIdentifier: $component.NetworkIdentifier,
        Disclosure : $component.Disclosure
    }
};
declare function local:components-by-disclosures($disclosures, $aids)
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    for $aid in $aids
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: $aid,
            "Profiles.SEC.Disclosure": { "$in" : [ $disclosures ] }
        })
};

declare function local:components-by-concepts($concepts, $aids)
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    let $ids := mongo:find($conn, "concepts",
        {|
            (
                { "Name" : { "$in" : [ $concepts ] } },
                { "Archive" : { "$in" : [ $aids ] } }
            )
        |}).Component
    return components:components($ids)
};

declare function local:filings(
    $ciks,
    $tags,
    $tickers,
    $sics,
    $fp,
    $fy)
{
    let $entities := (
        companies:companies($ciks),
        companies:companies-for-tags($tags),
        companies:companies-for-tickers($tickers),
        companies:companies-for-SIC($sics)
    )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                        case "ALL" return ()
                    default return $fy
                )
    for $fp in $fp
    return fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy cast as integer)
};

let $format      := lower-case((request:param-values("format"), substring-after(request:path(), ".jq."))[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y cast as integer
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then ("Q1", "Q2", "Q3", "FY")
                        else $fp)
let $aids        := archives:aid(request:param-values("aid"))
let $archives    := (
                        local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                        archives:archives($aids)
                    )
let $cid         := request:param-values("cid")
let $concepts    := distinct-values(request:param-values("concept"))
let $disclosures := request:param-values("disclosure")
let $components  := if (exists($cid))
                    then components:components($cid)
                    else if (exists($concepts) or exists($disclosures))
                    then (local:components-by-concepts($concepts, $archives._id), local:components-by-disclosures($disclosures, $archives._id))
                    else components:components-for-archives($archives)
let $component := $components[1] (: only one for know :)
let $archive   := archives:archives($component.Archive)
let $entity    := entities:entities($archives.Entity)
return
     if (session:only-dow30($entity) or session:valid())
     then {
        let $model := {|
                    { CIK : $entity._id },
                    { EntityRegistrantName : $entity.Profiles.SEC.CompanyName },
                    { ModelStructure : sec-networks:model-structures($component) },
                    { TableName : sec-networks:tables($component, {IncludeImpliedTable: true}).Name },
                    { Label : $component.Label },
                    { AccessionNumber : $component.Archive },
                    { FormType : $archive.Profiles.SEC.FormType },
                    { FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus },
                    { FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus },
                    { AcceptanceDatetime : filings:acceptance-dateTimes($archive) },
                    { NetworkIdentifier: $component.Role },
                    { Disclosure : $component.Profiles.SEC.Disclosure }
        |}
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($model)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=modelstructure-" || $cid || ".csv");
                local:to-csv($model)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=modelstructure-" || $cid || ".csv");
                local:to-csv($model)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    local:enrich-json($model),
                    session:comment("json")
                |}
            }
     } else {
        response:status-code(401);
        session:error("accessing fact table for an entity that is not in the DOW30", $format)
     }