Fixed report element maps.

Fixed report element maps.

import module namespace api = "http://apps.28.io/api";

import module namespace session = "http://apps.28.io/session";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concept-maps = "http://28.io/modules/xbrl/concept-maps";
import module namespace config = "http://apps.28.io/config";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace seq = "http://zorba.io/modules/sequence";

declare function local:to-csv($concepts as item*, $onlyNames as boolean) as string
{
    if ($onlyNames)
    then
        string-join(("Name", $concepts), "
")
    else
        string-join(csv:serialize($concepts, { serialize-null-as : "" }))
};

declare function local:to-xml($concepts as item*, $onlyNames as boolean) as element()*
{
    for $c in $concepts
    return
        <ReportElement>{
             if ($onlyNames)
             then
                <Name>{$c}</Name>
             else (
                <Name>{$c.Name}</Name>,
                <Label>{$c.Label}</Label>,
                <IsNillable>{$c.IsNillable}</IsNillable>,
                <IsAbstract>{$c.IsAbstract}</IsAbstract>,
                <PeriodType>{$c.PeriodType}</PeriodType>,
                <Balance>{$c.Balance}</Balance>,
                <SubstitutionGroup>{$c.SubstitutionGroup}</SubstitutionGroup>,
                <DataType>{$c.DataType}</DataType>,
                <BaseType>{$c.BaseType}</BaseType>,
                <ClosestSchemaBuiltinType>{$c.ClosestSchemaBuiltinType}</ClosestSchemaBuiltinType>,
                <IsTextBlock>{$c.IsTextBlock}</IsTextBlock>,
                <ComponentRole>{$c.ComponentRole}</ComponentRole>,
                <ComponentLabel>{$c.ComponentLabel}</ComponentLabel>,
                <AccessionNumber>{$c.AccessionNumber}</AccessionNumber>,
                <CIK>{$c.CIK}</CIK>,
                <EntityRegistrantName>{$c.EntityRegistrantName}</EntityRegistrantName>,
                <FiscalYear>{$c.FiscalYear}</FiscalYear>,
                <FiscalPeriod>{$c.FiscalPeriod}</FiscalPeriod>
             )
        }</ReportElement>
};

declare function local:concepts-for-archives(
    $aids as string*,
    $names as string*,
    $map as object?,
    $options as object?) as object*
{
    let $projection as object := if($options.OnlyNames eq true) then { Name: 1 } else {}
    let $concepts-computable-by-maps as object* :=
        switch(true)
            case not exists($map) return ()
            case not exists($names) return $map.Trees[]
            default return
                let $keys as string* := $map.Trees[].Name
                for $concept as string in $names[$$ = $keys]
                return ($map.Trees[])[$$.Name eq $concept]
    let $mapped-names as string* := $concepts-computable-by-maps.To[].Name
    let $concepts-not-computable-by-maps as string* := seq:value-except($names, $mapped-names)

    let $all-results as object* := mw:find($concepts:col,
        {
            "Archive": { "$in" : [ $aids ] }
        },
        $projection)
    let $results-not-computed-by-maps as object* := mw:find($concepts:col,
        {
            "Name" : { "$in" : [ $concepts-not-computable-by-maps ] },
            "Archive": { "$in" : [ $aids ] }
        },
        $projection)
    let $results-computed-by-maps as object* :=
        let $all-results as object* := mw:find($concepts:col,
            {
                "Name" : { "$in" : [ $mapped-names ] },
                "Archive": { "$in" : [ $aids ] }
            },
            $projection)
        for $concept as object in $concepts-computable-by-maps
        for $result as object in
            for $candidate-concept in $concept.To[].Name
            let $facts := $all-results[$$.Name = $candidate-concept]
            where exists($facts)
            count $c
            where $c eq 1
            return $facts
        let $map-concept := (for $candidate in $concepts-computable-by-maps
                            where $result.Name = (keys($candidate.To), $candidate.To[].Name)
                            return $candidate)[1]
        return
            copy $n := $result
            modify (
                replace value of json $n.Name with $map-concept.Name,
                insert json  { Origin : $result.Name } into $n)
            return $n
    return if(empty($map) and empty($names))
           then $all-results
           else ($results-not-computed-by-maps, $results-computed-by-maps)
};

declare function local:concepts-for-archives-and-labels($aids as string*, $labels as string) as object*
{
   mw:run-cmd-deterministic(
      {
        "text" : "concepts",
        "filter" : { "Archive" : { "$in" : [ $aids ] } },
        "search" : $labels,
        "limit" : 100,
        "score" : { "$meta" : "textScore" },
        "sort" : { score: { "$meta" : "textScore" } }
      }).results[].obj
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token         as string?  external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := $config:profile-name;
declare  %rest:env                              variable $request-uri   as string   external;
declare  %rest:case-insensitive                 variable $format        as string?  external;
declare  %rest:case-insensitive %rest:distinct  variable $cik           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $tag           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker        as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $sic           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear    as string*  external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod  as string*  external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $label         as string*  external;
declare  %rest:case-insensitive                 variable $map           as string?  external;
declare  %rest:case-insensitive                 variable $report        as string?  external;
declare  %rest:case-insensitive %rest:distinct  variable $name          as string*  external;
declare  %rest:case-insensitive                 variable $onlyNames     as boolean? external := false;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic)
let $archives as object* := fiscal-core:filings(
    $entities,
    $fiscalPeriod,
    $fiscalYear,
    $aid)
let $entities as object* :=
    ($entities[$$._id = $archives.Entity],
    let $not-found := $archives.Entity[not $entities._id = $$]
    where exists($not-found)
    return entities:entities($not-found))
let $map as object? :=
    if(exists($report))
    then reports:concept-map($report)
    else concept-maps:concept-maps($map)
let $concepts :=
    if($profile-name eq "sec")
    then
        if (exists($label))
            then local:concepts-for-archives-and-labels($archives._id, $label[1])
            else local:concepts-for-archives($archives._id, $name, $map, { OnlyNames: $onlyNames })
    else
        if (exists($label))
            then local:concepts-for-archives-and-labels($aid, $label[1])
            else local:concepts-for-archives($aid, $name, $map, { OnlyNames: $onlyNames })

let $result :=
    if($profile-name eq "sec")
    then {
        ReportElements : [
            if ($onlyNames)
            then distinct-values($concepts.Name)
            else
                let $all-aids := $concepts.Archive
                let $roles := $concepts.Role
                let $components := components:components-for-archives-and-roles($all-aids, $roles)
                return
                for $concept in $concepts
                group by $archive := $concept.Archive,  $role := $concept.Role
                let $component as object := $components[$$.Archive eq $archive and $$.Role eq $role]
                let $members as object* := $components.Concepts[]
                let $archive as object := $archives[$$._id eq $archive]
                let $entity as object := $entities[$$._id eq $archive.Entity]
                let $metadata := {
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label,
                    AccessionNumber : $archive._id,
                    CIK : $entity._id,
                    EntityRegistrantName : $entity.Profiles.SEC.CompanyName,
                    FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                    FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
                }
                for $concept in $concept
                let $original-name := ($concept.Origin, $concept.Name)[1]
                return {|
                    project($concept, ("Name", "Origin")),
                    trim($members.$original-name, "Name"),
                    $metadata
                |}
        ]
    }
    else {
        ReportElements : [
            if ($onlyNames)
            then distinct-values($concepts.Name)
            else
                let $all-aids := $concepts.Archive
                let $roles := $concepts.Role
                let $components := components:components-for-archives-and-roles($all-aids, $roles)
                return
                for $concept in $concepts
                group by $archive := $concept.Archive,  $role := $concept.Role
                let $component as object := $components[$$.Archive eq $archive and $$.Role eq $role]
                let $members as object* := $components.Concepts[]
                let $metadata := {
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label,
                    AccessionNumber : $archive._id,
                    FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                    FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
                }
                for $concept in $concept
                let $original-name := ($concept.Origin, $concept.Name)[1]
                return {|
                    project($concept, ("Name", "Origin")),
                    trim($members.$original-name, "Name"),
                    $metadata
                |}
        ]
    }
let $comment := {
    NumConcepts: count($concepts),
    TotalNumConcepts: session:num-concepts()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <ReportElements>{
                    local:to-xml($res.ReportElements[], $onlyNames)
                }</ReportElements>
    },
    to-csv : function($res as object) as string {
        local:to-csv($res.ReportElements[], $onlyNames)
    }
}
let $results := api:serialize($result, $comment, $serializers, $format, "report-elements")
return api:check-and-return-results($token, $results, $format)

import module namespace api = "http://apps.28.io/api";

import module namespace session = "http://apps.28.io/session";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concept-maps = "http://28.io/modules/xbrl/concept-maps";
import module namespace config = "http://apps.28.io/config";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace seq = "http://zorba.io/modules/sequence";

declare function local:to-csv($concepts as item*, $onlyNames as boolean) as string
{
    if ($onlyNames)
    then
        string-join(("Name", $concepts), "
")
    else
        string-join(csv:serialize($concepts, { serialize-null-as : "" }))
};

declare function local:to-xml($concepts as item*, $onlyNames as boolean) as element()*
{
    for $c in $concepts
    return
        <ReportElement>{
             if ($onlyNames)
             then
                <Name>{$c}</Name>
             else (
                <Name>{$c.Name}</Name>,
                <Label>{$c.Label}</Label>,
                <IsNillable>{$c.IsNillable}</IsNillable>,
                <IsAbstract>{$c.IsAbstract}</IsAbstract>,
                <PeriodType>{$c.PeriodType}</PeriodType>,
                <Balance>{$c.Balance}</Balance>,
                <SubstitutionGroup>{$c.SubstitutionGroup}</SubstitutionGroup>,
                <DataType>{$c.DataType}</DataType>,
                <BaseType>{$c.BaseType}</BaseType>,
                <ClosestSchemaBuiltinType>{$c.ClosestSchemaBuiltinType}</ClosestSchemaBuiltinType>,
                <IsTextBlock>{$c.IsTextBlock}</IsTextBlock>,
                <ComponentRole>{$c.ComponentRole}</ComponentRole>,
                <ComponentLabel>{$c.ComponentLabel}</ComponentLabel>,
                <AccessionNumber>{$c.AccessionNumber}</AccessionNumber>,
                <CIK>{$c.CIK}</CIK>,
                <EntityRegistrantName>{$c.EntityRegistrantName}</EntityRegistrantName>,
                <FiscalYear>{$c.FiscalYear}</FiscalYear>,
                <FiscalPeriod>{$c.FiscalPeriod}</FiscalPeriod>
             )
        }</ReportElement>
};

declare function local:concepts-for-archives(
    $aids as string*,
    $names as string*,
    $map as object?,
    $options as object?) as object*
{
    let $projection as object := if($options.OnlyNames eq true) then { Name: 1 } else {}
    let $concepts-computable-by-maps as object* :=
        switch(true)
            case not exists($map) return ()
            case not exists($names) return $map.Trees[]
            default return
                let $keys as string* := $map.Trees[].Name
                for $concept as string in $names[$$ = $keys]
                return ($map.Trees[])[$$.Name eq $concept]
    let $mapped-names as string* := (keys($concepts-computable-by-maps.To ), $concepts-computable-by-maps.To [].Name)
    let $concepts-not-computable-by-maps as string* := seq:value-except($names, $mapped-names)

    let $all-results as object* := mw:find($concepts:col,
        {
            "Archive": { "$in" : [ $aids ] }
        },
        $projection)
    let $results-not-computed-by-maps as object* := mw:find($concepts:col,
        {
            "Name" : { "$in" : [ $concepts-not-computable-by-maps ] },
            "Archive": { "$in" : [ $aids ] }
        },
        $projection)
    let $results-computed-by-maps as object* :=
        let $all-results as object* := mw:find($concepts:col,
            {
                "Name" : { "$in" : [ $mapped-names ] },
                "Archive": { "$in" : [ $aids ] }
            },
            $projection)
        for $concept as object in $concepts-computable-by-maps
        for $result as object in
            for $candidate-concept in (keys($concept.To), $concept[].Name)
            let $facts := $all-results[$$.Name = $candidate-concept]
            where exists($facts)
            count $c
            where $c eq 1
            return $facts
        let $map-concept := (for $candidate in $concepts-computable-by-maps
                            where $result.Name = (keys($candidate.To), $candidate.To[].Name)
                            return $candidate)[1]
        return
            copy $n := $result
            modify (
                replace value of json $n.Name with $map-concept.Name,
                insert json  { Origin : $result.Name } into $n)
            return $n
    return if(empty($map) and empty($names))
           then $all-results
           else ($results-not-computed-by-maps, $results-computed-by-maps)
};

declare function local:concepts-for-archives-and-labels($aids as string*, $labels as string) as object*
{
   mw:run-cmd-deterministic(
      {
        "text" : "concepts",
        "filter" : { "Archive" : { "$in" : [ $aids ] } },
        "search" : $labels,
        "limit" : 100,
        "score" : { "$meta" : "textScore" },
        "sort" : { score: { "$meta" : "textScore" } }
      }).results[].obj
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token         as string?  external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := $config:profile-name;
declare  %rest:env                              variable $request-uri   as string   external;
declare  %rest:case-insensitive                 variable $format        as string?  external;
declare  %rest:case-insensitive %rest:distinct  variable $cik           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $tag           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker        as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $sic           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear    as string*  external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod  as string*  external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $label         as string*  external;
declare  %rest:case-insensitive                 variable $map           as string?  external;
declare  %rest:case-insensitive                 variable $report        as string?  external;
declare  %rest:case-insensitive %rest:distinct  variable $name          as string*  external;
declare  %rest:case-insensitive                 variable $onlyNames     as boolean? external := false;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic)
let $archives as object* := fiscal-core:filings(
    $entities,
    $fiscalPeriod,
    $fiscalYear,
    $aid)
let $entities as object* :=
    ($entities[$$._id = $archives.Entity],
    let $not-found := $archives.Entity[not $entities._id = $$]
    where exists($not-found)
    return entities:entities($not-found))
let $map as object? :=
    if(exists($report))
    then reports:concept-map($report)
    else concept-maps:concept-maps($map)
let $concepts :=
    if($profile-name eq "sec")
    then
        if (exists($label))
            then local:concepts-for-archives-and-labels($archives._id, $label[1])
            else local:concepts-for-archives($archives._id, $name, $map, { OnlyNames: $onlyNames })
    else
        if (exists($label))
            then local:concepts-for-archives-and-labels($aid, $label[1])
            else local:concepts-for-archives($aid, $name, $map, { OnlyNames: $onlyNames })

let $result :=
    if($profile-name eq "sec")
    then {
        ReportElements : [
            if ($onlyNames)
            then distinct-values($concepts.Name)
            else
                let $all-aids := $concepts.Archive
                let $roles := $concepts.Role
                let $components := components:components-for-archives-and-roles($all-aids, $roles)
                return
                for $concept in $concepts
                group by $archive := $concept.Archive,  $role := $concept.Role
                let $component as object := $components[$$.Archive eq $archive and $$.Role eq $role]
                let $members as object* := $components.Concepts[]
                let $archive as object := $archives[$$._id eq $archive]
                let $entity as object := $entities[$$._id eq $archive.Entity]
                let $metadata := {
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label,
                    AccessionNumber : $archive._id,
                    CIK : $entity._id,
                    EntityRegistrantName : $entity.Profiles.SEC.CompanyName,
                    FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                    FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
                }
                for $concept in $concept
                let $original-name := ($concept.Origin, $concept.Name)[1]
                return {|
                    project($concept, ("Name", "Origin")),
                    trim($members.$original-name, "Name"),
                    $metadata
                |}
        ]
    }
    else {
        ReportElements : [
            if ($onlyNames)
            then distinct-values($concepts.Name)
            else
                let $all-aids := $concepts.Archive
                let $roles := $concepts.Role
                let $components := components:components-for-archives-and-roles($all-aids, $roles)
                return
                for $concept in $concepts
                group by $archive := $concept.Archive,  $role := $concept.Role
                let $component as object := $components[$$.Archive eq $archive and $$.Role eq $role]
                let $members as object* := $components.Concepts[]
                let $metadata := {
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label,
                    AccessionNumber : $archive._id,
                    FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                    FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
                }
                for $concept in $concept
                let $original-name := ($concept.Origin, $concept.Name)[1]
                return {|
                    project($concept, ("Name", "Origin")),
                    trim($members.$original-name, "Name"),
                    $metadata
                |}
        ]
    }
let $comment := {
    NumConcepts: count($concepts),
    TotalNumConcepts: session:num-concepts()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <ReportElements>{
                    local:to-xml($res.ReportElements[], $onlyNames)
                }</ReportElements>
    },
    to-csv : function($res as object) as string {
        local:to-csv($res.ReportElements[], $onlyNames)
    }
}
let $results := api:serialize($result, $comment, $serializers, $format, "report-elements")
return api:check-and-return-results($token, $results, $format)


Fixing concept names in report element endpoint.

import module namespace util = "http://secxbrl.info/modules/util";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concept-maps = "http://28.io/modules/xbrl/concept-maps";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace seq = "http://zorba.io/modules/sequence";

declare namespace concepts = "http://www.28msec.com/modules/bizql/concepts";

declare function local:to-csv($concepts as item*, $onlyNames as boolean) as string
{
    if ($onlyNames)
    then
        string-join(("Name", $concepts), "
")
    else
        string-join(csv:serialize($concepts, { serialize-null-as : "" }))
};

declare function local:to-xml($concepts as item*, $onlyNames as boolean) as element()*
{
    for $c in $concepts
    return
        <ReportElement>{
             if ($onlyNames)
             then
                <Name>{$c}</Name>
             else (
                <Name>{$c.Name}</Name>,
                <Label>{$c.Label}</Label>,
                <IsNillable>{$c.IsNillable}</IsNillable>,
                <IsAbstract>{$c.IsAbstract}</IsAbstract>,
                <PeriodType>{$c.PeriodType}</PeriodType>,
                <Balance>{$c.Balance}</Balance>,
                <SubstitutionGroup>{$c.SubstitutionGroup}</SubstitutionGroup>,
                <DataType>{$c.DataType}</DataType>,
                <BaseType>{$c.BaseType}</BaseType>,
                <ClosestSchemaBuiltinType>{$c.ClosestSchemaBuiltinType}</ClosestSchemaBuiltinType>,
                <IsTextBlock>{$c.IsTextBlock}</IsTextBlock>,
                <ComponentRole>{$c.ComponentRole}</ComponentRole>,
                <ComponentLabel>{$c.ComponentLabel}</ComponentLabel>,
                <AccessionNumber>{$c.AccessionNumber}</AccessionNumber>,
                <CIK>{$c.CIK}</CIK>,
                <EntityRegistrantName>{$c.EntityRegistrantName}</EntityRegistrantName>,
                <FiscalYear>{$c.FiscalYear}</FiscalYear>,
                <FiscalPeriod>{$c.FiscalPeriod}</FiscalPeriod>
             )
        }</ReportElement>
};

declare function local:concepts-for-archives($aids as string*, $projection as object) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("concepts:CONNECTION-FAILED"), $err:description)
        }
    return
        mongo:find($conn, "concepts",
        {
            "Archive": { "$in" : [ $aids ] }
        },
        $projection,
        {})
};

declare function local:concepts-for-archives($aids as string*, $names as string*, $map as string?) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("concepts:CONNECTION-FAILED"), $err:description)
        }
    let $map := if (exists($map)) then concept-maps:concept-maps($map) else ()

    let $all-concepts-computable-by-maps as string* := keys($map.Trees)

    let $concepts-computable-by-maps as object* :=
        for $c in $names[$$ = $all-concepts-computable-by-maps]
        return $map.Trees.$c

    let $mapped-names := (keys($concepts-computable-by-maps.To ), $concepts-computable-by-maps.To [].Name)

    let $concepts-not-computable-by-maps as string* := seq:value-except($names, $mapped-names)
    let $results-not-computed-by-maps := mongo:find($conn, "concepts",
        {
            "Name" : { "$in" : [ $concepts-not-computable-by-maps ] },
            "Archive": { "$in" : [ $aids ] }
        })
    let $results-computed-by-maps :=
        for $c in mongo:find($conn, "concepts",
            {
                "Name" : { "$in" : [ $mapped-names ] },
                "Archive": { "$in" : [ $aids ] }
            })
        group by $c.Component
        let $c := $c[1]
        let $map-concept := (for $candidate in $concepts-computable-by-maps
                            where $c.Name = (keys($candidate.To), $candidate.To[].Name)
                            return $candidate)[1]
        return
            copy $n := $c
            modify (
                replace value of json $n.Name with $map-concept.Name,
                insert json  { Origin : $c.Name } into $n)
            return $n
    return ($results-not-computed-by-maps, $results-computed-by-maps)
};

declare function local:concepts-for-archives-and-labels($aids as string*, $labels as string) as object*
{
    let $conn :=
        let $credentials := credentials:credentials("MongoDB", "xbrl")
        return
            try {
                mongo:connect($credentials)
            } catch mongo:* {
                error(QName("components:CONNECTION-FAILED"), $err:description)
            }
    return mongo:run-cmd-deterministic(
        $conn,
        {
            "text" : "concepts",
            "filter" : { "Archive" : { "$in" : [ $aids ] } },
            "search" : $labels,
            "limit" : 100,
            "score" : { "$meta" : "textScore" },
            "sort" : { score: { "$meta" : "textScore" } }
        }).results[].obj
};

session:audit-call();

(: Query parameters :)
let $format as string?         := request:param-values("format")
let $ciks as string*           := distinct-values(request:param-values("cik"))
let $tags as string*           := distinct-values(request:param-values("tag"))
let $tickers as string*        := distinct-values(request:param-values("ticker"))
let $sics as string*           := distinct-values(request:param-values("sic"))
let $fiscalYears as string*    := distinct-values(request:param-values("fiscalYear", "LATEST"))
let $fiscalPeriods as string*  := distinct-values(request:param-values("fiscalPeriod", "FY"))
let $aids as string*           := distinct-values(request:param-values("aid"))
let $labels as string*         := request:param-values("label")
let $map as string?            := request:param-values("map")
let $report as string?         := request:param-values("report")
let $names as string*          := request:param-values("name")

(: Post-processing :)
let $format as string? := (: backwards compatibility, to be deprecated  :)
    lower-case(($format, substring-after(request:path(), ".jq."))[1])
let $tags as string* := (: backwards compatibility, to be deprecated :)
    distinct-values($tags ! upper-case($$))
let $fiscalYears as integer* :=
    for $fy in $fiscalYears ! upper-case($$)
    return switch($fy)
           case "LATEST" return $fiscal-core:LATEST_FISCAL_YEAR
           case "ALL" return $fiscal-core:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
let $fiscalPeriods as string* :=
    for $fp in $fiscalPeriods ! upper-case($$)
    return switch($fp)
           case "ALL" return $fiscal-core:ALL_FISCAL_PERIODS
           default return $fp

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $ciks,
        $tags,
        $tickers,
        $sics)
let $archives as object* := fiscal-core:filings(
    $entities,
    $fiscalPeriods,
    $fiscalYears,
    $aids)
let $entities as object* :=
    ($entities[$$._id = $archives.Entity],
    let $not-found := $archives.Entity[not $entities._id = $$]
    where exists($not-found)
    return entities:entities($not-found))
let $onlyNames   := let $o := request:param-values("onlyNames")[1] return if (exists($o)) then ($o cast as boolean) else false
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map

let $concepts := if (exists($names))
                  then local:concepts-for-archives($archives._id, $names, $map)
                  else if (exists($labels))
                  then local:concepts-for-archives-and-labels($archives._id, $labels[1])
                  else if($onlyNames)
                  then local:concepts-for-archives($archives._id, { Name: 1 })
                  else local:concepts-for-archives($archives._id, {})
let $result := {
    ReportElements : [
        if ($onlyNames)
        then distinct-values($concepts.Name)
        else
            let $all-aids := $concepts.Archive
            let $roles := $concepts.Role
            let $components := components:components-for-archives-and-roles($all-aids, $roles)
            return
            for $concept in $concepts
            group by $archive := $concept.Archive,  $role := $concept.Role
            let $component as object := $components[$$.Archive eq $archive and $$.Role eq $role]
            let $default-hc as object := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube")
            let $members as object* := $default-hc.Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain".Members
            let $archive as object := $archives[$$._id eq $archive]
            let $entity as object := $entities[$$._id eq $archive.Entity]
            let $metadata := {
                ComponentRole : $component.Role,
                ComponentLabel : $component.Label,
                AccessionNumber : $archive._id,
                CIK : $entity._id,
                EntityRegistrantName : $entity.Profiles.SEC.CompanyName,
                FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
            }
            for $concept in $concept
            let $original-name := ($concept.Origin, $concept.Name)[1]
            return {|
                { Name: $concept.Name },
                { Origin : $concept.Origin }[exists($concept.Origin)],
                trim($members.$original-name, "Name"),
                $metadata
            |}
    ]
}
let $comment := {
    NumConcepts: count($concepts),
    TotalNumConcepts: session:num-concepts()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <ReportElements>{
                    local:to-xml($res.ReportElements[], $onlyNames)
                }</ReportElements>
    },
    to-csv : function($res as object) as string {
        local:to-csv($res.ReportElements[], $onlyNames)
    }
}
let $results := util:serialize($result, $comment, $serializers, $format, "report-elements")
return
    util:check-and-return-results($entities, $results, $format)

import module namespace util = "http://secxbrl.info/modules/util";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concept-maps = "http://28.io/modules/xbrl/concept-maps";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace seq = "http://zorba.io/modules/sequence";

declare namespace concepts = "http://www.28msec.com/modules/bizql/concepts";

declare function local:to-csv($concepts as item*, $onlyNames as boolean) as string
{
    if ($onlyNames)
    then
        string-join(("Name", $concepts), "
")
    else
        string-join(csv:serialize($concepts, { serialize-null-as : "" }))
};

declare function local:to-xml($concepts as item*, $onlyNames as boolean) as element()*
{
    for $c in $concepts
    return
        <ReportElement>{
             if ($onlyNames)
             then
                <Name>{$c}</Name>
             else (
                <Name>{$c.Name}</Name>,
                <Label>{$c.Label}</Label>,
                <IsNillable>{$c.IsNillable}</IsNillable>,
                <IsAbstract>{$c.IsAbstract}</IsAbstract>,
                <PeriodType>{$c.PeriodType}</PeriodType>,
                <Balance>{$c.Balance}</Balance>,
                <SubstitutionGroup>{$c.SubstitutionGroup}</SubstitutionGroup>,
                <DataType>{$c.DataType}</DataType>,
                <BaseType>{$c.BaseType}</BaseType>,
                <ClosestSchemaBuiltinType>{$c.ClosestSchemaBuiltinType}</ClosestSchemaBuiltinType>,
                <IsTextBlock>{$c.IsTextBlock}</IsTextBlock>,
                <ComponentRole>{$c.ComponentRole}</ComponentRole>,
                <ComponentLabel>{$c.ComponentLabel}</ComponentLabel>,
                <AccessionNumber>{$c.AccessionNumber}</AccessionNumber>,
                <CIK>{$c.CIK}</CIK>,
                <EntityRegistrantName>{$c.EntityRegistrantName}</EntityRegistrantName>,
                <FiscalYear>{$c.FiscalYear}</FiscalYear>,
                <FiscalPeriod>{$c.FiscalPeriod}</FiscalPeriod>
             )
        }</ReportElement>
};

declare function local:concepts-for-archives($aids as string*, $projection as object) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("concepts:CONNECTION-FAILED"), $err:description)
        }
    return
        mongo:find($conn, "concepts",
        {
            "Archive": { "$in" : [ $aids ] }
        },
        $projection,
        {})
};

declare function local:concepts-for-archives($aids as string*, $names as string*, $map as string?) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("concepts:CONNECTION-FAILED"), $err:description)
        }
    let $map := if (exists($map)) then concept-maps:concept-maps($map) else ()

    let $all-concepts-computable-by-maps as string* := keys($map.Trees)

    let $concepts-computable-by-maps as object* :=
        for $c in $names[$$ = $all-concepts-computable-by-maps]
        return $map.Trees.$c

    let $mapped-names := (keys($concepts-computable-by-maps.To ), $concepts-computable-by-maps.To [].Name)

    let $concepts-not-computable-by-maps as string* := seq:value-except($names, $mapped-names)
    let $results-not-computed-by-maps := mongo:find($conn, "concepts",
        {
            "Name" : { "$in" : [ $concepts-not-computable-by-maps ] },
            "Archive": { "$in" : [ $aids ] }
        })
    let $results-computed-by-maps :=
        for $c in mongo:find($conn, "concepts",
            {
                "Name" : { "$in" : [ $mapped-names ] },
                "Archive": { "$in" : [ $aids ] }
            })
        group by $c.Component
        let $c := $c[1]
        let $map-concept := (for $candidate in $concepts-computable-by-maps
                            where $c.Name = (keys($candidate.To), $candidate.To[].Name)
                            return $candidate)[1]
        return
            copy $n := $c
            modify (
                replace value of json $n.Name with $map-concept.Name,
                insert json  { Origin : $c.Name } into $n)
            return $n
    return ($results-not-computed-by-maps, $results-computed-by-maps)
};

declare function local:concepts-for-archives-and-labels($aids as string*, $labels as string) as object*
{
    let $conn :=
        let $credentials := credentials:credentials("MongoDB", "xbrl")
        return
            try {
                mongo:connect($credentials)
            } catch mongo:* {
                error(QName("components:CONNECTION-FAILED"), $err:description)
            }
    return mongo:run-cmd-deterministic(
        $conn,
        {
            "text" : "concepts",
            "filter" : { "Archive" : { "$in" : [ $aids ] } },
            "search" : $labels,
            "limit" : 100,
            "score" : { "$meta" : "textScore" },
            "sort" : { score: { "$meta" : "textScore" } }
        }).results[].obj
};

session:audit-call();

(: Query parameters :)
let $format as string?         := request:param-values("format")
let $ciks as string*           := distinct-values(request:param-values("cik"))
let $tags as string*           := distinct-values(request:param-values("tag"))
let $tickers as string*        := distinct-values(request:param-values("ticker"))
let $sics as string*           := distinct-values(request:param-values("sic"))
let $fiscalYears as string*    := distinct-values(request:param-values("fiscalYear", "LATEST"))
let $fiscalPeriods as string*  := distinct-values(request:param-values("fiscalPeriod", "FY"))
let $aids as string*           := distinct-values(request:param-values("aid"))
let $labels as string*         := request:param-values("label")
let $map as string?            := request:param-values("map")
let $report as string?         := request:param-values("report")
let $names as string*          := request:param-values("name")

(: Post-processing :)
let $format as string? := (: backwards compatibility, to be deprecated  :)
    lower-case(($format, substring-after(request:path(), ".jq."))[1])
let $tags as string* := (: backwards compatibility, to be deprecated :)
    distinct-values($tags ! upper-case($$))
let $fiscalYears as integer* :=
    for $fy in $fiscalYears ! upper-case($$)
    return switch($fy)
           case "LATEST" return $fiscal-core:LATEST_FISCAL_YEAR
           case "ALL" return $fiscal-core:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
let $fiscalPeriods as string* :=
    for $fp in $fiscalPeriods ! upper-case($$)
    return switch($fp)
           case "ALL" return $fiscal-core:ALL_FISCAL_PERIODS
           default return $fp

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $ciks,
        $tags,
        $tickers,
        $sics)
let $archives as object* := fiscal-core:filings(
    $entities,
    $fiscalPeriods,
    $fiscalYears,
    $aids)
let $entities as object* :=
    ($entities[$$._id = $archives.Entity],
    let $not-found := $archives.Entity[not $entities._id = $$]
    where exists($not-found)
    return entities:entities($not-found))
let $onlyNames   := let $o := request:param-values("onlyNames")[1] return if (exists($o)) then ($o cast as boolean) else false
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map

let $concepts := if (exists($names))
                  then local:concepts-for-archives($archives._id, $names, $map)
                  else if (exists($labels))
                  then local:concepts-for-archives-and-labels($archives._id, $labels[1])
                  else if($onlyNames)
                  then local:concepts-for-archives($archives._id, { Name: 1 })
                  else local:concepts-for-archives($archives._id, {})
let $result := {
    ReportElements : [
        if ($onlyNames)
        then distinct-values($concepts.Name)
        else
            let $all-aids := $concepts.Archive
            let $roles := $concepts.Role
            let $components := components:components-for-archives-and-roles($all-aids, $roles)
            return
            for $concept in $concepts
            group by $archive := $concept.Archive,  $role := $concept.Role
            let $component as object := $components[$$.Archive eq $archive and $$.Role eq $role]
            let $default-hc as object := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube")
            let $members as object* := $default-hc.Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain".Members
            let $archive as object := $archives[$$._id eq $archive]
            let $entity as object := $entities[$$._id eq $archive.Entity]
            let $metadata := {|
                { Origin : $concept.Origin }[exists($concept.Origin)],
                {
                    Name: $concept.Name,
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label,
                    AccessionNumber : $archive._id,
                    CIK : $entity._id,
                    EntityRegistrantName : $entity.Profiles.SEC.CompanyName,
                    FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                    FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
                }
            |}
            for $concept in $concept
            let $original-name := ($concept.Origin, $concept.Name)[1]
            return {|
                trim($members.$original-name, "Name"),
                $metadata
            |}
    ]
}
let $comment := {
    NumConcepts: count($concepts),
    TotalNumConcepts: session:num-concepts()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <ReportElements>{
                    local:to-xml($res.ReportElements[], $onlyNames)
                }</ReportElements>
    },
    to-csv : function($res as object) as string {
        local:to-csv($res.ReportElements[], $onlyNames)
    }
}
let $results := util:serialize($result, $comment, $serializers, $format, "report-elements")
return
    util:check-and-return-results($entities, $results, $format)


Add insertion function.

Add insertion function.

Instead of `setpath/2` and then `move_children_to_array`, we can now
directly insert objects at a path.

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# This is a predicate that returns true if an entry produced by `to_entries` is
# to be considered a child file, and false if it is to be considered a property
# of the file or directory.
def pred:
    .key != "meta" and (.value | type == "object")
;

# Given a JSON object that contains keys representing both child files and file
# properties, this function leaves the file properties intact and moves the
# child files to a separate array at the "contents" key.
def move_children_to_array:
    if type == "object"
    then
        ( to_entries | map(select(pred) | .value | move_children_to_array)) as $sub
        | with_entries(select(pred | not))
        | .contents = ($sub | sort_by(.modified) | reverse)
    else
        .
    end
;

# Given a stream of JSON objects of the form {"path":"a/b",…}, this function creates
# a single JSON object of the form {"path":"a", "a":{"path": "a/b"}}
def filetree:
    reduce inputs as $i
        ( {}
        ; ($i.path | split("/")) as $p
        | setpath($p; getpath($p) + ($i | .name |= $p[-1:][0]))
        )
;

# Merge together `filetree.json` and `*.meta.json` files, as given in the
# format produced by `file_entries`.
def combine_filetree_metadata:
    reduce (to_entries | .[]) as $entry
        (   {}
        ;   . * (
            if ($entry.key | endswith(".meta.json"))
            then
                ( $entry.key
                    | ltrimstr($prefix)
                    | rtrimstr(".meta.json")
                    | split("/")
                ) as $path
                | {}
                | setpath($path + ["meta"]; $entry.value)
            else
                $entry.value
            end
            )
        )
;

# Select the descendant described by the array of names in `$path`.
def descend($path):
    if ($path | length) > 0
    then
        .contents[]? | select(.name == $path[0]) | descend($path[1:])
    else
        .
    end
;

# Insert a child element at a particular path.
# Like `setpath/2`, but instead of making objects like `{"a":{"b":{…}}}`, this
# makes objects like `{"contents":[{"name":"a", "contents":[{"name":"b",…}]}]}`
# I don't really like this function, because it's so verbose. Making a path
# expression for updating with the |= operator (see `descend/1`) doesn't seem
# to work when it is recursive?
def insert($path; $child):
    if ($path | length) > 0
    then
        if ([.contents[]? | select(.path == $path[0])] | length) > 0
        then
            (.contents[] | select(.name == $path[0])) |= (. | insert($path[1:]; $child))
        else
            (.contents = [.contents[]?] + [{"name":$path[0]} | insert($path[1:];$child)])
        end
    else
        . * $child
    end
;

# Make an object out of a stream of files, associating the each filename with
# its contents. Use with the `--null-input` switch.
def file_entries:
    reduce inputs as $input
        ( {}
        ; . + { ($input | input_filename) : $input }
        )
;

# Adds links to each page object. The link should be the same as the path,
# except in the case of directories, which may either link to its index.md or
# have no link at all. If there is only one child to a page, perhaps it should
# collapse onto its parent.
def add_links:
    "work in progress"
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    file_entries
    | combine_filetree_metadata
    | move_children_to_array
;

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# This is a predicate that returns true if an entry produced by `to_entries` is
# to be considered a child file, and false if it is to be considered a property
# of the file or directory.
def pred:
    .key != "meta" and (.value | type == "object")
;

# Given a JSON object that contains keys representing both child files and file
# properties, this function leaves the file properties intact and moves the
# child files to a separate array at the "contents" key.
def move_children_to_array:
    if type == "object"
    then
        ( to_entries | map(select(pred) | .value | move_children_to_array)) as $sub
        | with_entries(select(pred | not))
        | .contents = ($sub | sort_by(.modified) | reverse)
    else
        .
    end
;

# Given a stream of JSON objects of the form {"path":"a/b",…}, this function creates
# a single JSON object of the form {"path":"a", "a":{"path": "a/b"}}
def filetree:
    reduce inputs as $i
        ( {}
        ; ($i.path | split("/")) as $p
        | setpath($p; getpath($p) + ($i | .name |= $p[-1:][0]))
        )
;

# Merge together `filetree.json` and `*.meta.json` files, as given in the
# format produced by `file_entries`.
def combine_filetree_metadata:
    reduce (to_entries | .[]) as $entry
        (   {}
        ;   . * (
            if ($entry.key | endswith(".meta.json"))
            then
                ( $entry.key
                    | ltrimstr($prefix)
                    | rtrimstr(".meta.json")
                    | split("/")
                ) as $path
                | {}
                | setpath($path + ["meta"]; $entry.value)
            else
                $entry.value
            end
            )
        )
;

# Make an object out of a stream of files, associating the each filename with
# its contents. Use with the `--null-input` switch.
def file_entries:
    reduce inputs as $input
        ( {}
        ; . + { ($input | input_filename) : $input }
        )
;

# Adds links to each page object. The link should be the same as the path,
# except in the case of directories, which may either link to its index.md or
# have no link at all. If there is only one child to a page, perhaps it should
# collapse onto its parent.
def add_links:
    "work in progress"
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    file_entries
    | combine_filetree_metadata
    | move_children_to_array
;


:green_heart: Fix error.

:green_heart: Fix error.

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace filings = "http://28.io/modules/xbrl/profiles/sec/filings";
import module namespace entities = "http://28.io/modules/xbrl/entities";

import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";

import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-xml-rec($o as object*, $level as integer) as element()*
{
    for $o in $o
    return
        element { if ($o.Kind eq "Domain") then "Member" else $o.Kind } {
            attribute { "name" } { $o.Name },
            attribute { "label" } { $o.Label },
            if (exists($o.Balance)) then attribute { "balance" } { $o.Balance } else (),
            if (exists($o.DataType)) then attribute { "dataType" } { $o.DataType } else (),
            if (exists($o.BaseType)) then attribute { "baseDataType" } { $o.BaseType } else (),
            if (exists($o.IsAbstract)) then attribute { "abstract" } { $o.IsAbstract } else (),
            if (exists($o.PeriodType)) then attribute { "periodType" } { $o.PeriodType } else (),
            attribute { "level" } { $level },
            local:to-xml-rec($o.Children[], $level + 1)
        }
};

declare function local:to-xml($model as object) as node()*
{
    <Component>
        <Network entityRegistrantName="{$model.EntityRegistrantName}"
                 accessionNumber="{$model.AccessionNumber}"
                 tableName="{$model.TableName}"
                 cik="{$model.CIK}"
                 label="{$model.Label}"
                 networkIdentifier="{$model.NetworkIdentifier}"
                 formType="{$model.FormType}"
                 fiscalPeriod="{$model.FiscalPeriod}"
                 fiscalYear="{$model.FiscalYear}"
                 acceptanceDatetime="{$model.AcceptanceDatetime}"
                 disclosure="{$model.Disclosure}"
                 >{
            local:to-xml-rec($model.ModelStructure[], 0)
        }</Network>
    </Component>
};

declare function local:to-csv-rec($objects as object*, $level as integer) as object*
{
    for $o in $objects
    let $object := {
        "Label" : $o.Label,
        "Name" : $o.Name,
        "ObjectClass" : if ($o.Kind eq "Domain") then "Member" else $o.Kind,
        "DataType" : $o.DataType,
        "BaseDataType" : $o.BaseType,
        "Balance" : $o.Balance,
        "Abstract" : $o.IsAbstract,
        "PeriodType" : $o.PeriodType,
        "Level" : $level
    }
    return (
        $object,
        local:to-csv-rec($o.Children[], $level + 1)
    )
};

declare function local:to-csv($model as object) as string
{
    let $lines := local:to-csv-rec($model.ModelStructure, 0)
    return
        if (exists($lines))
        then string-join(csv:serialize($lines, { serialize-null-as : "" }))
        else ""
};

declare function local:enrich-json-rec($objects as object*, $level as integer) as object*
{
    for $object in $objects
    return
        copy $o := $object
        modify (
            if (exists($o.Children)) then delete json $o("Children") else (),
            insert json { Level : $level } into $o,
            if ($o.Kind eq "Domain") then replace value of json $o("Kind") with "Member" else ()
        )
        return {|
            $o,
            let $children := local:enrich-json-rec($object.Children[], $level + 1)
            return if (exists($children)) then { Children : [ $children ] } else ()
        |}
};

declare function local:enrich-json($component as object) as object
{
    {
        ModelStructure : [ local:enrich-json-rec($component.ModelStructure, 0) ] ,
        CIK : $component.CIK,
        EntityRegistrantName : $component.EntityRegistrantName,
        Label : $component.Label,
        AccessionNumber : $component.AccessionNumber,
        TableName : $component.TableName,
        FormType : $component.FormType,
        FiscalPeriod : $component.FiscalPeriod,
        FiscalYear : $component.FiscalYear,
        AcceptanceDatetime : $component.AcceptanceDatetime,
        NetworkIdentifier: $component.NetworkIdentifier,
        Disclosure : $component.Disclosure
    }
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token              as string? external;
declare  %rest:env                              variable $request-uri        as string  external;
declare  %rest:case-insensitive                 variable $format             as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker             as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear         as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod       as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $filingKind         as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $eid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $networkIdentifier  as string* external;
declare  %rest:case-insensitive                 variable $cid                as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $reportElement      as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $concept            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $label              as string* external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := $config:profile-name;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $reportElement := ($reportElement, $concept)

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic, ())

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $filingKind,
  $aid)

let $components  := sec-networks:components(
    $archives,
    $cid,
    $reportElement,
    $disclosure,
    $networkIdentifier,
    $label)
let $component := $components[1] (: only one for now :)
let $archive   := archives:archives($component.Archive)
let $entity    := entities:entities($archive.Entity)

let $result := {
    CIK : entities:eid($entity),
    EntityRegistrantName : $entity.Profiles.SEC.CompanyName,
    ModelStructure : [ sec-networks:model-structures($component) ],
    TableName : components:hypercubes($component),
    Label : $component.Label,
    AccessionNumber : $component.Archive,
    FormType : $archive.Profiles.SEC.FormType,
    FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
    FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
    AcceptanceDatetime : filings:acceptance-dateTimes($archive),
    NetworkIdentifier: $component.Role,
    Disclosure : $component.Profiles.SEC.Disclosure
}
let $comment := {
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : local:to-xml#1,
    to-csv : local:to-csv#1
}
return if (exists($component))
    then api:serialize($result, $comment, $serializers, $format, "components")
    else {
        response:status-code(404);
        response:content-type("application/json");
        session:error("component not found", "json")
    }

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace filings = "http://28.io/modules/xbrl/profiles/sec/filings";
import module namespace entities = "http://28.io/modules/xbrl/entities";

import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";

import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-xml-rec($o as object*, $level as integer) as element()*
{
    for $o in $o
    return
        element { if ($o.Kind eq "Domain") then "Member" else $o.Kind } {
            attribute { "name" } { $o.Name },
            attribute { "label" } { $o.Label },
            if (exists($o.Balance)) then attribute { "balance" } { $o.Balance } else (),
            if (exists($o.DataType)) then attribute { "dataType" } { $o.DataType } else (),
            if (exists($o.BaseType)) then attribute { "baseDataType" } { $o.BaseType } else (),
            if (exists($o.IsAbstract)) then attribute { "abstract" } { $o.IsAbstract } else (),
            if (exists($o.PeriodType)) then attribute { "periodType" } { $o.PeriodType } else (),
            attribute { "level" } { $level },
            local:to-xml-rec($o.Children[], $level + 1)
        }
};

declare function local:to-xml($model as object) as node()*
{
    <Component>
        <Network entityRegistrantName="{$model.EntityRegistrantName}"
                 accessionNumber="{$model.AccessionNumber}"
                 tableName="{$model.TableName}"
                 cik="{$model.CIK}"
                 label="{$model.Label}"
                 networkIdentifier="{$model.NetworkIdentifier}"
                 formType="{$model.FormType}"
                 fiscalPeriod="{$model.FiscalPeriod}"
                 fiscalYear="{$model.FiscalYear}"
                 acceptanceDatetime="{$model.AcceptanceDatetime}"
                 disclosure="{$model.Disclosure}"
                 >{
            local:to-xml-rec($model.ModelStructure[], 0)
        }</Network>
    </Component>
};

declare function local:to-csv-rec($objects as object*, $level as integer) as object*
{
    for $o in $objects
    let $object := {
        "Label" : $o.Label,
        "Name" : $o.Name,
        "ObjectClass" : if ($o.Kind eq "Domain") then "Member" else $o.Kind,
        "DataType" : $o.DataType,
        "BaseDataType" : $o.BaseType,
        "Balance" : $o.Balance,
        "Abstract" : $o.IsAbstract,
        "PeriodType" : $o.PeriodType,
        "Level" : $level
    }
    return (
        $object,
        local:to-csv-rec($o.Children[], $level + 1)
    )
};

declare function local:to-csv($model as object) as string
{
    let $lines := local:to-csv-rec($model.ModelStructure, 0)
    return
        if (exists($lines))
        then string-join(csv:serialize($lines, { serialize-null-as : "" }))
        else ""
};

declare function local:enrich-json-rec($objects as object*, $level as integer) as object*
{
    for $object in $objects
    return
        copy $o := $object
        modify (
            if (exists($o.Children)) then delete json $o("Children") else (),
            insert json { Level : $level } into $o,
            if ($o.Kind eq "Domain") then replace value of json $o("Kind") with "Member" else ()
        )
        return {|
            $o,
            let $children := local:enrich-json-rec($object.Children[], $level + 1)
            return if (exists($children)) then { Children : [ $children ] } else ()
        |}
};

declare function local:enrich-json($component as object) as object
{
    {
        ModelStructure : [ local:enrich-json-rec($component.ModelStructure, 0) ] ,
        CIK : $component.CIK,
        EntityRegistrantName : $component.EntityRegistrantName,
        Label : $component.Label,
        AccessionNumber : $component.AccessionNumber,
        TableName : $component.TableName,
        FormType : $component.FormType,
        FiscalPeriod : $component.FiscalPeriod,
        FiscalYear : $component.FiscalYear,
        AcceptanceDatetime : $component.AcceptanceDatetime,
        NetworkIdentifier: $component.NetworkIdentifier,
        Disclosure : $component.Disclosure
    }
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token              as string? external;
declare  %rest:env                              variable $request-uri        as string  external;
declare  %rest:case-insensitive                 variable $format             as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker             as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear         as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod       as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $filingKind         as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $eid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $networkIdentifier  as string* external;
declare  %rest:case-insensitive                 variable $cid                as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $reportElement      as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $concept            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $label              as string* external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := $config:profile-name;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $reportElement := ($reportElement, $concept)

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic, ())

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $filingKind,
  $aid)

let $components  := sec-networks:components(
    $archives,
    $cid,
    $reportElement,
    $disclosure,
    $networkIdentifier,
    $label)
let $component := $components[1] (: only one for now :)
let $archive   := archives:archives($component.Archive)
let $entity    := entities:entities($archive.Entity)

let $result := {|
    { CIK : entities:eid($entity) },
    { EntityRegistrantName : $entity.Profiles.SEC.CompanyName },
    { ModelStructure : [ sec-networks:model-structures($component) ] },
    { TableName : sec-networks:tables($component, {IncludeImpliedTable: true}).Name },
    { Label : $component.Label },
    { AccessionNumber : $component.Archive },
    { FormType : $archive.Profiles.SEC.FormType },
    { FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus },
    { FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus },
    { AcceptanceDatetime : filings:acceptance-dateTimes($archive) },
    { NetworkIdentifier: $component.Role },
    { Disclosure : $component.Profiles.SEC.Disclosure }
|}
let $comment := {
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : local:to-xml#1,
    to-csv : local:to-csv#1
}
return if (exists($component))
    then api:serialize($result, $comment, $serializers, $format, "components")
    else {
        response:status-code(404);
        response:content-type("application/json");
        session:error("component not found", "json")
    }


Add links and index links to index.json.

Once again - not the nicest-looking code. I hope cleanup will make it
loo nice.

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Select the descendant described by the array of names in `$path`.
def descend($path):
    if ($path | length) > 0
    then
        .contents[]? | select(.name == $path[0]) | descend($path[1:])
    else
        .
    end
;

# Insert a child element at a particular path.
# Like `setpath/2`, but instead of making objects like `{"a":{"b":{…}}}`, this
# makes objects like `{"contents":[{"name":"a", "contents":[{"name":"b",…}]}]}`
# I don't really like this function, because it's so verbose. Making a path
# expression for updating with the |= operator (see `descend/1`) doesn't seem
# to work when it is recursive?
def insert($path; $child):
    if ($path | length) > 0
    then
        if ([.contents[]? | select(.name == $path[0])] | length) > 0
        then
            (.contents[] | select(.name == $path[0])) |= (. | insert($path[1:]; $child))
        else
            (.contents = [.contents[]?] + [{"name":$path[0]} | insert($path[1:];$child)])
        end
    else
        . * $child
    end
;

# Given a stream of JSON objects of the form {"path":"a/b",…}, this function creates
# a single JSON object of the form {"path":"a", "a":{"path": "a/b"}}
def filetree:
    reduce inputs as $i
        ( []
        ; . + [$i]
        )
;

# Merge together `filetree.json` and `*.meta.json` files, as given in the
# format produced by `file_entries`.
def combine_files:
    reduce (to_entries | .[]) as $entry
        (   {}
        ;   . * (
            if ($entry.key | endswith(".meta.json"))
            then
                ( $entry.key
                    | ltrimstr($prefix)
                    | rtrimstr(".meta.json")
                    | split("/")
                ) as $path
                | insert($path; {"meta": $entry.value})
            else
                reduce $entry.value[] as $f
                ( .
                ; insert($f.path | split("/"); $f)
                )
            end
            )
        )
;

# Make an object out of a stream of files, associating the each filename with
# its contents. Use with the `--null-input` switch.
def file_entries:
    reduce inputs as $input
        ( {}
        ; . + { ($input | input_filename) : $input }
        )
;

# Adds links to each page object. The link should be the same as the path,
# except in the case of directories, which may either link to its index.md or
# have no link at all. If there is only one child to a page, perhaps it should
# collapse onto its parent.
# Also not great.
def add_links:
    if (.path and (.path | endswith(".md")))
    then
        .link = (.path | rtrimstr(".md") | . + ".html")
    else
        [.contents[]? | add_links] as $newcontents
        |   [ $newcontents[] | select(.name=="index.md") ] as $indexfiles
        |   [ $newcontents[] | select(.name!="index.md") ] as $otherfiles
        |   if ($newcontents | length) == 0 then
                .
            elif ($indexfiles | length) == 0 then
                .contents = $newcontents
            else
                $indexfiles[0] as $index
                | .contents = $otherfiles + $indexfiles[1:]
                | .link = $index.link
            end
    end
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    file_entries
    | combine_files
    | add_links
;

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Select the descendant described by the array of names in `$path`.
def descend($path):
    if ($path | length) > 0
    then
        .contents[]? | select(.name == $path[0]) | descend($path[1:])
    else
        .
    end
;

# Insert a child element at a particular path.
# Like `setpath/2`, but instead of making objects like `{"a":{"b":{…}}}`, this
# makes objects like `{"contents":[{"name":"a", "contents":[{"name":"b",…}]}]}`
# I don't really like this function, because it's so verbose. Making a path
# expression for updating with the |= operator (see `descend/1`) doesn't seem
# to work when it is recursive?
def insert($path; $child):
    if ($path | length) > 0
    then
        if ([.contents[]? | select(.name == $path[0])] | length) > 0
        then
            (.contents[] | select(.name == $path[0])) |= (. | insert($path[1:]; $child))
        else
            (.contents = [.contents[]?] + [{"name":$path[0]} | insert($path[1:];$child)])
        end
    else
        . * $child
    end
;

# Given a stream of JSON objects of the form {"path":"a/b",…}, this function creates
# a single JSON object of the form {"path":"a", "a":{"path": "a/b"}}
def filetree:
    reduce inputs as $i
        ( []
        ; . + [$i]
        )
;

# Merge together `filetree.json` and `*.meta.json` files, as given in the
# format produced by `file_entries`.
def combine_files:
    reduce (to_entries | .[]) as $entry
        (   {}
        ;   . * (
            if ($entry.key | endswith(".meta.json"))
            then
                ( $entry.key
                    | ltrimstr($prefix)
                    | rtrimstr(".meta.json")
                    | split("/")
                ) as $path
                | insert($path; {"meta": $entry.value})
            else
                reduce $entry.value[] as $f
                ( .
                ; insert($f.path | split("/"); $f)
                )
            end
            )
        )
;

# Make an object out of a stream of files, associating the each filename with
# its contents. Use with the `--null-input` switch.
def file_entries:
    reduce inputs as $input
        ( {}
        ; . + { ($input | input_filename) : $input }
        )
;

# Adds links to each page object. The link should be the same as the path,
# except in the case of directories, which may either link to its index.md or
# have no link at all. If there is only one child to a page, perhaps it should
# collapse onto its parent.
def add_links:
    "work in progress"
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    file_entries
    | combine_files
;


Always asking for token in FYandFPByCIK.jq

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace sec-fiscal = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace session = "http://apps.28.io/session";
import module namespace api = "http://apps.28.io/api";

declare %an:sequential function local:filingPeriodInfo($archives as object*) as object*{
    for $archive in $archives
    return {
        fiscalPeriod: sec-fiscal:fiscal-period($archive),
        fiscalYear: sec-fiscal:fiscal-year($archive),
        endDate:  sec:end-date($archive)
    }
};

(: Query parameters :)
declare  %rest:case-insensitive  variable $token        as string? external;
declare  %rest:env               variable $request-uri  as string  external;
declare  %rest:case-insensitive  variable $format       as string? external;
declare  %rest:case-insensitive  variable $cik          as string  external := "0000354950";

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $cik := if (empty(entities:entities(companies:eid($cik))))
            then error(QName("local:INVALID-REQUEST"), "Given CIK:"||$cik|| " not found")
            else $cik

(: Object resolution :)
let $entity := entities:entities(companies:eid($cik))
let $archives :=  archives:archives-for-entities($entity)
return
    switch(session:has-access($token, "data_sec"))
    case $session:ACCESS-ALLOWED return {
            cik: $cik,
            companyName: $entity.Profiles.SEC.CompanyName,
            filings: [local:filingPeriodInfo($archives)]
       }
    case $session:ACCESS-DENIED return {
          response:status-code(403);
          session:error("accessing filings of an entity that is not in the DOW30", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          response:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace sec-fiscal = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace session = "http://apps.28.io/session";
import module namespace api = "http://apps.28.io/api";

declare %an:sequential function local:filingPeriodInfo($archives as object*) as object*{
    for $archive in $archives
    return {
        fiscalPeriod: sec-fiscal:fiscal-period($archive),
        fiscalYear: sec-fiscal:fiscal-year($archive),
        endDate:  sec:end-date($archive)
    }
};

(: Query parameters :)
declare  %rest:case-insensitive  variable $token        as string? external;
declare  %rest:env               variable $request-uri  as string  external;
declare  %rest:case-insensitive  variable $format       as string? external;
declare  %rest:case-insensitive  variable $cik          as string  external := "0000354950";

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $cik := if (empty(entities:entities(companies:eid($cik))))
            then error(QName("local:INVALID-REQUEST"), "Given CIK:"||$cik|| " not found")
            else $cik

(: Object resolution :)
let $entity := entities:entities(companies:eid($cik))
let $archives :=  archives:archives-for-entities($entity)
return
    switch(session:has-access($token, $entity, "data_sec"))
    case $session:ACCESS-ALLOWED return {
            cik: $cik,
            companyName: $entity.Profiles.SEC.CompanyName,
            filings: [local:filingPeriodInfo($archives)]
       }
    case $session:ACCESS-DENIED return {
          response:status-code(403);
          session:error("accessing filings of an entity that is not in the DOW30", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          response:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()