Adding eid parameter.

Adding eid parameter.

import module namespace api = "http://apps.28.io/api";

import module namespace session = "http://apps.28.io/session";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concept-maps = "http://28.io/modules/xbrl/concept-maps";
import module namespace config = "http://apps.28.io/config";

import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace seq = "http://zorba.io/modules/sequence";

declare function local:to-csv($concepts as item*, $onlyNames as boolean) as string
{
    if ($onlyNames)
    then
        string-join(("Name", $concepts), "
")
    else
        string-join(csv:serialize($concepts, { serialize-null-as : "" }))
};

declare function local:to-xml($concepts as item*, $onlyNames as boolean) as element()*
{
    for $c in $concepts
    return
        <ReportElement>{
             if ($onlyNames)
             then
                <Name>{$c}</Name>
             else (
                <Name>{$c.Name}</Name>,
                <Label>{$c.Label}</Label>,
                <IsNillable>{$c.IsNillable}</IsNillable>,
                <IsAbstract>{$c.IsAbstract}</IsAbstract>,
                <PeriodType>{$c.PeriodType}</PeriodType>,
                <Balance>{$c.Balance}</Balance>,
                <SubstitutionGroup>{$c.SubstitutionGroup}</SubstitutionGroup>,
                <DataType>{$c.DataType}</DataType>,
                <BaseType>{$c.BaseType}</BaseType>,
                <ClosestSchemaBuiltinType>{$c.ClosestSchemaBuiltinType}</ClosestSchemaBuiltinType>,
                <IsTextBlock>{$c.IsTextBlock}</IsTextBlock>,
                <ComponentRole>{$c.ComponentRole}</ComponentRole>,
                <ComponentLabel>{$c.ComponentLabel}</ComponentLabel>,
                <AccessionNumber>{$c.AccessionNumber}</AccessionNumber>,
                <CIK>{$c.CIK}</CIK>,
                <EntityRegistrantName>{$c.EntityRegistrantName}</EntityRegistrantName>,
                <FiscalYear>{$c.FiscalYear}</FiscalYear>,
                <FiscalPeriod>{$c.FiscalPeriod}</FiscalPeriod>
             )
        }</ReportElement>
};

declare function local:concepts-for-archives(
    $aids as string*,
    $names as string*,
    $map as object?,
    $options as object?) as object*
{
    let $projection as object := if($options.OnlyNames eq true) then { Name: 1 } else {}
    let $concepts-computable-by-maps as object* :=
        switch(true)
            case not exists($map) return ()
            case not exists($names) return values($map.Trees)
            default return
                let $keys as string* := keys($map.Trees)
                for $concept as string in $names[$$ = $keys]
                return $map.Trees.$concept
    let $mapped-names as string* := (keys($concepts-computable-by-maps.To ), $concepts-computable-by-maps.To [].Name)
    let $concepts-not-computable-by-maps as string* := seq:value-except($names, $mapped-names)

    let $all-results as object* := mw:find($concepts:col,
        {
            "Archive": { "$in" : [ $aids ] }
        },
        $projection)
    let $results-not-computed-by-maps as object* := mw:find($concepts:col,
        {
            "Name" : { "$in" : [ $concepts-not-computable-by-maps ] },
            "Archive": { "$in" : [ $aids ] }
        },
        $projection)
    let $results-computed-by-maps as object* :=
        let $all-results as object* := mw:find($concepts:col,
            {
                "Name" : { "$in" : [ $mapped-names ] },
                "Archive": { "$in" : [ $aids ] }
            },
            $projection)
        for $concept as object in $concepts-computable-by-maps
        for $result as object in
            for $candidate-concept in (keys($concept.To), $concept[].Name)
            let $facts := $all-results[$$.Name = $candidate-concept]
            where exists($facts)
            count $c
            where $c eq 1
            return $facts
        let $map-concept := (for $candidate in $concepts-computable-by-maps
                            where $result.Name = (keys($candidate.To), $candidate.To[].Name)
                            return $candidate)[1]
        return
            copy $n := $result
            modify (
                replace value of json $n.Name with $map-concept.Name,
                insert json  { Origin : $result.Name } into $n)
            return $n
    return if(empty($map) and empty($names))
           then $all-results
           else ($results-not-computed-by-maps, $results-computed-by-maps)
};

declare function local:concepts-for-archives-and-labels($aids as string*, $labels as string) as object*
{
   mw:run-cmd-deterministic(
      {
        "text" : "concepts",
        "filter" : { "Archive" : { "$in" : [ $aids ] } },
        "search" : $labels,
        "limit" : 100,
        "score" : { "$meta" : "textScore" },
        "sort" : { score: { "$meta" : "textScore" } }
      }).results[].obj
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token         as string?  external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := $config:profile-name;
declare  %rest:env                              variable $request-uri   as string   external;
declare  %rest:case-insensitive                 variable $format        as string?  external;
declare  %rest:case-insensitive %rest:distinct  variable $cik           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $tag           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker        as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $sic           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear    as string*  external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod  as string*  external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $eid           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $label         as string*  external;
declare  %rest:case-insensitive                 variable $map           as string?  external;
declare  %rest:case-insensitive                 variable $report        as string?  external;
declare  %rest:case-insensitive %rest:distinct  variable $name          as string*  external;
declare  %rest:case-insensitive                 variable $onlyNames     as boolean? external := false;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic)

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $aid)

let $entities as object* :=
    ($entities[$$._id = $archives.Entity],
    let $not-found := $archives.Entity[not $entities._id = $$]
    where exists($not-found)
    return entities:entities($not-found))
let $map as object? :=
    if(exists($report))
    then reports:concept-map($report)
    else concept-maps:concept-maps($map)
let $concepts :=
    if($profile-name eq "sec")
    then
        if (exists($label))
            then local:concepts-for-archives-and-labels($archives._id, $label[1])
            else local:concepts-for-archives($archives._id, $name, $map, { OnlyNames: $onlyNames })
    else
        if (exists($label))
            then local:concepts-for-archives-and-labels($aid, $label[1])
            else local:concepts-for-archives($aid, $name, $map, { OnlyNames: $onlyNames })

let $result :=
    if($profile-name eq "sec")
    then {
        ReportElements : [
            if ($onlyNames)
            then distinct-values($concepts.Name)
            else
                let $all-aids := $concepts.Archive
                let $roles := $concepts.Role
                let $components := components:components-for-archives-and-roles($all-aids, $roles)
                return
                for $concept in $concepts
                group by $archive := $concept.Archive,  $role := $concept.Role
                let $component as object := $components[$$.Archive eq $archive and $$.Role eq $role]
                let $default-hc as object := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube")
                let $members as object* := $default-hc.Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain".Members
                let $archive as object := $archives[$$._id eq $archive]
                let $entity as object := $entities[$$._id eq $archive.Entity]
                let $metadata := {
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label,
                    AccessionNumber : $archive._id,
                    CIK : $entity._id,
                    EntityRegistrantName : $entity.Profiles.SEC.CompanyName,
                    FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                    FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
                }
                for $concept in $concept
                let $original-name := ($concept.Origin, $concept.Name)[1]
                return {|
                    project($concept, ("Name", "Origin")),
                    trim($members.$original-name, "Name"),
                    $metadata
                |}
        ]
    }
    else {
        ReportElements : [
            if ($onlyNames)
            then distinct-values($concepts.Name)
            else
                let $all-aids := $concepts.Archive
                let $roles := $concepts.Role
                let $components := components:components-for-archives-and-roles($all-aids, $roles)
                return
                for $concept in $concepts
                group by $archive := $concept.Archive,  $role := $concept.Role
                let $component as object := $components[$$.Archive eq $archive and $$.Role eq $role]
                let $default-hc as object := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube")
                let $members as object* := $default-hc.Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain".Members
                let $metadata := {
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label,
                    AccessionNumber : $archive._id,
                    FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                    FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
                }
                for $concept in $concept
                let $original-name := ($concept.Origin, $concept.Name)[1]
                return {|
                    project($concept, ("Name", "Origin")),
                    trim($members.$original-name, "Name"),
                    $metadata
                |}
        ]
    }
let $comment := {
    NumConcepts: count($concepts),
    TotalNumConcepts: session:num-concepts()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <ReportElements>{
                    local:to-xml($res.ReportElements[], $onlyNames)
                }</ReportElements>
    },
    to-csv : function($res as object) as string {
        local:to-csv($res.ReportElements[], $onlyNames)
    }
}
let $results := api:serialize($result, $comment, $serializers, $format, "report-elements")
return api:check-and-return-results($token, $results, $format)

import module namespace api = "http://apps.28.io/api";

import module namespace session = "http://apps.28.io/session";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concept-maps = "http://28.io/modules/xbrl/concept-maps";
import module namespace config = "http://apps.28.io/config";

import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace seq = "http://zorba.io/modules/sequence";

declare function local:to-csv($concepts as item*, $onlyNames as boolean) as string
{
    if ($onlyNames)
    then
        string-join(("Name", $concepts), "
")
    else
        string-join(csv:serialize($concepts, { serialize-null-as : "" }))
};

declare function local:to-xml($concepts as item*, $onlyNames as boolean) as element()*
{
    for $c in $concepts
    return
        <ReportElement>{
             if ($onlyNames)
             then
                <Name>{$c}</Name>
             else (
                <Name>{$c.Name}</Name>,
                <Label>{$c.Label}</Label>,
                <IsNillable>{$c.IsNillable}</IsNillable>,
                <IsAbstract>{$c.IsAbstract}</IsAbstract>,
                <PeriodType>{$c.PeriodType}</PeriodType>,
                <Balance>{$c.Balance}</Balance>,
                <SubstitutionGroup>{$c.SubstitutionGroup}</SubstitutionGroup>,
                <DataType>{$c.DataType}</DataType>,
                <BaseType>{$c.BaseType}</BaseType>,
                <ClosestSchemaBuiltinType>{$c.ClosestSchemaBuiltinType}</ClosestSchemaBuiltinType>,
                <IsTextBlock>{$c.IsTextBlock}</IsTextBlock>,
                <ComponentRole>{$c.ComponentRole}</ComponentRole>,
                <ComponentLabel>{$c.ComponentLabel}</ComponentLabel>,
                <AccessionNumber>{$c.AccessionNumber}</AccessionNumber>,
                <CIK>{$c.CIK}</CIK>,
                <EntityRegistrantName>{$c.EntityRegistrantName}</EntityRegistrantName>,
                <FiscalYear>{$c.FiscalYear}</FiscalYear>,
                <FiscalPeriod>{$c.FiscalPeriod}</FiscalPeriod>
             )
        }</ReportElement>
};

declare function local:concepts-for-archives(
    $aids as string*,
    $names as string*,
    $map as object?,
    $options as object?) as object*
{
    let $projection as object := if($options.OnlyNames eq true) then { Name: 1 } else {}
    let $concepts-computable-by-maps as object* :=
        switch(true)
            case not exists($map) return ()
            case not exists($names) return values($map.Trees)
            default return
                let $keys as string* := keys($map.Trees)
                for $concept as string in $names[$$ = $keys]
                return $map.Trees.$concept
    let $mapped-names as string* := (keys($concepts-computable-by-maps.To ), $concepts-computable-by-maps.To [].Name)
    let $concepts-not-computable-by-maps as string* := seq:value-except($names, $mapped-names)

    let $all-results as object* := mw:find($concepts:col,
        {
            "Archive": { "$in" : [ $aids ] }
        },
        $projection)
    let $results-not-computed-by-maps as object* := mw:find($concepts:col,
        {
            "Name" : { "$in" : [ $concepts-not-computable-by-maps ] },
            "Archive": { "$in" : [ $aids ] }
        },
        $projection)
    let $results-computed-by-maps as object* :=
        let $all-results as object* := mw:find($concepts:col,
            {
                "Name" : { "$in" : [ $mapped-names ] },
                "Archive": { "$in" : [ $aids ] }
            },
            $projection)
        for $concept as object in $concepts-computable-by-maps
        for $result as object in
            for $candidate-concept in (keys($concept.To), $concept[].Name)
            let $facts := $all-results[$$.Name = $candidate-concept]
            where exists($facts)
            count $c
            where $c eq 1
            return $facts
        let $map-concept := (for $candidate in $concepts-computable-by-maps
                            where $result.Name = (keys($candidate.To), $candidate.To[].Name)
                            return $candidate)[1]
        return
            copy $n := $result
            modify (
                replace value of json $n.Name with $map-concept.Name,
                insert json  { Origin : $result.Name } into $n)
            return $n
    return if(empty($map) and empty($names))
           then $all-results
           else ($results-not-computed-by-maps, $results-computed-by-maps)
};

declare function local:concepts-for-archives-and-labels($aids as string*, $labels as string) as object*
{
   mw:run-cmd-deterministic(
      {
        "text" : "concepts",
        "filter" : { "Archive" : { "$in" : [ $aids ] } },
        "search" : $labels,
        "limit" : 100,
        "score" : { "$meta" : "textScore" },
        "sort" : { score: { "$meta" : "textScore" } }
      }).results[].obj
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token         as string?  external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := $config:profile-name;
declare  %rest:env                              variable $request-uri   as string   external;
declare  %rest:case-insensitive                 variable $format        as string?  external;
declare  %rest:case-insensitive %rest:distinct  variable $cik           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $tag           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker        as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $sic           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear    as string*  external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod  as string*  external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $label         as string*  external;
declare  %rest:case-insensitive                 variable $map           as string?  external;
declare  %rest:case-insensitive                 variable $report        as string?  external;
declare  %rest:case-insensitive %rest:distinct  variable $name          as string*  external;
declare  %rest:case-insensitive                 variable $onlyNames     as boolean? external := false;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic)

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $aid)

let $entities as object* :=
    ($entities[$$._id = $archives.Entity],
    let $not-found := $archives.Entity[not $entities._id = $$]
    where exists($not-found)
    return entities:entities($not-found))
let $map as object? :=
    if(exists($report))
    then reports:concept-map($report)
    else concept-maps:concept-maps($map)
let $concepts :=
    if($profile-name eq "sec")
    then
        if (exists($label))
            then local:concepts-for-archives-and-labels($archives._id, $label[1])
            else local:concepts-for-archives($archives._id, $name, $map, { OnlyNames: $onlyNames })
    else
        if (exists($label))
            then local:concepts-for-archives-and-labels($aid, $label[1])
            else local:concepts-for-archives($aid, $name, $map, { OnlyNames: $onlyNames })

let $result :=
    if($profile-name eq "sec")
    then {
        ReportElements : [
            if ($onlyNames)
            then distinct-values($concepts.Name)
            else
                let $all-aids := $concepts.Archive
                let $roles := $concepts.Role
                let $components := components:components-for-archives-and-roles($all-aids, $roles)
                return
                for $concept in $concepts
                group by $archive := $concept.Archive,  $role := $concept.Role
                let $component as object := $components[$$.Archive eq $archive and $$.Role eq $role]
                let $default-hc as object := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube")
                let $members as object* := $default-hc.Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain".Members
                let $archive as object := $archives[$$._id eq $archive]
                let $entity as object := $entities[$$._id eq $archive.Entity]
                let $metadata := {
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label,
                    AccessionNumber : $archive._id,
                    CIK : $entity._id,
                    EntityRegistrantName : $entity.Profiles.SEC.CompanyName,
                    FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                    FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
                }
                for $concept in $concept
                let $original-name := ($concept.Origin, $concept.Name)[1]
                return {|
                    project($concept, ("Name", "Origin")),
                    trim($members.$original-name, "Name"),
                    $metadata
                |}
        ]
    }
    else {
        ReportElements : [
            if ($onlyNames)
            then distinct-values($concepts.Name)
            else
                let $all-aids := $concepts.Archive
                let $roles := $concepts.Role
                let $components := components:components-for-archives-and-roles($all-aids, $roles)
                return
                for $concept in $concepts
                group by $archive := $concept.Archive,  $role := $concept.Role
                let $component as object := $components[$$.Archive eq $archive and $$.Role eq $role]
                let $default-hc as object := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube")
                let $members as object* := $default-hc.Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain".Members
                let $metadata := {
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label,
                    AccessionNumber : $archive._id,
                    FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                    FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
                }
                for $concept in $concept
                let $original-name := ($concept.Origin, $concept.Name)[1]
                return {|
                    project($concept, ("Name", "Origin")),
                    trim($members.$original-name, "Name"),
                    $metadata
                |}
        ]
    }
let $comment := {
    NumConcepts: count($concepts),
    TotalNumConcepts: session:num-concepts()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <ReportElements>{
                    local:to-xml($res.ReportElements[], $onlyNames)
                }</ReportElements>
    },
    to-csv : function($res as object) as string {
        local:to-csv($res.ReportElements[], $onlyNames)
    }
}
let $results := api:serialize($result, $comment, $serializers, $format, "report-elements")
return api:check-and-return-results($token, $results, $format)


Update entities.jq

Update entities.jq

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/test";

import module namespace http-request = "http://www.28msec.com/modules/http/request";
import module namespace csv = "http://zorba.io/modules/json-csv";

import module namespace entities = "http://28.io/modules/xbrl/entities";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token        as string? external;
declare  %rest:env                              variable $request-uri  as string  external;
declare  %rest:case-insensitive                 variable $format       as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $eid          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cik          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker       as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic          as string* external;
declare  %rest:case-insensitive                 variable $profile-name as string  external := $config:profile-name;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)
let $tag := if (exists(($cik, $tag, $ticker, $sic)))
             then $tag
             else "ALL"

(: Object resolution :)
let $entities :=
    switch($profile-name)
    case "sec" return
        for $entity in companies:companies(
            $cik,
            $tag,
            $ticker,
            $sic)
        order by $entity.Profiles.SEC.CompanyName
        return $entity
    default return
        for $entity in
            if(exists($eid)) then entities:entities($eid)
                                else entities:entities()
        return {
            EID: $entity._id
        }
let $comment :=
{
    NumEntities: count($entities),
    TotalNumEntities: session:num-entities()
}
let $entities :=
  switch($profile-name)
  case "sec" return
    for $entity in $entities
    return {|
      project($entity, "_id"),
      {
        Archives: backend:url("filings", {
          cik: tokenize($entity._id, " ")[2],
          fiscalYear: "ALL",
          fiscalPeriod: "ALL",
          format: $format,
          profile-name: $profile-name
        }, true),
      trim($entity, "_id")
    |}
  default return
    for $entity in $entities
    return {|
      $entity,
      {
        Archives: backend:url("filings", {
          eid: encode-for-uri($entity.EID),
          format: $format,
          profile-name: $profile-name
        }, true)
      }
  |}

let $result := { "Entities" : [ $entities ] }
let $serializers := {
    to-xml : function($res as object) as node() {
        switch($profile-name)
        case "sec" return
        <Entities>{
            companies:to-xml($res.Entities[])
        }</Entities>
        default return <Entities>
            {
                for $id in $res.Entities[].EID
                return <EID>{$id}</EID>
            }
        </Entities>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec"
        return string-join(companies:to-csv($res.Entities[]))
        default return
            string-join(
                csv:serialize($res.Entities[], { serialize-null-as : "" }),
        "")
    }
}
return api:serialize($result, $comment, $serializers, $format, "entities")

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/test";

import module namespace http-request = "http://www.28msec.com/modules/http/request";
import module namespace csv = "http://zorba.io/modules/json-csv";

import module namespace entities = "http://28.io/modules/xbrl/entities";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token        as string? external;
declare  %rest:env                              variable $request-uri  as string  external;
declare  %rest:case-insensitive                 variable $format       as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $eid          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cik          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker       as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic          as string* external;
declare  %rest:case-insensitive                 variable $profile-name as string  external := $config:profile-name;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)
let $tag := if (exists(($cik, $tag, $ticker, $sic)))
             then $tag
             else "ALL"

(: Object resolution :)
let $entities :=
    switch($profile-name)
    case "sec" return
        for $entity in companies:companies(
            $cik,
            $tag,
            $ticker,
            $sic)
        order by $entity.Profiles.SEC.CompanyName
        return $entity
    default return
        for $entity in
            if(exists($eid)) then entities:entities($eid)
                                else entities:entities()
        return {
            EID: $entity._id
        }
let $comment :=
{
    NumEntities: count($entities),
    TotalNumEntities: session:num-entities()
}
let $entities :=
  switch($profile-name)
  case "sec" return
    for $entity in $entities
    return {|
      project($entity, "_id"),
      {
        Archives: backend:url("filings", {
          cik: tokenize($entity._id, " ")[2],
          fiscalYear: "ALL",
          fiscalPeriod: "ALL",
          format: $format,
          profile-name: $profile-name
        }),
      trim($entity, "_id")
    |}
  default return
    for $entity in $entities
    return {|
      $entity,
      {
        Archives: backend:url("filings", {
          eid: encode-for-uri($entity.EID),
          format: $format,
          profile-name: $profile-name
        })
      }
  |}

let $result := { "Entities" : [ $entities ] }
let $serializers := {
    to-xml : function($res as object) as node() {
        switch($profile-name)
        case "sec" return
        <Entities>{
            companies:to-xml($res.Entities[])
        }</Entities>
        default return <Entities>
            {
                for $id in $res.Entities[].EID
                return <EID>{$id}</EID>
            }
        </Entities>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec"
        return string-join(companies:to-csv($res.Entities[]))
        default return
            string-join(
                csv:serialize($res.Entities[], { serialize-null-as : "" }),
        "")
    }
}
return api:serialize($result, $comment, $serializers, $format, "entities")


add bpm-release

add bpm-release

include "./concourse/pipeline-helpers";

{
  "jobs": [
    import_repo_release("github.com/dpb587/openvpn-bosh-release"),
    import_repo_release("github.com/dpb587/ssoca-bosh-release"),

    compile_boshio_release("cloudfoundry/bosh"; "ubuntu-trusty"),
    compile_boshio_release("concourse/concourse"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/garden-runc-release"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/uaa-release"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry-community/docker-registry-boshrelease"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry-community/prometheus-boshrelease"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/postgres-release"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/syslog-release"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/bosh-dns-release"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry-incubator/bpm-release"; "ubuntu-trusty"),
    compile_boshio_release("pivotal-cf/credhub-release"; "ubuntu-trusty"),
    compile_boshio_release("vito/grafana-boshrelease"; "ubuntu-trusty"),

    {
      "name": "import-cf-deployment",
      "plan": [
        {
          "get": "cf-deployment",
          "trigger": true
        },
        {
          "get": "repo"
        },
        {
          "task": "import-cf-deployment",
          "file": "repo/concourse/tasks/import-cf-deployment/config.yml",
          "params": {
            "git_user_email": "((maintainer_email))",
            "git_user_name": "((maintainer_name))"
          }
        },
        {
          "put": "repo",
          "params": {
            "rebase": true,
            "repository": "repo"
          }
        }
      ]
    },

    {
      "name": "push-app",
      "serial": true,
      "plan": [
        {
          "get": "repo",
          "trigger": true
        },
        {
          "task": "build-app",
          "file": "repo/concourse/tasks/build-app/config.yml"
        },
        {
          "put": "app",
          "params": {
            "manifest": "app/manifest.yml"
          }
        }
      ]
    }
  ],
  "resources": [
    {
      "name": "repo",
      "type": "git",
      "source": {
        "branch": "master",
        "private_key": "((git_private_key))",
        "uri": "git@github.com:dpb587/bosh-compiled-releases.git"
      }
    },
    {
      "name": "release-compiler",
      "type": "git",
      "source": {
        "uri": "https://github.com/dpb587/bosh-release-compiler.git"
      }
    },
    {
      "name": "ubuntu-trusty",
      "type": "metalink-repository",
      "source": {
        "uri": "https://dpb587.github.io/upstream-blob-mirror/repository/bosh.io/stemcell/bosh-warden-boshlite-ubuntu-trusty-go_agent/index.xml"
      }
    },

    repo_release("github.com/dpb587/openvpn-bosh-release"; {"uri": "https://github.com/dpb587/openvpn-bosh-release.git"}),
    repo_release("github.com/dpb587/ssoca-bosh-release"; {"uri": "https://github.com/dpb587/ssoca-bosh-release.git"}),

    boshio_release("cloudfoundry/bosh"),
    boshio_release("concourse/concourse"),
    boshio_release("cloudfoundry/garden-runc-release"),
    boshio_release("cloudfoundry/uaa-release"),
    boshio_release("cloudfoundry-community/docker-registry-boshrelease"),
    boshio_release("cloudfoundry-community/prometheus-boshrelease"),
    boshio_release("cloudfoundry/postgres-release"),
    boshio_release("cloudfoundry/syslog-release"),
    boshio_release("cloudfoundry/bosh-dns-release"),
    boshio_release("cloudfoundry-incubator/bpm-release"),
    boshio_release("pivotal-cf/credhub-release"),
    boshio_release("vito/grafana-boshrelease"),

    {
      "name": "cf-deployment",
      "type": "git",
      "source": {
        "uri": "https://github.com/cloudfoundry/cf-deployment.git",
        "paths": [
          "operations/use-compiled-releases.yml"
        ]
      }
    },

    {
      "name": "app",
      "type": "cf",
      "source": {
        "api": "((cf_api))",
        "username": "((cf_username))",
        "password": "((cf_password))",
        "organization": "((cf_organization))",
        "space": "((cf_space))"
      }
    }
  ],
  "resource_types": [
    {
      "name": "metalink-repository",
      "type": "docker-image",
      "source": {
        "repository": "dpb587/metalink-repository-resource"
      }
    }
  ]
}

include "./concourse/pipeline-helpers";

{
  "jobs": [
    import_repo_release("github.com/dpb587/openvpn-bosh-release"),
    import_repo_release("github.com/dpb587/ssoca-bosh-release"),

    compile_boshio_release("cloudfoundry/bosh"; "ubuntu-trusty"),
    compile_boshio_release("concourse/concourse"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/garden-runc-release"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/uaa-release"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry-community/docker-registry-boshrelease"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry-community/prometheus-boshrelease"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/postgres-release"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/syslog-release"; "ubuntu-trusty"),
    compile_boshio_release("cloudfoundry/bosh-dns-release"; "ubuntu-trusty"),
    compile_boshio_release("pivotal-cf/credhub-release"; "ubuntu-trusty"),
    compile_boshio_release("vito/grafana-boshrelease"; "ubuntu-trusty"),

    {
      "name": "import-cf-deployment",
      "plan": [
        {
          "get": "cf-deployment",
          "trigger": true
        },
        {
          "get": "repo"
        },
        {
          "task": "import-cf-deployment",
          "file": "repo/concourse/tasks/import-cf-deployment/config.yml",
          "params": {
            "git_user_email": "((maintainer_email))",
            "git_user_name": "((maintainer_name))"
          }
        },
        {
          "put": "repo",
          "params": {
            "rebase": true,
            "repository": "repo"
          }
        }
      ]
    },

    {
      "name": "push-app",
      "serial": true,
      "plan": [
        {
          "get": "repo",
          "trigger": true
        },
        {
          "task": "build-app",
          "file": "repo/concourse/tasks/build-app/config.yml"
        },
        {
          "put": "app",
          "params": {
            "manifest": "app/manifest.yml"
          }
        }
      ]
    }
  ],
  "resources": [
    {
      "name": "repo",
      "type": "git",
      "source": {
        "branch": "master",
        "private_key": "((git_private_key))",
        "uri": "git@github.com:dpb587/bosh-compiled-releases.git"
      }
    },
    {
      "name": "release-compiler",
      "type": "git",
      "source": {
        "uri": "https://github.com/dpb587/bosh-release-compiler.git"
      }
    },
    {
      "name": "ubuntu-trusty",
      "type": "metalink-repository",
      "source": {
        "uri": "https://dpb587.github.io/upstream-blob-mirror/repository/bosh.io/stemcell/bosh-warden-boshlite-ubuntu-trusty-go_agent/index.xml"
      }
    },

    repo_release("github.com/dpb587/openvpn-bosh-release"; {"uri": "https://github.com/dpb587/openvpn-bosh-release.git"}),
    repo_release("github.com/dpb587/ssoca-bosh-release"; {"uri": "https://github.com/dpb587/ssoca-bosh-release.git"}),

    boshio_release("cloudfoundry/bosh"),
    boshio_release("concourse/concourse"),
    boshio_release("cloudfoundry/garden-runc-release"),
    boshio_release("cloudfoundry/uaa-release"),
    boshio_release("cloudfoundry-community/docker-registry-boshrelease"),
    boshio_release("cloudfoundry-community/prometheus-boshrelease"),
    boshio_release("cloudfoundry/postgres-release"),
    boshio_release("cloudfoundry/syslog-release"),
    boshio_release("cloudfoundry/bosh-dns-release"),
    boshio_release("pivotal-cf/credhub-release"),
    boshio_release("vito/grafana-boshrelease"),

    {
      "name": "cf-deployment",
      "type": "git",
      "source": {
        "uri": "https://github.com/cloudfoundry/cf-deployment.git",
        "paths": [
          "operations/use-compiled-releases.yml"
        ]
      }
    },

    {
      "name": "app",
      "type": "cf",
      "source": {
        "api": "((cf_api))",
        "username": "((cf_username))",
        "password": "((cf_password))",
        "organization": "((cf_organization))",
        "space": "((cf_space))"
      }
    }
  ],
  "resource_types": [
    {
      "name": "metalink-repository",
      "type": "docker-image",
      "source": {
        "repository": "dpb587/metalink-repository-resource"
      }
    }
  ]
}


Fixed concept querying with FACs, and optimized O(#concepts) -> O(1) by reducing the number of underlying queries to MongoDB.

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace sec-fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";

import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-csv($o as object*) as string?
{
    if (exists($o)) (: bug in csv:serialize :)
    then
    string-join(
        csv:serialize(
            for $o in $o
            let $a := $o.Aspects
            return {|
                for $k in keys($a)
                return { $k : $a.$k },
                { "Value" : $o.Value },
                if (exists($o."Unit"))
                then { "Unit" :  $o."Unit" }
                else (),
                if (exists($o.Decimals))
                then { "Decimals" : $o.Decimals }
                else(),
                { "EntityRegistrantName" : $o.EntityRegistrantName },
                if (exists($o.ReportedConcept))
                then { "ReportedConcept" : $o.ReportedConcept }
                else ()
            |}
        )
    )
    else ()
};

declare function local:to-xml($o as object*)
{
    (session:comment("xml"),
    <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
        for $o in $o
        let $a := $o.Aspects
        return
            <Fact>{
                <Aspects>{
                    for $k in keys($a)
                    return
                        <Aspect>
                            <Name>{$k}</Name>
                            <Value>{$a.$k}</Value>
                        </Aspect>
                }</Aspects>,
                <Value>{$o.Value}</Value>,
                <Type>{$o.Type}</Type>,
                if (exists($o.Unit))
                then <Unit>{$o.Unit}</Unit>
                else(),
                if (exists($o.Decimals))
                then <Decimals>{$o.Decimals}</Decimals>
                else (),
                <EntityRegistrantName>{$o.EntityRegistrantName}</EntityRegistrantName>,
                if (exists($o.ReportedConcept))
                then <ReportedConcept>{$o."ReportedConcept"}</ReportedConcept>
                else ()
            }</Fact>
    }</FactTable>)
};

declare function local:facts($entities, $fiscalPeriods, $fiscalYears, $concepts, $dimensions, $map)
{
    for $entity in $entities
    (: compute latest reported fiscalYear if none is specified :)
    let $years  :=  distinct-values(
                        for $f in $fiscalYears
                        return
                            switch ($f)
                            case "LATEST" return
                                for $p in $fiscalPeriods
                                return
                                    if ($p eq "FY")
                                    then sec-fiscal:latest-reported-fiscal-period($entity, "10-K").year
                                    else sec-fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                            case "ALL" return $sec-fiscal:ALL_FISCAL_YEARS
                            default return $f
                    )
    let $fiscalPeriods := distinct-values(
                            $fiscalPeriods !
                                (switch($$)
                                case "Q1" return ("Q1","YTD1")
                                case "Q2" return ("Q2","YTD2")
                                case "Q3" return ("Q3","YTD3")
                                default return ("Q4","FY")))
    let $aspects :=
        {|
            { "xbrl:Concept" : $concepts },
            { "xbrl:Entity" : $entity._id },
            $dimensions
        |}
    let $hypercube := copy $h := hypercubes:dimensionless-hypercube({ Concepts : $concepts })
                      modify insert json {|
                        "dei:LegalEntityAxis" ! { $$ : { Name : $$, Default : "sec:DefaultLegalEntity" } }
                      |}
                      into $h.Aspects
                      return $h
    let $options :=
            {|
                if (exists($map))
                then { "concept-maps" : $map }
                else (),
                { Hypercube : $hypercube }
            |}
    for $fact in
        for $f in
            sec-fiscal:facts-for-aspects-and-fiscal-periods-and-years(
                $aspects, $fiscalPeriods, $years, $options)
        order by $f.Profiles.SEC.Fiscal.Acceptance descending
        group by $f.Profiles.SEC.Fiscal.Year,
                 $f.Profiles.SEC.Fiscal.Period,
                 $f.Profiles.SEC.Fiscal.Acceptance
        return $f[1]
    return {|
        { Aspects : {|
            { "xbrl:Entity" : $fact.Aspects."xbrl:Entity" },
            { "dei:LegalEntityAxis" : $fact.Aspects."dei:LegalEntityAxis" },
            { "bizql:FiscalPeriod" : $fact.Profiles.SEC.Fiscal.Period },
            { "bizql:FiscalYear" : $fact.Profiles.SEC.Fiscal.Year },
            { "xbrl:Concept" : $fact.Aspects."xbrl:Concept" }
        |} },
        { Type: $fact.Type },
        if (exists($fact.Aspects."xbrl:Unit"))
        then { Unit: $fact.Aspects."xbrl:Unit" }
        else  (),
        if (exists($fact.Decimals))
        then { Decimals: $fact.Decimals }
        else (),
        { Value: $fact.Value },
        { "EntityRegistrantName" : $entity.Profiles.SEC.CompanyName },
        if (exists($map))
        then { "ReportedConcept" : $fact.AuditTrails[].Data.OriginalConcept[1] }
        else ()
    |}
};

let $format      := lower-case(request:param-values("format")[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y cast as integer
                            else ()
                    )
let $fiscalPeriods := let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (lower-case($fp) eq "all")
                        then ("Q1", "Q2", "Q3", "FY")
                        else $fp
let $dimensions :=  for $p in request:param-names()
                    return
                        switch ($p)
                        case contains($p, ":") return
                            {
                                $p : request:param-values($p)[1]
                            }
                        default return ()
let $concepts := request:param-values("concept")
let $map      := request:param-values("map")[1]

return
  if (empty($concepts))
  then {
    response:status-code(400);
    session:error("concept: missing parameter", $format)
  } else
    let $entities := (
        companies:companies($ciks),
        companies:companies-for-tags($tags),
        companies:companies-for-tickers($tickers),
        companies:companies-for-SIC($sics)
    )
    return switch(true)
      case empty($entities) return {
        response:status-code(404);
        session:error("entity not found (valid parameters: cik, ticker, tag, sic)", $format)
      }
      case not (session:only-dow30($entities) or session:valid()) return {
        response:status-code(401);
        session:error("accessing facts of an entity that is not in the DOW30", $format)
      }
      default return
        let $facts := local:facts($entities, $fiscalPeriods, $fiscalYears, $concepts, $dimensions, $map)
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($facts)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=facts.csv");
                local:to-csv($facts)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=fact.csv");
                local:to-csv($facts)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { NetworkIdentifier : "http://bizql.io/facts" },
                    { TableName : "xbrl:Facts" },
                    { FactTable : [ $facts ] },
                    session:comment("json")
                |}
            }

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace sec-fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";

import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-csv($o as object*) as string?
{
    if (exists($o)) (: bug in csv:serialize :)
    then
    string-join(
        csv:serialize(
            for $o in $o
            let $a := $o.Aspects
            return {|
                for $k in keys($a)
                return { $k : $a.$k },
                { "Value" : $o.Value },
                if (exists($o."Unit"))
                then { "Unit" :  $o."Unit" }
                else (),
                if (exists($o.Decimals))
                then { "Decimals" : $o.Decimals }
                else(),
                { "EntityRegistrantName" : $o.EntityRegistrantName },
                if (exists($o.ReportedConcept))
                then { "ReportedConcept" : $o.ReportedConcept }
                else ()
            |}
        )
    )
    else ()
};

declare function local:to-xml($o as object*)
{
    (session:comment("xml"),
    <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
        for $o in $o
        let $a := $o.Aspects
        return
            <Fact>{
                <Aspects>{
                    for $k in keys($a)
                    return
                        <Aspect>
                            <Name>{$k}</Name>
                            <Value>{$a.$k}</Value>
                        </Aspect>
                }</Aspects>,
                <Value>{$o.Value}</Value>,
                <Type>{$o.Type}</Type>,
                if (exists($o.Unit))
                then <Unit>{$o.Unit}</Unit>
                else(),
                if (exists($o.Decimals))
                then <Decimals>{$o.Decimals}</Decimals>
                else (),
                <EntityRegistrantName>{$o.EntityRegistrantName}</EntityRegistrantName>,
                if (exists($o.ReportedConcept))
                then <ReportedConcept>{$o."ReportedConcept"}</ReportedConcept>
                else ()
            }</Fact>
    }</FactTable>)
};

declare function local:facts($entities, $fiscalPeriods, $fiscalYears, $concepts, $dimensions, $map)
{
    for $entity in $entities
    (: compute latest reported fiscalYear if none is specified :)
    let $years  :=  distinct-values(
                        for $f in $fiscalYears
                        return
                            switch ($f)
                            case "LATEST" return
                                for $p in $fiscalPeriods
                                return
                                    if ($p eq "FY")
                                    then sec-fiscal:latest-reported-fiscal-period($entity, "10-K").year
                                    else sec-fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                            case "ALL" return ()
                            default return $f
                    )
    let $fiscalPeriods := distinct-values(
                            $fiscalPeriods !
                                (switch($$)
                                case "Q1" return ("Q1","YTD1")
                                case "Q2" return ("Q2","YTD2")
                                case "Q3" return ("Q3","YTD3")
                                default return ("Q4","FY")))
    for $concept in $concepts
    let $aspects :=
        {|
            { "xbrl:Concept" : $concept },
            { "xbrl:Entity" : $entity._id },
            $dimensions
        |}
    let $hypercube := copy $h := hypercubes:dimensionless-hypercube()
                      modify insert json {|
                        "dei:LegalEntityAxis" ! { $$ : { Name : $$, Default : "sec:DefaultLegalEntity" } }
                      |}
                      into $h.Aspects
                      return $h
    let $options :=
            {|
                if (exists($map))
                then { "concept-maps" : $map }
                else (),
                { Hypercube : $hypercube }
            |}
    for $fact in
        for $f in
            sec-fiscal:facts-for-aspects-and-fiscal-periods-and-years(
                $aspects, $fiscalPeriods, $years, $options)
        order by $f.Profiles.SEC.Fiscal.Acceptance descending
        group by $f.Profiles.SEC.Fiscal.Year,
                 $f.Profiles.SEC.Fiscal.Period,
                 $f.Profiles.SEC.Fiscal.Acceptance
        return $f[1]
    return {|
        { Aspects : {|
            { "xbrl:Entity" : $fact.Aspects."xbrl:Entity" },
            { "dei:LegalEntityAxis" : $fact.Aspects."dei:LegalEntityAxis" },
            { "bizql:FiscalPeriod" : $fact.Profiles.SEC.Fiscal.Period },
            { "bizql:FiscalYear" : $fact.Profiles.SEC.Fiscal.Year },
            { "xbrl:Concept" : $fact.Aspects."xbrl:Concept" }
        |} },
        { Type: $fact.Type },
        if (exists($fact.Aspects."xbrl:Unit"))
        then { Unit: $fact.Aspects."xbrl:Unit" }
        else  (),
        if (exists($fact.Decimals))
        then { Decimals: $fact.Decimals }
        else (),
        { Value: $fact.Value },
        { "EntityRegistrantName" : $entity.Profiles.SEC.CompanyName },
        if (exists($map))
        then { "ReportedConcept" : $fact.AuditTrails[].Data.OriginalConcept[1] }
        else ()
    |}
};

let $format      := lower-case(request:param-values("format")[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y cast as integer
                            else ()
                    )
let $fiscalPeriods := let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (lower-case($fp) eq "all")
                        then ("Q1", "Q2", "Q3", "FY")
                        else $fp
let $dimensions :=  for $p in request:param-names()
                    return
                        switch ($p)
                        case contains($p, ":") return
                            {
                                $p : request:param-values($p)[1]
                            }
                        default return ()
let $concepts := request:param-values("concept")
let $map      := request:param-values("map")[1]

return
  if (empty($concepts))
  then {
    response:status-code(400);
    session:error("concept: missing parameter", $format)
  } else
    let $entities := (
        companies:companies($ciks),
        companies:companies-for-tags($tags),
        companies:companies-for-tickers($tickers),
        companies:companies-for-SIC($sics)
    )
    return switch(true)
      case empty($entities) return {
        response:status-code(404);
        session:error("entity not found (valid parameters: cik, ticker, tag, sic)", $format)
      }
      case not (session:only-dow30($entities) or session:valid()) return {
        response:status-code(401);
        session:error("accessing facts of an entity that is not in the DOW30", $format)
      }
      default return
        let $facts := local:facts($entities, $fiscalPeriods, $fiscalYears, $concepts, $dimensions, $map)
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($facts)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=facts.csv");
                local:to-csv($facts)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=fact.csv");
                local:to-csv($facts)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { NetworkIdentifier : "http://bizql.io/facts" },
                    { TableName : "xbrl:Facts" },
                    { FactTable : [ $facts ] },
                    session:comment("json")
                |}
            }


Update entities.jq

Update entities.jq

import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace request = "http://www.28msec.com/modules/http-request";
declare variable $db := request:param-values("db", "all-sec-filings");

declare %an:nondeterministic function local:test-entities($expected as integer*, $params as string) as atomic
{
    let $actual as integer := count(parse-json(http-client:get(
        "http://" || request:server-name() || ":" || request:server-port() || "/v1/_queries/public/api/entities.jq?_method=POST" || $params
    ).body.content).Entities[])
    return
        if ($actual = $expected)
        then true
        else "false [Actual="||$actual||", Expected="||$expected ||"]"
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    dow30: local:test-entities(30, "&tag=DOW30"),
    cik: local:test-entities(1, "&cik=4962"),
    ticker: local:test-entities(1, "&ticker=wmt"),
    ticker2: local:test-entities(2, "&ticker=wmt&ticker=ko"),
    sic: local:test-entities(switch($db) case "all-dow30" return 2 default return 74, "&sic=4813"),
    mixed: local:test-entities(switch($db) case "all-dow30" return 5 default return 77, "&cik=4962&sic=4813&ticker=wmt&ticker=ko")
})

import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

declare %an:nondeterministic function local:test-entities($expected as integer*, $params as string) as atomic
{
    let $actual as integer := count(parse-json(http-client:get(
        "http://" || request:server-name() || ":" || request:server-port() || "/v1/_queries/public/api/entities.jq?_method=POST" || $params
    ).body.content).Entities[])
    return
        if ($actual = $expected)
        then true
        else "false [Actual="||$actual||", Expected="||$expected ||"]"
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    dow30: local:test-entities(30, "&tag=DOW30"),
    cik: local:test-entities(1, "&cik=4962"),
    ticker: local:test-entities(1, "&ticker=wmt"),
    ticker2: local:test-entities(2, "&ticker=wmt&ticker=ko"),
    sic: local:test-entities((2, 74), "&sic=4813"),
    mixed: local:test-entities((5, 77), "&cik=4962&sic=4813&ticker=wmt&ticker=ko")
})


Update patientfilter.jq

Update patientfilter.jq

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv

# start from the .results array
.results

# remove .patient.drug array
|map(del(.patient.drug))

# remove .patient.reaction array
|map(del(.patient.reaction))

# flattens .patient object and renames child keys with patient_ prefix
|map(with_entries(select(.key != "patient")) +
       (.patient|with_entries(.key = "patient_" + .key)))

# flattens all the objects and renames child keys
|map(with_entries(select(.key != "primarysource" and
                         .key != "sender" and
                         .key != "receiver" and
                         .key != "patient_summary" and
                         .key != "reportduplicate")) +
       (.primarysource//{patientsource:{"NA":"NA"}}|with_entries(.key = "primarysource_" + .key)) +
       (.sender//{sender:{"NA":"NA"}}|with_entries(.key = "sender_" + .key)) +
       (.receiver//{receiver:{"NA":"NA"}}|with_entries(.key = "receiver_" + .key)) +
       (.patient_summary//{patient_summary:{"NA":"NA"}}|with_entries(.key = "patient_summary_" + .key)) +
       (.report_duplicate//{report_duplicate:{"NA":"NA"}}|with_entries(.key = "report_duplicate_" + .key)))

|map(del(.primarysource_primary_source))
|map(del(.sender_sender))
|map(del(.receiver_receiver))
|map(del(.report_duplicate_report_duplicate))
|map(del(.patient_summary_patient_summary))

# grabs all unique keys and spits out CSV
|(map(keys)|add|unique) as $keys| $keys, (.[]|[.[$keys[]]])|@csv

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv

# start from the .results array
.results

# remove .patient.drug array
|map(del(.patient.drug))

# remove .patient.reaction array
|map(del(.patient.reaction))

# flattens .patient object and renames child keys with patient_ prefix
|map(with_entries(select(.key != "patient")) +
       (.patient|with_entries(.key = "patient_" + .key)))

# flattens all the objects and renames child keys
|map(with_entries(select(.key != "primarysource" and
                         .key != "sender" and
                         .key != "receiver" and
                         .key != "patient_summary" and
                         .key != "reportduplicate")) +
       (.primarysource//{patientsource:{"NA":"NA"}}|with_entries(.key = "primarysource_" + .key)) +
       (.sender//{sender:{"NA":"NA"}}|with_entries(.key = "sender_" + .key)) +
       (.receiver//{receiver:{"NA":"NA"}}|with_entries(.key = "receiver_" + .key)) +
       (.patient_summary//{patient_summary:{"NA":"NA"}}|with_entries(.key = "patient_summary_" + .key)) +
       (.report_duplicate//{report_duplicate:{"NA":"NA"}}|with_entries(.key = "report_duplicate_" + .key)))

|map(del(.primarysource_primary_source))
|map(del(.sender_sender))
|map(del(.receiver_receiver))
|map(del(.report_duplicate_report_duplicate))
|map(del(.patient_summary_patient_summary))

#Iterates through array of objects to return a stream of objects instead of a single array
|.[]


:green_heart: Replace latest with 2013.

:green_heart: Replace latest with 2013.

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-spreadsheet($expected as integer, $params as object) as item
{
    let $endpoint := "spreadsheet-for-component"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].TableSet[].TableCells.Facts[][].Value)
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    cocacola: local:test-spreadsheet(1, {
        ticker:"ko",
        fiscalYear: "2013",
        fiscalPeriod: "FY"
    }),
    tickerrole: local:test-spreadsheet(60, {
        ticker:"ko",
        networkIdentifier:"http://www.thecocacolacompany.com/role/ConsolidatedStatementsOfIncome",
        fiscalYear: "2013",
        fiscalPeriod: "FY"
    }),
    tickerconcept: local:test-spreadsheet(76, {
        ticker:"ko",
        fiscalYear:"2013",
        concept:"us-gaap:Assets",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-spreadsheet(70, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        networkIdentifier:"http://www.thecoca-colacompany.com/role/CondensedConsolidatedBalanceSheets"
    }),
    generic: local:test-spreadsheet(76, {
        profile-name: "generic",
        aid: "0000021344-14-000008",
        role:"http://www.thecocacolacompany.com/role/ConsolidatedBalanceSheets"
    }),
    merge: local:test-spreadsheet(73, {
        ticker: "ko",
        fiscalYear: [ "2012", "2013" ],
        fiscalPeriod: "FY",
        disclosure: "BalanceSheet",
        merge: "true"
    })
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-spreadsheet($expected as integer, $params as object) as item
{
    let $endpoint := "spreadsheet-for-component"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].TableSet[].TableCells.Facts[][].Value)
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    cocacola: local:test-spreadsheet(1, {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tickerrole: local:test-spreadsheet(60, {
        ticker:"ko",
        networkIdentifier:"http://www.thecocacolacompany.com/role/ConsolidatedStatementsOfIncome",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tickerconcept: local:test-spreadsheet(76, {
        ticker:"ko",
        fiscalYear:"2013",
        concept:"us-gaap:Assets",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-spreadsheet(70, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        networkIdentifier:"http://www.thecoca-colacompany.com/role/CondensedConsolidatedBalanceSheets"
    }),
    generic: local:test-spreadsheet(76, {
        profile-name: "generic",
        aid: "0000021344-14-000008",
        role:"http://www.thecocacolacompany.com/role/ConsolidatedBalanceSheets"
    }),
    merge: local:test-spreadsheet(73, {
        ticker: "ko",
        fiscalYear: [ "2012", "2013" ],
        fiscalPeriod: "FY",
        disclosure: "BalanceSheet",
        merge: "true"
    })
})


Update facts.jq

Update facts.jq

import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace rules = "http://28.io/modules/xbrl/rules";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace request = "http://www.28msec.com/modules/http-request";

declare function local:param-values($name as string) as string*
{
    switch(true)
     case $name eq "xbrl:Concept"
        return (request:param-values("concept"), request:param-values("xbrl:Concept"))

     case $name eq "sec:FiscalYear" and $profile-name eq "sec"
        return ($fiscalYear, request:param-values("sec:FiscalYear"))[$$ ne "LATEST"]
     case $name eq "sec:FiscalYear::type" and $profile-name eq "sec"
        return "integer"
     case $name eq "sec:FiscalPeriod" and $profile-name eq "sec"
        return ($fiscalPeriod, request:param-values("sec:FiscalPeriod"))
     case $name eq "sec:FiscalPeriodType" and $profile-name eq "sec"
        return
         if(empty((request:param-values("sec:FiscalPeriodType"), request:param-values("sec:FiscalPeriodType::default"))))
         then ("instant", "YTD")
         else request:param-values("sec:FiscalPeriodType")
     case $name eq "dei:LegalEntityAxis" and $profile-name eq "sec"
        return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis")
     case $name eq "dei:LegalEntityAxis::default" return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis::default")
     case $name eq "xbrl:Entity" and $profile-name eq "sec" return (
         let $companies := companies:companies(
            $cik,
            api:preprocess-tags($tag),
            $ticker,
            $sic)._id
        return if(empty(($cik,$tag,$ticker,$sic)) or exists($companies))
               then $companies
               else "dummy",
        request:param-values("xbrl:Entity"))
     case $name eq "xbrl28:Archive" and $profile-name eq "sec" return (
            let $fiscalYears := ($fiscalYear, request:param-values("sec:FiscalYear"))
            let $fiscalPeriods := local:param-values("sec:FiscalPeriod")
            let $entities := entities:entities(local:param-values("xbrl:Entity"))
            return
                if($fiscalYears = "LATEST")
                then fiscal-core:latest-filings($entities, $fiscalPeriods)._id
                else (),
            $aid,
            request:param-values("xbrl28:Archive")
        )
     default return request:param-values($name)
};

declare function local:param-names() as string*
{
    let $names := request:param-names()
    return distinct-values((
        $names[contains($$, ":")],
        "xbrl:Concept"[$names = "concept"],

        "sec:Accepted"[$profile-name eq "sec"],
        "sec:FiscalPeriod"[$profile-name eq "sec"],
        "sec:FiscalPeriodType"[$profile-name eq "sec"],
        "sec:FiscalYear"[$profile-name eq "sec"],
        "xbrl:Entity"[$profile-name eq "sec" and $names = ("cik", "tag", "ticker", "sic")],
        "dei:LegalEntityAxis"[$profile-name eq "sec"],
        "dei:LegalEntityAxis::default"[$profile-name eq "sec"],
        "xbrl28:Archive"[$profile-name eq "sec"]))
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube() as object
{
    let $hypercube-spec :=
    {|
        for $parameter in local:param-names()
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (local:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $type as string? :=
            (local:param-values($dimension-name || "::type"), local:param-values($dimension-name || ":type"))[1]

        let $values := local:param-values($dimension-name)
        let $typed-values := if (exists($type)) then local:cast-sequence($values, $type) else $values

        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (local:param-values($dimension-name || "::default"), local:param-values($dimension-name || ":default"))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value

        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                { "Domain" : [ $typed-values ] }[(exists($typed-values) and not($all))],
                { "Default" : $typed-default-value }[$has-default]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($hypercube-spec)
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token             as string? external;
declare  %rest:case-insensitive                 variable $profile-name      as string  external := "sec";
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $validate          as boolean external := false;
declare  %rest:case-insensitive                 variable $labels            as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic,
        $aid)
let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $hypercube := local:hypercube()

let $facts :=
    let $options := {|
      {
        Hypercube : $hypercube,
        Validate: $validate
      },
      { "ConceptMaps" : $map }[exists($map)],
      { "Rules" : [ $rule ] }[exists($rule)]
    |}
    return if($profile-name eq "sec")
    then sec:facts-for($options)
    else facts:facts-for($options)

let $facts :=
  if(not $labels)
  then $facts
  else
    let $archives as string* := distinct-values($facts.Aspects."xbrl28:Archive")
    let $concept-names as string* := distinct-values($facts.Aspects."xbrl:Concept")
    let $concepts as object* :=
      (
          concepts:concepts($concept-names, $archives, $concepts:ANY_COMPONENT_LINK_ROLE),
          (reports:concepts(($report,$map)))[$$.Name = $concept-names]
      )
    let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $concepts:AMERICAN_ENGLISH )[1]
    let $roles as string* := ( $report.Role, $concepts:ANY_COMPONENT_LINK_ROLE )
    for $fact as object in $facts
    let $entityName as string := $entities[$$._id eq $fact.Aspects."xbrl:Entity"].Profiles.SEC.CompanyName
    return
    {|
      $fact,
      let $concept-labels as object? := facts:labels($fact, $roles, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
      let $standard-labels as object := conversion:get-standard-labels($fact, $entityName)
      return { Labels : {| $concept-labels, $standard-labels |} }
    |}

let $facts :=
  if($profile-name eq "sec")
  then api:normalize-facts($facts)
  else $facts

let $result := {
    NetworkIdentifier : "http://bizql.io/facts",
    TableName : "xbrl:Facts",
    FactTable : [ $facts ]
}
let $comment :=
{
    NumFacts: count($facts),
    TotalNumFacts: session:num-facts(),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "facts")
return api:check-and-return-results($token, $results, $format)

import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace rules = "http://28.io/modules/xbrl/rules";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace request = "http://www.28msec.com/modules/http-request";

declare function local:param-values($name as string) as string*
{
    switch(true)
     case $name eq "xbrl:Concept"
        return (request:param-values("concept"), request:param-values("xbrl:Concept"))

     case $name eq "sec:FiscalYear" and $profile-name eq "sec"
        return ($fiscalYear, request:param-values("sec:FiscalYear"))[$$ ne "LATEST"]
     case $name eq "sec:FiscalYear::type" and $profile-name eq "sec"
        return "integer"
     case $name eq "sec:FiscalPeriod" and $profile-name eq "sec"
        return ($fiscalPeriod, request:param-values("sec:FiscalPeriod"))
     case $name eq "sec:FiscalPeriodType" and $profile-name eq "sec"
        return
         if(empty(request:param-values("sec:FiscalPeriodType"))
         then ("instant", "YTD")
         else request:param-values("sec:FiscalPeriodType"))
     case $name eq "dei:LegalEntityAxis" and $profile-name eq "sec"
        return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis")
     case $name eq "dei:LegalEntityAxis::default" return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis::default")
     case $name eq "xbrl:Entity" and $profile-name eq "sec" return (
         let $companies := companies:companies(
            $cik,
            api:preprocess-tags($tag),
            $ticker,
            $sic)._id
        return if(empty(($cik,$tag,$ticker,$sic)) or exists($companies))
               then $companies
               else "dummy",
        request:param-values("xbrl:Entity"))
     case $name eq "xbrl28:Archive" and $profile-name eq "sec" return (
            let $fiscalYears := ($fiscalYear, request:param-values("sec:FiscalYear"))
            let $fiscalPeriods := local:param-values("sec:FiscalPeriod")
            let $entities := entities:entities(local:param-values("xbrl:Entity"))
            return
                if($fiscalYears = "LATEST")
                then fiscal-core:latest-filings($entities, $fiscalPeriods)._id
                else (),
            $aid,
            request:param-values("xbrl28:Archive")
        )
     default return request:param-values($name)
};

declare function local:param-names() as string*
{
    let $names := request:param-names()
    return distinct-values((
        $names[contains($$, ":")],
        "xbrl:Concept"[$names = "concept"],

        "sec:Accepted"[$profile-name eq "sec"],
        "sec:FiscalPeriod"[$profile-name eq "sec"],
        "sec:FiscalPeriodType"[$profile-name eq "sec"],
        "sec:FiscalYear"[$profile-name eq "sec"],
        "xbrl:Entity"[$profile-name eq "sec" and $names = ("cik", "tag", "ticker", "sic")],
        "dei:LegalEntityAxis"[$profile-name eq "sec"],
        "dei:LegalEntityAxis::default"[$profile-name eq "sec"],
        "xbrl28:Archive"[$profile-name eq "sec"]))
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube() as object
{
    let $hypercube-spec :=
    {|
        for $parameter in local:param-names()
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (local:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $type as string? :=
            (local:param-values($dimension-name || "::type"), local:param-values($dimension-name || ":type"))[1]

        let $values := local:param-values($dimension-name)
        let $typed-values := if (exists($type)) then local:cast-sequence($values, $type) else $values

        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (local:param-values($dimension-name || "::default"), local:param-values($dimension-name || ":default"))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value

        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                { "Domain" : [ $typed-values ] }[(exists($typed-values) and not($all))],
                { "Default" : $typed-default-value }[$has-default]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($hypercube-spec)
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token             as string? external;
declare  %rest:case-insensitive                 variable $profile-name      as string  external := "sec";
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $validate          as boolean external := false;
declare  %rest:case-insensitive                 variable $labels            as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic,
        $aid)
let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $hypercube := local:hypercube()

let $facts :=
    let $options := {|
      {
        Hypercube : $hypercube,
        Validate: $validate
      },
      { "ConceptMaps" : $map }[exists($map)],
      { "Rules" : [ $rule ] }[exists($rule)]
    |}
    return if($profile-name eq "sec")
    then sec:facts-for($options)
    else facts:facts-for($options)

let $facts :=
  if(not $labels)
  then $facts
  else
    let $archives as string* := distinct-values($facts.Aspects."xbrl28:Archive")
    let $concept-names as string* := distinct-values($facts.Aspects."xbrl:Concept")
    let $concepts as object* :=
      (
          concepts:concepts($concept-names, $archives, $concepts:ANY_COMPONENT_LINK_ROLE),
          (reports:concepts(($report,$map)))[$$.Name = $concept-names]
      )
    let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $concepts:AMERICAN_ENGLISH )[1]
    let $roles as string* := ( $report.Role, $concepts:ANY_COMPONENT_LINK_ROLE )
    for $fact as object in $facts
    let $entityName as string := $entities[$$._id eq $fact.Aspects."xbrl:Entity"].Profiles.SEC.CompanyName
    return
    {|
      $fact,
      let $concept-labels as object? := facts:labels($fact, $roles, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
      let $standard-labels as object := conversion:get-standard-labels($fact, $entityName)
      return { Labels : {| $concept-labels, $standard-labels |} }
    |}

let $facts :=
  if($profile-name eq "sec")
  then api:normalize-facts($facts)
  else $facts

let $result := {
    NetworkIdentifier : "http://bizql.io/facts",
    TableName : "xbrl:Facts",
    FactTable : [ $facts ]
}
let $comment :=
{
    NumFacts: count($facts),
    TotalNumFacts: session:num-facts(),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "facts")
return api:check-and-return-results($token, $results, $format)