fixed filings to be backwards compatible

import module namespace facts = "http://xbrl.io/modules/bizql/facts";

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace filings = "http://xbrl.io/modules/bizql/profiles/sec/filings";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace session = "http://apps.28.io/session";

declare function local:to-xml($filings as object*) as node()*
{
    ( session:comment("xml"),
    <Filings>{
        for $f in $filings
        return
        <Filing>
            <CIK>{$f.CIK}</CIK>
            <EntityRegistrantName>{$f.EntityRegistrantName}</EntityRegistrantName>
            <FormType>{$f.FormType}</FormType>
            <FiscalYear>{$f.FiscalYear}</FiscalYear>
            <FiscalPeriod>{$f.FiscalPeriod}</FiscalPeriod>
            <Accepted>{$f.Accepted}</Accepted>
            <Generator>{$f.Generator}</Generator>
            <AccessionNumber>{$f.AccessionNumber}</AccessionNumber>
            <SECFilingPage>{$f.SECFilingPage}</SECFilingPage>
            <XBRLInstanceURL>{$f.XBRLInstanceURL}</XBRLInstanceURL>
            <Networks>{$f.Networks}</Networks>
            <ReportElements>{$f.ReportElements}</ReportElements>
            <Tables>{$f.Tables}</Tables>
            <Axis>{$f.Axis}</Axis>
            <Members>{$f.Members}</Members>
            <LineItems>{$f.LineItems}</LineItems>
            <Concepts>{$f.Concepts}</Concepts>
            <Abstracts>{$f.Abstracts}</Abstracts>
            <Footnotes>{$f.Footnotes}</Footnotes>
            <Facts>{$f.Facts}</Facts>
            <ExtensionFacts>{$f.ExtensionFacts}</ExtensionFacts>
            <ExtensionConcepts>{$f.ExtensionConcepts}</ExtensionConcepts>
            <ExtensionAbstracts>{$f.ExtensionAbstracts}</ExtensionAbstracts>
        </Filing>
    }</Filings>)
};

declare function local:to-csv($filings as object*) as string*
{
    csv:serialize($filings, { serialize-null-as : "" })
};

declare function local:summary($a)
{
    {
        CIK : $a.Entity,
        EntityRegistrantName :
            facts:facts-for-archives-and-concepts($a, "dei:EntityRegistrantName").Value,
        FormType : filings:document-types($a),
        FiscalYear : fiscal:fiscal-year($a),
        FiscalPeriod : fiscal:fiscal-period($a),
        Accepted : filings:acceptance-dateTimes($a),
        Generator : filings:generators($a),
        AccessionNumber: $a._id,
        SECFilingPage : $a.Profiles.SEC.SECFilingPage,
        XBRLInstanceURL : $a.InstanceURL,
        Networks : filings:num-networks($a),
        ReportElements : filings:num-report-elements($a),
        Tables : filings:num-tables($a),
        Axis : filings:num-axes($a),
        Members : filings:num-members($a),
        LineItems : $a.Statistics.Profiles.SEC.NumDistinctReportElementNamesEndingWithLineItems,
        Concepts : filings:num-concepts($a),
        Abstracts : filings:num-abstracts($a),
        Footnotes : filings:num-footnotes($a),
        Facts: filings:num-facts($a),
        ExtensionFacts:  filings:num-extension-facts($a),
        ExtensionConcepts : filings:num-extension-concepts($a),
        ExtensionAbstracts : filings:num-extension-abstracts($a)
    }
};

let $format      := lower-case(request:param-values("format")[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalPeriods := for $fp in request:param-values("fiscalPeriod", "ALL")
                      return
                        if (lower-case($fp) eq "all")
                        then ("Q1", "Q2", "Q3", "FY", "YTD1", "YTD2", "YTD3")
                        else $fp
let $aids     := request:param-values("aid")
let $ciks := ($ciks,
    companies:companies-for-tags($tags),
    companies:companies-for-tickers($tickers),
    companies:companies-for-SIC($sics))
let $fiscalYears := distinct-values(
                    for $y in request:param-values("fiscalYear", "ALL")
                    return
                        if ($y eq "ALL")
                        then 2000 to 2014 (: hack that needs to be replaced by $fiscal:ALL_FISCAL_YEARS :)
                        else if ($y eq "LATEST")
                        then for $cik in $ciks
                             for $fp in $fiscalPeriods
                             return
                                (fiscal:latest-reported-fiscal-period($cik, $fp).year) cast as integer
                        else if ($y castable as integer)
                        then $y cast as integer
                        else ()
                )
let $archives := (archives:archives($aids),
                    for $fp in $fiscalPeriods, $fy in $fiscalYears
                    return
                      fiscal:filings-for-entities-and-fiscal-periods-and-years($ciks, $fp, $fy))
let $entities := companies:companies($archives.Entities)
return
    if (session:only-dow30($entities) or session:valid())
    then {
        let $archives :=
                for $a in $archives
                order by filings:acceptance-dateTimes($a) descending
                return local:summary($a)
        return
            switch ($format)
            case "xml"  return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($archives)
            }
            case "csv" case "text" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=filings.csv");
                string-join(local:to-csv($archives), "")
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=filings.csv");
                string-join(local:to-csv($archives))
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { "Archives" : [ $archives ] },
                    session:comment("json")
                |}
            }
    } else {
        response:status-code(401);
        session:error("accessing filings of an entity that is not in the DOW30", $format)
    }

import module namespace facts = "http://xbrl.io/modules/bizql/facts";

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace filings = "http://xbrl.io/modules/bizql/profiles/sec/filings";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace session = "http://apps.28.io/session";

declare function local:to-xml($filings as object*) as node()*
{
    ( session:comment("xml"),
    <Filings>{
        for $f in $filings
        return
        <Filing>
            <CIK>{$f.CIK}</CIK>
            <EntityRegistrantName>{$f.EntityRegistrantName}</EntityRegistrantName>
            <FormType>{$f.FormType}</FormType>
            <FiscalYear>{$f.FiscalYear}</FiscalYear>
            <FiscalPeriod>{$f.FiscalPeriod}</FiscalPeriod>
            <Accepted>{$f.Accepted}</Accepted>
            <Generator>{$f.Generator}</Generator>
            <AccessionNumber>{$f.AccessionNumber}</AccessionNumber>
            <SECFilingPage>{$f.SECFilingPage}</SECFilingPage>
            <XBRLInstanceURL>{$f.XBRLInstanceURL}</XBRLInstanceURL>
            <Networks>{$f.Networks}</Networks>
            <ReportElements>{$f.ReportElements}</ReportElements>
            <Tables>{$f.Tables}</Tables>
            <Axis>{$f.Axis}</Axis>
            <Members>{$f.Members}</Members>
            <LineItems>{$f.LineItems}</LineItems>
            <Concepts>{$f.Concepts}</Concepts>
            <Abstracts>{$f.Abstracts}</Abstracts>
            <Footnotes>{$f.Footnotes}</Footnotes>
            <Facts>{$f.Facts}</Facts>
            <ExtensionFacts>{$f.ExtensionFacts}</ExtensionFacts>
            <ExtensionConcepts>{$f.ExtensionConcepts}</ExtensionConcepts>
            <ExtensionAbstracts>{$f.ExtensionAbstracts}</ExtensionAbstracts>
        </Filing>
    }</Filings>)
};

declare function local:to-csv($filings as object*) as string*
{
    csv:serialize($filings, { serialize-null-as : "" })
};

declare function local:summary($a)
{
    {
        CIK : $a.Entity,
        EntityRegistrantName :
            facts:facts-for-archives-and-concepts($a, "dei:EntityRegistrantName").Value,
        FormType : filings:document-types($a),
        FiscalYear : fiscal:fiscal-year($a),
        FiscalPeriod : fiscal:fiscal-period($a),
        Accepted : filings:acceptance-dateTimes($a),
        Generator : filings:generators($a),
        AccessionNumber: $a._id,
        SECFilingPage : $a.Profiles.SEC.SECFilingPage,
        XBRLInstanceURL : $a.InstanceURL,
        Networks : filings:num-networks($a),
        ReportElements : filings:num-report-elements($a),
        Tables : filings:num-tables($a),
        Axis : filings:num-axes($a),
        Members : filings:num-members($a),
        LineItems : $a.Statistics.Profiles.SEC.NumDistinctReportElementNamesEndingWithLineItems,
        Concepts : filings:num-concepts($a),
        Abstracts : filings:num-abstracts($a),
        Footnotes : filings:num-footnotes($a),
        Facts: filings:num-facts($a),
        ExtensionFacts:  filings:num-extension-facts($a),
        ExtensionConcepts : filings:num-extension-concepts($a),
        ExtensionAbstracts : filings:num-extension-abstracts($a)
    }
};

let $format      := lower-case(request:param-values("format")[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalPeriods := let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (lower-case($fp) eq "all")
                        then ("Q1", "Q2", "Q3", "FY")
                        else $fp
let $aids     := request:param-values("aid")
let $ciks := ($ciks,
    companies:companies-for-tags($tags),
    companies:companies-for-tickers($tickers),
    companies:companies-for-SIC($sics))
let $fiscalYears := distinct-values(
                    for $y in request:param-values("fiscalYear", "LATEST")
                    return
                        if ($y eq "LATEST" or $y eq "ALL")
                        then for $cik in $ciks
                             for $fp in $fiscalPeriods
                             return
                                (fiscal:latest-reported-fiscal-period($cik, $fp).year) cast as integer
                        else if ($y castable as integer)
                        then $y cast as integer
                        else ()
                )
let $archives := (archives:archives($aids),
                    fiscal:filings-for-entities-and-fiscal-periods-and-years($ciks, $fiscalPeriods, $fiscalYears))
let $entities := companies:companies($archives.Entities)
return
    if (session:only-dow30($entities) or session:valid())
    then {
        let $archives :=
                for $a in $archives
                order by filings:acceptance-dateTimes($a) descending
                return local:summary($a)
        return
            switch ($format)
            case "xml"  return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($archives)
            }
            case "csv" case "text" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=filings.csv");
                string-join(local:to-csv($archives), "")
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=filings.csv");
                string-join(local:to-csv($archives))
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { "Archives" : [ $archives ] },
                    session:comment("json")
                |}
            }
    } else {
        response:status-code(401);
        session:error("accessing filings of an entity that is not in the DOW30", $format)
    }


:lipstick: Add a test for tag parameter.

import module namespace test = "http://apps.28.io/test";

declare variable $local:expected as object :=
{
  "all" : [
    "http://info.edinet-fsa.go.jp E01225-000",
    "http://info.edinet-fsa.go.jp E01264-000",
    "http://info.edinet-fsa.go.jp E02166-000",
    "http://info.edinet-fsa.go.jp E02274-000",
    "http://info.edinet-fsa.go.jp E02513-000",
    "http://info.edinet-fsa.go.jp E02529-000",
    "http://info.edinet-fsa.go.jp E04147-000",
    "http://info.edinet-fsa.go.jp E04425-000",
    "http://info.edinet-fsa.go.jp E04430-000",
    "http://info.edinet-fsa.go.jp EDINET-000"
  ],
  "NIKKEI" : [
    "http://info.edinet-fsa.go.jp E01225-000",
    "http://info.edinet-fsa.go.jp E01264-000",
    "http://info.edinet-fsa.go.jp E02166-000",
    "http://info.edinet-fsa.go.jp E02274-000",
    "http://info.edinet-fsa.go.jp E02513-000",
    "http://info.edinet-fsa.go.jp E02529-000",
    "http://info.edinet-fsa.go.jp E04147-000",
    "http://info.edinet-fsa.go.jp E04425-000",
    "http://info.edinet-fsa.go.jp E04430-000",
    "http://info.edinet-fsa.go.jp EDINET-000"
  ]
};

test:check-all-success({
    all: test:invoke-and-assert-deep-equal(
      "entities",
      {},
      function($b as item*) as item* { [ $b.Entities[]._id ] },
      $local:expected.all
    ),
    nikkei: test:invoke-and-assert-deep-equal(
      "entities",
      {tag:"NIKKEI"},
      function($b as item*) as item* { [ $b.Entities[]._id ] },
      $local:expected.NIKKEI
    ),
    example-json: test:invoke-and-assert-deep-equal(
      "entities",
      {cik:"E01225"},
      function($b as item*) as item* { $b.Entities[] },
      test:get-expected-result("edinet/entities-expected1.jq")
    ),
    example-xml: test:invoke-and-assert-deep-equal(
      "entities",
      {cik:"E01225", format: "xml"},
      function($b as item*) as item* { document { $b/Entities } },
      test:get-expected-result-xml("edinet/entities-expected1-xml.jq"),
      { Format: "xml" }
    )
})

import module namespace test = "http://apps.28.io/test";

declare variable $local:expected as object :=
{
  "all" : [
    "http://info.edinet-fsa.go.jp E01225-000",
    "http://info.edinet-fsa.go.jp E01264-000",
    "http://info.edinet-fsa.go.jp E02166-000",
    "http://info.edinet-fsa.go.jp E02274-000",
    "http://info.edinet-fsa.go.jp E02513-000",
    "http://info.edinet-fsa.go.jp E02529-000",
    "http://info.edinet-fsa.go.jp E04147-000",
    "http://info.edinet-fsa.go.jp E04425-000",
    "http://info.edinet-fsa.go.jp E04430-000",
    "http://info.edinet-fsa.go.jp EDINET-000"
  ]
};

test:check-all-success({
    all: test:invoke-and-assert-deep-equal(
      "entities",
      {},
      function($b as item*) as item* { [ $b.Entities[]._id ] },
      $local:expected.all
    ),
    example-json: test:invoke-and-assert-deep-equal(
      "entities",
      {cik:"E01225"},
      function($b as item*) as item* { $b.Entities[] },
      test:get-expected-result("edinet/entities-expected1.jq")
    ),
    example-xml: test:invoke-and-assert-deep-equal(
      "entities",
      {cik:"E01225", format: "xml"},
      function($b as item*) as item* { document { $b/Entities } },
      test:get-expected-result-xml("edinet/entities-expected1-xml.jq"),
      { Format: "xml" }
    )
})


:bug: Fix variable name.

:bug: Fix variable name.

jsoniq version "1.0";
module namespace japan = "http://28.io/modules/xbrl/profiles/japan/core";

import module namespace mw = "http://28.io/modules/xbrl/profiles/japan/mongo-wrapper";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace entities = "http://28.io/modules/xbrl/entities";

(:
 : Copyright 2012-2015 28msec Inc.
 :)
(:~
 :
This module provides functionality for querying against the Japan profile.

 :
 : @author Ghislain Fourny
 :
 :)

(:~
 :
Retrieves the entities with the given identifiers.

 :
 : @param $cik a sequence of shortcut codes, like E01225.
 : @param $eid a sequence of raw EIDs.
 : @param $tag a sequence of tags (e.g., indices).
 :
 : @return the entities that match those identifiers.
 :)
declare function japan:entities(
    $cik as string*,
    $eid as string*,
    $tag as string*
) as object*
{
  let $eid := distinct-values((
    $cik ! ("http://info.edinet-fsa.go.jp " || $$ || "-000"),
    $eid
  ))
  return
    switch(true)
    case $tag = "ALL" return entities:entities()
    case exists(($eid, $tag)) return
       for $entity in (
         for $tag in $tag
         return mw:find($entities:col, { "Profiles.FSA.Tags" : $tag }),
         entities:entities($eid)
       )
       group by $entity._id
       return $entity[1]
    default return ()
};

(:~
 :
Retrieves the filings with the given characteristics.

 :
 : @param $entities a sequence of entities or EIDs.
 : @param $fiscalYear a sequence of fiscal years.
 : @param $fiscalPeriod a sequence of fiscal periods.
 : @param $aid a sequence of AIDs.
 :
 : @return the filings that match those criteria.
 :)
declare function japan:filings(
    $entities as item*,
    $fiscalYear as integer*,
    $fiscalPeriod as string*,
    $aid as string*) as object*
{
    archives:archives($aid),
    if($fiscalYear = 1)
    then
        for $a as object in archives:archives-for-entities($entities)
        where (empty($fiscalPeriod) or ($fiscalPeriod = "ALL") or $a.Profiles.FSA.DocumentFiscalPeriodFocus = $fiscalPeriod)
        group by $a.Entity
        return
            for $filing in $a
            group by $fy := $filing.Profiles.FSA.DocumentFiscalYearFocus
            order by $fy descending
            count $i where $i eq 1
            return $filing
    else
        for $a as object in archives:archives-for-entities($entities)
        where (empty($fiscalYear) or
               $fiscalYear = 0 or
               $fiscalYear = $a.Profiles.FSA.DocumentFiscalYearFocus)
               and (empty($fiscalPeriod) or ($fiscalPeriod = "ALL") or $a.Profiles.FSA.DocumentFiscalPeriodFocus = $fiscalPeriod)
        return $a
};

jsoniq version "1.0";
module namespace japan = "http://28.io/modules/xbrl/profiles/japan/core";

import module namespace mw = "http://28.io/modules/xbrl/profiles/japan/mongo-wrapper";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace entities = "http://28.io/modules/xbrl/entities";

(:
 : Copyright 2012-2015 28msec Inc.
 :)
(:~
 :
This module provides functionality for querying against the Japan profile.

 :
 : @author Ghislain Fourny
 :
 :)

(:~
 :
Retrieves the entities with the given identifiers.

 :
 : @param $cik a sequence of shortcut codes, like E01225.
 : @param $eid a sequence of raw EIDs.
 : @param $tag a sequence of tags (e.g., indices).
 :
 : @return the entities that match those identifiers.
 :)
declare function japan:entities(
    $cik as string*,
    $eid as string*,
    $tag as string*
) as object*
{
  let $eid := distinct-values((
    $cik ! ("http://info.edinet-fsa.go.jp " || $$ || "-000"),
    $eid
  ))
  return
    switch(true)
    case $tag = "ALL" return entities:entities()
    case exists(($eid, $tag)) return
       for $entity in (
         for $tag in $tags
         return mw:find($entities:col, { "Profiles.FSA.Tags" : $tag }),
         entities:entities($eid)
       )
       group by $entity._id
       return $entity[1]
    default return ()
};

(:~
 :
Retrieves the filings with the given characteristics.

 :
 : @param $entities a sequence of entities or EIDs.
 : @param $fiscalYear a sequence of fiscal years.
 : @param $fiscalPeriod a sequence of fiscal periods.
 : @param $aid a sequence of AIDs.
 :
 : @return the filings that match those criteria.
 :)
declare function japan:filings(
    $entities as item*,
    $fiscalYear as integer*,
    $fiscalPeriod as string*,
    $aid as string*) as object*
{
    archives:archives($aid),
    if($fiscalYear = 1)
    then
        for $a as object in archives:archives-for-entities($entities)
        where (empty($fiscalPeriod) or ($fiscalPeriod = "ALL") or $a.Profiles.FSA.DocumentFiscalPeriodFocus = $fiscalPeriod)
        group by $a.Entity
        return
            for $filing in $a
            group by $fy := $filing.Profiles.FSA.DocumentFiscalYearFocus
            order by $fy descending
            count $i where $i eq 1
            return $filing
    else
        for $a as object in archives:archives-for-entities($entities)
        where (empty($fiscalYear) or
               $fiscalYear = 0 or
               $fiscalYear = $a.Profiles.FSA.DocumentFiscalYearFocus)
               and (empty($fiscalPeriod) or ($fiscalPeriod = "ALL") or $a.Profiles.FSA.DocumentFiscalPeriodFocus = $fiscalPeriod)
        return $a
};


jq: Introduce clean filter [ci skip]

jq: Introduce clean filter [ci skip]

So other filters may use it and be less verbose by default

# You can use jq to analyse multiple huge log files at once:
#
#   `yarn -s jq -c 'issues|no_gui|xls|short' path/to/logs.txt*`
#
# The `yarn jq` script doesn't use `--slurp` so JSON lines are filtered one by
# one and memory usage is kept below 100MB.
#
# But we should be able to use it in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
#
# TODO: Find a way to make aggregation work, e.g.:
#   `yarn jq 'find_issues|.msg' | yarn jq:frequencies`

# Remove fields that are almost never used while debugging
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v);

# Errors
def error_level: 50;
def is_error: .level >= error_level;
def select_error: select(is_error);
def find_errors: select_error|{component,path,msg,time};

# Warnings
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def select_warn: select(is_warn);
def select_warn_strict: select(is_warn_strict);
def find_warns: select_warn|{component,path,msg,time};
def find_warns_strict: select_warn_strict|{component,path,msg,time};

# Conflicts
def is_conflict: .msg == "resolveConflictAsync";
def select_conflict: select(is_conflict);
def find_conflicts: select_conflict|{path,time};

# Non-issues
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Issues
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def select_issue: select(is_issue);
def find_issues: select_issue|{component,path,msg,time,level};
def issues: select_issue;

# Path filtering
def filter_path(pattern): select((.path,.oldpath,"")|strings|test(pattern));

# GUI
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# Config info
def find_client_info:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# File extensions
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Global overview
def short:
  {time,component,msg,path};

# Utils
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );

# You can use jq to analyse multiple huge log files at once:
#
#   `yarn -s jq -c 'issues|no_gui|xls|short' path/to/logs.txt*`
#
# The `yarn jq` script doesn't use `--slurp` so JSON lines are filtered one by
# one and memory usage is kept below 100MB.
#
# But we should be able to use it in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
#
# TODO: Find a way to make aggregation work, e.g.:
#   `yarn jq 'find_issues|.msg' | yarn jq:frequencies`

# Errors
def error_level: 50;
def is_error: .level >= error_level;
def select_error: select(is_error);
def find_errors: select_error|{component,path,msg,time};

# Warnings
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def select_warn: select(is_warn);
def select_warn_strict: select(is_warn_strict);
def find_warns: select_warn|{component,path,msg,time};
def find_warns_strict: select_warn_strict|{component,path,msg,time};

# Conflicts
def is_conflict: .msg == "resolveConflictAsync";
def select_conflict: select(is_conflict);
def find_conflicts: select_conflict|{path,time};

# Non-issues
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Issues
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def select_issue: select(is_issue);
def find_issues: select_issue|{component,path,msg,time,level};
def issues: select_issue;

# Path filtering
def filter_path(pattern): select((.path,.oldpath,"")|strings|test(pattern));

# GUI
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# Config info
def find_client_info:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# File extensions
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Global overview
def short:
  {time,component,msg,path};

# Utils
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );


:green_heart: One more fix.

:green_heart: One more fix.

jsoniq version "1.0";

(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for querying fiscal reports (10-K, 10-Q)
 : submitted to the SEC.

 :
 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace facts = "http://28.io/modules/xbrl/facts";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Joker for all fiscal periods.
 :)
declare variable $fiscal-core:ALL_FISCAL_PERIODS as string := "ALL";

(:~
 : Joker for all fiscal period types.
 :)
declare variable $fiscal-core:ALL_FISCAL_PERIOD_TYPES as string := "ALL";

(:~
 : Joker for all fiscal years.
 :)
declare variable $fiscal-core:ALL_FISCAL_YEARS as integer := 0;

(:~
 : Joker for the latest fiscal years.
 :)
declare variable $fiscal-core:LATEST_FISCAL_YEAR as integer := 1;

(:~
 :
Return a distinct set of companies identified by either
 :   CIKs, tags, tickers, or sics.

 :
 : @param $ciks a set of CIKs.
 : @param $tags a set of tags (ALL retrieves all companies).
 : @param $tickers a set of tickers.
 : @param $sics a set of SIC codes.
 :
 : @return the companies with the given identifiers, tags, tickers, or sic codes.
 :)
declare function fiscal-core:filings(
    $entities-or-eids as item*,
    $fiscal-periods as string*,
    $fiscal-years as integer*,
    $archives-or-aids as item*) as object*
{
    let $entities := entities:entities($entities-or-eids)
    return
        if(exists(index-of($fiscal-years, $fiscal-core:LATEST_FISCAL_YEAR)))
        then
            for $entity in $entities
            return
            fiscal-core:latest-filings($entity, $fiscal-periods)
        else
            let $fiscal-years as integer* :=
                $fiscal-years[$$ ne $fiscal-core:LATEST_FISCAL_YEAR]
            return
                fiscal-core:filings-for-entities-and-fiscal-periods-and-years($entities, $fiscal-periods, $fiscal-years),
    archives:archives($archives-or-aids)
};

declare function fiscal-core:latest-filings(
    $entities as object*,
    $fiscal-periods as string*) as object*
{
    for $entity in $entities
    for $fiscal-period in $fiscal-periods
    let $latest-fiscal-year :=
        fiscal-core:latest-reported-fiscal-period($entity, $fiscal-period).year
    return fiscal-core:filings-for-entities-and-fiscal-periods-and-years(
        $entity,
        $fiscal-period,
        $latest-fiscal-year cast as integer?
    )
};

declare function fiscal-core:filter-override(
    $entities-or-eids as item*,
    $fiscal-years as integer*,
    $fiscal-periods as string*,
    $fiscal-period-types as string*,
    $archives-or-aids as item*
) as object?
{
    let $override := exists(($entities-or-eids, $fiscal-years, $fiscal-periods, $fiscal-period-types, $archives-or-aids))
    where $override
    let $eids := entities:eid($entities-or-eids)
    let $entities := entities:entities($entities-or-eids)
    let $aids := archives:aid($archives-or-aids)
    let $latest-filings := fiscal-core:latest-filings($entities, $fiscal-periods)
    let $is-latest := exists(index-of($fiscal-years, $fiscal-core:LATEST_FISCAL_YEAR))
    let $fiscal-years := if(exists(index-of($fiscal-years, $fiscal-core:ALL_FISCAL_YEARS)))
                         then ()
                         else $fiscal-years[$$ ne  $fiscal-core:LATEST_FISCAL_YEAR]
    let $fiscal-periods := if(exists(index-of($fiscal-periods, $fiscal-core:ALL_FISCAL_PERIODS)))
                           then ()
                           else $fiscal-periods
    let $override-fiscal-period-types := exists($fiscal-period-types)
    let $fiscal-period-types := if(exists(index-of($fiscal-period-types, $fiscal-core:ALL_FISCAL_PERIOD_TYPES)))
                                then ()
                                else $fiscal-period-types
    return
     {
        "xbrl:Entity" : {|
          { Type: "string" },
          { Domain: [ $eids ] }[exists($eids)]
        |},

        "sec:Accepted" : {|
          { Type: "string" }
        |},

        "xbrl28:Archive" : {|
          { Type: "string" },
          { Domain : [ if($is-latest) then archives:aid($latest-filings) else $aids ] }
          [exists(if($is-latest) then archives:aid($latest-filings) else $aids)]
        |},

        "sec:FiscalYear" : {|
          { Type: "integer" },
          { Domain: [ $fiscal-years ] }[exists($fiscal-years)]
        |},

        "sec:FiscalPeriod" : {|
          { Type: "string" },
          { Domain: [ $fiscal-periods ] }[exists($fiscal-periods)]
        |},

        "sec:FiscalPeriodType" : {|
          { Type: "string" },
          if($override-fiscal-period-types)
          then { Domain: [ $fiscal-period-types ]}[exists($fiscal-period-types)]
          else { Domain: [ "instant", "YTD" ] }
        |}
    }
};

(:~
 :
Retrieves the fiscal period of a filing.

 :
 : @param $filing-or-id a filing or its id, or the empty sequence.
 :
 : @return the fiscal period (FY, YTD2, YTD3, Q1, Q2 or Q3) if any.
 :)
declare function fiscal-core:fiscal-period-for-filing(
    $filing-or-id as item?) as string?
{
  archives:archives($filing-or-id).Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
};

(:~
 :
Retrieves the fiscal year of a filing.

 :
 : @param $filing-or-id a filing or its id, or the empty sequence.
 :
 : @return the fiscal year (e.g., 2012) if any.
 :)
declare function fiscal-core:fiscal-year-for-filing(
    $filing-or-id as item?) as integer?
{
  archives:archives($filing-or-id).Profiles.SEC.Fiscal.DocumentFiscalYearFocus
};

(:~
 :
Retrieves the fiscal period of a fact.

 :
 : @param $fact a fact object or the empty sequence.
 :
 : @return the fiscal period (FY, YTD2, YTD3, Q1, Q2 or Q3) if any.
 :)
declare function fiscal-core:fiscal-period-for-fact(
    $fact as object?) as integer?
{
  $fact.Aspects."sec:FiscalPeriod"
};

(:~
 :
Retrieves the fiscal year of a fact.

 :
 : @param $fact a fact object or the empty sequence.
 :
 : @return the fiscal year (e.g., 2012) if any.
 :)
declare function fiscal-core:fiscal-year-for-fact(
    $fact as object?) as integer?
{
  $fact.Aspects."sec:FiscalYear"
};

(:~
 :
Retrieves all filings by entities and for given fiscal periods and years.

 :
 : @param $entities-or-ids entities or their ids.
 : @param $fiscal-period-focus fiscal periods (FY, YTD1, YTD2, YTD3, Q1, Q2, Q3, Q4) or $fiscal-core:ALL_FISCAL_PERIODS to accept all..
 : @param $fiscal-year-focus fiscal years or $fiscal-core:ALL_FISCAL_YEARS to accept all..
 :
 : @return all filings that match the criteria.
 :)
declare function
fiscal-core:filings-for-entities-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  let $fiscal-period-focus := (
    "FY"[$fiscal-period-focus = ("10-K", "FY", "Q4")],
    "Q1"[$fiscal-period-focus = ("10-Q", "YTD1", "Q1")],
    "Q2"[$fiscal-period-focus = ("10-Q", "YTD2", "Q2")],
    "Q3"[$fiscal-period-focus = ("10-Q", "YTD3", "Q3")],
    $fiscal-period-focus[not $$ = ("10-K", "10-Q", "YTD1", "YTD2", "YTD3", "Q1", "Q2", "Q3", "Q4")]
  )
  for $cik-or-entity in $entities-or-ids
  let $entity-id as xs:string := companies:eid($cik-or-entity)
  let $query := {|
      { "Entity" : $entity-id },
      if(deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS) and
         deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS))
      then ()
      else {
        "Profiles" : {
          "FSA" : {
            "SEC" : {
              "Fiscal" : {|
                if(deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS))
                then ()
                else { "DocumentFiscalPeriodFocus": [ $fiscal-period-focus ] },
                if(deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS))
                then ()
                else { "DocumentFiscalYearFocus": [ $fiscal-year-focus ] }
              |}
            }
          }
        }
      }
    |}
  return mw:find($archives:col, $query)
};

(:~
 :
Retrieves facts for the given aspects, fiscal periods and years.

 :
 : @param $aspects an object containing aspects to filter, among which
 :                 xbrl:Concept, xbrl:Entity and xbrl:Period (all optional).
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY).
 : @param $fiscal-year-focus the years (2012)
 :
 : @return all facts matching the given aspects, fiscal period and year.
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects as object?,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects,
    $fiscal-period-focus,
    $fiscal-year-focus,
    ())
};

(:~
 :
Retrieves facts for the given aspects, fiscal periods and years.

 :
 : @param $aspects an object containing aspects to filter, among which
 :                 xbrl:Concept, xbrl:Entity and xbrl:Period (all optional).
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 : @param $options standard SEC BizQL options.
 :
 : @return all facts matching the given aspects, fiscal period and year.
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects as object?,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*,
    $options as object?) as object*
{
  let $filter as object := {|
    {
      "Filter":
        {|
          if(exists($aspects))
          then { $facts:ASPECTS : $aspects }
          else (),
          if( deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS) and deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS))
          then { "Profiles" : { "SEC" : { "Fiscal" : { "$exists" : true } } } } }
          else { "Profiles" : { "SEC" : { "Fiscal":
                 {|
                   if(deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS) )
                   then ()
                   else { "Period" : [ $fiscal-period-focus ] },
                   if( deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS) )
                   then ()
                   else { "Year": [ $fiscal-year-focus ] }
                 |}
               }}} }
        |}
    },
    if (empty($options.Hypercube))
    then { Hypercube: sec:dimensionless-hypercube(()) }
    else ()
  |}
  let $facts :=
    facts:facts-for(
      facts:merge-objects($filter, $options, true (: giving parameters higher priority :))
    )
  return sec:hide-amended-facts($facts, $options)
};

(:~
 :
Retrieves facts for the given entities, concepts, fiscal periods and years.

 :
 : @param $entities-or-ids a list of entities or their EIDs.
 : @param $concepts a list of concept names (us-gaap:Assets)
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 :
 : @error entities:INVALID_PARAMETER if the CIK or entity is not valid.
 : @return all facts matching entities, concepts, fiscal periods and years
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $concepts as string*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids,
    $concepts,
    $fiscal-period-focus,
    $fiscal-year-focus,
    ())
};

(:~
 :
Retrieves facts for the given entities, concepts, fiscal periods and years.

 :
 : @param $entities-or-ids a list of entities or their EIDs.
 : @param $concepts a list of concept names (us-gaap:Assets)
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 : @param $options standard SEC BizQL options.
 :
 : @error entities:INVALID_PARAMETER if the CIK or entity is not valid.
 : @return all facts matching entities, concepts, fiscal periods and years
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $concepts as string*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*,
    $options as object?) as object*
{
  let $entity-ids as xs:string* := $entities-or-ids ! companies:eid($$)
  let $aspects := {|
    if(exists($entity-ids))
    then { "xbrl:Entity": [ $entity-ids ] }
    else (),
    if(exists($concepts))
    then { "xbrl:Concept": [ $concepts ] }
    else ()
  |}
  return
    fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
      $aspects,
      $fiscal-period-focus,
      $fiscal-year-focus,
      $options)
};

(:~
 :
Retrieves the latest reported fiscal period and year for the
 : supplied entity.

 :
 : @param $entity-or-id an entity or its EID.
 :
 : @return an object with two fields: period and year.
 :)
declare function fiscal-core:latest-reported-fiscal-period(
    $entity-or-id as item) as object?
{
  (
  let $entity-or-id := companies:eid($entity-or-id)
  for $archive in archives:archives-for-entities($entity-or-id)
  let $archive-fiscal-period := fiscal-core:fiscal-period-for-filing($archive)
  let $archive-fiscal-year := fiscal-core:fiscal-year-for-filing($archive)
  group by $archive-fiscal-year
  order by $archive-fiscal-year descending empty least
  return {
    year: $archive-fiscal-year,
    period: switch(true)
            case $archive-fiscal-period = ("Q4", "FY") return "FY"
            case $archive-fiscal-period = ("Q3", "YTD3") return "Q3"
            case $archive-fiscal-period = ("Q2", "YTD2") return "Q2"
            case $archive-fiscal-period = ("Q1", "YTD1") return "Q1"
            default return () }
  )[1]
};

(:~
 :
Retrieves the latest fiscal period and year for the
 : supplied entity.

 :
 : @param $entity-or-id an entity or its EID.
 : @param $type 10-K or 10-Q or FY, YTD1, YTD2, YTD3, Q1, Q2, or Q3, or $fiscal-core:ALL_FISCAL_PERIODS
 : to accept all.
 :
 : @return an object with two fields: period and year.
 :)
declare function fiscal-core:latest-reported-fiscal-period(
    $entity-or-id as item,
    $type as string) as object?
{
  if(deep-equal($type, $fiscal-core:ALL_FISCAL_PERIODS))
  then fiscal-core:latest-reported-fiscal-period($entity-or-id)
  else
  (
  let $type-filter :=
    switch(true)
    case $type = ( "10-K", "FY", "Q4") return "FY"
    case $type eq "10-Q" return ( "Q3", "Q2", "Q1" )
    case $type eq "YTD1" return "Q1"
    case $type eq "YTD2" return "Q2"
    case $type eq "YTD3" return "Q3"
    default return $type
  let $entity-or-id := companies:eid($entity-or-id)
  for $archive in archives:archives-for-entities($entity-or-id)
  let $archive-fiscal-period := fiscal-core:fiscal-period-for-filing($archive)
  let $archive-fiscal-year := fiscal-core:fiscal-year-for-filing($archive)
  where $archive-fiscal-period = $type-filter
  group by $archive-fiscal-year
  order by $archive-fiscal-year descending empty least
  return {
    year: $archive-fiscal-year,
    period:
      switch(true)
      case $archive-fiscal-period = "FY" and ($type eq "10-K" or $type eq "FY") return "FY"
      case $archive-fiscal-period = "FY" and ($type eq "10-K" or $type eq "Q4") return "FY"
      case $archive-fiscal-period = "Q3" and ($type eq "10-Q" or $type eq "Q3") return "Q3"
      case $archive-fiscal-period = "Q2" and ($type eq "10-Q" or $type eq "Q2") return "Q2"
      case $archive-fiscal-period = "Q1" and ($type eq "10-Q" or $type eq "Q1") return "Q1"
      default return () }
  )[1]
};

jsoniq version "1.0";

(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for querying fiscal reports (10-K, 10-Q)
 : submitted to the SEC.

 :
 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace facts = "http://28.io/modules/xbrl/facts";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Joker for all fiscal periods.
 :)
declare variable $fiscal-core:ALL_FISCAL_PERIODS as string := "ALL";

(:~
 : Joker for all fiscal period types.
 :)
declare variable $fiscal-core:ALL_FISCAL_PERIOD_TYPES as string := "ALL";

(:~
 : Joker for all fiscal years.
 :)
declare variable $fiscal-core:ALL_FISCAL_YEARS as integer := 0;

(:~
 : Joker for the latest fiscal years.
 :)
declare variable $fiscal-core:LATEST_FISCAL_YEAR as integer := 1;

(:~
 :
Return a distinct set of companies identified by either
 :   CIKs, tags, tickers, or sics.

 :
 : @param $ciks a set of CIKs.
 : @param $tags a set of tags (ALL retrieves all companies).
 : @param $tickers a set of tickers.
 : @param $sics a set of SIC codes.
 :
 : @return the companies with the given identifiers, tags, tickers, or sic codes.
 :)
declare function fiscal-core:filings(
    $entities-or-eids as item*,
    $fiscal-periods as string*,
    $fiscal-years as integer*,
    $archives-or-aids as item*) as object*
{
    let $entities := entities:entities($entities-or-eids)
    return
        if(exists(index-of($fiscal-years, $fiscal-core:LATEST_FISCAL_YEAR)))
        then
            for $entity in $entities
            return
            fiscal-core:latest-filings($entity, $fiscal-periods)
        else
            let $fiscal-years as integer* :=
                $fiscal-years[$$ ne $fiscal-core:LATEST_FISCAL_YEAR]
            return
                fiscal-core:filings-for-entities-and-fiscal-periods-and-years($entities, $fiscal-periods, $fiscal-years),
    archives:archives($archives-or-aids)
};

declare function fiscal-core:latest-filings(
    $entities as object*,
    $fiscal-periods as string*) as object*
{
    for $entity in $entities
    for $fiscal-period in $fiscal-periods
    let $latest-fiscal-year :=
        fiscal-core:latest-reported-fiscal-period($entity, $fiscal-period).year
    return fiscal-core:filings-for-entities-and-fiscal-periods-and-years(
        $entity,
        $fiscal-period,
        $latest-fiscal-year cast as integer?
    )
};

declare function fiscal-core:filter-override(
    $entities-or-eids as item*,
    $fiscal-years as integer*,
    $fiscal-periods as string*,
    $fiscal-period-types as string*,
    $archives-or-aids as item*
) as object?
{
    let $override := exists(($entities-or-eids, $fiscal-years, $fiscal-periods, $fiscal-period-types, $archives-or-aids))
    where $override
    let $eids := entities:eid($entities-or-eids)
    let $entities := entities:entities($entities-or-eids)
    let $aids := archives:aid($archives-or-aids)
    let $latest-filings := fiscal-core:latest-filings($entities, $fiscal-periods)
    let $is-latest := exists(index-of($fiscal-years, $fiscal-core:LATEST_FISCAL_YEAR))
    let $fiscal-years := if(exists(index-of($fiscal-years, $fiscal-core:ALL_FISCAL_YEARS)))
                         then ()
                         else $fiscal-years[$$ ne  $fiscal-core:LATEST_FISCAL_YEAR]
    let $fiscal-periods := if(exists(index-of($fiscal-periods, $fiscal-core:ALL_FISCAL_PERIODS)))
                           then ()
                           else $fiscal-periods
    let $override-fiscal-period-types := exists($fiscal-period-types)
    let $fiscal-period-types := if(exists(index-of($fiscal-period-types, $fiscal-core:ALL_FISCAL_PERIOD_TYPES)))
                                then ()
                                else $fiscal-period-types
    return
     {
        "xbrl:Entity" : {|
          { Type: "string" },
          { Domain: [ $eids ] }[exists($eids)]
        |},

        "sec:Accepted" : {|
          { Type: "string" }
        |},

        "xbrl28:Archive" : {|
          { Type: "string" },
          { Domain : [ if($is-latest) then archives:aid($latest-filings) else $aids ] }
          [exists(if($is-latest) then archives:aid($latest-filings) else $aids)]
        |},

        "sec:FiscalYear" : {|
          { Type: "integer" },
          { Domain: [ $fiscal-years ] }[exists($fiscal-years)]
        |},

        "sec:FiscalPeriod" : {|
          { Type: "string" },
          { Domain: [ $fiscal-periods ] }[exists($fiscal-periods)]
        |},

        "sec:FiscalPeriodType" : {|
          { Type: "string" },
          if($override-fiscal-period-types)
          then { Domain: [ $fiscal-period-types ]}[exists($fiscal-period-types)]
          else { Domain: [ "instant", "YTD" ] }
        |}
    }
};

(:~
 :
Retrieves the fiscal period of a filing.

 :
 : @param $filing-or-id a filing or its id, or the empty sequence.
 :
 : @return the fiscal period (FY, YTD2, YTD3, Q1, Q2 or Q3) if any.
 :)
declare function fiscal-core:fiscal-period-for-filing(
    $filing-or-id as item?) as string?
{
  archives:archives($filing-or-id).Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
};

(:~
 :
Retrieves the fiscal year of a filing.

 :
 : @param $filing-or-id a filing or its id, or the empty sequence.
 :
 : @return the fiscal year (e.g., 2012) if any.
 :)
declare function fiscal-core:fiscal-year-for-filing(
    $filing-or-id as item?) as integer?
{
  archives:archives($filing-or-id).Profiles.SEC.Fiscal.DocumentFiscalYearFocus
};

(:~
 :
Retrieves the fiscal period of a fact.

 :
 : @param $fact a fact object or the empty sequence.
 :
 : @return the fiscal period (FY, YTD2, YTD3, Q1, Q2 or Q3) if any.
 :)
declare function fiscal-core:fiscal-period-for-fact(
    $fact as object?) as integer?
{
  $fact.Aspects."sec:FiscalPeriod"
};

(:~
 :
Retrieves the fiscal year of a fact.

 :
 : @param $fact a fact object or the empty sequence.
 :
 : @return the fiscal year (e.g., 2012) if any.
 :)
declare function fiscal-core:fiscal-year-for-fact(
    $fact as object?) as integer?
{
  $fact.Aspects."sec:FiscalYear"
};

(:~
 :
Retrieves all filings by entities and for given fiscal periods and years.

 :
 : @param $entities-or-ids entities or their ids.
 : @param $fiscal-period-focus fiscal periods (FY, YTD1, YTD2, YTD3, Q1, Q2, Q3, Q4) or $fiscal-core:ALL_FISCAL_PERIODS to accept all..
 : @param $fiscal-year-focus fiscal years or $fiscal-core:ALL_FISCAL_YEARS to accept all..
 :
 : @return all filings that match the criteria.
 :)
declare function
fiscal-core:filings-for-entities-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  let $fiscal-period-focus := (
    "FY"[$fiscal-period-focus = ("10-K", "FY", "Q4")],
    "Q1"[$fiscal-period-focus = ("10-Q", "YTD1", "Q1")],
    "Q2"[$fiscal-period-focus = ("10-Q", "YTD2", "Q2")],
    "Q3"[$fiscal-period-focus = ("10-Q", "YTD3", "Q3")],
    $fiscal-period-focus[not $$ = ("10-K", "10-Q", "YTD1", "YTD2", "YTD3", "Q1", "Q2", "Q3", "Q4")]
  )
  for $cik-or-entity in $entities-or-ids
  let $entity-id as xs:string := companies:eid($cik-or-entity)
  let $query := {|
      { "Entity" : $entity-id },
      if(deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS) and
         deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS))
      then ()
      else {
        "Profiles" : {
          "FSA" : {
            "SEC" : {
              "Fiscal" : {|
                if(deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS))
                then ()
                else { "DocumentFiscalPeriodFocus": [ $fiscal-period-focus ] },
                if(deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS))
                then ()
                else { "DocumentFiscalYearFocus": [ $fiscal-year-focus ] }
              |}
            }
          }
        }
      }
    |}
  return mw:find($archives:col, $query)
};

(:~
 :
Retrieves facts for the given aspects, fiscal periods and years.

 :
 : @param $aspects an object containing aspects to filter, among which
 :                 xbrl:Concept, xbrl:Entity and xbrl:Period (all optional).
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY).
 : @param $fiscal-year-focus the years (2012)
 :
 : @return all facts matching the given aspects, fiscal period and year.
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects as object?,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects,
    $fiscal-period-focus,
    $fiscal-year-focus,
    ())
};

(:~
 :
Retrieves facts for the given aspects, fiscal periods and years.

 :
 : @param $aspects an object containing aspects to filter, among which
 :                 xbrl:Concept, xbrl:Entity and xbrl:Period (all optional).
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 : @param $options standard SEC BizQL options.
 :
 : @return all facts matching the given aspects, fiscal period and year.
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects as object?,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*,
    $options as object?) as object*
{
  let $filter as object := {|
    {
      "Filter":
        {|
          if(exists($aspects))
          then { $facts:ASPECTS : $aspects }
          else (),
          if( deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS) and deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS))
          then { "Profiles" : { "FSA" : { ".SEC.Fiscal" : { "$exists" : true } } } }
          else { "Profiles" : { "FSA" : { "": { "SEC": { "Fiscal":
                 {|
                   if(deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS) )
                   then ()
                   else { "Period" : [ $fiscal-period-focus ] },
                   if( deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS) )
                   then ()
                   else { "Year": [ $fiscal-year-focus ] }
                 |}
               }}} } }
        |}
    },
    if (empty($options.Hypercube))
    then { Hypercube: sec:dimensionless-hypercube(()) }
    else ()
  |}
  let $facts :=
    facts:facts-for(
      facts:merge-objects($filter, $options, true (: giving parameters higher priority :))
    )
  return sec:hide-amended-facts($facts, $options)
};

(:~
 :
Retrieves facts for the given entities, concepts, fiscal periods and years.

 :
 : @param $entities-or-ids a list of entities or their EIDs.
 : @param $concepts a list of concept names (us-gaap:Assets)
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 :
 : @error entities:INVALID_PARAMETER if the CIK or entity is not valid.
 : @return all facts matching entities, concepts, fiscal periods and years
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $concepts as string*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids,
    $concepts,
    $fiscal-period-focus,
    $fiscal-year-focus,
    ())
};

(:~
 :
Retrieves facts for the given entities, concepts, fiscal periods and years.

 :
 : @param $entities-or-ids a list of entities or their EIDs.
 : @param $concepts a list of concept names (us-gaap:Assets)
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 : @param $options standard SEC BizQL options.
 :
 : @error entities:INVALID_PARAMETER if the CIK or entity is not valid.
 : @return all facts matching entities, concepts, fiscal periods and years
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $concepts as string*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*,
    $options as object?) as object*
{
  let $entity-ids as xs:string* := $entities-or-ids ! companies:eid($$)
  let $aspects := {|
    if(exists($entity-ids))
    then { "xbrl:Entity": [ $entity-ids ] }
    else (),
    if(exists($concepts))
    then { "xbrl:Concept": [ $concepts ] }
    else ()
  |}
  return
    fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
      $aspects,
      $fiscal-period-focus,
      $fiscal-year-focus,
      $options)
};

(:~
 :
Retrieves the latest reported fiscal period and year for the
 : supplied entity.

 :
 : @param $entity-or-id an entity or its EID.
 :
 : @return an object with two fields: period and year.
 :)
declare function fiscal-core:latest-reported-fiscal-period(
    $entity-or-id as item) as object?
{
  (
  let $entity-or-id := companies:eid($entity-or-id)
  for $archive in archives:archives-for-entities($entity-or-id)
  let $archive-fiscal-period := fiscal-core:fiscal-period-for-filing($archive)
  let $archive-fiscal-year := fiscal-core:fiscal-year-for-filing($archive)
  group by $archive-fiscal-year
  order by $archive-fiscal-year descending empty least
  return {
    year: $archive-fiscal-year,
    period: switch(true)
            case $archive-fiscal-period = ("Q4", "FY") return "FY"
            case $archive-fiscal-period = ("Q3", "YTD3") return "Q3"
            case $archive-fiscal-period = ("Q2", "YTD2") return "Q2"
            case $archive-fiscal-period = ("Q1", "YTD1") return "Q1"
            default return () }
  )[1]
};

(:~
 :
Retrieves the latest fiscal period and year for the
 : supplied entity.

 :
 : @param $entity-or-id an entity or its EID.
 : @param $type 10-K or 10-Q or FY, YTD1, YTD2, YTD3, Q1, Q2, or Q3, or $fiscal-core:ALL_FISCAL_PERIODS
 : to accept all.
 :
 : @return an object with two fields: period and year.
 :)
declare function fiscal-core:latest-reported-fiscal-period(
    $entity-or-id as item,
    $type as string) as object?
{
  if(deep-equal($type, $fiscal-core:ALL_FISCAL_PERIODS))
  then fiscal-core:latest-reported-fiscal-period($entity-or-id)
  else
  (
  let $type-filter :=
    switch(true)
    case $type = ( "10-K", "FY", "Q4") return "FY"
    case $type eq "10-Q" return ( "Q3", "Q2", "Q1" )
    case $type eq "YTD1" return "Q1"
    case $type eq "YTD2" return "Q2"
    case $type eq "YTD3" return "Q3"
    default return $type
  let $entity-or-id := companies:eid($entity-or-id)
  for $archive in archives:archives-for-entities($entity-or-id)
  let $archive-fiscal-period := fiscal-core:fiscal-period-for-filing($archive)
  let $archive-fiscal-year := fiscal-core:fiscal-year-for-filing($archive)
  where $archive-fiscal-period = $type-filter
  group by $archive-fiscal-year
  order by $archive-fiscal-year descending empty least
  return {
    year: $archive-fiscal-year,
    period:
      switch(true)
      case $archive-fiscal-period = "FY" and ($type eq "10-K" or $type eq "FY") return "FY"
      case $archive-fiscal-period = "FY" and ($type eq "10-K" or $type eq "Q4") return "FY"
      case $archive-fiscal-period = "Q3" and ($type eq "10-Q" or $type eq "Q3") return "Q3"
      case $archive-fiscal-period = "Q2" and ($type eq "10-Q" or $type eq "Q2") return "Q2"
      case $archive-fiscal-period = "Q1" and ($type eq "10-Q" or $type eq "Q1") return "Q1"
      default return () }
  )[1]
};


updated entities query (rename CompanyName=>EntityName; better response content-type and indentation)

import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-xml($entities as object*) as element()*
{
  <Entities>{
    for $e in $entities
    return
    <Entity>
        <ID>{$e._id}</ID>
        <Profile name="{$e.Profiles.SEC.Name}">
            <EntityName>{$e.Profiles.SEC.CompanyName}</EntityName>
            <CompanyType>{$e.Profiles.SEC.CompanyType}</CompanyType>
            <SIC>{$e.Profiles.SEC.SIC}</SIC>
            <SICDescription>{$e.Profiles.SEC.SICDescription}</SICDescription>
            <SICGroup>{$e.Profiles.SEC.SICGroup}</SICGroup>
            <Sector>{$e.Profiles.SEC.Sector}</Sector>
            <IsTrust>{$e.Profiles.SEC.IsTrust}</IsTrust>
            <Tickers>{
                for $t in $e.Profiles.SEC.Tickers[]
                return <Ticker>{$t}</Ticker>
            }</Tickers>
            <Tags>
            {
                for $t in $e.Profiles.SEC.Tags[]
                return <Tag>{$t}</Tag>
            }
            </Tags>
        </Profile>
    </Entity>
  }</Entities>
};

declare function local:to-csv($entities as object*) as string*
{
    csv:serialize(
        for $e in $entities
        return  {
            ID : $e._id,
            Profile : "SEC",
            EntityName : $e.Profiles.SEC.CompanyName,
            CompanyType : $e.Profiles.SEC.CompanyType,
            SIC : $e.Profiles.SEC.SIC,
            SICDescription : $e.Profiles.SEC.SICDescription,
            SICGroup :$e.Profiles.SEC.SICGroup,
            Sector : $e.Profiles.SEC.Sector,
            IsTrust : $e.Profiles.SEC.IsTrust,
            Tickers : string-join($e.Profiles.SEC.Tickers[], " "),
            Tags : string-join($e.Profiles.SEC.Tags[], " ")
        }
    )
};

let $format  := lower-case(substring-after(request:path(), ".jq.")) (: text, xml, or json (default) :)
let $ciks    := request:param-values("cik")
let $tags    := request:param-values("tag") ! upper-case($$) (: DOW30, SP500, FORTUNE100 :)
let $tickers := request:param-values("ticker")
let $entities :=
    for $entity in
        if (exists(($ciks, $tags, $tickers)))
        then
            for $entity in (companies:companies($ciks),
                       companies:companies-for-tags($tags),
                       companies:companies-for-tickers($tickers))
            group by companies:eid($entity) (: duplicate elimination :)
            return $entity[1]
        else
            entities:entities() (: companies:companies() ? :)
    order by $entity.Profiles.SEC.CompanyName (: companies:name() ? :)
    return $entity
return
    switch ($format)
        case "xml"  return {
            response:content-type("application/xml");
            response:serialization-parameters({"omit-xml-declaration" : false});
            local:to-xml($entities)
        }
        case "text" return {
            response:content-type("application/csv");
            string-join(local:to-csv($entities))
        }
        default return {
            response:content-type("application/json");
            response:serialization-parameters({"indent" : true});
            [ $entities ]
        }

import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-xml($entities as object*) as element()*
{
  <Entities>{
    for $e in $entities
    return
    <Entity>
        <ID>{$e._id}</ID>
        <Profile name="{$e.Profiles.SEC.Name}">
            <CompanyName>{$e.Profiles.SEC.CompanyName}</CompanyName>
            <CompanyType>{$e.Profiles.SEC.CompanyType}</CompanyType>
            <SIC>{$e.Profiles.SEC.SIC}</SIC>
            <SICDescription>{$e.Profiles.SEC.SICDescription}</SICDescription>
            <SICGroup>{$e.Profiles.SEC.SICGroup}</SICGroup>
            <Sector>{$e.Profiles.SEC.Sector}</Sector>
            <IsTrust>{$e.Profiles.SEC.IsTrust}</IsTrust>
            <Tickers>{
                for $t in $e.Profiles.SEC.Tickers[]
                return <Ticker>{$t}</Ticker>
            }</Tickers>
            <Tags>
            {
                for $t in $e.Profiles.SEC.Tags[]
                return <Tag>{$t}</Tag>
            }
            </Tags>
        </Profile>
    </Entity>
  }</Entities>
};

declare function local:to-csv($entities as object*) as string*
{
    csv:serialize(
        for $e in $entities
        return  {
            ID : $e._id,
            Profile : "SEC",
            CompanyName : $e.Profiles.SEC.CompanyName,
            CompanyType : $e.Profiles.SEC.CompanyType,
            SIC : $e.Profiles.SEC.SIC,
            SICDescription : $e.Profiles.SEC.SICDescription,
            SICGroup :$e.Profiles.SEC.SICGroup,
            Sector : $e.Profiles.SEC.Sector,
            IsTrust : $e.Profiles.SEC.IsTrust,
            Tickers : string-join($e.Profiles.SEC.Tickers[], " "),
            Tags : string-join($e.Profiles.SEC.Tags[], " ")
        }
    )
};

let $format  := lower-case(substring-after(request:path(), ".jq.")) (: text, xml, or json (default) :)
let $ciks    := request:param-values("cik")
let $tags    := request:param-values("tag") ! upper-case($$) (: DOW30, SP500, FORTUNE100 :)
let $tickers := request:param-values("ticker")
let $entities :=
    for $entity in
        if (exists(($ciks, $tags, $tickers)))
        then
            for $entity in (companies:companies($ciks),
                       companies:companies-for-tags($tags),
                       companies:companies-for-tickers($tickers))
            group by companies:eid($entity) (: duplicate elimination :)
            return $entity[1]
        else
            entities:entities() (: companies:companies() ? :)
    order by $entity.Profiles.SEC.CompanyName (: companies:name() ? :)
    return $entity
return
    switch ($format)
        case "xml"  return local:to-xml($entities)
        case "text"  return string-join(local:to-csv($entities))
        default return [ $entities ]


Refactored model structure.

Refactored model structure.

jsoniq version "1.0";

import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace filings = "http://xbrl.io/modules/bizql/profiles/sec/filings";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";

import module namespace sec-networks = "http://xbrl.io/modules/bizql/profiles/sec/networks";

import module namespace util = "http://secxbrl.info/modules/util";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-xml-rec($o as object*, $level as integer) as element()*
{
    for $o in $o
    return
        element { if ($o.Kind eq "Domain") then "Member" else $o.Kind } {
            attribute { "name" } { $o.Name },
            attribute { "label" } { $o.Label },
            if (exists($o.Balance)) then attribute { "balance" } { $o.Balance } else (),
            if (exists($o.DataType)) then attribute { "dataType" } { $o.DataType } else (),
            if (exists($o.BaseType)) then attribute { "baseDataType" } { $o.BaseType } else (),
            if (exists($o.IsAbstract)) then attribute { "abstract" } { $o.IsAbstract } else (),
            if (exists($o.PeriodType)) then attribute { "periodType" } { $o.PeriodType } else (),
            attribute { "level" } { $level },
            local:to-xml-rec($o.Children[], $level + 1)
        }
};

declare function local:to-xml($model as object) as node()*
{
    ((session:comment("xml", {
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })),
    <Component>
        <Network entityRegistrantName="{$model.EntityRegistrantName}"
                 accessionNumber="{$model.AccessionNumber}"
                 tableName="{$model.TableName}"
                 cik="{$model.CIK}"
                 label="{$model.Label}"
                 networkIdentifier="{$model.NetworkIdentifier}"
                 formType="{$model.FormType}"
                 fiscalPeriod="{$model.FiscalPeriod}"
                 fiscalYear="{$model.FiscalYear}"
                 acceptanceDatetime="{$model.AcceptanceDatetime}"
                 disclosure="{$model.Disclosure}"
                 >{
            local:to-xml-rec($model.ModelStructure, 0)
        }</Network>
    </Component>)

};

declare function local:to-csv-rec($objects as object*, $level as integer) as object*
{
    for $o in $objects
    let $object := {
        "Label" : $o.Label,
        "Name" : $o.Name,
        "ObjectClass" : if ($o.Kind eq "Domain") then "Member" else $o.Kind,
        "DataType" : $o.DataType,
        "BaseDataType" : $o.BaseType,
        "Balance" : $o.Balance,
        "Abstract" : $o.IsAbstract,
        "PeriodType" : $o.PeriodType,
        "Level" : $level
    }
    return (
        $object,
        local:to-csv-rec($o.Children[], $level + 1)
    )
};

declare function local:to-csv($model as object) as string
{
    let $lines := local:to-csv-rec($model.ModelStructure, 0)
    return
        if (exists($lines))
        then string-join(csv:serialize($lines, { serialize-null-as : "" }))
        else ""
};

declare function local:enrich-json-rec($objects as object*, $level as integer) as object*
{
    for $object in $objects
    return
        copy $o := $object
        modify (
            if (exists($o.Children)) then delete json $o("Children") else (),
            insert json { Level : $level } into $o,
            if ($o.Kind eq "Domain") then replace value of json $o("Kind") with "Member" else ()
        )
        return {|
            $o,
            let $children := local:enrich-json-rec($object.Children[], $level + 1)
            return if (exists($children)) then { Children : [ $children ] } else ()
        |}
};

declare function local:enrich-json($component as object) as object
{
    {
        ModelStructure : [ local:enrich-json-rec($component.ModelStructure, 0) ] ,
        CIK : $component.CIK,
        EntityRegistrantName : $component.EntityRegistrantName,
        Label : $component.Label,
        AccessionNumber : $component.AccessionNumber,
        TableName : $component.TableName,
        FormType : $component.FormType,
        FiscalPeriod : $component.FiscalPeriod,
        FiscalYear : $component.FiscalYear,
        AcceptanceDatetime : $component.AcceptanceDatetime,
        NetworkIdentifier: $component.NetworkIdentifier,
        Disclosure : $component.Disclosure
    }
};

session:audit-call();

(: Query parameters :)
let $format as string?         := request:param-values("format")
let $ciks as string*           := distinct-values(request:param-values("cik"))
let $tags as string*           := distinct-values(request:param-values("tag"))
let $tickers as string*        := distinct-values(request:param-values("ticker"))
let $sics as string*           := distinct-values(request:param-values("sic"))
let $fiscalYears as string*    := distinct-values(request:param-values("fiscalYear", "LATEST"))
let $fiscalPeriods as string*  := distinct-values(request:param-values("fiscalPeriod", "FY"))
let $aids as string*           := distinct-values(request:param-values("aid"))
let $roles as string*          := request:param-values("networkIdentifier")
let $cid as string?            := request:param-values("cid")
let $reportElements as string* := distinct-values(request:param-values("reportElement"))
let $concepts as string*       := distinct-values(request:param-values("concept"))
let $rollups as string*        := distinct-values(request:param-values("rollup"))
let $map as string?            := request:param-values("map")
let $disclosures as string*    := request:param-values("disclosure")
let $validate as string        := request:param-values("validate", "false")
let $eliminate as string       := request:param-values("eliminate", "false")
let $report as string?         := request:param-values("report")
let $search as string*         := request:param-values("label")
let $parameters := {|
    {
        CIKs: [ $ciks ],
        Tags: [ $tags ],
        Tickers: [ $tickers ],
        SICs: [ $sics ],
        FiscalYears: [ $fiscalYears ],
        FiscalPeriods: [ $fiscalPeriods ],
        AIDs: [ $aids ],
        Roles: [ $roles ],
        CIDs: [ $cid ],
        ReportElements: [ $reportElements ],
        Concepts: [ $concepts ],
        RollUps: [ $rollups ],
        Disclosures: [ $disclosures ],
        Search: [ $search ]
    },
    { Format: $format }[exists($format)],
    { Map: $map }[exists($map)],
    { Validate: $validate }[exists($validate)],
    { Eliminate: $eliminate }[exists($eliminate)],
    { Report: $report }[exists($report)]
|}

(: Object resolution :)
let $parameters as object := util:process-parameters($parameters)
let $components  := util:components-from-parameters($parameters, ())
let $component := $components[1] (: only one for now :)
let $archive   := archives:archives($component.Archive)
let $entity    := entities:entities($archive.Entity)

let $results :=
    let $model := {|
        { CIK : $entity._id },
        { EntityRegistrantName : $entity.Profiles.SEC.CompanyName },
        { ModelStructure : sec-networks:model-structures($component) },
        { TableName : sec-networks:tables($component, {IncludeImpliedTable: true}).Name },
        { Label : $component.Label },
        { AccessionNumber : $component.Archive },
        { FormType : $archive.Profiles.SEC.FormType },
        { FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus },
        { FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus },
        { AcceptanceDatetime : filings:acceptance-dateTimes($archive) },
        { NetworkIdentifier: $component.Role },
        { Disclosure : $component.Profiles.SEC.Disclosure }
    |}
    return
    switch ($format)
    case "xml" return {
        response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
        local:to-xml($model)
    }
    case "text" case "csv" return {
        response:content-type("text/csv");
        response:header("Content-Disposition", "attachment; filename=modelstructure-" || $cid || ".csv");
        local:to-csv($model)
    }
    case "excel" return {
        response:content-type("application/vnd.ms-excel");
        response:header("Content-Disposition", "attachment; filename=modelstructure-" || $cid || ".csv");
        local:to-csv($model)
    }
    default return {
        response:content-type("application/json");
        response:serialization-parameters({"indent" : true});
        {|
            local:enrich-json($model),
            session:comment("json", {
                    TotalNumArchives: session:num-archives(),
                    TotalNumEntities: session:num-entities()
                })
        |}
    }
return util:check-and-return-results($entity, $results, $parameters.Format)

jsoniq version "1.0";

import module namespace components = "http://xbrl.io/modules/bizql/components";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace filings = "http://xbrl.io/modules/bizql/profiles/sec/filings";
import module namespace fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

import module namespace sec-networks = "http://xbrl.io/modules/bizql/profiles/sec/networks";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-xml-rec($o as object*, $level as integer) as element()*
{
    for $o in $o
    return
        element { if ($o.Kind eq "Domain") then "Member" else $o.Kind } {
            attribute { "name" } { $o.Name },
            attribute { "label" } { $o.Label },
            if (exists($o.Balance)) then attribute { "balance" } { $o.Balance } else (),
            if (exists($o.DataType)) then attribute { "dataType" } { $o.DataType } else (),
            if (exists($o.BaseType)) then attribute { "baseDataType" } { $o.BaseType } else (),
            if (exists($o.IsAbstract)) then attribute { "abstract" } { $o.IsAbstract } else (),
            if (exists($o.PeriodType)) then attribute { "periodType" } { $o.PeriodType } else (),
            attribute { "level" } { $level },
            local:to-xml-rec($o.Children[], $level + 1)
        }
};

declare function local:to-xml($model as object) as node()*
{
    ((session:comment("xml", {
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })),
    <Component>
        <Network entityRegistrantName="{$model.EntityRegistrantName}"
                 accessionNumber="{$model.AccessionNumber}"
                 tableName="{$model.TableName}"
                 cik="{$model.CIK}"
                 label="{$model.Label}"
                 networkIdentifier="{$model.NetworkIdentifier}"
                 formType="{$model.FormType}"
                 fiscalPeriod="{$model.FiscalPeriod}"
                 fiscalYear="{$model.FiscalYear}"
                 acceptanceDatetime="{$model.AcceptanceDatetime}"
                 disclosure="{$model.Disclosure}"
                 >{
            local:to-xml-rec($model.ModelStructure, 0)
        }</Network>
    </Component>)

};

declare function local:to-csv-rec($objects as object*, $level as integer) as object*
{
    for $o in $objects
    let $object := {
        "Label" : $o.Label,
        "Name" : $o.Name,
        "ObjectClass" : if ($o.Kind eq "Domain") then "Member" else $o.Kind,
        "DataType" : $o.DataType,
        "BaseDataType" : $o.BaseType,
        "Balance" : $o.Balance,
        "Abstract" : $o.IsAbstract,
        "PeriodType" : $o.PeriodType,
        "Level" : $level
    }
    return (
        $object,
        local:to-csv-rec($o.Children[], $level + 1)
    )
};

declare function local:to-csv($model as object) as string
{
    let $lines := local:to-csv-rec($model.ModelStructure, 0)
    return
        if (exists($lines))
        then string-join(csv:serialize($lines, { serialize-null-as : "" }))
        else ""
};

declare function local:enrich-json-rec($objects as object*, $level as integer) as object*
{
    for $object in $objects
    return
        copy $o := $object
        modify (
            if (exists($o.Children)) then delete json $o("Children") else (),
            insert json { Level : $level } into $o,
            if ($o.Kind eq "Domain") then replace value of json $o("Kind") with "Member" else ()
        )
        return {|
            $o,
            let $children := local:enrich-json-rec($object.Children[], $level + 1)
            return if (exists($children)) then { Children : [ $children ] } else ()
        |}
};

declare function local:enrich-json($component as object) as object
{
    {
        ModelStructure : [ local:enrich-json-rec($component.ModelStructure, 0) ] ,
        CIK : $component.CIK,
        EntityRegistrantName : $component.EntityRegistrantName,
        Label : $component.Label,
        AccessionNumber : $component.AccessionNumber,
        TableName : $component.TableName,
        FormType : $component.FormType,
        FiscalPeriod : $component.FiscalPeriod,
        FiscalYear : $component.FiscalYear,
        AcceptanceDatetime : $component.AcceptanceDatetime,
        NetworkIdentifier: $component.NetworkIdentifier,
        Disclosure : $component.Disclosure
    }
};
declare function local:components-by-disclosures($disclosures as string*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    for $aid in $aids
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: $aid,
            "Profiles.SEC.Disclosure": { "$in" : [ $disclosures ] }
        })
};

declare function local:components-by-roles($roles as string*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: { "$in" : [ $aids ] },
            "Role": { "$in" : [ $roles ] }
        })
};

declare function local:components-by-concepts($concepts as object*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    let $ids := mongo:find($conn, "concepts",
        {|
            (
                { "Name" : { "$in" : [ $concepts ] } },
                { "Archive" : { "$in" : [ $aids ] } }
            )
        |}).Component
    return components:components($ids)
};

declare function local:filings(
    $ciks as string*,
    $tags as string*,
    $tickers as string*,
    $sics as string*,
    $fp as string*,
    $fy as string*) as object*
{
    let $entities := if ($tags = "ALL") then companies:companies()
                                        else (
                                            companies:companies($ciks),
                                            companies:companies-for-tags($tags),
                                            companies:companies-for-tickers($tickers),
                                            companies:companies-for-SIC($sics)
                                        )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                        case "ALL" return $fiscal:ALL_FISCAL_YEARS
                    default return $fy cast as integer
                )
    for $fp in $fp
    return fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy)
};

session:audit-call();

let $format      := lower-case((request:param-values("format"), substring-after(request:path(), ".jq."))[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids        := archives:aid(request:param-values("aid"))
let $roles       := request:param-values("networkIdentifier")
let $archives    := (
                        local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                        archives:archives($aids)
                    )
let $cid         := request:param-values("cid")
let $concepts    := distinct-values(request:param-values("concept"))
let $disclosures := request:param-values("disclosure")
let $components  := (if (exists($cid))
                    then components:components($cid)
                    else (),
                    if (exists($concepts))
                    then local:components-by-concepts($concepts, $archives._id)
                    else if (exists($disclosures))
                    then local:components-by-disclosures($disclosures, $archives._id)
                    else if (exists($roles))
                    then local:components-by-roles($roles, $archives._id)
                    else components:components-for-archives($archives._id))
let $component := $components[1] (: only one for know :)
let $archive   := archives:archives($component.Archive)
let $entity    := entities:entities($archive.Entity)
return
    switch(session:check-access($entity, "data_sec"))
    case $session:ACCESS-ALLOWED return {
        let $model := {|
                    { CIK : $entity._id },
                    { EntityRegistrantName : $entity.Profiles.SEC.CompanyName },
                    { ModelStructure : sec-networks:model-structures($component) },
                    { TableName : sec-networks:tables($component, {IncludeImpliedTable: true}).Name },
                    { Label : $component.Label },
                    { AccessionNumber : $component.Archive },
                    { FormType : $archive.Profiles.SEC.FormType },
                    { FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus },
                    { FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus },
                    { AcceptanceDatetime : filings:acceptance-dateTimes($archive) },
                    { NetworkIdentifier: $component.Role },
                    { Disclosure : $component.Profiles.SEC.Disclosure }
        |}
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($model)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=modelstructure-" || $cid || ".csv");
                local:to-csv($model)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=modelstructure-" || $cid || ".csv");
                local:to-csv($model)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    local:enrich-json($model),
                    session:comment("json", {
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })
                |}
            }
       }
    case $session:ACCESS-DENIED return {
          response:status-code(403);
          session:error("accessing filings of an entity that is not in the DOW30", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          response:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()


:green_heart: Fix test.

:green_heart: Fix test.

機械装置及び運搬具（純額）

その他有価証券評価差額金

退職給付に係る調整累計額

その他の包括利益累計額合計

機械装置及び運搬具（純額）

その他有価証券評価差額金

退職給付に係る調整累計額

その他の包括利益累計額合計


:fire: Remove migration.

:fire: Remove migration.

import module namespace export = "http://apps.28.io/reports-export";

for $report in collection("reports")
where $report.Hypercubes instance of object
return {
  replace value of json $report.Hypercubes
  with [ values($report.Hypercubes) ];

  let $rep := export:cleanup($report)
  return db:edit($report, $rep);
}