allow for multiple report elements

allow for multiple report elements

import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";

import module namespace components = "http://xbrl.io/modules/bizql/components";
import module namespace sec-networks = "http://xbrl.io/modules/bizql/profiles/sec/networks";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace session = "http://apps.28.io/session";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

declare function local:summary-to-xml($components as object*) as element()*
{
    for $c in $components
    return
    <Component id="{$c._id}">
        <NetworkLabel>{$c.Label}</NetworkLabel>
        <NetworkIdentifier>{$c.Role}</NetworkIdentifier>
        <Category>{sec-networks:categories($c)}</Category>
        <SubCategory>{sec-networks:sub-categories($c)}</SubCategory>
        <Table>{sec-networks:tables($c, { IncludeImpliedTable: true}).Name[1]}</Table>
        <Disclosure>{sec-networks:disclosures($c)}</Disclosure>
        <Tables>{sec-networks:num-tables($c)}</Tables>
        <Axis>{sec-networks:num-axes($c)}</Axis>
        <Members>{sec-networks:num-members($c)}</Members>
        <LineItems>{sec-networks:num-line-items($c)}</LineItems>
        <Concepts>{sec-networks:num-concepts($c)}</Concepts>
        <Abstracts>{count(sec-networks:abstracts($c))}</Abstracts>
    </Component>
};

declare function local:to-csv($components as object*) as string*
{
    csv:serialize(
        for $c in $components
        let $disclosure := sec-networks:disclosures($c)
        where $disclosure ne "DefaultComponent" and
                exists(sec-networks:model-structures($c))
        order by $c.Label
        let $a := archives:archives($c.Archive)
        let $e := entities:entities($a.Entity)
        return {|
            ({ CIK : $e._id },
            { EntityRegistrantName : $e.Profiles.SEC.CompanyName },
            { FiscalYear : $a.Profiles.SEC.Fiscal.DocumentFiscalYearFocus },
            { FiscalPeriod : $a.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus },
            { AcceptanceDateTime : $a.Profiles.SEC.AcceptanceDatetime },
            { FormType : $a.Profiles.SEC.FormType },
            { AcessionNumber : $a._id },
            { ComponentId : $c._id },
            { NetworkLabel : $c.Label},
            { NetworkIdentifier : $c.Role},
            { Category : sec-networks:categories($c)},
            { SubCategory : sec-networks:sub-categories($c)},
            { Table : sec-networks:tables($c, { IncludeImpliedTable: true}).Name[1]},
            { Disclosure : sec-networks:disclosures($c)},
            { ReportElements : sec-networks:num-report-elements($c)},
            { Tables : sec-networks:num-tables($c)},
            { Axis : sec-networks:num-axes($c)},
            { Members : sec-networks:num-members($c)},
            { LineItems : sec-networks:num-line-items($c)},
            { Concepts : sec-networks:num-concepts($c) },
            { Abstracts : count(sec-networks:abstracts($c)) })
        |},
    { serialize-null-as : "" })
};

declare function local:component-summary($component as object) as object
{
    {
        ComponentId : $component._id,
        NetworkLabel : $component.Label,
        NetworkIdentifier : $component.Role,
        Category : sec-networks:categories($component),
        SubCategory : sec-networks:sub-categories($component),
        Table : sec-networks:tables($component, { IncludeImpliedTable: true}).Name[1],
        Disclosure : sec-networks:disclosures($component),
        ReportElements : sec-networks:num-report-elements($component),
        Tables : sec-networks:num-tables($component),
        Axis : sec-networks:num-axes($component),
        Members : sec-networks:num-members($component),
        LineItems : sec-networks:num-line-items($component),
        Concepts : sec-networks:num-concepts($component),
        Abstracts : count(sec-networks:abstracts($component))
    }
};

declare function local:components-by-disclosures($disclosures as string, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    for $aid in $aids
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: $aid,
            "Profiles.SEC.Disclosure": { "$in" : [ $disclosures ] }
        })
};

declare function local:components-by-roles($roles as string, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: { "$in" : [ $aids ] },
            "Role": { "$in" : [ $roles ] }
        })
};

declare function local:components-by-reportElements($reportElements as string*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    let $ids := mongo:find($conn, "concepts",
        {|
            (
                { "Name" : { "$in" : [ $reportElements ] } },
                { "Archive" : { "$in" : [ $aids ] } }
            )
        |}).Component
    return components:components($ids)
};

declare function local:components-by-label($search-term as string, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    return mongo:run-cmd-deterministic(
           $conn,
           {
             "text" : "components",
             "filter" : { "Archive" : { "$in" : [ $aids ] } },
             "search" : $search-term,
             "limit" : 100,
             "score" : { "$meta" : "textScore" },
             "sort" : { score: { "$meta" : "textScore" } }
           }).results[].obj
};

declare function local:filings(
    $ciks as string*,
    $tags as string*,
    $tickers as string*,
    $sics as string*,
    $fp as string*,
    $fy as string*) as object*
{
    let $entities := (
        companies:companies($ciks),
        companies:companies-for-tags($tags),
        companies:companies-for-tickers($tickers),
        companies:companies-for-SIC($sics)
    )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                    case "ALL" return  $fiscal:ALL_FISCAL_YEARS
                    default return $fy cast as integer
                )
    for $fp in $fp
    return fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy)
};

let $format      := lower-case((request:param-values("format"), substring-after(request:path(), ".jq."))[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y
                            else  ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids        := archives:aid(request:param-values("aid"))
let $roles       := request:param-values("networkIdentifier")
let $archives    := (
                        local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                        archives:archives($aids)
                    )
let $cid         := request:param-values("cid")
let $reportElements    := distinct-values(request:param-values("reportElement"))
let $disclosures := request:param-values("disclosure")
let $search := request:param-values("label")
let $components  := (if (exists($cid))
                    then components:components($cid)
                    else (),
                    if (exists($reportElements))
                    then local:components-by-reportElements($reportElements, $archives._id)
                    else if (exists($disclosures))
                    then local:components-by-disclosures($disclosures, $archives._id)
                    else if (exists($roles))
                    then local:components-by-roles($roles, $archives._id)
                    else if (exists($search))
                    then local:components-by-label($search, $archives._id)
                    else components:components-for-archives($archives._id))
let $entities    := entities:entities($archives.Entity)
return
    if (session:only-dow30($entities) or session:valid())
    then {
        switch ($format)
        case "xml" return {
            response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
            (session:comment("xml", {
                    NumComponents : count($components),
                    TotalNumComponents: session:num-components(),
                    TotalNumArchives: session:num-archives()
                }),
             <Archives>{
                 for $r in $components
                 let $disclosure := sec-networks:disclosures($r)
                 where $disclosure ne "DefaultComponent" and
                        exists(sec-networks:model-structures($r))
                 order by $r.Label
                 group by $archive := $r.Archive
                 let $archive := archives:archives($archive)
                 let $e := entities:entities($archive.Entity)
                 return
                     <Archive id="{$archive._id}">
                        <EntityRegistrantName>{$e.Profiles.SEC.CompanyName}</EntityRegistrantName>
                        <CIK>{$e._id}</CIK>
                        <FiscalYear>{$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus}</FiscalYear>
                        <FiscalPeriod>{$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus}</FiscalPeriod>
                        <AcceptanceDatetime>{$archive.Profiles.SEC.AcceptanceDatetime}</AcceptanceDatetime>
                        <FormType>{$archive.Profiles.SEC.FormType}</FormType>
                        <Components>{
                            local:summary-to-xml($r)
                        }</Components>
                    </Archive>
            }</Archives>)
        }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=components.csv");
                string-join(local:to-csv($components), "")
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=components.csv");
                string-join(local:to-csv($components), "")
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    session:comment("json", {
                        NumComponents : count($components),
                        TotalNumComponents: session:num-components(),
                        TotalNumArchives: session:num-archives()
                    }),
                    { "Archives" : [
                        for $r in $components
                        let $disclosure := sec-networks:disclosures($r)
                        where $disclosure ne "DefaultComponent" and
                            exists(sec-networks:model-structures($r))
                        order by $r.Label
                        group by $archive := $r.Archive
                        let $archive := archives:archives($archive)
                        let $e := entities:entities($archive.Entity)
                        return
                        {
                          AccessionNumber : $archive._id,
                          EntityRegistrantName : $e.Profiles.SEC.CompanyName,
                          CIK : $e._id,
                          FiscalYear :$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                          FiscalPeriod :$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
                          AcceptanceDatetime : $archive.Profiles.SEC.AcceptanceDatetime,
                          FormType : $archive.Profiles.SEC.FormType,
                          Components : [
                            $r ! local:component-summary($$)
                          ]
                        }
                    ]}
                |}
            }
    } else {
        response:status-code(401);
        let $res := session:error("accessing components of an entity that is not in the DOW30", $format)
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                (session:comment("xml"), $res)
            }
            case "text" case "csv" return {
                response:content-type("text/plain");
                $res
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=components.csv");
                $res
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    session:comment("json"),
                    $res
                |}
            }
    }

import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";

import module namespace components = "http://xbrl.io/modules/bizql/components";
import module namespace sec-networks = "http://xbrl.io/modules/bizql/profiles/sec/networks";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace session = "http://apps.28.io/session";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

declare function local:summary-to-xml($components as object*) as element()*
{
    for $c in $components
    return
    <Component id="{$c._id}">
        <NetworkLabel>{$c.Label}</NetworkLabel>
        <NetworkIdentifier>{$c.Role}</NetworkIdentifier>
        <Category>{sec-networks:categories($c)}</Category>
        <SubCategory>{sec-networks:sub-categories($c)}</SubCategory>
        <Table>{sec-networks:tables($c, { IncludeImpliedTable: true}).Name[1]}</Table>
        <Disclosure>{sec-networks:disclosures($c)}</Disclosure>
        <Tables>{sec-networks:num-tables($c)}</Tables>
        <Axis>{sec-networks:num-axes($c)}</Axis>
        <Members>{sec-networks:num-members($c)}</Members>
        <LineItems>{sec-networks:num-line-items($c)}</LineItems>
        <Concepts>{sec-networks:num-concepts($c)}</Concepts>
        <Abstracts>{count(sec-networks:abstracts($c))}</Abstracts>
    </Component>
};

declare function local:to-csv($components as object*) as string*
{
    csv:serialize(
        for $c in $components
        let $disclosure := sec-networks:disclosures($c)
        where $disclosure ne "DefaultComponent" and
                exists(sec-networks:model-structures($c))
        order by $c.Label
        let $a := archives:archives($c.Archive)
        let $e := entities:entities($a.Entity)
        return {|
            ({ CIK : $e._id },
            { EntityRegistrantName : $e.Profiles.SEC.CompanyName },
            { FiscalYear : $a.Profiles.SEC.Fiscal.DocumentFiscalYearFocus },
            { FiscalPeriod : $a.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus },
            { AcceptanceDateTime : $a.Profiles.SEC.AcceptanceDatetime },
            { FormType : $a.Profiles.SEC.FormType },
            { AcessionNumber : $a._id },
            { ComponentId : $c._id },
            { NetworkLabel : $c.Label},
            { NetworkIdentifier : $c.Role},
            { Category : sec-networks:categories($c)},
            { SubCategory : sec-networks:sub-categories($c)},
            { Table : sec-networks:tables($c, { IncludeImpliedTable: true}).Name[1]},
            { Disclosure : sec-networks:disclosures($c)},
            { ReportElements : sec-networks:num-report-elements($c)},
            { Tables : sec-networks:num-tables($c)},
            { Axis : sec-networks:num-axes($c)},
            { Members : sec-networks:num-members($c)},
            { LineItems : sec-networks:num-line-items($c)},
            { Concepts : sec-networks:num-concepts($c) },
            { Abstracts : count(sec-networks:abstracts($c)) })
        |},
    { serialize-null-as : "" })
};

declare function local:component-summary($component as object) as object
{
    {
        ComponentId : $component._id,
        NetworkLabel : $component.Label,
        NetworkIdentifier : $component.Role,
        Category : sec-networks:categories($component),
        SubCategory : sec-networks:sub-categories($component),
        Table : sec-networks:tables($component, { IncludeImpliedTable: true}).Name[1],
        Disclosure : sec-networks:disclosures($component),
        ReportElements : sec-networks:num-report-elements($component),
        Tables : sec-networks:num-tables($component),
        Axis : sec-networks:num-axes($component),
        Members : sec-networks:num-members($component),
        LineItems : sec-networks:num-line-items($component),
        Concepts : sec-networks:num-concepts($component),
        Abstracts : count(sec-networks:abstracts($component))
    }
};

declare function local:components-by-disclosures($disclosures as string, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    for $aid in $aids
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: $aid,
            "Profiles.SEC.Disclosure": { "$in" : [ $disclosures ] }
        })
};

declare function local:components-by-roles($roles as string, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: { "$in" : [ $aids ] },
            "Role": { "$in" : [ $roles ] }
        })
};

declare function local:components-by-reportElements($reportElements as string, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    let $ids := mongo:find($conn, "concepts",
        {|
            (
                { "Name" : { "$in" : [ $reportElements ] } },
                { "Archive" : { "$in" : [ $aids ] } }
            )
        |}).Component
    return components:components($ids)
};

declare function local:components-by-label($search-term as string, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    return mongo:run-cmd-deterministic(
           $conn,
           {
             "text" : "components",
             "filter" : { "Archive" : { "$in" : [ $aids ] } },
             "search" : $search-term,
             "limit" : 100,
             "score" : { "$meta" : "textScore" },
             "sort" : { score: { "$meta" : "textScore" } }
           }).results[].obj
};

declare function local:filings(
    $ciks as string*,
    $tags as string*,
    $tickers as string*,
    $sics as string*,
    $fp as string*,
    $fy as string*) as object*
{
    let $entities := (
        companies:companies($ciks),
        companies:companies-for-tags($tags),
        companies:companies-for-tickers($tickers),
        companies:companies-for-SIC($sics)
    )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                    case "ALL" return  $fiscal:ALL_FISCAL_YEARS
                    default return $fy cast as integer
                )
    for $fp in $fp
    return fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy)
};

let $format      := lower-case((request:param-values("format"), substring-after(request:path(), ".jq."))[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y
                            else  ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids        := archives:aid(request:param-values("aid"))
let $roles       := request:param-values("networkIdentifier")
let $archives    := (
                        local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                        archives:archives($aids)
                    )
let $cid         := request:param-values("cid")
let $reportElements    := distinct-values(request:param-values("reportElement"))
let $disclosures := request:param-values("disclosure")
let $search := request:param-values("label")
let $components  := (if (exists($cid))
                    then components:components($cid)
                    else (),
                    if (exists($reportElements))
                    then local:components-by-reportElements($reportElements, $archives._id)
                    else if (exists($disclosures))
                    then local:components-by-disclosures($disclosures, $archives._id)
                    else if (exists($roles))
                    then local:components-by-roles($roles, $archives._id)
                    else if (exists($search))
                    then local:components-by-label($search, $archives._id)
                    else components:components-for-archives($archives._id))
let $entities    := entities:entities($archives.Entity)
return
    if (session:only-dow30($entities) or session:valid())
    then {
        switch ($format)
        case "xml" return {
            response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
            (session:comment("xml", {
                    NumComponents : count($components),
                    TotalNumComponents: session:num-components(),
                    TotalNumArchives: session:num-archives()
                }),
             <Archives>{
                 for $r in $components
                 let $disclosure := sec-networks:disclosures($r)
                 where $disclosure ne "DefaultComponent" and
                        exists(sec-networks:model-structures($r))
                 order by $r.Label
                 group by $archive := $r.Archive
                 let $archive := archives:archives($archive)
                 let $e := entities:entities($archive.Entity)
                 return
                     <Archive id="{$archive._id}">
                        <EntityRegistrantName>{$e.Profiles.SEC.CompanyName}</EntityRegistrantName>
                        <CIK>{$e._id}</CIK>
                        <FiscalYear>{$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus}</FiscalYear>
                        <FiscalPeriod>{$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus}</FiscalPeriod>
                        <AcceptanceDatetime>{$archive.Profiles.SEC.AcceptanceDatetime}</AcceptanceDatetime>
                        <FormType>{$archive.Profiles.SEC.FormType}</FormType>
                        <Components>{
                            local:summary-to-xml($r)
                        }</Components>
                    </Archive>
            }</Archives>)
        }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=components.csv");
                string-join(local:to-csv($components), "")
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=components.csv");
                string-join(local:to-csv($components), "")
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    session:comment("json", {
                        NumComponents : count($components),
                        TotalNumComponents: session:num-components(),
                        TotalNumArchives: session:num-archives()
                    }),
                    { "Archives" : [
                        for $r in $components
                        let $disclosure := sec-networks:disclosures($r)
                        where $disclosure ne "DefaultComponent" and
                            exists(sec-networks:model-structures($r))
                        order by $r.Label
                        group by $archive := $r.Archive
                        let $archive := archives:archives($archive)
                        let $e := entities:entities($archive.Entity)
                        return
                        {
                          AccessionNumber : $archive._id,
                          EntityRegistrantName : $e.Profiles.SEC.CompanyName,
                          CIK : $e._id,
                          FiscalYear :$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                          FiscalPeriod :$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
                          AcceptanceDatetime : $archive.Profiles.SEC.AcceptanceDatetime,
                          FormType : $archive.Profiles.SEC.FormType,
                          Components : [
                            $r ! local:component-summary($$)
                          ]
                        }
                    ]}
                |}
            }
    } else {
        response:status-code(401);
        let $res := session:error("accessing components of an entity that is not in the DOW30", $format)
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                (session:comment("xml"), $res)
            }
            case "text" case "csv" return {
                response:content-type("text/plain");
                $res
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=components.csv");
                $res
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    session:comment("json"),
                    $res
                |}
            }
    }


Making components module generic.

Making components module generic.

import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec-filings = "http://28.io/modules/xbrl/profiles/sec/filings";
import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-csv($res as object*) as string*
{
    csv:serialize(
        for $a in $res
        for $c in $a.Components[]
        return {
            AcessionNumber : $a.AccessionNumber,
            EntityRegistrantName : $a.EntityRegistrantName,
            CIK : $a.CIK,
            FiscalYear : $a.FiscalYear,
            FiscalPeriod : $a.FiscalPeriod,
            AcceptanceDateTime : $a.AcceptanceDatetime,
            FormType : $a.FormType,
            NetworkLabel : $c.NetworkLabel,
            NetworkIdentifier : $c.NetworkRole,
            Category : $c.Category,
            SubCategory : $c.SubCategory,
            Table : flatten($c.Table),
            Disclosure : $c.Disclosure,
            ReportElements : $c.ReportElements,
            Tables : $c.Tables,
            Axis : $c.Axis,
            Members : $c.Members,
            LineItems : $c.LineItems,
            Concepts : $c.Concepts,
            Abstracts : $c.Abstracts
        },
    { serialize-null-as : "" })
};

declare function local:to-csv-generic($res as object*) as string*
{
    csv:serialize(
        for $a in $res
        return {
            Archive: $a.Archive,
            Role: $a.Role,
            NumRules: $a.NumRules,
            NumNetworks: $a.NumNetworks,
            NumHypercubes: size($a.Hypercubes)
        },
    { serialize-null-as : "" })
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token              as string? external;
declare  %rest:env                              variable $request-uri        as string  external;
declare  %rest:case-insensitive                 variable $format             as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker             as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear         as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod       as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $networkIdentifier  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $role               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $reportElement      as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $concept            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $label              as string* external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := "generic";

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $reportElement := ($reportElement, $concept)
let $networkIdentifier := distinct-values(($networkIdentifier, $role))

(: Object resolution :)
let $entities :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic)
let $archives as object* := fiscal-core:filings(
    $entities,
    $fiscalPeriod,
    $fiscalYear,
    $aid)
let $entities    := entities:entities($archives.Entity)
let $components  :=
    switch($profile-name)
    case "sec" return sec-networks:components(
        $archives,
        $cid,
        $reportElement,
        $disclosure,
        $networkIdentifier,
        $label)
    default return
        switch(true)
        case (exists($networkIdentifier) and exists($aid))
        return components:components-for-archives-and-roles($aid, $networkIdentifier)
        case exists($aid)
        return components:components-for-archives($aid)
        default
        return components:components()
let $res         :=
    switch($profile-name)
    case "sec" return
        for $r in $components
        let $disclosure := sec-networks:disclosures($r)
        where $disclosure ne "DefaultComponent"
        order by $r.Label
        group by $archive := $r.Archive
        let $archive := $archives[$$._id eq $archive]
        let $e := $entities[$$._id eq $archive.Entity]
        return
            {
               AccessionNumber : $archive._id,
               EntityRegistrantName : $e.Profiles.SEC.CompanyName,
               CIK : $e._id,
               FiscalYear :$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
               FiscalPeriod :$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
               AcceptanceDatetime : sec-filings:acceptance-dateTimes($archive),
               FormType : $archive.Profiles.SEC.FormType,
               Components : [
                   sec-networks:summaries($r)
               ]
           }
    default return
        for $r in $components
        return {
            Archive: $r.Archive,
            Role: $r.Role,
            NumRules: size($r.Rules),
            NumNetworks: size($r.Networks),
            Hypercubes: [ keys($r.Hypercubes) ]
        }
let $result := switch($profile-name) case "sec" return { Archives: [ $res ] } default return { Components : [ $res ] }
let $comment :=
 {
    NumComponents : count($components),
    TotalNumComponents: session:num-components(),
    TotalNumArchives: session:num-archives()
}
let $serializers := {
    to-xml : switch($profile-name)
        case "sec"
        return function($res as object) as node() {
        <Archives>{
                  for $r in flatten($res.Archives)
                  return
                      <Archive id="{$r.AccessionNumber}">
                         <EntityRegistrantName>{$r.EntityRegistrantName}</EntityRegistrantName>
                         <CIK>{$r.CIK}</CIK>
                         <FiscalYear>{$r.FiscalYear}</FiscalYear>
                         <FiscalPeriod>{$r.FiscalPeriod}</FiscalPeriod>
                         <AcceptanceDatetime>{$r.AcceptanceDatetime}</AcceptanceDatetime>
                         <FormType>{$r.FormType}</FormType>
                         <Components>{
                             sec-networks:summaries-to-xml(flatten($r.Components))
                         }</Components>
                     </Archive>
             }</Archives>
        }
        default return function($res as object) as node() {
        <Components>{
                  for $r in $res.Components[]
                  return
                    <Component>
                         <Archive>{$r.Archive}</Archive>
                         <Role>{$r.Role}</Role>
                         <NumRules>{$r.NumRules}</NumRules>
                         <NumNetworks>{$r.NumNetworks}</NumNetworks>
                         <Hypercubes>{$r.Hypercubes[] ! <Hypercube>{$$}</Hypercube>}</Hypercubes>
                    </Component>
             }</Components>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec" return string-join(local:to-csv($res.Archives[]), "")
        default return string-join(local:to-csv-generic($res.Components[]), "")
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "components")
return api:check-and-return-results($token, $results, $format)

import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace entities = "http://28.io/modules/xbrl/entities";

import module namespace sec-filings = "http://28.io/modules/xbrl/profiles/sec/filings";
import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-csv($res as object*) as string*
{
    csv:serialize(
        for $a in $res
        for $c in $a.Components[]
        return {
            AcessionNumber : $a.AccessionNumber,
            EntityRegistrantName : $a.EntityRegistrantName,
            CIK : $a.CIK,
            FiscalYear : $a.FiscalYear,
            FiscalPeriod : $a.FiscalPeriod,
            AcceptanceDateTime : $a.AcceptanceDatetime,
            FormType : $a.FormType,
            NetworkLabel : $c.NetworkLabel,
            NetworkIdentifier : $c.NetworkRole,
            Category : $c.Category,
            SubCategory : $c.SubCategory,
            Table : flatten($c.Table),
            Disclosure : $c.Disclosure,
            ReportElements : $c.ReportElements,
            Tables : $c.Tables,
            Axis : $c.Axis,
            Members : $c.Members,
            LineItems : $c.LineItems,
            Concepts : $c.Concepts,
            Abstracts : $c.Abstracts
        },
    { serialize-null-as : "" })
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token              as string? external;
declare  %rest:env                              variable $request-uri        as string  external;
declare  %rest:case-insensitive                 variable $format             as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker             as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear         as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod       as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $networkIdentifier  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $reportElement      as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $concept            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $label              as string* external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $reportElement := ($reportElement, $concept)

(: Object resolution :)
let $entities :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic)
let $archives as object* := fiscal-core:filings(
    $entities,
    $fiscalPeriod,
    $fiscalYear,
    $aid)
let $entities    := entities:entities($archives.Entity)
let $components  := sec-networks:components(
    $archives,
    $cid,
    $reportElement,
    $disclosure,
    $networkIdentifier,
    $label)
let $res         :=
    for $r in $components
    let $disclosure := sec-networks:disclosures($r)
    where $disclosure ne "DefaultComponent"
    order by $r.Label
    group by $archive := $r.Archive
    let $archive := $archives[$$._id eq $archive]
    let $e := $entities[$$._id eq $archive.Entity]
    return
        {
           AccessionNumber : $archive._id,
           EntityRegistrantName : $e.Profiles.SEC.CompanyName,
           CIK : $e._id,
           FiscalYear :$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
           FiscalPeriod :$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
           AcceptanceDatetime : sec-filings:acceptance-dateTimes($archive),
           FormType : $archive.Profiles.SEC.FormType,
           Components : [
               sec-networks:summaries($r)
           ]
       }

let $result := { Archives: [ $res ] }
let $comment :=
 {
    NumComponents : count($components),
    TotalNumComponents: session:num-components(),
    TotalNumArchives: session:num-archives()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <Archives>{
                  for $r in flatten($res.Archives)
                  return
                      <Archive id="{$r.AccessionNumber}">
                         <EntityRegistrantName>{$r.EntityRegistrantName}</EntityRegistrantName>
                         <CIK>{$r.CIK}</CIK>
                         <FiscalYear>{$r.FiscalYear}</FiscalYear>
                         <FiscalPeriod>{$r.FiscalPeriod}</FiscalPeriod>
                         <AcceptanceDatetime>{$r.AcceptanceDatetime}</AcceptanceDatetime>
                         <FormType>{$r.FormType}</FormType>
                         <Components>{
                             sec-networks:summaries-to-xml(flatten($r.Components))
                         }</Components>
                     </Archive>
             }</Archives>
    },
    to-csv : function($res as object) as string {
        string-join(local:to-csv($res.Archives[]), "")
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "components")
return api:check-and-return-results($token, $results, $format)


:green_heart: Fix typo.

:green_heart: Fix typo.

jsoniq version "1.0";

(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for querying financial reports (filings)
 : submitted to the SEC.

 :
SEC Filings are nothing else than XBRL archives. For XBRL-generic requests on archives,
 : use the generic archives module.

 :
 :
With this module, you can access a number of properties of an SEC filing, such as
 : its document type. You can also retrieve an SEC filing given an SEC company (or its CIK).

 :
You can also access SEC filings statistics (the number of SEC Networks, of SEC Axes, etc).

 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace filings = "http://28.io/modules/xbrl/profiles/sec/filings";

import module namespace datetime = "http://zorba.io/modules/datetime";

import module namespace csv = "http://zorba.io/modules/json-csv";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 :
Retrieves all filings.

 :
 : @return all filings.
 :)
declare function filings:filings() as object*
{
  archives:archives()
};

(:~
 :
Retrieves the filings with the given IDs.

 :
 : @param $id the ids of the filings (accession numbers) or the filings themselves.
 :
 : @return the fiinngs with the given IDs
 :)
declare function filings:filings($filings as item*) as object*
{
  archives:archives($filings)
};

(:~
 :
Return all filings submitted by the supplied companies.

 :
 : @param $companies-or-ciks arbitrary number of company objects or CIKs.
 :
 : @return all filings submitted by these companies.
 :)
declare function filings:filings-for-companies($companies-or-ciks as item*) as object*
{
  let $companies := companies:companies($companies-or-ciks)
  return archives:archives-for-entities($companies)
};

(:~
 :
Retrieves the document type of the filings.

 :
 : @param $filings-or-ids a sequence of filings or their ids.
 :
 : @return the document types (10-K, 10-Q).
 :)
declare function filings:document-types(
    $filings-or-ids as item*) as string?
{
  archives:archives($filings-or-ids).Profiles.SEC.FormType
};

(:~
 :
 :
Retrieves the acceptance date of filings.

 :
 : @param $filings-or-ids a sequence of filings or their IDs.
 : @return the acceptance dateTimes.
 :)
declare function filings:acceptance-dateTimes($filings-or-ids as item*) as dateTime*
{
  archives:archives($filings-or-ids) !
    datetime:parse-dateTime($$.Profiles.SEC.AcceptanceDatetime, "%Y%m%d%H%M%S")
};

(:~
 :
 :
Retrieves the submission date of filings.

 :
 : @param $filings-or-ids a sequence of filings or their IDs.
 : @return the submission dates.
 :)
declare function filings:filing-dates($filings-or-ids as item*) as date*
{
  archives:archives($filings-or-ids) !
  date(string-join(tokenize($$.Profiles.SEC.FilingDate, ("/")), "-"))
};

(:~
 :
 :
Retrieves the generators of filings.

 :
 : @param $filings-or-ids a sequence of filings or their IDs.
 : @return the generators used.
 :)
declare function filings:generators($filings-or-ids as item*) as string*
{
  archives:archives($filings-or-ids).Profiles.SEC.Generator
};

(:~
 : Return the number of extension facts in each of the given filings.
 :
 : @param $filings-or-ids list of filings or IDs
 :
 : @return the said number of facts
 :)
declare function filings:num-extension-facts($filings-or-ids as item*) as integer*
{
  archives:archives($filings-or-ids) ! $$.Statistics.Profiles.SEC.NumExtensionFacts
};

(:~
 : Return the number of extension concepts in each of the given filings.
 :
 : @param $filings-or-ids list of filings or IDs
 :
 : @return the said number of concepts
 :)
declare function filings:num-extension-concepts($filings-or-ids as item*) as integer*
{
  archives:archives($filings-or-ids) ! $$.Statistics.Profiles.SEC.NumExtensionConcepts
};

(:~
 : Return the number of extension abstracts in each of the given filings.
 :
 : @param $filings-or-ids list of filings or IDs
 :
 : @return the said number of abstracts
 :)
declare function filings:num-extension-abstracts($filings-or-ids as item*) as integer*
{
  archives:archives($filings-or-ids) ! $$.Statistics.Profiles.SEC.NumExtensionAbstracts
};

(:~
 :
Returns summaries of the given filings or IDs.

 :
 :
The function eliminates duplicates and returns only one
 : summary object per filing.

 :
 : @param $filings-or-ids list of filings or IDs
 :
 : @return a sequence of objects, each one representing one summary
 :)
declare function filings:summaries($filings-or-ids as item*) as object*
{
  for $a in archives:archives($filings-or-ids)
  group by $a._id
  let $a := $a[1]
  return
    {
        CIK : $a.Entity,
        EntityRegistrantName : companies:companies($a.Entity).Profiles.SEC.CompanyName,
        FormType : filings:document-types($a),
        FiscalYear : fiscal:fiscal-year-for-filing($a),
        FiscalPeriod : fiscal:fiscal-period($a),
        Accepted : filings:acceptance-dateTimes($a),
        Generator : filings:generators($a),
        AccessionNumber: $a._id,
        SECFilingPage : $a.Profiles.SEC.SECFilingPage,
        XBRLInstanceURL : $a.InstanceURL,
        ExtensionFacts:  filings:num-extension-facts($a),
        ExtensionConcepts : filings:num-extension-concepts($a),
        ExtensionAbstracts : filings:num-extension-abstracts($a),
        Statistics: $a.Statistics
    }
};

(:~
 :
Returns an XML representation of the given filing summaries.

 :
 : @param $summaries summaries of the filings to convert to XML.
 :
 : @return a sequence of elements, each one representing one filing summary as XML
 :)
declare function filings:summaries-to-xml($summaries as object*) as element()*
{
  for $s in $summaries
  return
    <Filing>
        <CIK>{$s.CIK}</CIK>
        <EntityRegistrantName>{$s.EntityRegistrantName}</EntityRegistrantName>
        <FormType>{$s.FormType}</FormType>
        <FiscalYear>{$s.FiscalYear}</FiscalYear>
        <FiscalPeriod>{$s.FiscalPeriod}</FiscalPeriod>
        <Accepted>{$s.Accepted}</Accepted>
        <Generator>{$s.Generator}</Generator>
        <AccessionNumber>{$s.AccessionNumber}</AccessionNumber>
        <SECFilingPage>{$s.SECFilingPage}</SECFilingPage>
        <XBRLInstanceURL>{$s.XBRLInstanceURL}</XBRLInstanceURL>
        <Networks>{$s.Networks}</Networks>
        <ReportElements>{$s.ReportElements}</ReportElements>
        <Tables>{$s.Tables}</Tables>
        <Axis>{$s.Axis}</Axis>
        <Members>{$s.Members}</Members>
        <LineItems>{$s.LineItems}</LineItems>
        <Concepts>{$s.Concepts}</Concepts>
        <Abstracts>{$s.Abstracts}</Abstracts>
        <Footnotes>{$s.Footnotes}</Footnotes>
        <Facts>{$s.Facts}</Facts>
        <ExtensionFacts>{$s.ExtensionFacts}</ExtensionFacts>
        <ExtensionConcepts>{$s.ExtensionConcepts}</ExtensionConcepts>
        <ExtensionAbstracts>{$s.ExtensionAbstracts}</ExtensionAbstracts>
    </Filing>
};

(:~
 :
Returns a CSV representation of the given filing summaries.

 :
 :
Commas are used as column separators. Abscent values are serialized
 : as empty strings.

 :
 : @param $summaries summaries of the filings to convert to CSV.
 :
 : @return a sequence of strings, each one representing one filings summary as CSV
 :)
declare function filings:summaries-to-csv($summaries as object*) as string*
{
  csv:serialize($summaries, { serialize-null-as : "" })
};

jsoniq version "1.0";

(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for querying financial reports (filings)
 : submitted to the SEC.

 :
SEC Filings are nothing else than XBRL archives. For XBRL-generic requests on archives,
 : use the generic archives module.

 :
 :
With this module, you can access a number of properties of an SEC filing, such as
 : its document type. You can also retrieve an SEC filing given an SEC company (or its CIK).

 :
You can also access SEC filings statistics (the number of SEC Networks, of SEC Axes, etc).

 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace filings = "http://28.io/modules/xbrl/profiles/sec/filings";

import module namespace datetime = "http://zorba.io/modules/datetime";

import module namespace csv = "http://zorba.io/modules/json-csv";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 :
Retrieves all filings.

 :
 : @return all filings.
 :)
declare function filings:filings() as object*
{
  archives:archives()
};

(:~
 :
Retrieves the filings with the given IDs.

 :
 : @param $id the ids of the filings (accession numbers) or the filings themselves.
 :
 : @return the fiinngs with the given IDs
 :)
declare function filings:filings($filings as item*) as object*
{
  archives:archives($filings)
};

(:~
 :
Return all filings submitted by the supplied companies.

 :
 : @param $companies-or-ciks arbitrary number of company objects or CIKs.
 :
 : @return all filings submitted by these companies.
 :)
declare function filings:filings-for-companies($companies-or-ciks as item*) as object*
{
  let $companies := companies:companies($companies-or-ciks)
  return archives:archives-for-entities($companies)
};

(:~
 :
Retrieves the document type of the filings.

 :
 : @param $filings-or-ids a sequence of filings or their ids.
 :
 : @return the document types (10-K, 10-Q).
 :)
declare function filings:document-types(
    $filings-or-ids as item*) as string?
{
  archives:archives($filings-or-ids).Profiles.SEC.FormType
};

(:~
 :
 :
Retrieves the acceptance date of filings.

 :
 : @param $filings-or-ids a sequence of filings or their IDs.
 : @return the acceptance dateTimes.
 :)
declare function filings:acceptance-dateTimes($filings-or-ids as item*) as dateTime*
{
  archives:archives($filings-or-ids) !
    datetime:parse-dateTime($$.Profiles.SEC.AcceptanceDatetime, "%Y%m%d%H%M%S")
};

(:~
 :
 :
Retrieves the submission date of filings.

 :
 : @param $filings-or-ids a sequence of filings or their IDs.
 : @return the submission dates.
 :)
declare function filings:filing-dates($filings-or-ids as item*) as date*
{
  archives:archives($filings-or-ids) !
  date(string-join(tokenize($$.Profiles.SEC.FilingDate, ("/")), "-"))
};

(:~
 :
 :
Retrieves the generators of filings.

 :
 : @param $filings-or-ids a sequence of filings or their IDs.
 : @return the generators used.
 :)
declare function filings:generators($filings-or-ids as item*) as string*
{
  archives:archives($filings-or-ids).Profiles.SEC.Generator
};

(:~
 : Return the number of extension facts in each of the given filings.
 :
 : @param $filings-or-ids list of filings or IDs
 :
 : @return the said number of facts
 :)
declare function filings:num-extension-facts($filings-or-ids as item*) as integer*
{
  archives:archives($filings-or-ids) ! $$.Statistics.Profiles.SEC.NumExtensionFacts
};

(:~
 : Return the number of extension concepts in each of the given filings.
 :
 : @param $filings-or-ids list of filings or IDs
 :
 : @return the said number of concepts
 :)
declare function filings:num-extension-concepts($filings-or-ids as item*) as integer*
{
  archives:archives($filings-or-ids) ! $$.Statistics.Profiles.SEC.NumExtensionConcepts
};

(:~
 : Return the number of extension abstracts in each of the given filings.
 :
 : @param $filings-or-ids list of filings or IDs
 :
 : @return the said number of abstracts
 :)
declare function filings:num-extension-abstracts($filings-or-ids as item*) as integer*
{
  archives:archives($filings-or-ids) ! $$.Statistics.Profiles.SEC.NumExtensionAbstracts
};

(:~
 :
Returns summaries of the given filings or IDs.

 :
 :
The function eliminates duplicates and returns only one
 : summary object per filing.

 :
 : @param $filings-or-ids list of filings or IDs
 :
 : @return a sequence of objects, each one representing one summary
 :)
declare function filings:summaries($filings-or-ids as item*) as object*
{
  for $a in archives:archives($filings-or-ids)
  group by $a._id
  let $a := $a[1]
  return
    {
        CIK : $a.Entity,
        EntityRegistrantName : companies:companies($a.Entity).Profiles.SEC.CompanyName,
        FormType : filings:document-types($a),
        FiscalYear : fiscal:fiscal-year-for-filings($a),
        FiscalPeriod : fiscal:fiscal-period($a),
        Accepted : filings:acceptance-dateTimes($a),
        Generator : filings:generators($a),
        AccessionNumber: $a._id,
        SECFilingPage : $a.Profiles.SEC.SECFilingPage,
        XBRLInstanceURL : $a.InstanceURL,
        ExtensionFacts:  filings:num-extension-facts($a),
        ExtensionConcepts : filings:num-extension-concepts($a),
        ExtensionAbstracts : filings:num-extension-abstracts($a),
        Statistics: $a.Statistics
    }
};

(:~
 :
Returns an XML representation of the given filing summaries.

 :
 : @param $summaries summaries of the filings to convert to XML.
 :
 : @return a sequence of elements, each one representing one filing summary as XML
 :)
declare function filings:summaries-to-xml($summaries as object*) as element()*
{
  for $s in $summaries
  return
    <Filing>
        <CIK>{$s.CIK}</CIK>
        <EntityRegistrantName>{$s.EntityRegistrantName}</EntityRegistrantName>
        <FormType>{$s.FormType}</FormType>
        <FiscalYear>{$s.FiscalYear}</FiscalYear>
        <FiscalPeriod>{$s.FiscalPeriod}</FiscalPeriod>
        <Accepted>{$s.Accepted}</Accepted>
        <Generator>{$s.Generator}</Generator>
        <AccessionNumber>{$s.AccessionNumber}</AccessionNumber>
        <SECFilingPage>{$s.SECFilingPage}</SECFilingPage>
        <XBRLInstanceURL>{$s.XBRLInstanceURL}</XBRLInstanceURL>
        <Networks>{$s.Networks}</Networks>
        <ReportElements>{$s.ReportElements}</ReportElements>
        <Tables>{$s.Tables}</Tables>
        <Axis>{$s.Axis}</Axis>
        <Members>{$s.Members}</Members>
        <LineItems>{$s.LineItems}</LineItems>
        <Concepts>{$s.Concepts}</Concepts>
        <Abstracts>{$s.Abstracts}</Abstracts>
        <Footnotes>{$s.Footnotes}</Footnotes>
        <Facts>{$s.Facts}</Facts>
        <ExtensionFacts>{$s.ExtensionFacts}</ExtensionFacts>
        <ExtensionConcepts>{$s.ExtensionConcepts}</ExtensionConcepts>
        <ExtensionAbstracts>{$s.ExtensionAbstracts}</ExtensionAbstracts>
    </Filing>
};

(:~
 :
Returns a CSV representation of the given filing summaries.

 :
 :
Commas are used as column separators. Abscent values are serialized
 : as empty strings.

 :
 : @param $summaries summaries of the filings to convert to CSV.
 :
 : @return a sequence of strings, each one representing one filings summary as CSV
 :)
declare function filings:summaries-to-csv($summaries as object*) as string*
{
  csv:serialize($summaries, { serialize-null-as : "" })
};


jq: Add filter to remove Notes metadata (#1870)

Cozy Notes content and schema are stored in the record's metadata and
are displayed when parsing the logs.
This data can be quite long and is irrelevant to what the Desktop
client does so we can remove them when using the `clean` filter to
make for a more compact output.

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove notes content
def delNoteMetadata(obj):
  obj |=
    (.
    | del(.metadata.content)
    | del(.metadata.schema)
    | if .moveFrom then delNoteMetadata(.moveFrom) else . end
    )
;

def cleanNote:
  .
  | if .doc then delNoteMetadata(.doc) else . end
  | if .was then delNoteMetadata(.was) else . end
  | if .remoteDoc then delNoteMetadata(.remoteDoc) else . end
  | if .change then .
    | .change |= (. | delNoteMetadata(.doc))
    | .change |= (. | delNoteMetadata(.was))
    else . end
;

  #del(.doc.metadata.content) |
  #del(.doc.moveFrom.metadata.content) |
  #del(.was.metadata.content) |
  #del(.was.moveFrom.metadata.content) |
  #del(.change.doc.metadata.content) |
  #del(.change.doc.moveFrom.metadata.content) |
  #del(.change.was.metadata.content) |
  #del(.change.was.moveFrom.metadata.content);

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v) |
  del(.level) |
  del(.local) |
  del(.remote) |
  cleanNote |
  {time,component,msg,path}+(del(.time)|del(.component)|del(.msg)|del(.path));

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: select(is_error) | clean;
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: select(is_warn) | clean;
def warn_strict: select(is_warn_strict) | clean;
def info_level: 30;
def is_info: .level >= info_level;
def info: select(is_info) | clean;
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: select(is_debug) | clean;

# Components:
def component(pattern): select(.component | test(pattern));
def App: component("^App$");
def Chokidar: component("^Chokidar|chokidar$");
def Fs: component("^Fs|FS$");
def LocalAnalysis: component("^LocalAnalysis|local/analysis$");
def LocalChange: component("^LocalChange|local/change$");
def LocalWatcher: component("^LocalWatcher$");
def LocalWriter: component("^LocalWriter$");
def Merge: component("^Merge$");
def Metadata: component("^Metadata$");
def Perfs: component("^Perfs|PerfReports$");
def Pouch: component("^Pouch$");
def Prep: component("^Prep$");
def RemoteCozy: component("^RemoteCozy$");
def RemoteUserActionRequired: component("^RemoteUserActionRequired|remote/user_action_required$");
def RemoteWarningPoller: component("^RemoteWarningPoller$");
def RemoteWatcher: component("^RemoteWatcher$");
def RemoteWriter: component("^RemoteWriter$");
def Sync: component("^Sync$");

def Local: select(.component | test("^local|chokidar"; "i"));
def Remote: select(.component | test("^remote"; "i"));

# Exclude "up-to-date" messages
#
#     yarn jq nouptodate path/to/logs*
#
def nouptodate: select(.msg | test("up[- ]to[- ]date") | not);

# Exclude initial scan messages
#
#     yarn jq noscan path/to/logs*
#
def noscan: select(.action != "scan");

# Exclude Proxy stuff:
#
#     yarn jq noproxy path/to/logs*
#
def noproxy: select(.component | test("GUI:proxy") | not);

# Find conflicts:
#
#     yarn jq -c conflict path/to/logs*
#
def is_conflict: .msg == "resolveConflictAsync";
def conflict: clean | select(is_conflict);

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
# FIXME: Should not output the same line twice when both path & oldpath match.
def path(pattern):
  clean | select(
    (
      try (.path // (.doc | .path) // (.change | .doc | .path) // (.idConflict | .newDoc | .path) // (.event | .path)),
      try (.oldpath // (.was | .path) // (.idConflict | .existingDoc | .path) // (.event | .oldPath)),
      ""
    )
      | strings
      | test(pattern)
  );

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#    yarn jq 'before("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#    yarn jq 'after("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#
def time(pattern):
  select(.time | test(pattern));
def before(time): select(.time <= time);
def after(time): select(.time >= time);

# Drop time properties
def notime: del(.time);

# Filter inode number(s):
#
#    yarn jq 'ino(7852458)' path/to/logs*
#    yarn jq 'ino(7852458; 464672)' path/to/logs*
#
def ino(n): select((.ino // (.doc|.ino) // (.change|.doc|.ino)) as $ino | [n] | flatten | map(. == $ino) | any);

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path,oldpath,event}
    | if (.event | not) then del(.event) else . end
    | if (.event | type == "object") then del(.event) + (.event|{path, oldpath: .oldPath, action}) else . end
    | if .oldpath then . else del(.oldpath) end
    ;

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v) |
  del(.level) |
  del(.local) |
  del(.remote) |
  {time,component,msg,path}+(del(.time)|del(.component)|del(.msg)|del(.path));

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: select(is_error) | clean;
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: select(is_warn) | clean;
def warn_strict: select(is_warn_strict) | clean;
def info_level: 30;
def is_info: .level >= info_level;
def info: select(is_info) | clean;
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: select(is_debug) | clean;

# Components:
def component(pattern): select(.component | test(pattern));
def App: component("^App$");
def Chokidar: component("^Chokidar|chokidar$");
def Fs: component("^Fs|FS$");
def LocalAnalysis: component("^LocalAnalysis|local/analysis$");
def LocalChange: component("^LocalChange|local/change$");
def LocalWatcher: component("^LocalWatcher$");
def LocalWriter: component("^LocalWriter$");
def Merge: component("^Merge$");
def Metadata: component("^Metadata$");
def Perfs: component("^Perfs|PerfReports$");
def Pouch: component("^Pouch$");
def Prep: component("^Prep$");
def RemoteCozy: component("^RemoteCozy$");
def RemoteUserActionRequired: component("^RemoteUserActionRequired|remote/user_action_required$");
def RemoteWarningPoller: component("^RemoteWarningPoller$");
def RemoteWatcher: component("^RemoteWatcher$");
def RemoteWriter: component("^RemoteWriter$");
def Sync: component("^Sync$");

def Local: select(.component | test("^local|chokidar"; "i"));
def Remote: select(.component | test("^remote"; "i"));

# Exclude "up-to-date" messages
#
#     yarn jq nouptodate path/to/logs*
#
def nouptodate: select(.msg | test("up[- ]to[- ]date") | not);

# Exclude initial scan messages
#
#     yarn jq noscan path/to/logs*
#
def noscan: select(.action != "scan");

# Exclude Proxy stuff:
#
#     yarn jq noproxy path/to/logs*
#
def noproxy: select(.component | test("GUI:proxy") | not);

# Find conflicts:
#
#     yarn jq -c conflict path/to/logs*
#
def is_conflict: .msg == "resolveConflictAsync";
def conflict: clean | select(is_conflict);

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
# FIXME: Should not output the same line twice when both path & oldpath match.
def path(pattern):
  clean | select(
    (
      try (.path // (.doc | .path) // (.change | .doc | .path) // (.idConflict | .newDoc | .path) // (.event | .path)),
      try (.oldpath // (.was | .path) // (.idConflict | .existingDoc | .path) // (.event | .oldPath)),
      ""
    )
      | strings
      | test(pattern)
  );

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#    yarn jq 'before("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#    yarn jq 'after("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#
def time(pattern):
  select(.time | test(pattern));
def before(time): select(.time <= time);
def after(time): select(.time >= time);

# Drop time properties
def notime: del(.time);

# Filter inode number(s):
#
#    yarn jq 'ino(7852458)' path/to/logs*
#    yarn jq 'ino(7852458; 464672)' path/to/logs*
#
def ino(n): select((.ino // (.doc|.ino) // (.change|.doc|.ino)) as $ino | [n] | flatten | map(. == $ino) | any);

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path,oldpath,event}
    | if (.event | not) then del(.event) else . end
    | if (.event | type == "object") then del(.event) + (.event|{path, oldpath: .oldPath, action}) else . end
    | if .oldpath then . else del(.oldpath) end
    ;

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );