Update facttable-for-report.jq

Update facttable-for-report.jq

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace schema = "http://zorba.io/modules/schema";
import module namespace test = "http://apps.28.io/test";
import module namespace http-client = "http://zorba.io/modules/http-client";

declare function local:expected-OtherOperatingIncomeExpenses-ATnT-2013() as object {
    {
        "Aspects": {
            "xbrl28:Archive": "0000732717-14-000010",
            "xbrl:Concept": "fac:OtherOperatingIncomeExpenses",
            "xbrl:Entity": "http://www.sec.gov/CIK 0000732717",
            "xbrl:Period": "2013-01-01/2013-12-31",
            "sec:FiscalPeriod": "FY",
            "sec:FiscalPeriodType": "YTD",
            "sec:FiscalYear": 2013,
            "sec:Accepted": "20140221162554",
            "dei:LegalEntityAxis": "sec:DefaultLegalEntity"
        },
        "KeyAspects": [
            "xbrl:Concept",
            "xbrl:Entity",
            "xbrl:Period",
            "xbrl:Unit",
            "sec:Accepted"
        ],
        "Type": "NumericValue",
        "Value": -145082000000,
        "Decimals": -6,
        "Concept": {
            "Balance": "CREDIT",
            "PeriodType": "duration",
            "DataType": "xbrli:monetaryItemType"
        },
        "AuditTrails": [
            {
                "Id": "bf2d1587-491c-4492-b587-ddd00d762f2d",
                "Type": "xbrl28:formula",
                "Label": "Other Operating Income (Expenses)",
                "Message": "fac:OtherOperatingIncomeExpenses[-145,082,000,000 USD] = fac:OperatingIncomeLoss[30,479,000,000 USD] - fac:GrossProfit[77,288,000,000 USD] - fac:OperatingExpenses[98,273,000,000 USD]",
                "Data": {
                    "OutputConcept": "fac:OtherOperatingIncomeExpenses"
                }
            }
        ],
        "xbrl28:Type": "xbrl28:formula",
        "EntityRegistrantName": "AT&T INC.",
        "Unit": "iso4217:USD"
    }
};

declare function local:compare-audit-trails($expected as array?, $actual as array?) as object* {
    for $exp as object at $pos in $expected[]
    let $act as object? := $actual[][$pos]
    return
        if (exists($exp.Data.ValidatedFacts))
        then
            let $expM := copy $e := $exp modify ( delete json $e.Data.ValidatedFacts ) return $e
            let $actM := copy $a := $act modify ( delete json $a.Data.ValidatedFacts ) return $a
            return  (
                if(deep-equal($expM, $actM))
                then ()
                else
                    {
                        type: "unexpected-audittrails-value",
                        expected: $expM,
                        actual: $actM
                    },
                if(count($exp.Data.ValidatedFacts[]) eq count($act.Data.ValidatedFacts[]))
                then ()
                else
                    {
                        type: "number-of-validated-facts-differs",
                        expected: $exp,
                        actual: $act
                    }
            )
        else
            if(deep-equal($exp, $act))
            then ()
            else
                {
                    type: "unexpected-audittrails-value",
                    expected: $exp,
                    actual: $act
                }
};

declare function local:diff-facts($fact-expected as object, $fact-actual as object?) as object* {
    if(empty($fact-actual))
    then
        {
            type: "missing-fact-in-result"
        }
    else
        (
            for $key in distinct-values((keys($fact-expected), keys($fact-actual)))
            where not($key = ("Aspects", "KeyAspects", "Profiles", "AuditTrails"))
            let $exp-value := $fact-expected.$key
            let $act-value := $fact-actual.$key
            return
                if(deep-equal($exp-value, $act-value) or
                   (schema:schema-type($exp-value) eq schema:schema-type($act-value) and schema:schema-type($exp-value) eq xs:QName("xs:decimal") and fn:format-number(xs:decimal($exp-value),"#,##0.0000000000000") eq fn:format-number(xs:decimal($act-value),"#,##0.0000000000000"))
                )
                then ()
                else
                    {
                        type: "unexpected-fact-field-value",
                        field: $key,
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare aspects :)
            for $key in distinct-values((keys($fact-expected.Aspects), keys($fact-actual.Aspects)))
            let $exp-value := $fact-expected.Aspects.$key
            let $act-value := $fact-actual.Aspects.$key
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-aspect-value",
                        aspect: $key,
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare keyaspects :)
            let $exp-value := $fact-expected.KeyAspects
            let $act-value := $fact-actual.KeyAspects
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-keyaspects-value",
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare Profiles :)
            let $exp-value := $fact-expected.Profiles
            let $act-value := $fact-actual.Profiles
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-profiles-value",
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare Audittrails :)
            let $exp-value := $fact-expected.AuditTrails
            let $act-value := $fact-actual.AuditTrails
            return
                local:compare-audit-trails($exp-value, $act-value)
        )
};

declare function local:compare-fact-tables($fact-table-expected as object, $fact-table-actual as object) as object*{
    let $actual-facts as object+ := $fact-table-actual.FactTable[]
    let $expected-facts as object+ := $fact-table-expected.FactTable[]
    let $errors :=
        (
            for $exp in $expected-facts
            let $act := $actual-facts[$$.Aspects."xbrl:Concept" eq $exp.Aspects."xbrl:Concept"
                                      and $$.Aspects."xbrl:Entity" eq $exp.Aspects."xbrl:Entity"
                                      and $$.Aspects."xbrl:Period" eq $exp.Aspects."xbrl:Period"
                                      and $$.Aspects."xbrl28:Archive" eq $exp.Aspects."xbrl28:Archive"]
            let $errors as object* := local:diff-facts($exp, $act)
            return
                if(empty($errors))
                then ()
                else {
                    expectedFact: $exp,
                    actualFact: $act,
                    errors: [$errors]
                },

            for $act in $actual-facts
            let $exp := $expected-facts[$$.Aspects."xbrl:Concept" eq $act.Aspects."xbrl:Concept"
                                      and $$.Aspects."xbrl:Entity" eq $act.Aspects."xbrl:Entity"
                                      and $$.Aspects."xbrl:Period" eq $act.Aspects."xbrl:Period"
                                      and $$.Aspects."xbrl28:Archive" eq $act.Aspects."xbrl28:Archive"]
            where empty($exp)
            return
                {
                    unexpectedFact: $act
                }
        )
    return
        if(empty($errors))
        then ()
        else (
                {
                    expectedNumberOfFacts: count($expected-facts),
                    actualNumberOfFacts: count($actual-facts)
                },
                $errors
            )
};

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-facttable-fact($concept as string, $expected as object, $params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $facts as object* := $request[2].FactTable[]
    let $actual := $facts[$$.Aspects."xbrl:Concept" eq $concept]
    let $diff :=
      for $f in $actual return local:diff-facts($expected, $f)
    return if (empty($diff)) then true else { url: test:url($endpoint, $params), factDiffErrors: [ $diff ], expectedFact: $expected, actualFact: $actual }
};

declare %an:nondeterministic function local:test-report-does-not-exist($params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $status as integer := $request[1]
    return if ($status eq 404) then true else { url: test:url($endpoint, $params), unexpectedResponse: $request[2] }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-values() as item*
{
    let $endpoint := "facttable-for-report"
    let $params := {ticker:"ko",fiscalYear:"2013",fiscalPeriod:"FY",fiscalPeriodType: ["instant", "YTD", "QTD"],report:"FundamentalAccountingConcepts"}
    let $request := test:invoke($endpoint, $params)
    let $actual as object := $request[2]
    let $expected := parse-json(
        http-client:get("http://" || request:server-name() || ":" || request:server-port() ||
                        "/test/facttable-for-report-expected1.jq").body.content)
    let $diff := local:compare-fact-tables($expected, $actual)
    return if (empty($diff)) then true else { url: test:url($endpoint, $params), factTableDiff: [ ({ params : $params },$diff) ], expectedFactTable: $expected, actualFactTable: $actual }
};

let $dow30 := test:is-dow30()
local:check({
    cocacola: local:test-facttable(95, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2013",
        fiscalPeriod:"Q1"}),
    aid: local:test-facttable(if($dow30) then 0 else 95, {
        aid:"0001193125-14-157120",
        report:"FundamentalAccountingConcepts"}),
    aid2: local:test-facttable(19, {
        aid:"0000732717-14-000022",
        report:"FundamentalAccountingConcepts"}),
    aid2-qtd: local:test-facttable(96, {
        aid:"0000732717-14-000022",
        report:"FundamentalAccountingConcepts",
        fiscalPeriodType: [ "instant", "QTD" ]
    }),
    tickerrole: local:test-facttable(95, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1"}),
    tickerfyfprole: local:test-facttable(246, {
        report:"FundamentalAccountingConcepts",
        ticker:["ko","wmt"],
        fiscalYear:"2013",
        fiscalPeriod:"FY",
        fiscalPeriodType:["instant", "QTD", "YTD"]
    }),
    allvalues : local:test-values(),
    otheroperatingincometest: local:test-facttable-fact("fac:OtherOperatingIncomeExpenses",
                                                        local:expected-OtherOperatingIncomeExpenses-ATnT-2013(),
                                                        {
        report:"FundamentalAccountingConcepts",
        ticker:"t",
        fiscalYear:"2013",
        fiscalPeriod:["FY"]}),
    reportDoesntExist: local:test-report-does-not-exist({
        report:"report-not-found",
        ticker:"MSFT",
        fiscalYear:"ALL",
        fiscalPeriod:"FY"})
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace schema = "http://zorba.io/modules/schema";
import module namespace test = "http://apps.28.io/test";
import module namespace http-client = "http://zorba.io/modules/http-client";

declare function local:expected-OtherOperatingIncomeExpenses-ATnT-2013() as object {
    {
        "Aspects": {
            "xbrl28:Archive": "0000732717-14-000010",
            "xbrl:Concept": "fac:OtherOperatingIncomeExpenses",
            "xbrl:Entity": "http://www.sec.gov/CIK 0000732717",
            "xbrl:Period": "2013-01-01/2013-12-31",
            "sec:FiscalPeriod": "FY",
            "sec:FiscalPeriodType": "YTD",
            "sec:FiscalYear": 2013,
            "sec:Accepted": "20140221162554",
            "dei:LegalEntityAxis": "sec:DefaultLegalEntity"
        },
        "KeyAspects": [
            "xbrl:Concept",
            "xbrl:Entity",
            "xbrl:Period",
            "xbrl:Unit",
            "sec:Accepted"
        ],
        "Type": "NumericValue",
        "Value": -145082000000,
        "Decimals": -6,
        "Concept": {
            "Balance": "CREDIT",
            "PeriodType": "duration",
            "DataType": "xbrli:monetaryItemType"
        },
        "AuditTrails": [
            {
                "Id": "bf2d1587-491c-4492-b587-ddd00d762f2d",
                "Type": "xbrl28:formula",
                "Label": "Other Operating Income (Expenses)",
                "Message": "fac:OtherOperatingIncomeExpenses[-145,082,000,000 USD] = fac:OperatingIncomeLoss[30,479,000,000 USD] - fac:GrossProfit[77,288,000,000 USD] - fac:OperatingExpenses[98,273,000,000 USD]",
                "Data": {
                    "OutputConcept": "fac:OtherOperatingIncomeExpenses"
                }
            }
        ],
        "xbrl28:Type": "xbrl28:formula",
        "EntityRegistrantName": "AT&T INC.",
        "Unit": "iso4217:USD"
    }
};

declare function local:compare-audit-trails($expected as array?, $actual as array?) as object* {
    for $exp as object at $pos in $expected[]
    let $act as object? := $actual[][$pos]
    return
        if (exists($exp.Data.ValidatedFacts))
        then
            let $expM := copy $e := $exp modify ( delete json $e.Data.ValidatedFacts ) return $e
            let $actM := copy $a := $act modify ( delete json $a.Data.ValidatedFacts ) return $a
            return  (
                if(deep-equal($expM, $actM))
                then ()
                else
                    {
                        type: "unexpected-audittrails-value",
                        expected: $expM,
                        actual: $actM
                    },
                if(count($exp.Data.ValidatedFacts[]) eq count($act.Data.ValidatedFacts[]))
                then ()
                else
                    {
                        type: "number-of-validated-facts-differs",
                        expected: $exp,
                        actual: $act
                    }
            )
        else
            if(deep-equal($exp, $act))
            then ()
            else
                {
                    type: "unexpected-audittrails-value",
                    expected: $exp,
                    actual: $act
                }
};

declare function local:diff-facts($fact-expected as object, $fact-actual as object?) as object* {
    if(empty($fact-actual))
    then
        {
            type: "missing-fact-in-result"
        }
    else
        (
            for $key in distinct-values((keys($fact-expected), keys($fact-actual)))
            where not($key = ("Aspects", "KeyAspects", "Profiles", "AuditTrails"))
            let $exp-value := $fact-expected.$key
            let $act-value := $fact-actual.$key
            return
                if(deep-equal($exp-value, $act-value) or
                   (schema:schema-type($exp-value) eq schema:schema-type($act-value) and schema:schema-type($exp-value) eq xs:QName("xs:decimal") and fn:format-number(xs:decimal($exp-value),"#,##0.0000000000000") eq fn:format-number(xs:decimal($act-value),"#,##0.0000000000000"))
                )
                then ()
                else
                    {
                        type: "unexpected-fact-field-value",
                        field: $key,
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare aspects :)
            for $key in distinct-values((keys($fact-expected.Aspects), keys($fact-actual.Aspects)))
            let $exp-value := $fact-expected.Aspects.$key
            let $act-value := $fact-actual.Aspects.$key
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-aspect-value",
                        aspect: $key,
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare keyaspects :)
            let $exp-value := $fact-expected.KeyAspects
            let $act-value := $fact-actual.KeyAspects
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-keyaspects-value",
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare Profiles :)
            let $exp-value := $fact-expected.Profiles
            let $act-value := $fact-actual.Profiles
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-profiles-value",
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare Audittrails :)
            let $exp-value := $fact-expected.AuditTrails
            let $act-value := $fact-actual.AuditTrails
            return
                local:compare-audit-trails($exp-value, $act-value)
        )
};

declare function local:compare-fact-tables($fact-table-expected as object, $fact-table-actual as object) as object*{
    let $actual-facts as object+ := $fact-table-actual.FactTable[]
    let $expected-facts as object+ := $fact-table-expected.FactTable[]
    let $errors :=
        (
            for $exp in $expected-facts
            let $act := $actual-facts[$$.Aspects."xbrl:Concept" eq $exp.Aspects."xbrl:Concept"
                                      and $$.Aspects."xbrl:Entity" eq $exp.Aspects."xbrl:Entity"
                                      and $$.Aspects."xbrl:Period" eq $exp.Aspects."xbrl:Period"
                                      and $$.Aspects."xbrl28:Archive" eq $exp.Aspects."xbrl28:Archive"]
            let $errors as object* := local:diff-facts($exp, $act)
            return
                if(empty($errors))
                then ()
                else {
                    expectedFact: $exp,
                    actualFact: $act,
                    errors: [$errors]
                },

            for $act in $actual-facts
            let $exp := $expected-facts[$$.Aspects."xbrl:Concept" eq $act.Aspects."xbrl:Concept"
                                      and $$.Aspects."xbrl:Entity" eq $act.Aspects."xbrl:Entity"
                                      and $$.Aspects."xbrl:Period" eq $act.Aspects."xbrl:Period"
                                      and $$.Aspects."xbrl28:Archive" eq $act.Aspects."xbrl28:Archive"]
            where empty($exp)
            return
                {
                    unexpectedFact: $act
                }
        )
    return
        if(empty($errors))
        then ()
        else (
                {
                    expectedNumberOfFacts: count($expected-facts),
                    actualNumberOfFacts: count($actual-facts)
                },
                $errors
            )
};

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-facttable-fact($concept as string, $expected as object, $params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $facts as object* := $request[2].FactTable[]
    let $actual := $facts[$$.Aspects."xbrl:Concept" eq $concept]
    let $diff :=
      for $f in $actual return local:diff-facts($expected, $f)
    return if (empty($diff)) then true else { url: test:url($endpoint, $params), factDiffErrors: [ $diff ], expectedFact: $expected, actualFact: $actual }
};

declare %an:nondeterministic function local:test-report-does-not-exist($params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $status as integer := $request[1]
    return if ($status eq 404) then true else { url: test:url($endpoint, $params), unexpectedResponse: $request[2] }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-values() as item*
{
    let $endpoint := "facttable-for-report"
    let $params := {ticker:"ko",fiscalYear:"2013",fiscalPeriod:"FY",fiscalPeriodType: ["instant", "YTD", "QTD"],report:"FundamentalAccountingConcepts"}
    let $request := test:invoke($endpoint, $params)
    let $actual as object := $request[2]
    let $expected := parse-json(
        http-client:get("http://" || request:server-name() || ":" || request:server-port() ||
                        "/test/facttable-for-report-expected1.jq").body.content)
    let $diff := local:compare-fact-tables($expected, $actual)
    return if (empty($diff)) then true else { url: test:url($endpoint, $params), factTableDiff: [ ({ params : $params },$diff) ], expectedFactTable: $expected, actualFactTable: $actual }
};

let $dow30 := test:is-dow30()
local:check({
    cocacola: local:test-facttable(95, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2013",
        fiscalPeriod:"Q1"}),
    aid: local:test-facttable(if($dow30) then 0 else 95, {
        aid:"0001193125-14-157120",
        report:"FundamentalAccountingConcepts"}),
    aid2: local:test-facttable(19, {
        aid:"0000732717-14-000022",
        report:"FundamentalAccountingConcepts"}),
    aid2-qtd: local:test-facttable(96, {
        aid:"0000732717-14-000022",
        report:"FundamentalAccountingConcepts",
        fiscalPeriodType: [ "instant", "QTD" ]
    }),
    tickerrole: local:test-facttable(95, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1"}),
    tickerfyfprole: local:test-facttable(246, {
        report:"FundamentalAccountingConcepts",
        ticker:["ko","wmt"],
        fiscalYear:"2013",
        fiscalPeriod:"FY",
        fiscalPeriodType:["instant", "QTD"]
    }),
    allvalues : local:test-values(),
    otheroperatingincometest: local:test-facttable-fact("fac:OtherOperatingIncomeExpenses",
                                                        local:expected-OtherOperatingIncomeExpenses-ATnT-2013(),
                                                        {
        report:"FundamentalAccountingConcepts",
        ticker:"t",
        fiscalYear:"2013",
        fiscalPeriod:["FY"]}),
    reportDoesntExist: local:test-report-does-not-exist({
        report:"report-not-found",
        ticker:"MSFT",
        fiscalYear:"ALL",
        fiscalPeriod:"FY"})
})


Improve readability.

Improve readability.

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    (. == {} or . == [] or . == false or . == null or . == 0) | not
;

# Group an array of objects into an object of arrays, such that the key-value
# pairs of the new object are an accumulation of those of the old object. For
# example, `[{"a":1, "b":2}, {"a":3}]` turns into `{"a":[1, 3], "b":[3]}`. This
# can be used to partition an array.
# Try: `[1,2,3] | map({(.%2|tostring):.}) | group`
def group:
    reduce ([.[] | to_entries[]] | group_by(.key))[] as $group
    ( {}
    ; .[ $group[0].key ] |= (. // []) + [ $group[].value ]
    )
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`.
def tree($path):
    def tree_aux($future):
        { "name": $future[0] } +
        if ($future | length) > 1 then
            { "contents": [ tree_aux($future[1:]) ] }
        else
            .
        end
    ;
    tree_aux($path)
;

# Add paths to each object, that is, the names of the ancestors.
def add_paths:
    def add_path_aux($history):
        ($history + [.name]) as $present |
        .path = ($present | join("/")) |
        (.contents[]? |= add_path_aux($present))
    ;
    add_path_aux([])
;

# Merge two values. If the values are both objects, the values at equal indices
# will be recursively merged; if the values are both arrays, the arrays are
# concatenated; if the values are some other type, we only check if the values
# do not conflict.
def merge(other):
    if (. | type) == "object" and (other | type) == "object" then
        (. | to_entries) + (other | to_entries)
        | group_by(.key)
        | map({ "key": (.[0].key), "value" : (reduce .[1:][].value as $x (.[0].value; merge($x)))})
        | from_entries
    elif (. | type) == "array" and (other | type) == "array" then
        . + other
    elif . == other then
        .
    else
        error("trying to merge incompatible objects " + (.|type) + " and " + (other|type))
    end
;

# Determine whether a page should be uploaded and included in the table of
# contents: it should either be marked as `publish`, or be *not* marked as
# `draft` and have children that *are* marked as `publish`.
def is_publication:
    ( .publish or .meta.publish | bool)
    or
    ( (.meta.draft | bool | not) and ((.contents // []) | any(is_publication)) )
;

# Merge filetrees by grouping them by name and then merging all the pages with
# the same name. This is necessary because simply merging content will
# concatenate pages.
def merge_content:
    if has("contents") then
        .contents |= (
            group_by(.name)
            | map(reduce .[] as $x ({}; merge($x)) | merge_content)
        )
    else
        .
    end
;

# Any page gets a link to its HTML.
def add_links:
    if (has("path") and (.path | endswith(".md"))) then
        .link = (.path | rtrimstr(".md") | . + ".html")
    else
        .contents[]? |= add_links
    end
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def add_frontmatter:
    if has("contents") then
        ( .contents | map({(.name=="index.md" or .meta.frontmatter|tostring):.}) | group) as {$true, $false}
        | .frontmatter = $true[0]
        | .contents = ($false + $true[1:])
        | (.contents[] |= add_frontmatter)
    else
        .
    end
;

# If marked as such, drafts can be excluded from being uploaded and included
# in the table of contents.
def add_drafts:
    if has("contents") then
        ( .contents | map({(is_publication|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .drafts = ($false // [])
        | (.contents[] |= add_drafts)
    else
        .
    end
;

# A resource is anything that is neither a directory nor a page with metadata
# nor a forced link.
def add_resources:
    if has("contents") then
        ( .contents | map({(has("contents") or has("meta") or has("publish")|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .resources = ($false // [])
        | (.contents[]? |= add_resources)
    else
        .
    end;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    reduce .[] as $x ({}; merge($x))
    | merge_content
    | add_paths
    | add_links
    | add_drafts
    | add_frontmatter
    | add_resources
;

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    (. == {} or . == [] or . == false or . == null or . == 0) | not
;

# Determine whether an index entry was manually entered into the table of
# contents "base".
def is_manual:
    .type == "hyperlink"
;

# Determine whether a page should be uploaded and included in the table of
# contents: it should either be marked as `publish`, or be *not* marked as
# `draft` and have children that *are* marked as `publish`.
def is_publication:
    (.publish or .meta.publish|bool)
    or
    ((.meta.draft|bool|not) and ((.contents // [])|any(is_publication or is_manual))
    )
;

# Group the values of properties of an array of objects. For example, [{"a":1,
# "b":2}, {"a":3}] turns into {"a":[1, 3], "b":[3]}. This can be used to
# partition an array; example: `[1,2,3] | map({(.%2|tostring):.}) | group`
def group:
    reduce ([.[] | to_entries[]] | group_by(.key))[] as $group
    ( {}
    ; .[ $group[0].key ] |= (. // []) + [ $group[].value ]
    )
;

# Put parent objects so that the current object exists at a particular path.
# Like `setpath/2`, but instead of making objects like `{"a":{"b":{…}}}`, this
# makes objects like `{"contents":[{"name":"a", "contents":[{"name":"b",…}]}]}`
def tree($path):
    def tree_aux($future):
        { "name": $future[0] } +
        if ($future | length) > 1 then
            { "contents": [ tree_aux($future[1:]) ] }
        else
            .
        end
    ;
    tree_aux($path)
;

# Merge two values. If the values are both objects, the values at equal indices
# will be merged; if the values are both arrays, the arrays are concatenated;
# if the values are some other type, we only check if the values do not
# conflict.
def merge(other):
    if (. | type) == "object" and (other | type) == "object" then
        (. | to_entries) + (other | to_entries)
        | group_by(.key)
        | map({ "key": (.[0].key), "value" : (reduce .[1:][].value as $x (.[0].value; merge($x)))})
        | from_entries
    elif (. | type) == "array" and (other | type) == "array" then
        . + other
    elif . == other then
        .
    else
        error("trying to merge incompatible objects " + (.|tostring) + " and " + (other|tostring))
    end
;

# Merge an array of pages by grouping them by name and then merging all the
# pages with the same name.
def combine_pages:
    if has("contents") then
        .contents |= (
            group_by(.name)
            |   map(reduce .[] as $x ({}; merge($x)) | combine_pages)
        )
    else
        .
    end
;

# Add paths to each object, that is, the names of the ancestors.
def add_path:
    def add_path_aux($history):
        ($history + [.name]) as $present |
        .path = ($present | join("/")) |
        (.contents[]? |= add_path_aux($present))
    ;
    add_path_aux([])
;

# Any page gets a link to its HTML.
def add_links:
    if (has("path") and (.path | endswith(".md"))) then
        .link = (.path | rtrimstr(".md") | . + ".html")
    else
        .contents[]? |= add_links
    end
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def add_frontmatter:
    if has("contents") then
        ( .contents | map({(.name=="index.md" or .meta.frontmatter|tostring):.}) | group) as {$true, $false}
        | .frontmatter = $true[0]
        | .contents = ($false + $true[1:])
        | (.contents[]? |= add_frontmatter)
    else
        .
    end
;

# If marked as such, drafts can be excluded from being uploaded and included
# in the table of contents.
def add_drafts:
    if has("contents") then
        ( .contents | map({(is_publication or is_manual|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .drafts = ($false // [])
        | (.contents[]? |= add_drafts)
    else
        .
    end
;

# A resource is anything that is neither a directory nor a page with metadata
# nor a forced link.
def add_resources:
    if has("contents") then
        ( .contents | map({(has("contents") or has("meta") or has("publish")|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .resources = ($false // [])
        | (.contents[]? |= add_resources)
    else
        .
    end;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    reduce .[] as $x ({}; merge($x))
    | combine_pages
    | add_path
    | add_links
    | add_drafts
    | add_frontmatter
    | add_resources
;


More documentation.

More documentation.

xquery version "3.0";

(:
 : Copyright 2006-2011 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

(:===========================================================================:)

(:~
 : This module provides an API for parsing and serializing CSV (comma-separated
 : values) files.
 : See RFC 4180,
 : "Common Format and MIME Type for Comma-Separated Values (CSV) Files."
 : @project Zorba/CSV
 :)

module namespace csv = "http://zorba.io/modules/csv";

declare namespace err = "http://www.w3.org/2005/xqt-errors";
declare namespace zerr = "http://zorba.io/errors";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:===========================================================================:)

(:~
 : Parses a CSV (comma-separated values) string using the given options.
 : A newline (U+000A), optionally preceeded by a carriage-return (U+000D),
 : terminates lines, aka, "records."
 :
 : @param $csv The CSV string to parse.
 : @param $options The options to use:
 :

 :
field-names

 :

 :        A JSON array of strings denoting field names;
 :        default: none.
 :        If this option is given, then the first CSV line is assumed not to be
 :        a header line; if omitted, then the first CSV line is assumed to be a
 :        header line and the field names are taken from this line.
 :

 :
quote-char

 :

 :        The single ASCII character that may be used to quote values;
 :        default: " (U+0022).
 :

 :
quote-esc

 :

 :        The single ASCII character used to escape quote-char;
 :        default: same as quote-char.
 :        If quote-esc equals quote-char,
 :        it means that quote-char must be doubled to escape it.
 :        If quote-esc does not equal quote-char,
 :        it means that quote-esc is used to escape
 :        quote-char.
 :        For example, a quote-char of " (U+0022)
 :        and a quote-esc of \ (U+005C) means that
 :        quotes will be escaped by \".
 :

 :
separator

 :

 :        The single ASCII character used to separate values;
 :        default: , (U+002C).
 :

 :
missing-value

 :

 :        Either error or null.
 :

 :

 : @return a sequence of zero or more JSON objects where each key is a field
 : name and each value is a parsed value.
 : @error zerr:ZCSV0001 if the quote-char, quote-esc,
 : or separator option is given and it's not a single ASCII
 : character.
 :)
declare function csv:parse( $csv as string, $options as object() )
  as object()* external;

(:~
 : Parses a CSV (comma-separated values) string using the default options.
 : A newline (U+000A), optionally preceeded by a carriage-return (U+000D),
 : terminates lines, aka, "records."
 : The default options are:
 :

 :
field-names

 :

 :        A JSON array of strings denoting field names;
 :        default: none.
 :        The first CSV line is assumed to be a header line
 :        and the field names are taken from this line.
 :

 :
quote-char

 :

 :        The single ASCII character that may be used to quote values;
 :        default: " (U+0022).
 :

 :
quote-esc

 :

 :        The single ASCII character used to escape quote-char;
 :        default: same as quote-char.
 :        This means that an escaped quote is doubled as "".
 :

 :
separator

 :

 :        The single ASCII character used to separate values;
 :        default: , (U+002C).
 :

 :
missing-value

 :

 :        Default: null.
 :

 :

 :
 : @param $csv The CSV string to parse.
 : @return a sequence of zero or more JSON objects where each key is a field
 : name and each value is a parsed value.
 :)
declare function csv:parse( $csv as string )
  as object()*
{
  csv:parse( $csv, {} )
};

(:~
 : TODO
 :
 : @param $obj TODO
 : @param $options TODO
 : @return TODO
 :)
declare function csv:serialize( $obj as object()*, $options as object() )
  as string* external;

(:~
 : TODO
 :
 : @param $obj TODO
 : @return TODO
 :)
declare function csv:serialize( $obj as object()* )
  as string*
{
  csv:serialize( $obj, {} )
};

(:===========================================================================:)

(: vim:set syntax=xquery et sw=2 ts=2: :)

xquery version "3.0";

(:
 : Copyright 2006-2011 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

(:===========================================================================:)

(:~
 : This module provides an API for reading and serializing CSV (comma-separated
 : values) files.
 : See RFC 4180,
 : "Common Format and MIME Type for Comma-Separated Values (CSV) Files."
 : @project Zorba/CSV
 :)

module namespace csv = "http://zorba.io/modules/csv";

declare namespace err = "http://www.w3.org/2005/xqt-errors";
declare namespace zerr = "http://zorba.io/errors";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:===========================================================================:)

(:~
 : TODO
 :
 : @param $csv TODO
 : @param $options TODO
 : @return TODO
 : @error zerr:ZCSV0001 if the \c separator option is given and it's not a
 : single ASCII character.
 :)
declare function csv:parse( $csv as string, $options as object() )
  as object()* external;

(:~
 : TODO
 :
 : @param $csv TODO
 : @return TODO
 :)
declare function csv:parse( $csv as string )
  as object()*
{
  csv:parse( $csv, {} )
};

(:~
 : TODO
 :
 : @param $obj TODO
 : @param $options TODO
 : @return TODO
 :)
declare function csv:serialize( $obj as object()*, $options as object() )
  as string* external;

(:~
 : TODO
 :
 : @param $obj TODO
 : @return TODO
 :)
declare function csv:serialize( $obj as object()* )
  as string*
{
  csv:serialize( $obj, {} )
};

(:===========================================================================:)

(: vim:set syntax=xquery et sw=2 ts=2: :)


:bug: Catching components that may have no hypercube.

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/backend";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec-filings = "http://28.io/modules/xbrl/profiles/sec/filings";
import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

declare option rest:response "first-item";

declare function local:links($component as object, $hypercube as string?) as object
{
  {
    FactTable: backend:url("facttable-for-component", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    hypercube: $hypercube,
                    profile-name: $profile-name,
                    labels: "true"
                    }),
    SpreadSheet: backend:url("spreadsheet-for-component", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    hypercube: $hypercube,
                    profile-name: $profile-name,
                    eliminate: "true",
                    validate: "true"
                }),
    ReportElements: backend:url("report-elements", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    hypercube: $hypercube,
                    profile-name: $profile-name
                }),
    ModelStructure: backend:url("modelstructure-for-component", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    hypercube: $hypercube,
                    profile-name: $profile-name
                })
  }
};

declare function local:statistics($component as object, $hypercube as string?) as object
{
  {
    NumRules: size($component.Rules),
    NumNetworks: components:num-networks($component),
    NumReportElements: components:num-report-elements($component, $hypercube),
    NumHypercubes: components:num-hypercubes($component, $hypercube),
    NumDimensions: components:num-dimensions($component, $hypercube),
    NumMembers: components:num-members($component, $hypercube),
    NumLineItems: components:num-line-items($component, $hypercube),
    NumAbstracts: components:num-abstracts($component, $hypercube),
    NumConcepts: components:num-concepts($component, $hypercube)
  }
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token              as string? external;
declare  %rest:env                              variable $request-uri        as string  external;
declare  %rest:case-insensitive                 variable $format             as string  external := "json";
declare  %rest:case-insensitive %rest:distinct  variable $cik                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode   as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker             as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear         as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod       as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $filingKind         as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $eid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $networkIdentifier  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $role               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $reportElement      as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $concept            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $label              as string* external;
declare  %rest:case-insensitive                 variable $profile-name       as string  external := $config:profile-name;
declare  %rest:case-insensitive                 variable $count              as boolean external := false;

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $reportElement := ($reportElement, $concept)
let $networkIdentifier := distinct-values(($networkIdentifier, $role))

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic,
  (),
  (),
  (),
  ())

let $entities-not-found as boolean :=
  exists(($eid, $cik, $tag, $ticker, $sic)) and empty($entities)

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $filingKind,
  $aid)

let $archives-not-found as boolean :=
  exists(($entities, $fiscalPeriod, $fiscalYear, $filingKind, $aid)) and empty($archives)

let $entities as object*  := entities:entities($archives.Entity)
let $components as object* :=
    multiplexer:components(
      $profile-name,
      $archives,
      $cid,
      $reportElement,
      $disclosure,
      $networkIdentifier,
      $label[$profile-name ne "sec"],
      $label[$profile-name eq "sec"]
    )

let $res as object* :=
    switch($profile-name)
    case "sec" return
        for $component-group in $components
        let $disclosure := sec-networks:disclosures($component-group)
        where $disclosure ne "DefaultComponent"
        order by $component-group.Label
        group by $archive := $component-group.Archive
        let $archive := $archives[archives:aid($$) eq $archive]
        let $e := $entities[entities:eid($$) eq $archive.Entity]
        return
            {
               AccessionNumber : archives:aid($archive),
               EntityRegistrantName : $e.Profiles.SEC.CompanyName,
               CIK : entities:eid($e),
               FiscalYear :$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
               FiscalPeriod :$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
               AcceptanceDatetime : sec-filings:acceptance-dateTimes($archive),
               FormType : $archive.Profiles.SEC.FormType,
               Components : [
                 for $component in $component-group
                 for $hypercube allowing empty in components:hypercubes($component)
                 return {|
                   sec-networks:summary($component),
                   {
                     Table: $hypercube
                   },
                   local:links($component, $hypercube),
                   local:statistics($component, $hypercube),
                   {
                     ValidationErrors: [ components:validation-errors($component, $hypercube) ]
                   }
                 |}
               ]
           }
    default return
        for $component in $components
        for $hypercube allowing empty in components:hypercubes($component)
        return {|
          project($component, ("Archive", "Label", "Role")),
          {
            Hypercube: $hypercube
          },
          local:links($component, $hypercube),
          local:statistics($component, $hypercube),
          {
            ValidationErrors: [ components:validation-errors($component, $hypercube) ]
          },
          { Disclosure: $component.Profiles.FSA.Disclosure}[$profile-name eq "japan"]
        |}
let $result := switch($profile-name)
               case "sec"
               return { Archives: [ $res ] }
               default return { Components : [ if($count) then () else $res ] }
let $comment :=
 {
    NumComponents : count($components),
    TotalNumComponents: session:num-components(),
    TotalNumArchives: session:num-archives()
}
let $serializers := {
    to-xml : switch($profile-name)
        case "sec"
        return function($res as object) as node() {
        <Archives>{
                  for $r in flatten($res.Archives)
                  return
                      <Archive id="{$r.AccessionNumber}">
                         <EntityRegistrantName>{$r.EntityRegistrantName}</EntityRegistrantName>
                         <CIK>{$r.CIK}</CIK>
                         <FiscalYear>{$r.FiscalYear}</FiscalYear>
                         <FiscalPeriod>{$r.FiscalPeriod}</FiscalPeriod>
                         <AcceptanceDatetime>{$r.AcceptanceDatetime}</AcceptanceDatetime>
                         <FormType>{$r.FormType}</FormType>
                         <Components>{
                             api:json-to-xml($r.Components[], "Component")
                         }</Components>
                     </Archive>
             }</Archives>
        }
        default return function($res as object) as node() {
        <Components>{
          api:json-to-xml($res.Components[], "Component")
        }</Components>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec" return api:json-to-csv($res.Archives[].Components[])
        default return api:json-to-csv($res.Components[])
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "components")
return switch(true)
       case $entities-not-found return api:not-found("entity")
       case $archives-not-found return api:not-found("archive")
       default return api:check-and-return-results($token, $results, $format)

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/backend";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec-filings = "http://28.io/modules/xbrl/profiles/sec/filings";
import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

declare option rest:response "first-item";

declare function local:links($component as object, $hypercube as string) as object
{
  {
    FactTable: backend:url("facttable-for-component", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    hypercube: $hypercube,
                    profile-name: $profile-name,
                    labels: "true"
                    }),
    SpreadSheet: backend:url("spreadsheet-for-component", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    hypercube: $hypercube,
                    profile-name: $profile-name,
                    eliminate: "true",
                    validate: "true"
                }),
    ReportElements: backend:url("report-elements", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    hypercube: $hypercube,
                    profile-name: $profile-name
                }),
    ModelStructure: backend:url("modelstructure-for-component", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    hypercube: $hypercube,
                    profile-name: $profile-name
                })
  }
};

declare function local:statistics($component as object, $hypercube as string) as object
{
  {
    NumRules: size($component.Rules),
    NumNetworks: components:num-networks($component),
    NumReportElements: components:num-report-elements($component, $hypercube),
    NumHypercubes: components:num-hypercubes($component, $hypercube),
    NumDimensions: components:num-dimensions($component, $hypercube),
    NumMembers: components:num-members($component, $hypercube),
    NumLineItems: components:num-line-items($component, $hypercube),
    NumAbstracts: components:num-abstracts($component, $hypercube),
    NumConcepts: components:num-concepts($component, $hypercube)
  }
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token              as string? external;
declare  %rest:env                              variable $request-uri        as string  external;
declare  %rest:case-insensitive                 variable $format             as string  external := "json";
declare  %rest:case-insensitive %rest:distinct  variable $cik                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode   as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker             as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear         as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod       as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $filingKind         as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $eid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $networkIdentifier  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $role               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $reportElement      as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $concept            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $label              as string* external;
declare  %rest:case-insensitive                 variable $profile-name       as string  external := $config:profile-name;
declare  %rest:case-insensitive                 variable $count              as boolean external := false;

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $reportElement := ($reportElement, $concept)
let $networkIdentifier := distinct-values(($networkIdentifier, $role))

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic,
  (),
  (),
  (),
  ())

let $entities-not-found as boolean :=
  exists(($eid, $cik, $tag, $ticker, $sic)) and empty($entities)

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $filingKind,
  $aid)

let $archives-not-found as boolean :=
  exists(($entities, $fiscalPeriod, $fiscalYear, $filingKind, $aid)) and empty($archives)

let $entities as object*  := entities:entities($archives.Entity)
let $components as object* :=
    multiplexer:components(
      $profile-name,
      $archives,
      $cid,
      $reportElement,
      $disclosure,
      $networkIdentifier,
      $label[$profile-name ne "sec"],
      $label[$profile-name eq "sec"]
    )

let $res as object* :=
    switch($profile-name)
    case "sec" return
        for $component-group in $components
        let $disclosure := sec-networks:disclosures($component-group)
        where $disclosure ne "DefaultComponent"
        order by $component-group.Label
        group by $archive := $component-group.Archive
        let $archive := $archives[archives:aid($$) eq $archive]
        let $e := $entities[entities:eid($$) eq $archive.Entity]
        return
            {
               AccessionNumber : archives:aid($archive),
               EntityRegistrantName : $e.Profiles.SEC.CompanyName,
               CIK : entities:eid($e),
               FiscalYear :$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
               FiscalPeriod :$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
               AcceptanceDatetime : sec-filings:acceptance-dateTimes($archive),
               FormType : $archive.Profiles.SEC.FormType,
               Components : [
                 for $component in $component-group
                 for $hypercube in components:hypercubes($component)
                 return {|
                   sec-networks:summary($component),
                   {
                     Table: $hypercube
                   },
                   local:links($component, $hypercube),
                   local:statistics($component, $hypercube),
                   {
                     ValidationErrors: [ components:validation-errors($component, $hypercube) ]
                   }
                 |}
               ]
           }
    default return
        for $component in $components
        for $hypercube in components:hypercubes($component)
        return {|
          project($component, ("Archive", "Label", "Role")),
          {
            Hypercube: $hypercube
          },
          local:links($component, $hypercube),
          local:statistics($component, $hypercube),
          {
            ValidationErrors: [ components:validation-errors($component, $hypercube) ]
          },
          { Disclosure: $component.Profiles.FSA.Disclosure}[$profile-name eq "japan"]
        |}
let $result := switch($profile-name)
               case "sec"
               return { Archives: [ $res ] }
               default return { Components : [ if($count) then () else $res ] }
let $comment :=
 {
    NumComponents : count($components),
    TotalNumComponents: session:num-components(),
    TotalNumArchives: session:num-archives()
}
let $serializers := {
    to-xml : switch($profile-name)
        case "sec"
        return function($res as object) as node() {
        <Archives>{
                  for $r in flatten($res.Archives)
                  return
                      <Archive id="{$r.AccessionNumber}">
                         <EntityRegistrantName>{$r.EntityRegistrantName}</EntityRegistrantName>
                         <CIK>{$r.CIK}</CIK>
                         <FiscalYear>{$r.FiscalYear}</FiscalYear>
                         <FiscalPeriod>{$r.FiscalPeriod}</FiscalPeriod>
                         <AcceptanceDatetime>{$r.AcceptanceDatetime}</AcceptanceDatetime>
                         <FormType>{$r.FormType}</FormType>
                         <Components>{
                             api:json-to-xml($r.Components[], "Component")
                         }</Components>
                     </Archive>
             }</Archives>
        }
        default return function($res as object) as node() {
        <Components>{
          api:json-to-xml($res.Components[], "Component")
        }</Components>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec" return api:json-to-csv($res.Archives[].Components[])
        default return api:json-to-csv($res.Components[])
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "components")
return switch(true)
       case $entities-not-found return api:not-found("entity")
       case $archives-not-found return api:not-found("archive")
       default return api:check-and-return-results($token, $results, $format)


Make _mask public

Make _mask public

module {
    name: "math/bitwise",
    description: "Bitwise operations on numbers",
    namespace: "fadado.github.io",
    author: {
        name: "Joan Josep Ordinas Rosa",
        email: "jordinas@gmail.com"
    }
};

########################################################################
# Bitwise operations

# BitField: 0..(2^53-1) (53 bits unsigned integer)
# Bit:      2^0, 2^1, 2^2, ...2^52
# Position: 0..52
# New empty bitset: 0e0

# def _bit($position): $position|exp2;
# def _pos($bit): $bit|log2;

# Build mask with $size bits (mask(53) == -1 in signed integers)
def mask($size): #:: (number) => BitField
    $size|exp2-1
;

# Get bit at $position as 0 or 1
def getbit($position; $n): #:: (Position;BitField) => 0^1
    fmod($n/($position|exp2)|floor;2) | fabs
;

# Set bit at $position
def setbit($position; $n): #:: (Position;BitField) => BitField
    if fmod($n/($position|exp2)|floor;2) != 0
    then $n
    else $n + ($position|exp2)
    end
;

# Clear bit at $position
def clrbit($position; $n): #:: (Position;BitField) => BitField
    if fmod($n/($position|exp2)|floor;2) != 0
    then $n - ($position|exp2)
    else $n
    end
;

# Toggle bit at $position
def tglbit($position; $n): #:: (Position;BitField) => BitField
    if fmod($n/($position|exp2)|floor;2) != 0
    then $n - ($position|exp2)
    else $n + ($position|exp2)
    end
;

########################################################################
# Common LISP borrowed functions
#
# CLTL: 12.7. Logical Operations on Numbers
#       http://www.cs.cmu.edu/Groups/AI/html/cltl/clm/node131.html
########################################################################

# See: http://clhs.lisp.se/Body/f_logand.htm
def lognot($n): #:: (BitField) => BitField
    9007199254740991 - $n # use precalculated constant 53|exp2-1
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def logand($m; $n): #:: (BitField;BitField) => BitField
    def r($m; $n; $result; $pow2):
        if $m < 1 or $n < 1
        then $result
        elif fmod($m;2)!=0 and fmod($n;2)!=0
        then r($m/2|floor; $n/2|floor; $result+$pow2; $pow2*2)
        else r($m/2|floor; $n/2|floor; $result; $pow2*2)
        end
    ;
    r($m; $n; 0e0; 1)
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def logior($m; $n): #:: (BitField;BitField) => BitField
    def r($m; $n; $result; $pow2):
        if $m < 1 and $n < 1
        then $result
        elif fmod($m;2)!=0 or fmod($n;2)!=0
        then r($m/2|floor; $n/2|floor; $result+$pow2; $pow2*2)
        else r($m/2|floor; $n/2|floor; $result; $pow2*2)
        end
    ;
    r($m; $n; 0e0; 1)
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def logxor($m; $n): #:: (BitField;BitField) => BitField
    def r($m; $n; $result; $pow2):
        if $m < 1 and $n < 1
        then $result
        elif (fmod($m;2)!=0) != (fmod($n;2)!=0)
        then r($m/2|floor; $n/2|floor; $result+$pow2; $pow2*2)
        else r($m/2|floor; $n/2|floor; $result; $pow2*2)
        end
    ;
    r($m; $n; 0e0; 1)
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def logeqv($m; $n): #:: (BitField;BitField) => BitField
    lognot(logxor($m; $n))
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def lognand($m; $n): #:: (BitField;BitField) => BitField
    lognot(logand($m; $n))
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def lognor($m; $n): #:: (BitField;BitField) => BitField
    lognot(logior($m; $n))
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def logandc1($m; $n): #:: (BitField;BitField) => BitField
    logand(lognot($m); $n)
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def logandc2($m; $n): #:: (BitField;BitField) => BitField
    logand($m; lognot($n))
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def logorc1($m; $n): #:: (BitField;BitField) => BitField
    logior(lognot($m); $n)
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def logorc2($m; $n): #:: (BitField;BitField) => BitField
    logior($m; lognot($n))
;

# See: http://clhs.lisp.se/Body/f_logtes.htm
def logtest($m; $n): #:: (BitField;BitField) => boolean
    logand($m; $n) != 0e0
;

# See: http://clhs.lisp.se/Body/f_logbtp.htm
def logbitp($position; $n): #:: (Position;BitField) => boolean
    fmod($n/($position|exp2)|floor;2) != 0
;

# See: http://clhs.lisp.se/Body/f_ash.htm
def ash($count; $n): #:: (number;BitField) => BitField
    $n*($count|exp2)|floor
;

# See: http://clhs.lisp.se/Body/f_intege.htm
def integer_length($n): #:: (BitField) => number
    if $n < 0
    then -$n
    else $n+1
    end | log2 | ceil
;

# See: http://clhs.lisp.se/Body/f_logcou.htm
def logcount($n): #:: (BitField) => number
    def r($n; $result):
        if $n < 1
        then $result
        elif fmod($n;2) != 0
        then r($n/2|floor; $result+1)
        else r($n/2|floor; $result)
        end
    ;
    r($n; 0)
;

########################################################################
# Common LISP borrowed functions
#
# CLTL 12.8. Byte Manipulation Functions
#       http://www.cs.cmu.edu/Groups/AI/html/cltl/clm/node132.html
########################################################################

# See: http://clhs.lisp.se/Body/f_by_by.htm
def byte($size; $position):
    {
        siz: $size,
        pos: $position,
        msk: ash($position; $size|exp2-1), # precomputed mask
        neg: null # precomputed negated mask
    } | .neg = lognot(.msk)
;

# See: http://clhs.lisp.se/Body/f_by_by.htm
def byte_size($bytespec):
    $bytespec.siz
;

# See: http://clhs.lisp.se/Body/f_by_by.htm
def byte_position($bytespec):
    $bytespec.pos
;

# See: http://clhs.lisp.se/Body/f_ldb.htm
def ldb($bytespec; $n):
    ash(-$bytespec.pos; logand($bytespec.msk; $n))
;

# See: http://clhs.lisp.se/Body/f_ldb_te.htm
def ldb_test($bytespec; $n):
    logand($bytespec.msk; $n) != 0
;

# See: http://clhs.lisp.se/Body/f_mask_f.htm
def mask_field($bytespec; $n):
    logand($bytespec.msk; $n)
;

# See: http://clhs.lisp.se/Body/f_dpb.htm
def dpb($newbyte; $bytespec; $n):
    logior(logand($bytespec.neg; $n);
           logand($bytespec.msk; ash($bytespec.pos; $newbyte)))
;

# See: http://clhs.lisp.se/Body/f_deposi.htm
def deposit_field($newbyte; $bytespec; $n):
    logior(logand($bytespec.neg; $n);
           logand($bytespec.msk; $newbyte))
;

# vim:ai:sw=4:ts=4:et:syntax=jq

module {
    name: "math/bitwise",
    description: "Bitwise operations on numbers",
    namespace: "fadado.github.io",
    author: {
        name: "Joan Josep Ordinas Rosa",
        email: "jordinas@gmail.com"
    }
};

########################################################################
# Bitwise operations

# BitField: 0..(2^53-1) (53 bits unsigned integer)
# Bit:      2^0, 2^1, 2^2, ...2^52
# Position: 0..52
# New empty bitset: 0e0

# def _bit($position): $position|exp2;
# def _pos($bit): $bit|log2;

# Build mask with $size bits (mask(53) == -1 in signed integers)
def mask($size): #:: (number) => BitField
    $size|exp2-1
;

# Get bit at $position as 0 or 1
def getbit($position; $n): #:: (Position;BitField) => 0^1
    fmod($n/($position|exp2)|floor;2) | fabs
;

# Set bit at $position
def setbit($position; $n): #:: (Position;BitField) => BitField
    if fmod($n/($position|exp2)|floor;2) != 0
    then $n
    else $n + ($position|exp2)
    end
;

# Clear bit at $position
def clrbit($position; $n): #:: (Position;BitField) => BitField
    if fmod($n/($position|exp2)|floor;2) != 0
    then $n - ($position|exp2)
    else $n
    end
;

# Toggle bit at $position
def tglbit($position; $n): #:: (Position;BitField) => BitField
    if fmod($n/($position|exp2)|floor;2) != 0
    then $n - ($position|exp2)
    else $n + ($position|exp2)
    end
;

########################################################################
# Common LISP borrowed functions
#
# CLTL: 12.7. Logical Operations on Numbers
#       http://www.cs.cmu.edu/Groups/AI/html/cltl/clm/node131.html
########################################################################

# See: http://clhs.lisp.se/Body/f_logand.htm
def lognot($n): #:: (BitField) => BitField
    9007199254740991 - $n # use precalculated constant 53|exp2-1
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def logand($m; $n): #:: (BitField;BitField) => BitField
    def r($x; $y; $result; $pow2):
        if $x < 1 or $y < 1
        then $result
        elif fmod($x;2)!=0 and fmod($y;2)!=0
        then r($x/2|floor; $y/2|floor; $result+$pow2; $pow2*2)
        else r($x/2|floor; $y/2|floor; $result; $pow2*2)
        end
    ;
    r($m; $n; 0e0; 1)
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def logior($m; $n): #:: (BitField;BitField) => BitField
    def r($x; $y; $result; $pow2):
        if $x < 1 and $y < 1
        then $result
        elif fmod($x;2)!=0 or fmod($y;2)!=0
        then r($x/2|floor; $y/2|floor; $result+$pow2; $pow2*2)
        else r($x/2|floor; $y/2|floor; $result; $pow2*2)
        end
    ;
    r($m; $n; 0e0; 1)
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def logxor($m; $n): #:: (BitField;BitField) => BitField
    def r($x; $y; $result; $pow2):
        if $x < 1 and $y < 1
        then $result
        elif (fmod($x;2)!=0) != (fmod($y;2)!=0)
        then r($x/2|floor; $y/2|floor; $result+$pow2; $pow2*2)
        else r($x/2|floor; $y/2|floor; $result; $pow2*2)
        end
    ;
    r($m; $n; 0e0; 1)
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def logeqv($m; $n): #:: (BitField;BitField) => BitField
    lognot(logxor($m; $n))
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def lognand($m; $n): #:: (BitField;BitField) => BitField
    lognot(logand($m; $n))
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def lognor($m; $n): #:: (BitField;BitField) => BitField
    lognot(logior($m; $n))
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def logandc1($m; $n): #:: (BitField;BitField) => BitField
    logand(lognot($m); $n)
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def logandc2($m; $n): #:: (BitField;BitField) => BitField
    logand($m; lognot($n))
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def logorc1($m; $n): #:: (BitField;BitField) => BitField
    logior(lognot($m); $n)
;

# See: http://clhs.lisp.se/Body/f_logand.htm
def logorc2($m; $n): #:: (BitField;BitField) => BitField
    logior($m; lognot($n))
;

# See: http://clhs.lisp.se/Body/f_logtes.htm
def logtest($m; $n): #:: (BitField;BitField) => boolean
    logand($m; $n) != 0e0
;

# See: http://clhs.lisp.se/Body/f_logbtp.htm
def logbitp($position; $n): #:: (Position;BitField) => boolean
    fmod($n/($position|exp2)|floor;2) != 0
;

# See: http://clhs.lisp.se/Body/f_ash.htm
def ash($count; $n): #:: (number;BitField) => BitField
    $n*($count|exp2)|floor
;

# See: http://clhs.lisp.se/Body/f_intege.htm
def integer_length($n): #:: (BitField) => number
    if $n < 0 then -$n else $n+1 end
    | log2 | ceil
;

# See: http://clhs.lisp.se/Body/f_logcou.htm
def logcount($n): #:: (BitField) => number
    def r($x; $result):
        if $x < 1
        then $result
        elif fmod($x;2) != 0
        then r($x/2|floor; $result+1)
        else r($x/2|floor; $result)
        end
    ;
    r($n; 0)
;

########################################################################
# Common LISP borrowed functions
#
# CLTL 12.8. Byte Manipulation Functions
#       http://www.cs.cmu.edu/Groups/AI/html/cltl/clm/node132.html
########################################################################

# See: http://clhs.lisp.se/Body/f_by_by.htm
def byte($size; $position):
    {
        siz: $size,
        pos: $position,
        msk: ash($position; $size|exp2-1), # precomputed mask
        neg: null # precomputed negated mask
    } | .neg = lognot(.msk)
;

# See: http://clhs.lisp.se/Body/f_by_by.htm
def byte_size($bytespec):
    $bytespec.siz
;

# See: http://clhs.lisp.se/Body/f_by_by.htm
def byte_position($bytespec):
    $bytespec.pos
;

# See: http://clhs.lisp.se/Body/f_ldb.htm
def ldb($bytespec; $n):
    ash(-$bytespec.pos; logand($bytespec.msk; $n))
;

# See: http://clhs.lisp.se/Body/f_ldb_te.htm
def ldb_test($bytespec; $n):
    logand($bytespec.msk; $n) != 0
;

# See: http://clhs.lisp.se/Body/f_mask_f.htm
def mask_field($bytespec; $n):
    logand($bytespec.msk; $n)
;

# See: http://clhs.lisp.se/Body/f_dpb.htm
def dpb($newbyte; $bytespec; $n):
    logior(logand($bytespec.neg; $n);
           logand($bytespec.msk; ash($bytespec.pos; $newbyte)))
;

# See: http://clhs.lisp.se/Body/f_deposi.htm
def deposit_field($newbyte; $bytespec; $n):
    logior(logand($bytespec.neg; $n);
           logand($bytespec.msk; $newbyte))
;

# vim:ai:sw=4:ts=4:et:syntax=jq


:green_heart: Fix typo

:green_heart: Fix typo

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/backend";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec-filings = "http://28.io/modules/xbrl/profiles/sec/filings";
import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare option rest:response "first-item";

declare function local:to-csv($res as object*) as string*
{
    csv:serialize(
        for $a in $res
        for $c in $a.Components[]
        return {
            AcessionNumber : $a.AccessionNumber,
            NetworkIdentifier : $c.NetworkIdentifier,
            FactTable: $c.FactTable,
            SpreadSheet: $c.SpreadSheet,
            EntityRegistrantName : $a.EntityRegistrantName,
            CIK : $a.CIK,
            FiscalYear : $a.FiscalYear,
            FiscalPeriod : $a.FiscalPeriod,
            AcceptanceDateTime : $a.AcceptanceDatetime,
            FormType : $a.FormType,
            NetworkLabel : $c.NetworkLabel,
            Category : $c.Category,
            SubCategory : $c.SubCategory,
            Table : flatten($c.Table),
            Disclosure : $c.Disclosure,
            ReportElements : $c.ReportElements,
            Tables : $c.Tables,
            Axis : $c.Axis,
            Members : $c.Members,
            LineItems : $c.LineItems,
            Concepts : $c.Concepts,
            Abstracts : $c.Abstracts
        },
    { serialize-null-as : "" })
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token              as string? external;
declare  %rest:env                              variable $request-uri        as string  external;
declare  %rest:case-insensitive                 variable $format             as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode   as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker             as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear         as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod       as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $filingKind         as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $eid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $networkIdentifier  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $role               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $reportElement      as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $concept            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $label              as string* external;
declare  %rest:case-insensitive                 variable $profile-name       as string  external := $config:profile-name;

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $reportElement := ($reportElement, $concept)
let $networkIdentifier := distinct-values(($networkIdentifier, $role))

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic, ())

let $entities-not-found as boolean :=
  exists(($eid, $cik, $tag, $ticker, $sic)) and empty($entities)

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $filingKind,
  $aid)

let $archives-not-found as boolean :=
  exists(($entities, $fiscalPeriod, $fiscalYear, $filingKind, $aid)) and empty($archives)

let $entities as object*  := entities:entities($archives.Entity)
let $components as object* :=
    multiplexer:components(
      $profile-name,
      $archives,
      $cid,
      $reportElement,
      $disclosure,
      $networkIdentifier,
      $label[$profile-name ne "sec"],
      $label[$profile-name eq "sec"]
    )

let $res as object* :=
    switch($profile-name)
    case "sec" return
        for $r in $components
        let $disclosure := sec-networks:disclosures($r)
        where $disclosure ne "DefaultComponent"
        order by $r.Label
        group by $archive := $r.Archive
        let $archive := $archives[archives:aid($$) eq $archive]
        let $e := $entities[entities:eid($$) eq $archive.Entity]
        return
            {
               AccessionNumber : archives:aid($archive),
               EntityRegistrantName : $e.Profiles.SEC.CompanyName,
               CIK : entities:eid($e),
               FiscalYear :$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
               FiscalPeriod :$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
               AcceptanceDatetime : sec-filings:acceptance-dateTimes($archive),
               FormType : $archive.Profiles.SEC.FormType,
               Components : [
                    for $component in sec-networks:summaries($r)
                    return copy $c := $component
                    modify insert json {
                        FactTable: backend:url("facttable-for-component", {
                            token: $token,
                            aid: archives:aid($archive),
                            format: $format,
                            role: $component.NetworkIdentifier,
                            profile-name: $profile-name
                            }),
                        SpreadSheet: "http://rendering.secxbrl.info/#?url=" || encode-for-uri(
                            backend:url("spreadsheet-for-component", {
                            token: $token,
                            aid: archives:aid($archive),
                            format: $format,
                            role: $component.NetworkIdentifier,
                            profile-name: $profile-name
                            })
                        )
                    } into $c
                    return $c
               ]
           }
    default return
        for $r in $components
        return {|
          {
            Archive: $r.Archive,
            Label: $r.Label,
            Role: $r.Role,
            FactTable: backend:url("facttable-for-component", {
                            token: $token,
                            aid: $r.Archive,
                            format: $format,
                            role: $r.Role,
                            profile-name: $profile-name
                            }),
            SpreadSheet: "http://rendering.secxbrl.info/#?url=" || encode-for-uri(
                        backend:url("spreadsheet-for-component", {
                            token: $token,
                            aid: $r.Archive,
                            format: $format,
                            role: $r.Role,
                            profile-name: $profile-name,
                            eliminate: "true"
                        })),
            ReportElements: backend:url("report-elements", {
                            token: $token,
                            aid: $r.Archive,
                            format: $format,
                            role: $r.Role,
                            profile-name: $profile-name
                        }),
            NumRules: size($r.Rules),
            NumNetworks: components:num-networks($r),
            NumReportElements: components:num-report-elements($r),
            NumHypercubes: components:num-hypercubes($r),
            NumDimensions: components:num-dimensions($r),
            NumMembers: components:num-members($r),
            NumLineItems: components:num-line-items($r),
            NumAbstracts: components:num-abstracts($r),
            NumConcepts: components:num-concepts($r),
            Hypercubes: [ keys($r.Hypercubes) ],
            ValidationErrors: [ components:validation-errors($r) ]
          },
          { Disclosure: $r.Profiles.FSA.Disclosure}[$profile-name eq "japan"]
        |}
let $result := switch($profile-name)
               case "sec"
               return { Archives: [ $res ] }
               default return { Components : [ $res ] }
let $comment :=
 {
    NumComponents : count($components),
    TotalNumComponents: session:num-components(),
    TotalNumArchives: session:num-archives()
}
let $serializers := {
    to-xml : switch($profile-name)
        case "sec"
        return function($res as object) as node() {
        <Archives>{
                  for $r in flatten($res.Archives)
                  return
                      <Archive id="{$r.AccessionNumber}">
                         <EntityRegistrantName>{$r.EntityRegistrantName}</EntityRegistrantName>
                         <CIK>{$r.CIK}</CIK>
                         <FiscalYear>{$r.FiscalYear}</FiscalYear>
                         <FiscalPeriod>{$r.FiscalPeriod}</FiscalPeriod>
                         <AcceptanceDatetime>{$r.AcceptanceDatetime}</AcceptanceDatetime>
                         <FormType>{$r.FormType}</FormType>
                         <Components>{
                             sec-networks:summaries-to-xml(flatten($r.Components))
                         }</Components>
                     </Archive>
             }</Archives>
        }
        default return function($res as object) as node() {
        <Components>{
          api:json-to-xml($res.Components[], "Component")
        }</Components>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec" return string-join(local:to-csv($res.Archives[]), "")
        default return api:json-to-csv($res.Components[])
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "components")
return switch(true)
       case $entities-not-found return api:not-found("entity")
       case $archives-not-found return api:not-found("archive")
       default return api:check-and-return-results($token, $results, $format)

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/backend";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec-filings = "http://28.io/modules/xbrl/profiles/sec/filings";
import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare option rest:response "first-item";

declare function local:to-csv($res as object*) as string*
{
    csv:serialize(
        for $a in $res
        for $c in $a.Components[]
        return {
            AcessionNumber : $a.AccessionNumber,
            NetworkIdentifier : $c.NetworkIdentifier,
            FactTable: $c.FactTable,
            SpreadSheet: $c.SpreadSheet,
            EntityRegistrantName : $a.EntityRegistrantName,
            CIK : $a.CIK,
            FiscalYear : $a.FiscalYear,
            FiscalPeriod : $a.FiscalPeriod,
            AcceptanceDateTime : $a.AcceptanceDatetime,
            FormType : $a.FormType,
            NetworkLabel : $c.NetworkLabel,
            Category : $c.Category,
            SubCategory : $c.SubCategory,
            Table : flatten($c.Table),
            Disclosure : $c.Disclosure,
            ReportElements : $c.ReportElements,
            Tables : $c.Tables,
            Axis : $c.Axis,
            Members : $c.Members,
            LineItems : $c.LineItems,
            Concepts : $c.Concepts,
            Abstracts : $c.Abstracts
        },
    { serialize-null-as : "" })
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token              as string? external;
declare  %rest:env                              variable $request-uri        as string  external;
declare  %rest:case-insensitive                 variable $format             as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode   as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker             as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear         as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod       as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $filingKind         as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $eid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $networkIdentifier  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $role               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $reportElement      as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $concept            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $label              as string* external;
declare  %rest:case-insensitive                 variable $profile-name       as string  external := $config:profile-name;

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $reportElement := ($reportElement, $concept)
let $networkIdentifier := distinct-values(($networkIdentifier, $role))

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic, ())

let $entities-not-found as boolean :=
  exists(($eid, $cik, $tag, $ticker, $sic)) and empty($entities)

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $filingKind,
  $aid)

let $archives-not-found as boolean :=
  exists(($entities, $fiscalPeriod, $fiscalYear, $filingKind, $aid)) and empty($archives)

let $entities as object*  := entities:entities($archives.Entity)
let $components as object* :=
    multiplexer:components(
      $profile-name,
      $archives,
      $cid,
      $reportElement,
      $disclosure,
      $networkIdentifier,
      $label[$profile-name ne "sec"],
      $label[$profile-name eq "sec"]
    )

let $res as object* :=
    switch($profile-name)
    case "sec" return
        for $r in $components
        let $disclosure := sec-networks:disclosures($r)
        where $disclosure ne "DefaultComponent"
        order by $r.Label
        group by $archive := $r.Archive
        let $archive := $archives[archives:aid($$) eq $archive]
        let $e := $entities[entities:eid($$) eq $archive.Entity]
        return
            {
               AccessionNumber : archives:aid($archive),
               EntityRegistrantName : $e.Profiles.SEC.CompanyName,
               CIK : entities:eid($e),
               FiscalYear :$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
               FiscalPeriod :$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
               AcceptanceDatetime : sec-filings:acceptance-dateTimes($archive),
               FormType : $archive.Profiles.SEC.FormType,
               Components : [
                    for $component in sec-networks:summaries($r)
                    return copy $c := $component
                    modify insert json {
                        FactTable: backend:url("facttable-for-component", {
                            token: $token,
                            aid: archives:aid($archive),
                            format: $format,
                            role: $component.NetworkIdentifier,
                            profile-name: $profile-name
                            }),
                        SpreadSheet: "http://rendering.secxbrl.info/#?url=" || encode-for-uri(
                            backend:url("spreadsheet-for-component", {
                            token: $token,
                            aid: archives:aid($archive),
                            format: $format,
                            role: $component.NetworkIdentifier,
                            profile-name: $profile-name
                            })
                        )
                    } into $c
                    return $c
               ]
           }
    default return
        for $r in $components
        return {|
          {
            Archive: $r.Archive,
            Label: $r.Label,
            Role: $r.Role,
            FactTable: backend:url("facttable-for-component", {
                            token: $token,
                            aid: $r.Archive,
                            format: $format,
                            role: $r.Role,
                            profile-name: $profile-name
                            }),
            SpreadSheet: "http://rendering.secxbrl.info/#?url=" || encode-for-uri(
                        backend:url("spreadsheet-for-component", {
                            token: $token,
                            aid: $r.Archive,
                            format: $format,
                            role: $r.Role,
                            profile-name: $profile-name,
                            eliminate: "true"
                        })),
            ReportElements: backend:url("report-elements", {
                            token: $token,
                            aid: $r.Archive,
                            format: $format,
                            role: $r.Role,
                            profile-name: $profile-name
                        }),
            NumRules: size($r.Rules),
            NumNetworks: components:num-networks($r),
            NumReportElements: components:num-report-elements($r),
            NumHypercubes: components:num-hypercubes($r),
            NumDimensions: components:num-dimensions($r),
            NumMembers: components:num-members($r),
            NumLineItems: components:num-line-items($r),
            NumAbstracts: components:num-abstracts($r),
            NumConcepts: components:num-concepts($r),
            Hypercubes: [ keys($r.Hypercubes) ],
            ValidationErrors: [ components:validation-errors($r) ]
          },
          { Disclosure: $r.Profiles.FSA.Disclosure}[$profile-name eq "japan"]
        |}
let $result := switch($profile-name)
               case "sec"
               return { Archives: [ $res ] }
               default return { Components : [ $res ] }
let $comment :=
 {
    NumComponents : count($components),
    TotalNumComponents: session:num-components(),
    TotalNumArchives: session:num-archives()
}
let $serializers := {
    to-xml : switch($profile-name)
        case "sec"
        return function($res as object) as node() {
        <Archives>{
                  for $r in flatten($res.Archives)
                  return
                      <Archive id="{$r.AccessionNumber}">
                         <EntityRegistrantName>{$r.EntityRegistrantName}</EntityRegistrantName>
                         <CIK>{$r.CIK}</CIK>
                         <FiscalYear>{$r.FiscalYear}</FiscalYear>
                         <FiscalPeriod>{$r.FiscalPeriod}</FiscalPeriod>
                         <AcceptanceDatetime>{$r.AcceptanceDatetime}</AcceptanceDatetime>
                         <FormType>{$r.FormType}</FormType>
                         <Components>{
                             sec-networks:summaries-to-xml(flatten($r.Components))
                         }</Components>
                     </Archive>
             }</Archives>
        }
        default return function($res as object) as node() {
        <Components>{
          api:json-to-xml($res.Components[], "Component")
        }</Components>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec" return string-join(local:to-csv($res.Archives[]), "")
        default return api:json-to-csv($res.Components[])
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "components")
return switch(true)
       case $entities-not-found return api:not-found("entity")
       case $archives-not-found return api:not-found("archive")
       case api:check-and-return-results($token, $results, $format)