jq: Add msg & time filters [ci skip]

jq: Add msg & time filters [ci skip]

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v);

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: clean | select(is_error);
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: clean | select(is_warn);
def warn_strict: clean | select(is_warn_strict);
def info_level: 30;
def is_info: .level >= info_level;
def info: clean | select(is_info);
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: clean | select(is_debug);

# Components:
def chokidar: select(.component == "Chokidar");
def LocalChange: select(.component == "local/change");
def LocalWatcher: select(.component == "LocalWatcher");
def Merge: select(.component == "Merge");
def Metadata: select(.component == "Metadata");
def Pouch: select(.component == "Pouch");
def Sync: select(.component == "Sync");

# Find conflicts:
#
#     yarn jq -c conflicts path/to/logs*
#
def is_conflict: .msg == "resolveConflictAsync";
def conflicts: clean | select(is_conflict) | {time,path};

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
def path(pattern): clean | select((.path,.oldpath,"") | strings | test(pattern));

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#
def time(pattern):
    select(.time | test(pattern));

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path};

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v);

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: clean | select(is_error);
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: clean | select(is_warn);
def warn_strict: clean | select(is_warn_strict);
def info_level: 30;
def is_info: .level >= info_level;
def info: clean | select(is_info);
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: clean | select(is_debug);

# Components:
def chokidar: select(.component == "Chokidar");
def LocalChange: select(.component == "local/change");
def LocalWatcher: select(.component == "LocalWatcher");
def Merge: select(.component == "Merge");
def Metadata: select(.component == "Metadata");
def Pouch: select(.component == "Pouch");
def Sync: select(.component == "Sync");

# Find conflicts:
#
#     yarn jq -c conflicts path/to/logs*
#
def is_conflict: .msg == "resolveConflictAsync";
def conflicts: clean | select(is_conflict) | {time,path};

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
def path(pattern): clean | select((.path,.oldpath,"") | strings | test(pattern));

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path};

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );


Update facttable-for-report.jq

Update facttable-for-report.jq

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace schema = "http://zorba.io/modules/schema";
import module namespace test = "http://apps.28.io/test";
import module namespace http-client = "http://zorba.io/modules/http-client";

declare function local:expected-OtherOperatingIncomeExpenses-ATnT-2013() as object {
    {
        "Aspects": {
            "xbrl28:Archive": "0000732717-14-000010",
            "xbrl:Concept": "fac:OtherOperatingIncomeExpenses",
            "xbrl:Entity": "http://www.sec.gov/CIK 0000732717",
            "xbrl:Period": "2013-01-01/2013-12-31",
            "sec:FiscalPeriod": "FY",
            "sec:FiscalYear": 2013,
            "sec:Accepted": "20140221162554",
            "dei:LegalEntityAxis": "sec:DefaultLegalEntity"
        },
        "KeyAspects": [
            "xbrl:Concept",
            "xbrl:Entity",
            "xbrl:Period",
            "xbrl:Unit",
            "sec:Accepted"
        ],
        "Type": "NumericValue",
        "Value": -145082000000,
        "Decimals": -6,
        "Concept": {
            "Balance": "CREDIT",
            "PeriodType": "duration",
            "DataType": "xbrli:monetaryItemType"
        },
        "AuditTrails": [
            {
                "Id": "bf2d1587-491c-4492-b587-ddd00d762f2d",
                "Type": "xbrl28:formula",
                "Label": "Other Operating Income (Expenses)",
                "Message": "fac:OtherOperatingIncomeExpenses[-145,082,000,000 USD] = fac:OperatingIncomeLoss[30,479,000,000 USD] - fac:GrossProfit[77,288,000,000 USD] - fac:OperatingExpenses[98,273,000,000 USD]",
                "Data": {
                    "OutputConcept": "fac:OtherOperatingIncomeExpenses"
                }
            }
        ],
        "xbrl28:Type": "xbrl28:formula",
        "EntityRegistrantName": "AT&T INC.",
        "Unit": "iso4217:USD"
    }
};

declare function local:compare-audit-trails($expected as array?, $actual as array?) as object* {
    for $exp as object at $pos in $expected[]
    let $act as object? := $actual[][$pos]
    return
        if (exists($exp.Data.ValidatedFacts))
        then
            let $expM := copy $e := $exp modify ( delete json $e.Data.ValidatedFacts ) return $e
            let $actM := copy $a := $act modify ( delete json $a.Data.ValidatedFacts ) return $a
            return  (
                if(deep-equal($expM, $actM))
                then ()
                else
                    {
                        type: "unexpected-audittrails-value",
                        expected: $expM,
                        actual: $actM
                    },
                if(count($exp.Data.ValidatedFacts[]) eq count($act.Data.ValidatedFacts[]))
                then ()
                else
                    {
                        type: "number-of-validated-facts-differs",
                        expected: $exp,
                        actual: $act
                    }
            )
        else
            if(deep-equal($exp, $act))
            then ()
            else
                {
                    type: "unexpected-audittrails-value",
                    expected: $exp,
                    actual: $act
                }
};

declare function local:diff-facts($fact-expected as object, $fact-actual as object?) as object* {
    if(empty($fact-actual))
    then
        {
            type: "missing-fact-in-result"
        }
    else
        (
            for $key in distinct-values((keys($fact-expected), keys($fact-actual)))
            where not($key = ("Aspects", "KeyAspects", "Profiles", "AuditTrails"))
            let $exp-value := $fact-expected.$key
            let $act-value := $fact-actual.$key
            return
                if(deep-equal($exp-value, $act-value) or
                   (schema:schema-type($exp-value) eq schema:schema-type($act-value) and schema:schema-type($exp-value) eq xs:QName("xs:decimal") and fn:format-number(xs:decimal($exp-value),"#,##0.0000000000000") eq fn:format-number(xs:decimal($act-value),"#,##0.0000000000000"))
                )
                then ()
                else
                    {
                        type: "unexpected-fact-field-value",
                        field: $key,
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare aspects :)
            for $key in distinct-values((keys($fact-expected.Aspects), keys($fact-actual.Aspects)))
            let $exp-value := $fact-expected.Aspects.$key
            let $act-value := $fact-actual.Aspects.$key
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-aspect-value",
                        aspect: $key,
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare keyaspects :)
            let $exp-value := $fact-expected.KeyAspects
            let $act-value := $fact-actual.KeyAspects
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-keyaspects-value",
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare Profiles :)
            let $exp-value := $fact-expected.Profiles
            let $act-value := $fact-actual.Profiles
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-profiles-value",
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare Audittrails :)
            let $exp-value := $fact-expected.AuditTrails
            let $act-value := $fact-actual.AuditTrails
            return
                local:compare-audit-trails($exp-value, $act-value)
        )
};

declare function local:compare-fact-tables($fact-table-expected as object, $fact-table-actual as object) as object*{
    let $actual-facts as object+ := $fact-table-actual.FactTable[]
    let $expected-facts as object+ := $fact-table-expected.FactTable[]
    let $errors :=
        (
            for $exp in $expected-facts
            let $act := $actual-facts[$$.Aspects."xbrl:Concept" eq $exp.Aspects."xbrl:Concept"
                                      and $$.Aspects."xbrl:Entity" eq $exp.Aspects."xbrl:Entity"
                                      and $$.Aspects."xbrl:Period" eq $exp.Aspects."xbrl:Period"
                                      and $$.Aspects."xbrl28:Archive" eq $exp.Aspects."xbrl28:Archive"]
            let $errors as object* := local:diff-facts($exp, $act)
            return
                if(empty($errors))
                then ()
                else {
                    expectedFact: $exp,
                    actualFact: $act,
                    errors: [$errors]
                },

            for $act in $actual-facts
            let $exp := $expected-facts[$$.Aspects."xbrl:Concept" eq $act.Aspects."xbrl:Concept"
                                      and $$.Aspects."xbrl:Entity" eq $act.Aspects."xbrl:Entity"
                                      and $$.Aspects."xbrl:Period" eq $act.Aspects."xbrl:Period"
                                      and $$.Aspects."xbrl28:Archive" eq $act.Aspects."xbrl28:Archive"]
            where empty($exp)
            return
                {
                    unexpectedFact: $act
                }
        )
    return
        if(empty($errors))
        then ()
        else (
                {
                    expectedNumberOfFacts: count($expected-facts),
                    actualNumberOfFacts: count($actual-facts)
                },
                $errors
            )
};

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-facttable-fact($concept as string, $expected as object, $params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $facts as object* := $request[2].FactTable[]
    let $actual := $facts[$$.Aspects."xbrl:Concept" eq $concept]
    let $diff :=
      for $f in $actual return local:diff-facts($expected, $f)
    return if (empty($diff)) then true else { url: test:url($endpoint, $params), factDiffErrors: [ $diff ], expectedFact: $expected, actualFact: $actual }
};

declare %an:nondeterministic function local:test-report-does-not-exist($params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $status as integer := $request[1]
    return if ($status eq 404) then true else { url: test:url($endpoint, $params), unexpectedResponse: $request[2] }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-values() as item*
{
    let $endpoint := "facttable-for-report"
    let $params := {ticker:"ko",fiscalYear:"2013",fiscalPeriod:["FY", "YTD4", "QTD4"],report:"FundamentalAccountingConcepts"}
    let $request := test:invoke($endpoint, $params)
    let $actual as object := $request[2]
    let $expected := parse-json(
        http-client:get("http://" || request:server-name() || ":" || request:server-port() ||
                        "/test/facttable-for-report-expected1.jq").body.content)
    let $diff := local:compare-fact-tables($expected, $actual)
    return if (empty($diff)) then true else { url: test:url($endpoint, $params), factTableDiff: [ ({ params : $params },$diff) ], expectedFactTable: $expected, actualFactTable: $actual }
};

local:check({
    cocacola: local:test-facttable(95, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2013",
        fiscalPeriod:["Q1", "YTD1"]}),
    aid: local:test-facttable(95, {
        aid:"0001193125-14-157120",
        report:"FundamentalAccountingConcepts"}),
    aid2: local:test-facttable(96, {
        aid:"0000732717-14-000022",
        report:"FundamentalAccountingConcepts"}),
    tickerrole: local:test-facttable(95, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:["Q1", "YTD1"]}),
    tickerfyfprole: local:test-facttable(191, {
        report:"FundamentalAccountingConcepts",
        ticker:["ko","wmt"],
        fiscalYear:"2013",
        fiscalPeriod:["FY", "QTD4", "YTD4"]}),
    allvalues : local:test-values(),
    otheroperatingincometest: local:test-facttable-fact("fac:OtherOperatingIncomeExpenses",
                                                        local:expected-OtherOperatingIncomeExpenses-ATnT-2013(),
                                                        {
        report:"FundamentalAccountingConcepts",
        ticker:"t",
        fiscalYear:"2013",
        fiscalPeriod:["FY", "QTD4", "YTD4"]}),
    reportDoesntExist: local:test-report-does-not-exist({
        report:"report-not-found",
        ticker:"MSFT",
        fiscalYear:"ALL",
        fiscalPeriod:"FY"})
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace schema = "http://zorba.io/modules/schema";
import module namespace test = "http://apps.28.io/test";
import module namespace http-client = "http://zorba.io/modules/http-client";

declare function local:expected-OtherOperatingIncomeExpenses-ATnT-2013() as object {
    {
        "Aspects": {
            "xbrl28:Archive": "0000732717-14-000010",
            "xbrl:Concept": "fac:OtherOperatingIncomeExpenses",
            "xbrl:Entity": "http://www.sec.gov/CIK 0000732717",
            "xbrl:Period": "2013-01-01/2013-12-31",
            "sec:FiscalPeriod": "FY",
            "sec:FiscalYear": 2013,
            "sec:Accepted": "20140221162554",
            "dei:LegalEntityAxis": "sec:DefaultLegalEntity"
        },
        "KeyAspects": [
            "xbrl:Concept",
            "xbrl:Entity",
            "xbrl:Period",
            "xbrl:Unit",
            "sec:Accepted"
        ],
        "Type": "NumericValue",
        "Value": -145082000000,
        "Decimals": -6,
        "Concept": {
            "Balance": "CREDIT",
            "PeriodType": "duration",
            "DataType": "xbrli:monetaryItemType"
        },
        "AuditTrails": [
            {
                "Id": "bf2d1587-491c-4492-b587-ddd00d762f2d",
                "Type": "xbrl28:formula",
                "Label": "Other Operating Income (Expenses)",
                "Message": "fac:OtherOperatingIncomeExpenses[-145,082,000,000 USD] = fac:OperatingIncomeLoss[30,479,000,000 USD] - fac:GrossProfit[77,288,000,000 USD] - fac:OperatingExpenses[98,273,000,000 USD]",
                "Data": {
                    "OutputConcept": "fac:OtherOperatingIncomeExpenses"
                }
            }
        ],
        "xbrl28:Type": "xbrl28:formula",
        "EntityRegistrantName": "AT&T INC.",
        "Unit": "iso4217:USD"
    }
};

declare function local:compare-audit-trails($expected as array?, $actual as array?) as object* {
    for $exp as object at $pos in $expected[]
    let $act as object? := $actual[][$pos]
    return
        if (exists($exp.Data.ValidatedFacts))
        then
            let $expM := copy $e := $exp modify ( delete json $e.Data.ValidatedFacts ) return $e
            let $actM := copy $a := $act modify ( delete json $a.Data.ValidatedFacts ) return $a
            return  (
                if(deep-equal($expM, $actM))
                then ()
                else
                    {
                        type: "unexpected-audittrails-value",
                        expected: $expM,
                        actual: $actM
                    },
                if(count($exp.Data.ValidatedFacts[]) eq count($act.Data.ValidatedFacts[]))
                then ()
                else
                    {
                        type: "number-of-validated-facts-differs",
                        expected: $exp,
                        actual: $act
                    }
            )
        else
            if(deep-equal($exp, $act))
            then ()
            else
                {
                    type: "unexpected-audittrails-value",
                    expected: $exp,
                    actual: $act
                }
};

declare function local:diff-facts($fact-expected as object, $fact-actual as object?) as object* {
    if(empty($fact-actual))
    then
        {
            type: "missing-fact-in-result"
        }
    else
        (
            for $key in distinct-values((keys($fact-expected), keys($fact-actual)))
            where not($key = ("Aspects", "KeyAspects", "Profiles", "AuditTrails"))
            let $exp-value := $fact-expected.$key
            let $act-value := $fact-actual.$key
            return
                if(deep-equal($exp-value, $act-value) or
                   (schema:schema-type($exp-value) eq schema:schema-type($act-value) and schema:schema-type($exp-value) eq xs:QName("xs:decimal") and fn:format-number(xs:decimal($exp-value),"#,##0.0000000000000") eq fn:format-number(xs:decimal($act-value),"#,##0.0000000000000"))
                )
                then ()
                else
                    {
                        type: "unexpected-fact-field-value",
                        field: $key,
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare aspects :)
            for $key in distinct-values((keys($fact-expected.Aspects), keys($fact-actual.Aspects)))
            let $exp-value := $fact-expected.Aspects.$key
            let $act-value := $fact-actual.Aspects.$key
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-aspect-value",
                        aspect: $key,
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare keyaspects :)
            let $exp-value := $fact-expected.KeyAspects
            let $act-value := $fact-actual.KeyAspects
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-keyaspects-value",
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare Profiles :)
            let $exp-value := $fact-expected.Profiles
            let $act-value := $fact-actual.Profiles
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-profiles-value",
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare Audittrails :)
            let $exp-value := $fact-expected.AuditTrails
            let $act-value := $fact-actual.AuditTrails
            return
                local:compare-audit-trails($exp-value, $act-value)
        )
};

declare function local:compare-fact-tables($fact-table-expected as object, $fact-table-actual as object) as object*{
    let $actual-facts as object+ := $fact-table-actual.FactTable[]
    let $expected-facts as object+ := $fact-table-expected.FactTable[]
    let $errors :=
        (
            for $exp in $expected-facts
            let $act := $actual-facts[$$.Aspects."xbrl:Concept" eq $exp.Aspects."xbrl:Concept"
                                      and $$.Aspects."xbrl:Entity" eq $exp.Aspects."xbrl:Entity"
                                      and $$.Aspects."xbrl:Period" eq $exp.Aspects."xbrl:Period"
                                      and $$.Aspects."xbrl28:Archive" eq $exp.Aspects."xbrl28:Archive"]
            let $errors as object* := local:diff-facts($exp, $act)
            return
                if(empty($errors))
                then ()
                else {
                    expectedFact: $exp,
                    actualFact: $act,
                    errors: [$errors]
                },

            for $act in $actual-facts
            let $exp := $expected-facts[$$.Aspects."xbrl:Concept" eq $act.Aspects."xbrl:Concept"
                                      and $$.Aspects."xbrl:Entity" eq $act.Aspects."xbrl:Entity"
                                      and $$.Aspects."xbrl:Period" eq $act.Aspects."xbrl:Period"
                                      and $$.Aspects."xbrl28:Archive" eq $act.Aspects."xbrl28:Archive"]
            where empty($exp)
            return
                {
                    unexpectedFact: $act
                }
        )
    return
        if(empty($errors))
        then ()
        else (
                {
                    expectedNumberOfFacts: count($expected-facts),
                    actualNumberOfFacts: count($actual-facts)
                },
                $errors
            )
};

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-facttable-fact($concept as string, $expected as object, $params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $facts as object* := $request[2].FactTable[]
    let $actual := $facts[$$.Aspects."xbrl:Concept" eq $concept]
    let $diff :=
      for $f in $actual return local:diff-facts($expected, $f)
    return if (empty($diff)) then true else { url: test:url($endpoint, $params), factDiffErrors: [ $diff ], expectedFact: $expected, actualFact: $actual }
};

declare %an:nondeterministic function local:test-report-does-not-exist($params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $status as integer := $request[1]
    return if ($status eq 404) then true else { url: test:url($endpoint, $params), unexpectedResponse: $request[2] }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-values() as item*
{
    let $endpoint := "facttable-for-report"
    let $params := {ticker:"ko",fiscalYear:"2013",fiscalPeriod:["FY", "YTD4", "QTD4"],report:"FundamentalAccountingConcepts"}
    let $request := test:invoke($endpoint, $params)
    let $actual as object := $request[2]
    let $expected := parse-json(
        http-client:get("http://" || request:server-name() || ":" || request:server-port() ||
                        "/test/facttable-for-report-expected1.json").body.content)
    let $diff := local:compare-fact-tables($expected, $actual)
    return if (empty($diff)) then true else { url: test:url($endpoint, $params), factTableDiff: [ ({ params : $params },$diff) ], expectedFactTable: $expected, actualFactTable: $actual }
};

local:check({
    cocacola: local:test-facttable(95, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2013",
        fiscalPeriod:["Q1", "YTD1"]}),
    aid: local:test-facttable(95, {
        aid:"0001193125-14-157120",
        report:"FundamentalAccountingConcepts"}),
    aid2: local:test-facttable(96, {
        aid:"0000732717-14-000022",
        report:"FundamentalAccountingConcepts"}),
    tickerrole: local:test-facttable(95, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:["Q1", "YTD1"]}),
    tickerfyfprole: local:test-facttable(191, {
        report:"FundamentalAccountingConcepts",
        ticker:["ko","wmt"],
        fiscalYear:"2013",
        fiscalPeriod:["FY", "QTD4", "YTD4"]}),
    allvalues : local:test-values(),
    otheroperatingincometest: local:test-facttable-fact("fac:OtherOperatingIncomeExpenses",
                                                        local:expected-OtherOperatingIncomeExpenses-ATnT-2013(),
                                                        {
        report:"FundamentalAccountingConcepts",
        ticker:"t",
        fiscalYear:"2013",
        fiscalPeriod:["FY", "QTD4", "YTD4"]}),
    reportDoesntExist: local:test-report-does-not-exist({
        report:"report-not-found",
        ticker:"MSFT",
        fiscalYear:"ALL",
        fiscalPeriod:"FY"})
})


Add "next" and "prev" entries to every page.

Only in the `index.json`; it is not actually inserted into templates
yet, since I'm not yet sure how to do that elegantly.

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    [. == (null,false,0,{},[])] | any | not
;

# Is a given array at least length $n?
def at_least($n):
    (. | length) >= $n
;

# Generate this object and all its children.
def all_children:
    ., recurse(.contents?[]?)
;

# To remove an object, we first mark it for removal, then actually remove it
# later. It would be nicer if we could just do `all_children ... |= empty`,
# which works in some cases but not all. I suppose it has to do with changing
# objects as we are iterating over them.
def mark:
    .remove = true
;

# Remove all objects marked for removal.
def remove_marked:
    if .remove == true
    then empty
    else (.contents // empty) |= map(remove_marked)
    end
;

# Enumerate the `make` formats of a page.
def target_formats($all_formats):
    (.make // empty)
    |   if . == "null" then empty else . end # fix for wrong YAML parse
    |   if (. | type) == "array" then .[] else . end
    |   tostring
    |   ascii_downcase
    |   if [. == $all_formats[]] | any then .
        elif . == "all" then $all_formats[]
        else error("unrecognized format: \(.)")
        end
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`. We use `name` and `contents` to be compatible
# with the output of the `tree` program on the shell.
def tree($path):
    if $path | at_least(2)
    then { "contents": [ tree($path[1:]) ] }
    else .
    end | .name = $path[0]
;

# Merge an array of page objects into a single page object.
def merge:
    map(to_entries) | add | group_by(.key) | map(
        { "key": (.[0].key)
        , "value": (
            if .[0].key == "contents"
            then [ .[].value ] | add | group_by(.name) | map(merge)
            elif [ .[].value == .[0].value ] | all
            then .[0].value
            else error("can't merge incompatible values")
            end
        ) }
    ) | from_entries
;

# Add paths to each object, that is, the names of the ancestors.
def directory:
    def f($d):
        .dir = ($d | join("/") | ltrimstr("./"))
        | .name as $n | .contents[]? |= f($d+[$n])
    ;
    f([])
;

# Every leaf node gets a basename, eg name without extension.
def basename:
    (all_children | select(has("contents") | not)) |= (
        .basename = (.name | sub(".([A-z]*)$";""))
    )
;

# Any page gets its target formats.
def formats($all_formats):
    all_children |= (
        .formats = [ target_formats($all_formats) ]
    )
;

# Any page is linked to its first target format.
def link:
    (all_children | select(has("dir") and has("basename") and (.formats | at_least(1)))) |= (
        .link = "\(.dir)/\(.basename).\(.formats[0])"
    )
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def frontmatter:
    (all_children | select(has("contents"))) |= (
        (.frontmatter = [.contents[] | select(.frontmatter | bool)][0]) |
        (.contents = .contents - [.frontmatter])
    )
;

# Drafts can be excluded from being uploaded and included in the table of
# contents. To not count as a draft, a document should be explicitly marked as
# "make" in the metadata.
# Anything that does not have a link, nor has any children, should be removed
# from the index.
def remove_unlinked:
    def linked: has("contents") or has("link");
    (( all_children | select(linked | not)) |= mark) | remove_marked
;

# Add links to the next and previous page for every linked page.
def navigation:
    def f($paths; $i; $j; $key):
        setpath($paths[$i]+[$key];
            getpath($paths[$j]) | {link,title,date}
        )
    ;
    [ path(all_children | select(has("link"))) ] as $paths
    | ($paths | length) as $n
    | reduce range(0; $n-1) as $i (.; f($paths; $i; $i+1; "next"))
    | reduce range(1, $n)   as $i (.; f($paths; $i; $i-1; "prev"))
;

# Sort content first according to sort order in metadata.
def sort_content:
    (all_children | select(has("contents")) | .contents) |= (
        sort_by(.sort // .title // .name)
    )
;

# Add a note that tells us whether this has only children who have no more
# subchildren.
def annotate_leaves:
    (all_children | select(has("contents"))) |= (
        .only_leaves = (.contents | all(has("contents") | not))
    )
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index($all_formats):
    merge
    | directory
    | basename
    | formats($all_formats)
    | link
    | remove_unlinked
    | navigation
    | frontmatter
    | sort_content
    | annotate_leaves
;

# Get target files and their dependencies as an enumeration of {"target":...,
# "dependencies":...} objects.
def targets($dest; $default_style):
    [ all_children
        | ., (.frontmatter // empty)
        | .formats[] as $format
        | "\($dest)/\(.dir)/\(.basename).\($format)" as $doc
        | ["\($dest)/\(.dir)/\(.resources?[])"] as $external
        | ["\($dest)/\(.style // $default_style).css"] as $css
        | (if $format == "html" then ($css + $external) else [] end) as $linked
        | (if $format == "pdf"  then ($css + $external) else [] end) as $embedded
        | {"target": $format, "dependencies": ([$doc] + $linked) }
        , {"target": $doc, "dependencies": $embedded }
    ]
    | group_by(.target)
    | map(
        {"target": (.[0].target)
        , "dependencies": (map(.dependencies) | add | unique) }
    )
    | .[]?
;

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    [. == (null,false,0,{},[])] | any | not
;

# Is a given array at least length $n?
def at_least($n):
    (. | length) >= $n
;

# Generate this object and all its children.
def all_children:
    ., recurse(.contents?[]?)
;

# To remove an object, we first mark it for removal, then actually remove it
# later. It would be nicer if we could just do `all_children ... |= empty`,
# which works in some cases but not all. I suppose it has to do with changing
# objects as we are iterating over them.
def mark:
    .remove = true
;

# Remove all objects marked for removal.
def remove_marked:
    if .remove == true
    then empty
    else (.contents // empty) |= map(remove_marked)
    end
;

# Enumerate the `make` formats of a page.
def target_formats($all_formats):
    (.make // empty)
    |   if . == "null" then empty else . end # fix for wrong YAML parse
    |   if (. | type) == "array" then .[] else . end
    |   tostring
    |   ascii_downcase
    |   if [. == $all_formats[]] | any then .
        elif . == "all" then $all_formats[]
        else error("unrecognized format: \(.)")
        end
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`. We use `name` and `contents` to be compatible
# with the output of the `tree` program on the shell.
def tree($path):
    if $path | at_least(2)
    then { "contents": [ tree($path[1:]) ] }
    else .
    end | .name = $path[0]
;

# Merge an array of page objects into a single page object.
def merge:
    map(to_entries) | add | group_by(.key) | map(
        { "key": (.[0].key)
        , "value": (
            if .[0].key == "contents"
            then [ .[].value ] | add | group_by(.name) | map(merge)
            elif [ .[].value == .[0].value ] | all
            then .[0].value
            else error("can't merge incompatible values")
            end
        ) }
    ) | from_entries
;

# Add paths to each object, that is, the names of the ancestors.
def directory:
    def f($d):
        .dir = ($d | join("/") | ltrimstr("./"))
        | .name as $n | .contents[]? |= f($d+[$n])
    ;
    f([])
;

# Every leaf node gets a basename, eg name without extension.
def basename:
    (all_children | select(has("contents") | not)) |= (
        .basename = (.name | sub(".([A-z]*)$";""))
    )
;

# Any page gets its target formats.
def formats($all_formats):
    all_children |= (
        .formats = [ target_formats($all_formats) ]
    )
;

# Any page is linked to its first target format.
def link:
    (all_children | select(has("dir") and has("basename") and (.formats | at_least(1)))) |= (
        .link = "\(.dir)/\(.basename).\(.formats[0])"
    )
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def frontmatter:
    (all_children | select(has("contents"))) |= (
        (.frontmatter = [.contents[] | select(.frontmatter | bool)][0]) |
        (.contents = .contents - [.frontmatter])
    )
;

# Drafts can be excluded from being uploaded and included in the table of
# contents. To not count as a draft, a document should be explicitly marked as
# "make" in the metadata.
# Anything that does not have a link, nor has any children, should be removed
# from the index.
def remove_unlinked:
    def linked: has("contents") or has("link");
    (( all_children | select(linked | not)) |= mark) | remove_marked
;

# Sort content first according to sort order in metadata.
def sort_content:
    (all_children | select(has("contents")) | .contents) |= (
        sort_by(.sort // .title // .name)
    )
;

# Add a note that tells us whether this has only children who have no more
# subchildren.
def annotate_leaves:
    (all_children | select(has("contents"))) |= (
        .only_leaves = (.contents | all(has("contents") | not))
    )
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index($all_formats):
    merge
    | directory
    | basename
    | formats($all_formats)
    | link
    | remove_unlinked
    | frontmatter
    | sort_content
    | annotate_leaves
;

# Get target files and their dependencies as an enumeration of {"target":...,
# "dependencies":...} objects.
def targets($dest; $default_style):
    [ all_children
        | ., (.frontmatter // empty)
        | .formats[] as $format
        | "\($dest)/\(.dir)/\(.basename).\($format)" as $doc
        | ["\($dest)/\(.dir)/\(.resources?[])"] as $external
        | ["\($dest)/\(.style // $default_style).css"] as $css
        | (if $format == "html" then ($css + $external) else [] end) as $linked
        | (if $format == "pdf"  then ($css + $external) else [] end) as $embedded
        | {"target": $format, "dependencies": ([$doc] + $linked) }
        , {"target": $doc, "dependencies": $embedded }
    ]
    | group_by(.target)
    | map(
        {"target": (.[0].target)
        , "dependencies": (map(.dependencies) | add | unique) }
    )
    | .[]?
;


:green_heart: Fix test.

:green_heart: Fix test.

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";
import module namespace config = "http://apps.28.io/config";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facts"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-404($params as object) as item
{
    let $endpoint := "facts"
    let $res as object := test:invoke-raw($endpoint, $params)
    return if($res.status eq 404)
           then true
           else {
              url: test:url($endpoint, $params),
              unexpectedResponse: $res
           }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw($endpoint, $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "Archive [Axis],Fiscal Period [Axis],Fiscal Year [Axis],Fiscal Period Type [Axis],Period [Axis],Reporting Entity [Axis],Concept [Axis],Unit [Axis],Legal Entity [Axis],Value,Decimals,Balance,Period Type,Data Type\r\n0000021344-13-000050,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),Assets,USD,Default Legal Entity [Member],89432000000,-6,Debit,instant,Monetary\r\n0000021344-14-000036,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),\"Cash and Cash Equivalents, at Carrying Value\",USD,Default Legal Entity [Member],11118000000,-6,Debit,instant,Monetary\r\n"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res,
        expected: $expectedLines
    }
};

declare %an:nondeterministic function local:test-labels-aids() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: "disc:AdvertisingCostsPolicyTextBlock",
        report: "Disclosures",
        format: "json",
        fiscalYear: 2014,
        fiscalPeriod: "FY",
        labels: true,
        aid: "0000858877-14-000029"
    }
    let $res := test:invoke($endpoint, $params)
    let $actual := [
            for $labels in $res[2].FactTable[].Labels
            return (keys($labels) ! $labels.$$)
        ]
    let $expected := [ "Archive [Axis]", "Fiscal Year [Axis]", "Non Numeric", "Default Legal Entity [Member]", "Unit [Axis]", "Advertising Costs, Policy", "Fiscal Period [Axis]", "Concept [Axis]", "Reporting Entity [Axis]", "Legal Entity [Axis]", "Fiscal Period Type [Axis]", "Period [Axis]", "CISCO SYSTEMS, INC. (858877)", "From July 28, 2013 to July 26, 2014", "Period Type", "Data Type" ]
    let $status as integer := $res[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

if(not $config:content[] = ("SEC-big", "SEC-small"))
then true
else
local:check({
    cocacola-latest:
    (: this test will fail and needs to be updated if a newer report has been filed.
       current latest filing: 2014 :)
    local:test-facttable(462, {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    cocacola-all: local:test-facttable(477, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-all-q3: local:test-facttable(341, {
        ticker:"ko",
        fiscalYear: 2014,
        fiscalPeriod: [ "Q3" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-instant: local:test-facttable(170, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant" ]
    }),
    cocacola-ytd: local:test-facttable(307, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-ytd-several-years: local:test-facttable(607, {
        ticker:"ko",
        fiscalYear: [ 2013, 2014 ],
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-qtd: local:test-facttable(102, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "Q2" ],
        fiscalPeriodType: [ "QTD" ]
    }),
    cocacolaCSVLabels: local:test-labels(),
    ciscoLabelsByAid: local:test-labels-aids(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(30, { (: AT&T wrongly stamped a filing as FY, but it's now FIXED :-):)
        tag:"DOW30",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-404({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    }),
    quartersfact: local:test-facttable(0, { (: this fact is not reported in the raw XML filing :)
        aid: "0000051143-13-000007",
        concept: "disc:AssetRetirementObligationsPolicy",
        fiscalPeriod: "Q3",
        fiscalYear: "2013",
        map: "Disclosures"
    }),
    quartersfact2: local:test-facttable(1, {
        aid: "0001193125-12-207154",
        concept: "disc:CashFlowOperatingCapitalTableTextBlock",
        fiscalPeriod: "Q1",
        fiscalYear: "2012",
        map: "Disclosures"
    }),
    generic: local:test-facttable(2, {
        "xbrl:Entity":"http://www.sec.gov/CIK 0000021344",
        "sec:Accepted":"20140227132423",
        concept:"us-gaap:Assets",
        profile-name: "generic"
    }),
    labels: test:invoke-and-assert-deep-equal(
          "facts",
          {
            tag : "DOW30",
            concept:"us-gaap:Assets",
            labels: "true",
            fiscalPeriod: "FY",
            fiscalYear: "2014"
          },
          function($res as item*) as item* {
            count(distinct-values($res.FactTable[].Labels."dei:LegalEntityAxis")),
            count($res.FactTable[].Labels."dei:LegalEntityAxis")
          },
          (1, 31)
      ),
    count: test:invoke-and-assert-deep-equal(
          "facts",
          {
              "ticker" : "ko",
              "fiscalYear" : ["2012", "2013", "2014"],
              "fiscalPeriod" : "FY",
              "count" : "true"
          },
          function($res as item*) as item* { $res.Statistics.NumFacts },
          1444
        ),
    odata: test:invoke-and-assert-deep-equal(
      "facts",
      {
          "ticker" : "ko",
          "fiscalYear" : "ALL",
          "fiscalPeriod" : "FY",
          "concept" : [ "us-gaap:Assets", "us-gaap:LiabilitiesCurrent", "us-gaap:StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" ],
          "format" : "atom-xml"
      },
      function($res as item*) as item* { count($res/*:feed/*:entry) },
      18,
      { Format: "xml" }
    ),
    slicers-columns: test:invoke-and-assert-deep-equal(
          "facts",
          {
            tag : "DOW30",
            concept:"us-gaap:Assets",
            labels: "true",
            fiscalPeriod: "FY",
            fiscalYear: "2014",
            "xbrl:Period::slicer": "true",
            "xbrl28:Archive::slicer": "true",
            "dei:LegalEntityAxis::slicer": "true",
            "sec:FiscalPeriod::slicer": "true",
            "sec:FiscalYear::slicer": "true",
            "sec:FiscalPeriodType::slicer": "true",
            "xbrl:Concept::column": "1",
            "xbrl:Entity::column": "2",
            "xbrl:Unit::column": "3"
          },
          function($res as item*) as item* {
            keys($res.FactTable[[1]].Aspects)
          },
          ("xbrl:Concept", "xbrl:Entity", "xbrl:Unit")
      )
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";
import module namespace config = "http://apps.28.io/config";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facts"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-404($params as object) as item
{
    let $endpoint := "facts"
    let $res as object := test:invoke-raw($endpoint, $params)
    return if($res.status eq 404)
           then true
           else {
              url: test:url($endpoint, $params),
              unexpectedResponse: $res
           }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw($endpoint, $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "Archive [Axis],Fiscal Period [Axis],Fiscal Year [Axis],Fiscal Period Type [Axis],Period [Axis],Reporting Entity [Axis],Concept [Axis],Unit [Axis],Legal Entity [Axis],Value,Decimals,Balance,Period Type,Data Type\r\n0000021344-13-000050,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),Assets,USD,Default Legal Entity [Member],89432000000,-6,DEBIT,instant,xbrli:monetaryItemType\r\n0000021344-14-000036,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),\"Cash and Cash Equivalents, at Carrying Value\",USD,Default Legal Entity [Member],11118000000,-6,Debit,instant,Monetary\r\n"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res,
        expected: $expectedLines
    }
};

declare %an:nondeterministic function local:test-labels-aids() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: "disc:AdvertisingCostsPolicyTextBlock",
        report: "Disclosures",
        format: "json",
        fiscalYear: 2014,
        fiscalPeriod: "FY",
        labels: true,
        aid: "0000858877-14-000029"
    }
    let $res := test:invoke($endpoint, $params)
    let $actual := [
            for $labels in $res[2].FactTable[].Labels
            return (keys($labels) ! $labels.$$)
        ]
    let $expected := [ "Archive [Axis]", "Fiscal Year [Axis]", "Non Numeric", "Default Legal Entity [Member]", "Unit [Axis]", "Advertising Costs, Policy", "Fiscal Period [Axis]", "Concept [Axis]", "Reporting Entity [Axis]", "Legal Entity [Axis]", "Fiscal Period Type [Axis]", "Period [Axis]", "CISCO SYSTEMS, INC. (858877)", "From July 28, 2013 to July 26, 2014", "Period Type", "Data Type" ]
    let $status as integer := $res[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

if(not $config:content[] = ("SEC-big", "SEC-small"))
then true
else
local:check({
    cocacola-latest:
    (: this test will fail and needs to be updated if a newer report has been filed.
       current latest filing: 2014 :)
    local:test-facttable(462, {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    cocacola-all: local:test-facttable(477, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-all-q3: local:test-facttable(341, {
        ticker:"ko",
        fiscalYear: 2014,
        fiscalPeriod: [ "Q3" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-instant: local:test-facttable(170, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant" ]
    }),
    cocacola-ytd: local:test-facttable(307, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-ytd-several-years: local:test-facttable(607, {
        ticker:"ko",
        fiscalYear: [ 2013, 2014 ],
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-qtd: local:test-facttable(102, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "Q2" ],
        fiscalPeriodType: [ "QTD" ]
    }),
    cocacolaCSVLabels: local:test-labels(),
    ciscoLabelsByAid: local:test-labels-aids(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(30, { (: AT&T wrongly stamped a filing as FY, but it's now FIXED :-):)
        tag:"DOW30",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-404({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    }),
    quartersfact: local:test-facttable(0, { (: this fact is not reported in the raw XML filing :)
        aid: "0000051143-13-000007",
        concept: "disc:AssetRetirementObligationsPolicy",
        fiscalPeriod: "Q3",
        fiscalYear: "2013",
        map: "Disclosures"
    }),
    quartersfact2: local:test-facttable(1, {
        aid: "0001193125-12-207154",
        concept: "disc:CashFlowOperatingCapitalTableTextBlock",
        fiscalPeriod: "Q1",
        fiscalYear: "2012",
        map: "Disclosures"
    }),
    generic: local:test-facttable(2, {
        "xbrl:Entity":"http://www.sec.gov/CIK 0000021344",
        "sec:Accepted":"20140227132423",
        concept:"us-gaap:Assets",
        profile-name: "generic"
    }),
    labels: test:invoke-and-assert-deep-equal(
          "facts",
          {
            tag : "DOW30",
            concept:"us-gaap:Assets",
            labels: "true",
            fiscalPeriod: "FY",
            fiscalYear: "2014"
          },
          function($res as item*) as item* {
            count(distinct-values($res.FactTable[].Labels."dei:LegalEntityAxis")),
            count($res.FactTable[].Labels."dei:LegalEntityAxis")
          },
          (1, 31)
      ),
    count: test:invoke-and-assert-deep-equal(
          "facts",
          {
              "ticker" : "ko",
              "fiscalYear" : ["2012", "2013", "2014"],
              "fiscalPeriod" : "FY",
              "count" : "true"
          },
          function($res as item*) as item* { $res.Statistics.NumFacts },
          1444
        ),
    odata: test:invoke-and-assert-deep-equal(
      "facts",
      {
          "ticker" : "ko",
          "fiscalYear" : "ALL",
          "fiscalPeriod" : "FY",
          "concept" : [ "us-gaap:Assets", "us-gaap:LiabilitiesCurrent", "us-gaap:StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" ],
          "format" : "atom-xml"
      },
      function($res as item*) as item* { count($res/*:feed/*:entry) },
      18,
      { Format: "xml" }
    ),
    slicers-columns: test:invoke-and-assert-deep-equal(
          "facts",
          {
            tag : "DOW30",
            concept:"us-gaap:Assets",
            labels: "true",
            fiscalPeriod: "FY",
            fiscalYear: "2014",
            "xbrl:Period::slicer": "true",
            "xbrl28:Archive::slicer": "true",
            "dei:LegalEntityAxis::slicer": "true",
            "sec:FiscalPeriod::slicer": "true",
            "sec:FiscalYear::slicer": "true",
            "sec:FiscalPeriodType::slicer": "true",
            "xbrl:Concept::column": "1",
            "xbrl:Entity::column": "2",
            "xbrl:Unit::column": "3"
          },
          function($res as item*) as item* {
            keys($res.FactTable[[1]].Aspects)
          },
          ("xbrl:Concept", "xbrl:Entity", "xbrl:Unit")
      )
})


Update spreadsheet-for-report.jq

Update spreadsheet-for-report.jq

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-spreadsheet($expected as integer, $params as object) as item
{
    let $endpoint := "spreadsheet-for-report"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].TableSet[].TableCells.Facts[][].Value)
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-report-does-not-exist($params as object) as item
{
    let $endpoint := "spreadsheet-for-report"
    let $request := test:invoke($endpoint, $params)
    let $status as integer := $request[1]
    return if ($status eq 404) then true else { url: test:url($endpoint, $params), unexpectedResponse: $request[2] }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    cocacola: local:test-spreadsheet(96, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2013",
        fiscalPeriod:"Q1"}),
    tickerrole: local:test-spreadsheet(96, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1"}),
    tickerconcept: local:test-spreadsheet(410, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"ALL"}),
    tickerfyfprole: local:test-spreadsheet(193, {
        report:"FundamentalAccountingConcepts",
        ticker:["ko","wmt"],
        fiscalYear:"2013",
        fiscalPeriod:"FY",
        eliminate:"true"}),
    reportDoesntExist: local:test-report-does-not-exist({
        report:"report-not-found",
        ticker:"MSFT",
        fiscalYear:"ALL",
        fiscalPeriod:"FY"})
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-spreadsheet($expected as integer, $params as object) as item
{
    let $endpoint := "spreadsheet-for-report"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].TableSet[].TableCells.Facts[][].Value)
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-report-does-not-exist($params as object) as item
{
    let $endpoint := "spreadsheet-for-report"
    let $request := test:invoke($endpoint, $params)
    let $status as integer := $request[1]
    return if ($status eq 404) then true else { url: test:url($endpoint, $params), unexpectedResponse: $request[2] }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    cocacola: local:test-spreadsheet(96, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2013",
        fiscalPeriod:["Q1"]}),
    tickerrole: local:test-spreadsheet(96, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:["Q1"]}),
    tickerconcept: local:test-spreadsheet(508, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"ALL"}),
    tickerfyfprole: local:test-spreadsheet(249, {
        report:"FundamentalAccountingConcepts",
        ticker:["ko","wmt"],
        fiscalYear:"2013",
        fiscalPeriod:["FY"],
        eliminate:"true"}),
    reportDoesntExist: local:test-report-does-not-exist({
        report:"report-not-found",
        ticker:"MSFT",
        fiscalYear:"ALL",
        fiscalPeriod:["FY"]})
})


removed setup failure

removed setup failure

import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

declare %an:nondeterministic function local:test-concepts($expected as integer, $params as string) as atomic
{
    let $elements as item* := parse-json(http-client:get("http://" || request:server-name() || ":" || request:server-port() || "/v1/_queries/public/api/report-elements.jq?_method=POST" || $params).body.content).ReportElements[]
    let $actual as integer := count($elements)
    let $duplicates := for $element in $elements where $element.CIK instance of array return $element
    return
        if (exists($duplicates)) then
            "false [CIK " || serialize($duplicates[1].CIK, ()) || " is not unique]"
        else if ($actual eq $expected) then
            true
        else
            "false [Actual="||$actual||", Expected="||$expected ||"]"
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    all: local:test-concepts(3, "&cik=4962&name=us-gaap:Assets"),
    onlyNames: local:test-concepts(16143, "&tag=DOW30&fiscalYear=2012&fiscalPeriod=FY&onlyNames=true"),
    disclosures: local:test-concepts(1855, "&ticker=ko&fiscalYear=2012&fiscalPeriod=FY&report=Disclosure&concept=disc:ConsolidationVariableInterestEntityPolicy")
})

import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

declare %an:nondeterministic function local:test-concepts($expected as integer, $params as string) as atomic
{
    let $elements as item* := parse-json(http-client:get("http://" || request:server-name() || ":" || request:server-port() || "/v1/_queries/public/api/report-elements.jq?_method=POST" || $params).body.content).ReportElements[]
    let $actual as integer := count($elements)
    let $duplicates := for $element in $elements where $element.CIK instance of array return $element
        if (exists($duplicates)) then
            "false [CIK " || serialize($duplicates[1].CIK, ()) || " is not unique]"
        else if ($actual eq $expected) then
            true
        else
            "false [Actual="||$actual||", Expected="||$expected ||"]"
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    all: local:test-concepts(3, "&cik=4962&name=us-gaap:Assets"),
    onlyNames: local:test-concepts(16143, "&tag=DOW30&fiscalYear=2012&fiscalPeriod=FY&onlyNames=true"),
    disclosures: local:test-concepts(1855, "&ticker=ko&fiscalYear=2012&fiscalPeriod=FY&report=Disclosure&concept=disc:ConsolidationVariableInterestEntityPolicy")
})


:green_heart: Fix build.

:green_heart: Fix build.

jsoniq version "1.0";

(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for querying financial reports (filings)
 : submitted to the SEC.

 :
SEC Filings are nothing else than XBRL archives. For XBRL-generic requests on archives,
 : use the generic archives module.

 :
 :
With this module, you can access a number of properties of an SEC filing, such as
 : its document type. You can also retrieve an SEC filing given an SEC company (or its CIK).

 :
You can also access SEC filings statistics (the number of SEC Networks, of SEC Axes, etc).

 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace filings = "http://28.io/modules/xbrl/profiles/sec/filings";

import module namespace datetime = "http://zorba.io/modules/datetime";

import module namespace csv = "http://zorba.io/modules/json-csv";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 :
Retrieves all filings.

 :
 : @return all filings.
 :)
declare function filings:filings() as object*
{
  archives:archives()
};

(:~
 :
Retrieves the filings with the given IDs.

 :
 : @param $id the ids of the filings (accession numbers) or the filings themselves.
 :
 : @return the fiinngs with the given IDs
 :)
declare function filings:filings($filings as item*) as object*
{
  archives:archives($filings)
};

(:~
 :
Return all filings submitted by the supplied companies.

 :
 : @param $companies-or-ciks arbitrary number of company objects or CIKs.
 :
 : @return all filings submitted by these companies.
 :)
declare function filings:filings-for-companies($companies-or-ciks as item*) as object*
{
  let $companies := companies:companies($companies-or-ciks)
  return archives:archives-for-entities($companies)
};

(:~
 :
Retrieves the document type of the filings.

 :
 : @param $filings-or-ids a sequence of filings or their ids.
 :
 : @return the document types (10-K, 10-Q).
 :)
declare function filings:document-types(
    $filings-or-ids as item*) as string?
{
  archives:archives($filings-or-ids).Profiles.SEC.FormType
};

(:~
 :
 :
Retrieves the acceptance date of filings.

 :
 : @param $filings-or-ids a sequence of filings or their IDs.
 : @return the acceptance dateTimes.
 :)
declare function filings:acceptance-dateTimes($filings-or-ids as item*) as dateTime*
{
  archives:archives($filings-or-ids) !
    datetime:parse-dateTime($$.Profiles.SEC.AcceptanceDatetime, "%Y%m%d%H%M%S")
};

(:~
 :
 :
Retrieves the submission date of filings.

 :
 : @param $filings-or-ids a sequence of filings or their IDs.
 : @return the submission dates.
 :)
declare function filings:filing-dates($filings-or-ids as item*) as date*
{
  archives:archives($filings-or-ids) !
  date(string-join(tokenize($$.Profiles.SEC.FilingDate, ("/")), "-"))
};

(:~
 :
 :
Retrieves the generators of filings.

 :
 : @param $filings-or-ids a sequence of filings or their IDs.
 : @return the generators used.
 :)
declare function filings:generators($filings-or-ids as item*) as string*
{
  archives:archives($filings-or-ids).Profiles.SEC.Generator
};

(:~
 : Return the number of extension facts in each of the given filings.
 :
 : @param $filings-or-ids list of filings or IDs
 :
 : @return the said number of facts
 :)
declare function filings:num-extension-facts($filings-or-ids as item*) as integer*
{
  archives:archives($filings-or-ids) ! $$.Statistics.Profiles.SEC.NumExtensionFacts
};

(:~
 : Return the number of extension concepts in each of the given filings.
 :
 : @param $filings-or-ids list of filings or IDs
 :
 : @return the said number of concepts
 :)
declare function filings:num-extension-concepts($filings-or-ids as item*) as integer*
{
  archives:archives($filings-or-ids) ! $$.Statistics.Profiles.SEC.NumExtensionConcepts
};

(:~
 : Return the number of extension abstracts in each of the given filings.
 :
 : @param $filings-or-ids list of filings or IDs
 :
 : @return the said number of abstracts
 :)
declare function filings:num-extension-abstracts($filings-or-ids as item*) as integer*
{
  archives:archives($filings-or-ids) ! $$.Statistics.Profiles.SEC.NumExtensionAbstracts
};

(:~
 :
Returns summaries of the given filings or IDs.

 :
 :
The function eliminates duplicates and returns only one
 : summary object per filing.

 :
 : @param $filings-or-ids list of filings or IDs
 :
 : @return a sequence of objects, each one representing one summary
 :)
declare function filings:summaries($filings-or-ids as item*) as object*
{
  for $a in archives:archives($filings-or-ids)
  group by $a._id
  let $a := $a[1]
  return
    {
        CIK : $a.Entity,
        EntityRegistrantName : companies:companies($a.Entity).Profiles.SEC.CompanyName,
        FormType : filings:document-types($a),
        FiscalYear : fiscal:fiscal-year-for-filings($a),
        FiscalPeriod : fiscal:fiscal-period($a),
        Accepted : filings:acceptance-dateTimes($a),
        Generator : filings:generators($a),
        AccessionNumber: $a._id,
        SECFilingPage : $a.Profiles.SEC.SECFilingPage,
        XBRLInstanceURL : $a.InstanceURL,
        ExtensionFacts:  filings:num-extension-facts($a),
        ExtensionConcepts : filings:num-extension-concepts($a),
        ExtensionAbstracts : filings:num-extension-abstracts($a),
        Statistics: $a.Statistics
    }
};

(:~
 :
Returns an XML representation of the given filing summaries.

 :
 : @param $summaries summaries of the filings to convert to XML.
 :
 : @return a sequence of elements, each one representing one filing summary as XML
 :)
declare function filings:summaries-to-xml($summaries as object*) as element()*
{
  for $s in $summaries
  return
    <Filing>
        <CIK>{$s.CIK}</CIK>
        <EntityRegistrantName>{$s.EntityRegistrantName}</EntityRegistrantName>
        <FormType>{$s.FormType}</FormType>
        <FiscalYear>{$s.FiscalYear}</FiscalYear>
        <FiscalPeriod>{$s.FiscalPeriod}</FiscalPeriod>
        <Accepted>{$s.Accepted}</Accepted>
        <Generator>{$s.Generator}</Generator>
        <AccessionNumber>{$s.AccessionNumber}</AccessionNumber>
        <SECFilingPage>{$s.SECFilingPage}</SECFilingPage>
        <XBRLInstanceURL>{$s.XBRLInstanceURL}</XBRLInstanceURL>
        <Networks>{$s.Networks}</Networks>
        <ReportElements>{$s.ReportElements}</ReportElements>
        <Tables>{$s.Tables}</Tables>
        <Axis>{$s.Axis}</Axis>
        <Members>{$s.Members}</Members>
        <LineItems>{$s.LineItems}</LineItems>
        <Concepts>{$s.Concepts}</Concepts>
        <Abstracts>{$s.Abstracts}</Abstracts>
        <Footnotes>{$s.Footnotes}</Footnotes>
        <Facts>{$s.Facts}</Facts>
        <ExtensionFacts>{$s.ExtensionFacts}</ExtensionFacts>
        <ExtensionConcepts>{$s.ExtensionConcepts}</ExtensionConcepts>
        <ExtensionAbstracts>{$s.ExtensionAbstracts}</ExtensionAbstracts>
    </Filing>
};

(:~
 :
Returns a CSV representation of the given filing summaries.

 :
 :
Commas are used as column separators. Abscent values are serialized
 : as empty strings.

 :
 : @param $summaries summaries of the filings to convert to CSV.
 :
 : @return a sequence of strings, each one representing one filings summary as CSV
 :)
declare function filings:summaries-to-csv($summaries as object*) as string*
{
  csv:serialize($summaries, { serialize-null-as : "" })
};

jsoniq version "1.0";

(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for querying financial reports (filings)
 : submitted to the SEC.

 :
SEC Filings are nothing else than XBRL archives. For XBRL-generic requests on archives,
 : use the generic archives module.

 :
 :
With this module, you can access a number of properties of an SEC filing, such as
 : its document type. You can also retrieve an SEC filing given an SEC company (or its CIK).

 :
You can also access SEC filings statistics (the number of SEC Networks, of SEC Axes, etc).

 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace filings = "http://28.io/modules/xbrl/profiles/sec/filings";

import module namespace datetime = "http://zorba.io/modules/datetime";

import module namespace csv = "http://zorba.io/modules/json-csv";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 :
Retrieves all filings.

 :
 : @return all filings.
 :)
declare function filings:filings() as object*
{
  archives:archives()
};

(:~
 :
Retrieves the filings with the given IDs.

 :
 : @param $id the ids of the filings (accession numbers) or the filings themselves.
 :
 : @return the fiinngs with the given IDs
 :)
declare function filings:filings($filings as item*) as object*
{
  archives:archives($filings)
};

(:~
 :
Return all filings submitted by the supplied companies.

 :
 : @param $companies-or-ciks arbitrary number of company objects or CIKs.
 :
 : @return all filings submitted by these companies.
 :)
declare function filings:filings-for-companies($companies-or-ciks as item*) as object*
{
  let $companies := companies:companies($companies-or-ciks)
  return archives:archives-for-entities($companies)
};

(:~
 :
Retrieves the document type of the filings.

 :
 : @param $filings-or-ids a sequence of filings or their ids.
 :
 : @return the document types (10-K, 10-Q).
 :)
declare function filings:document-types(
    $filings-or-ids as item*) as string?
{
  archives:archives($filings-or-ids).Profiles.SEC.FormType
};

(:~
 :
 :
Retrieves the acceptance date of filings.

 :
 : @param $filings-or-ids a sequence of filings or their IDs.
 : @return the acceptance dateTimes.
 :)
declare function filings:acceptance-dateTimes($filings-or-ids as item*) as dateTime*
{
  archives:archives($filings-or-ids) !
    datetime:parse-dateTime($$.Profiles.SEC.AcceptanceDatetime, "%Y%m%d%H%M%S")
};

(:~
 :
 :
Retrieves the submission date of filings.

 :
 : @param $filings-or-ids a sequence of filings or their IDs.
 : @return the submission dates.
 :)
declare function filings:filing-dates($filings-or-ids as item*) as date*
{
  archives:archives($filings-or-ids) !
  date(string-join(tokenize($$.Profiles.SEC.FilingDate, ("/")), "-"))
};

(:~
 :
 :
Retrieves the generators of filings.

 :
 : @param $filings-or-ids a sequence of filings or their IDs.
 : @return the generators used.
 :)
declare function filings:generators($filings-or-ids as item*) as string*
{
  archives:archives($filings-or-ids).Profiles.SEC.Generator
};

(:~
 : Return the number of extension facts in each of the given filings.
 :
 : @param $filings-or-ids list of filings or IDs
 :
 : @return the said number of facts
 :)
declare function filings:num-extension-facts($filings-or-ids as item*) as integer*
{
  archives:archives($filings-or-ids) ! $$.Statistics.Profiles.SEC.NumExtensionFacts
};

(:~
 : Return the number of extension concepts in each of the given filings.
 :
 : @param $filings-or-ids list of filings or IDs
 :
 : @return the said number of concepts
 :)
declare function filings:num-extension-concepts($filings-or-ids as item*) as integer*
{
  archives:archives($filings-or-ids) ! $$.Statistics.Profiles.SEC.NumExtensionConcepts
};

(:~
 : Return the number of extension abstracts in each of the given filings.
 :
 : @param $filings-or-ids list of filings or IDs
 :
 : @return the said number of abstracts
 :)
declare function filings:num-extension-abstracts($filings-or-ids as item*) as integer*
{
  archives:archives($filings-or-ids) ! $$.Statistics.Profiles.SEC.NumExtensionAbstracts
};

(:~
 :
Returns summaries of the given filings or IDs.

 :
 :
The function eliminates duplicates and returns only one
 : summary object per filing.

 :
 : @param $filings-or-ids list of filings or IDs
 :
 : @return a sequence of objects, each one representing one summary
 :)
declare function filings:summaries($filings-or-ids as item*) as object*
{
  for $a in archives:archives($filings-or-ids)
  group by $a._id
  let $a := $a[1]
  return
    {
        CIK : $a.Entity,
        EntityRegistrantName : companies:companies($a.Entity).Profiles.SEC.CompanyName,
        FormType : filings:document-types($a),
        FiscalYear : fiscal:fiscal-year($a),
        FiscalPeriod : fiscal:fiscal-period($a),
        Accepted : filings:acceptance-dateTimes($a),
        Generator : filings:generators($a),
        AccessionNumber: $a._id,
        SECFilingPage : $a.Profiles.SEC.SECFilingPage,
        XBRLInstanceURL : $a.InstanceURL,
        ExtensionFacts:  filings:num-extension-facts($a),
        ExtensionConcepts : filings:num-extension-concepts($a),
        ExtensionAbstracts : filings:num-extension-abstracts($a),
        Statistics: $a.Statistics
    }
};

(:~
 :
Returns an XML representation of the given filing summaries.

 :
 : @param $summaries summaries of the filings to convert to XML.
 :
 : @return a sequence of elements, each one representing one filing summary as XML
 :)
declare function filings:summaries-to-xml($summaries as object*) as element()*
{
  for $s in $summaries
  return
    <Filing>
        <CIK>{$s.CIK}</CIK>
        <EntityRegistrantName>{$s.EntityRegistrantName}</EntityRegistrantName>
        <FormType>{$s.FormType}</FormType>
        <FiscalYear>{$s.FiscalYear}</FiscalYear>
        <FiscalPeriod>{$s.FiscalPeriod}</FiscalPeriod>
        <Accepted>{$s.Accepted}</Accepted>
        <Generator>{$s.Generator}</Generator>
        <AccessionNumber>{$s.AccessionNumber}</AccessionNumber>
        <SECFilingPage>{$s.SECFilingPage}</SECFilingPage>
        <XBRLInstanceURL>{$s.XBRLInstanceURL}</XBRLInstanceURL>
        <Networks>{$s.Networks}</Networks>
        <ReportElements>{$s.ReportElements}</ReportElements>
        <Tables>{$s.Tables}</Tables>
        <Axis>{$s.Axis}</Axis>
        <Members>{$s.Members}</Members>
        <LineItems>{$s.LineItems}</LineItems>
        <Concepts>{$s.Concepts}</Concepts>
        <Abstracts>{$s.Abstracts}</Abstracts>
        <Footnotes>{$s.Footnotes}</Footnotes>
        <Facts>{$s.Facts}</Facts>
        <ExtensionFacts>{$s.ExtensionFacts}</ExtensionFacts>
        <ExtensionConcepts>{$s.ExtensionConcepts}</ExtensionConcepts>
        <ExtensionAbstracts>{$s.ExtensionAbstracts}</ExtensionAbstracts>
    </Filing>
};

(:~
 :
Returns a CSV representation of the given filing summaries.

 :
 :
Commas are used as column separators. Abscent values are serialized
 : as empty strings.

 :
 : @param $summaries summaries of the filings to convert to CSV.
 :
 : @return a sequence of strings, each one representing one filings summary as CSV
 :)
declare function filings:summaries-to-csv($summaries as object*) as string*
{
  csv:serialize($summaries, { serialize-null-as : "" })
};


Update api.jq

Update api.jq

jsoniq version "1.0";

module namespace api = "http://apps.28.io/api";

import module namespace session    = "http://apps.28.io/session";
import module namespace resp       = "http://www.28msec.com/modules/http-response";
import module namespace sec-fiscal = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

declare function api:validate-regexp($name as string, $value as string?, $regexp as string)
as ()
{
  if (exists($value))
  then
    if (matches($value, "^" || $regexp || "$"))
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " does not match reg. expression " || $regexp || ".");
  else ();
};

declare function api:validate-enum($name as string, $value as string?, $enum as string*)
as ()
{
  if (exists($value))
  then
    if ($value = $enum)
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " is not one of these: " || string-join($enum,", ") || ".");
  else ();
};

declare %private function api:sgpl($count as integer, $singular as string, $plural as string)
as string
{
    if ($count eq 1)
    then $count || " " || $singular
    else $count || " " || $plural
};

declare function api:passed($timestamp as xs:dateTime?)
as string
{
    if (empty($timestamp) or $timestamp eq null)
    then "Never"
    else
        let $now := current-dateTime()
        let $difference := $now - $timestamp
        let $years := years-from-duration($difference)
        let $month := months-from-duration($difference)
        let $days := days-from-duration($difference)
        let $hours := hours-from-duration($difference)
        let $minutes := minutes-from-duration($difference)
        return
            if ($years gt 0) then api:sgpl($years,"year ago","years ago")
            else if ($month gt 0) then api:sgpl($month,"month ago","month ago")
            else if ($days gt 0) then api:sgpl($days, "day ago", "days ago")
            else if ($hours gt 0) then api:sgpl($hours, "hour ago", "hours ago")
            else if ($minutes gt 0) then api:sgpl($minutes, "minute ago", "minutes ago")
            else if ($difference div xs:dayTimeDuration("PT1S") lt 0) then "future"
            else "Just Now"
};

declare function api:success() as object
{
  { "success" : true }
};

declare function api:success($data as object()) as object
{
  {|
     {"success" : true },
     $data
  |}
};

declare %an:sequential function api:check-and-return-results(
    $token as string?,
    $results as item*,
    $format as string?
) as item*
{
    switch(session:has-access($token, "data_sec"))
    case $session:ACCESS-ALLOWED return
        $results
    case $session:ACCESS-DENIED return {
          resp:status-code(403);
          session:error("access denied", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          resp:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()
};

declare function api:normalize-facts(
    $facts as object*) as object*
{
    for $fact in $facts
    return {|
        {
            "Aspects" : {|
                trim($fact.Aspects, ("xbrl:Unit"))
            |}
        },
        trim($fact, ("Aspects", "_id")),
        { Unit: $fact.Aspects."xbrl:Unit" }[exists($fact.Aspects."xbrl:Unit")]
    |}
};

declare %an:sequential function api:serialize(
    $result as json-item,
    $comment as object,
    $serializers as object,
    $format as string?,
    $file-name as string) as item*
{
    switch ($format)
    case "xml" return {
        resp:serialization-parameters({"omit-xml-declaration" : false, indent : true });
        session:comment("xml", $comment),
        $serializers.to-xml($result)
    }
    case "text" case "csv" return {
        resp:content-type("text/csv");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    case "excel" return {
        resp:content-type("application/vnd.ms-excel");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    default return {
        resp:content-type("application/json");
        resp:serialization-parameters({"indent" : true});
        {|
            session:comment("json", $comment),
            $result
        |}
    }
};

declare function api:preprocess-fiscal-years($fiscal-years as string*) as integer*
{
  distinct-values(
    for $fy in $fiscal-years ! upper-case($$)
    return switch($fy)
           case "LATEST" return $sec-fiscal:LATEST_FISCAL_YEAR
           case "ALL" return $sec-fiscal:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
  )
};

declare function api:preprocess-fiscal-periods($fiscal-periods as string*) as string*
{
  distinct-values(
    for $fp in $fiscal-periods ! upper-case($$)
    return switch($fp)
           case "Q1"
           case "Q2"
           case "Q3"
           case "FY"
             return $fp
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIODS
           default
             return error(xs:QName("local:INVALID-PERIOD"),
               $fp || ": fiscalPeriod values must be one or more of Q1, Q2, Q3, FY, ALL")
  )
};

declare function api:preprocess-fiscal-period-types($fiscal-period-types as string*) as string*
{
  distinct-values(
    for $fpt in $fiscal-period-types ! upper-case($$)
    return switch($fpt)
           case "instant"
           case "QTD"
           case "YTD"
             return $fpt
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIOD_TYPES
           default
             return error(xs:QName("local:INVALID-PERIOD-TYPE"),
               $fpt || ": fiscalPeriodType values must be one or more of instant, YTD, QTD")
  )
};

declare function api:preprocess-format($format as string?, $request-uri as string) as string?
{
  let $request-path := tokenize($request-uri, "\\?")[1]
  return lower-case(($format, substring-after($request-path, ".jq."))[1])
};

declare function api:preprocess-tags($tags as string*) as string*
{
  distinct-values($tags ! upper-case($$))
};

declare function api:preprocess-boolean($name as string, $value as string)
as boolean
{
  if ($value eq "")
  then true
  else
  {
      api:validate-enum($name, lower-case($value), ("true", "false"));
      boolean($value)
  }
};

jsoniq version "1.0";

module namespace api = "http://apps.28.io/api";

import module namespace session    = "http://apps.28.io/session";
import module namespace resp       = "http://www.28msec.com/modules/http-response";
import module namespace sec-fiscal = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

declare function api:validate-regexp($name as string, $value as string?, $regexp as string)
as ()
{
  if (exists($value))
  then
    if (matches($value, "^" || $regexp || "$"))
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " does not match reg. expression " || $regexp || ".");
  else ();
};

declare function api:validate-enum($name as string, $value as string?, $enum as string*)
as ()
{
  if (exists($value))
  then
    if ($value = $enum)
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " is not one of these: " || string-join($enum,", ") || ".");
  else ();
};

declare %private function api:sgpl($count as integer, $singular as string, $plural as string)
as string
{
    if ($count eq 1)
    then $count || " " || $singular
    else $count || " " || $plural
};

declare function api:passed($timestamp as xs:dateTime?)
as string
{
    if (empty($timestamp) or $timestamp eq null)
    then "Never"
    else
        let $now := current-dateTime()
        let $difference := $now - $timestamp
        let $years := years-from-duration($difference)
        let $month := months-from-duration($difference)
        let $days := days-from-duration($difference)
        let $hours := hours-from-duration($difference)
        let $minutes := minutes-from-duration($difference)
        return
            if ($years gt 0) then api:sgpl($years,"year ago","years ago")
            else if ($month gt 0) then api:sgpl($month,"month ago","month ago")
            else if ($days gt 0) then api:sgpl($days, "day ago", "days ago")
            else if ($hours gt 0) then api:sgpl($hours, "hour ago", "hours ago")
            else if ($minutes gt 0) then api:sgpl($minutes, "minute ago", "minutes ago")
            else if ($difference div xs:dayTimeDuration("PT1S") lt 0) then "future"
            else "Just Now"
};

declare function api:success() as object
{
  { "success" : true }
};

declare function api:success($data as object()) as object
{
  {|
     {"success" : true },
     $data
  |}
};

declare %an:sequential function api:check-and-return-results(
    $token as string?,
    $results as item*,
    $format as string?
) as item*
{
    switch(session:has-access($token, "data_sec"))
    case $session:ACCESS-ALLOWED return
        $results
    case $session:ACCESS-DENIED return {
          resp:status-code(403);
          session:error("access denied", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          resp:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()
};

declare function api:normalize-facts(
    $facts as object*) as object*
{
    for $fact in $facts
    return {|
        {
            "Aspects" : {|
                trim($fact.Aspects, ("xbrl:Unit"))
            |}
        },
        trim($fact, ("Aspects", "_id")),
        { Unit: $fact.Aspects."xbrl:Unit" }[exists($fact.Aspects."xbrl:Unit")]
    |}
};

declare %an:sequential function api:serialize(
    $result as json-item,
    $comment as object,
    $serializers as object,
    $format as string?,
    $file-name as string) as item*
{
    switch ($format)
    case "xml" return {
        resp:serialization-parameters({"omit-xml-declaration" : false, indent : true });
        session:comment("xml", $comment),
        $serializers.to-xml($result)
    }
    case "text" case "csv" return {
        resp:content-type("text/csv");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    case "excel" return {
        resp:content-type("application/vnd.ms-excel");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    default return {
        resp:content-type("application/json");
        resp:serialization-parameters({"indent" : true});
        {|
            session:comment("json", $comment),
            $result
        |}
    }
};

declare function api:preprocess-fiscal-years($fiscal-years as string*) as integer*
{
  distinct-values(
    for $fy in $fiscal-years ! upper-case($$)
    return switch($fy)
           case "LATEST" return $sec-fiscal:LATEST_FISCAL_YEAR
           case "ALL" return $sec-fiscal:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
  )
};

declare function api:preprocess-fiscal-periods($fiscal-periods as string*) as string*
{
  distinct-values(
    for $fp in $fiscal-periods ! upper-case($$)
    return switch($fp)
           case "Q1"
           case "Q2"
           case "Q3"
           case "FY"
             return $fp
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIODS
           default
             return error(xs:QName("local:INVALID-PERIOD"),
               $fp || ": fiscalPeriod values must be one or more of Q1, Q2, Q3, FY, ALL")
  )
};

declare function api:preprocess-format($format as string?, $request-uri as string) as string?
{
  let $request-path := tokenize($request-uri, "\\?")[1]
  return lower-case(($format, substring-after($request-path, ".jq."))[1])
};

declare function api:preprocess-tags($tags as string*) as string*
{
  distinct-values($tags ! upper-case($$))
};

declare function api:preprocess-boolean($name as string, $value as string)
as boolean
{
  if ($value eq "")
  then true
  else
  {
      api:validate-enum($name, lower-case($value), ("true", "false"));
      boolean($value)
  }
};