dev/jq: Redact Notes in `remote` attribute

  Avoid displaying the full content of a Note when manipulating logs
  with `jq` as it can be quite long.

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove notes content
def redactNote(obj):
  obj |=
    (.
    | if isempty(.metadata.content | objects) | not then .metadata.content |= "[redacted]" else . end
    | if isempty(.metadata.schema | objects) | not then .metadata.schema |= "[redacted]" else . end
    | if isempty(.remote | objects) | not then redactNote(.remote) else . end
    | if isempty(.moveFrom | objects) | not then redactNote(.moveFrom) else . end
    | if isempty(.overwrite | objects) | not then redactNote(.overwrite) else . end
    )
;

def cleanNote:
  .
  | if isempty(.doc | objects) |not then redactNote(.doc) else . end
  | if isempty(.was | objects) |not then redactNote(.was) else . end
  | if isempty(.remoteDoc | objects) |not then redactNote(.remoteDoc) else . end
  | if isempty(.change | objects) | not then .
    | .change |= (. | redactNote(.doc))
    | .change |= (. | redactNote(.was))
    else . end
;

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v) |
  del(.level) |
  del(.local) |
  del(.remote) |
  cleanNote |
  {time,component,msg,path}+(del(.time)|del(.component)|del(.msg)|del(.path));

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: select(is_error) | clean;
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: select(is_warn) | clean;
def warn_strict: select(is_warn_strict) | clean;
def info_level: 30;
def is_info: .level >= info_level;
def info: select(is_info) | clean;
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: select(is_debug) | clean;

# Components:
def component(pattern): select(.component | test(pattern));
def App: component("^App$");
def Chokidar: component("^Chokidar|chokidar$");
def Fs: component("^Fs|FS$");
def LocalAnalysis: component("^LocalAnalysis|local/analysis$");
def LocalChange: component("^LocalChange|local/change$");
def LocalWatcher: component("^LocalWatcher$");
def LocalWriter: component("^LocalWriter$");
def Merge: component("^Merge$");
def Metadata: component("^Metadata$");
def Perfs: component("^Perfs|PerfReports$");
def Pouch: component("^Pouch$");
def Prep: component("^Prep$");
def RemoteCozy: component("^RemoteCozy$");
def RemoteUserActionRequired: component("^RemoteUserActionRequired|remote/user_action_required$");
def RemoteWarningPoller: component("^RemoteWarningPoller$");
def RemoteWatcher: component("^RemoteWatcher$");
def RemoteWriter: component("^RemoteWriter$");
def Sync: component("^Sync$");

def Local: select(.component | test("^local|chokidar"; "i"));
def Remote: select(.component | test("^remote"; "i"));

# Exclude "up-to-date" messages
#
#     yarn jq nouptodate path/to/logs*
#
def nouptodate: select(.msg | test("up[- ]to[- ]date") | not);

# Exclude initial scan messages
#
#     yarn jq noscan path/to/logs*
#
def noscan: select(.action != "scan");

# Exclude Proxy stuff:
#
#     yarn jq noproxy path/to/logs*
#
def noproxy: select(.component | test("GUI:proxy") | not);

# Find conflicts:
#
#     yarn jq -c conflict path/to/logs*
#
def is_conflict: .msg | test("resolveConflictAsync|Resolving conflict");
def conflict: clean | select(is_conflict);

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
# FIXME: Should not output the same line twice when both path & oldpath match.
def path(pattern):
  clean | select(
    (
      try (.path // (.doc | .path) // (.change | .doc | .path) // (.idConflict | .newDoc | .path) // (.event | .path)),
      try (.oldpath // (.was | .path) // (.idConflict | .existingDoc | .path) // (.event | .oldPath)),
      ""
    )
      | strings
      | test(pattern)
  );

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#    yarn jq 'before("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#    yarn jq 'after("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#
def time(pattern):
  select(.time | test(pattern));
def before(time): select(.time <= time);
def after(time): select(.time >= time);

# Drop time properties
def notime: del(.time);

# Filter inode number(s):
#
#    yarn jq 'ino(7852458)' path/to/logs*
#    yarn jq 'ino(7852458; 464672)' path/to/logs*
#
def ino(n): select((.ino // (.doc|.ino) // (.change|.doc|.ino)) as $ino | [n] | flatten | map(. == $ino) | any);

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path,oldpath,event}
    | if (.event | not) then del(.event) else . end
    | if (.event | type == "object") then del(.event) + (.event|{path, oldpath: .oldPath, action}) else . end
    | if .oldpath then . else del(.oldpath) end
    ;

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove notes content
def redactNote(obj):
  obj |=
    (.
    | if isempty(.metadata.content | objects) | not then .metadata.content |= "[redacted]" else . end
    | if isempty(.metadata.schema | objects) | not then .metadata.schema |= "[redacted]" else . end
    | if isempty(.moveFrom | objects) | not then redactNote(.moveFrom) else . end
    | if isempty(.overwrite | objects) | not then redactNote(.overwrite) else . end
    )
;

def cleanNote:
  .
  | if isempty(.doc | objects) |not then redactNote(.doc) else . end
  | if isempty(.was | objects) |not then redactNote(.was) else . end
  | if isempty(.remoteDoc | objects) |not then redactNote(.remoteDoc) else . end
  | if isempty(.change | objects) | not then .
    | .change |= (. | redactNote(.doc))
    | .change |= (. | redactNote(.was))
    else . end
;

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v) |
  del(.level) |
  del(.local) |
  del(.remote) |
  cleanNote |
  {time,component,msg,path}+(del(.time)|del(.component)|del(.msg)|del(.path));

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: select(is_error) | clean;
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: select(is_warn) | clean;
def warn_strict: select(is_warn_strict) | clean;
def info_level: 30;
def is_info: .level >= info_level;
def info: select(is_info) | clean;
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: select(is_debug) | clean;

# Components:
def component(pattern): select(.component | test(pattern));
def App: component("^App$");
def Chokidar: component("^Chokidar|chokidar$");
def Fs: component("^Fs|FS$");
def LocalAnalysis: component("^LocalAnalysis|local/analysis$");
def LocalChange: component("^LocalChange|local/change$");
def LocalWatcher: component("^LocalWatcher$");
def LocalWriter: component("^LocalWriter$");
def Merge: component("^Merge$");
def Metadata: component("^Metadata$");
def Perfs: component("^Perfs|PerfReports$");
def Pouch: component("^Pouch$");
def Prep: component("^Prep$");
def RemoteCozy: component("^RemoteCozy$");
def RemoteUserActionRequired: component("^RemoteUserActionRequired|remote/user_action_required$");
def RemoteWarningPoller: component("^RemoteWarningPoller$");
def RemoteWatcher: component("^RemoteWatcher$");
def RemoteWriter: component("^RemoteWriter$");
def Sync: component("^Sync$");

def Local: select(.component | test("^local|chokidar"; "i"));
def Remote: select(.component | test("^remote"; "i"));

# Exclude "up-to-date" messages
#
#     yarn jq nouptodate path/to/logs*
#
def nouptodate: select(.msg | test("up[- ]to[- ]date") | not);

# Exclude initial scan messages
#
#     yarn jq noscan path/to/logs*
#
def noscan: select(.action != "scan");

# Exclude Proxy stuff:
#
#     yarn jq noproxy path/to/logs*
#
def noproxy: select(.component | test("GUI:proxy") | not);

# Find conflicts:
#
#     yarn jq -c conflict path/to/logs*
#
def is_conflict: .msg | test("resolveConflictAsync|Resolving conflict");
def conflict: clean | select(is_conflict);

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
# FIXME: Should not output the same line twice when both path & oldpath match.
def path(pattern):
  clean | select(
    (
      try (.path // (.doc | .path) // (.change | .doc | .path) // (.idConflict | .newDoc | .path) // (.event | .path)),
      try (.oldpath // (.was | .path) // (.idConflict | .existingDoc | .path) // (.event | .oldPath)),
      ""
    )
      | strings
      | test(pattern)
  );

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#    yarn jq 'before("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#    yarn jq 'after("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#
def time(pattern):
  select(.time | test(pattern));
def before(time): select(.time <= time);
def after(time): select(.time >= time);

# Drop time properties
def notime: del(.time);

# Filter inode number(s):
#
#    yarn jq 'ino(7852458)' path/to/logs*
#    yarn jq 'ino(7852458; 464672)' path/to/logs*
#
def ino(n): select((.ino // (.doc|.ino) // (.change|.doc|.ino)) as $ino | [n] | flatten | map(. == $ino) | any);

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path,oldpath,event}
    | if (.event | not) then del(.event) else . end
    | if (.event | type == "object") then del(.event) + (.event|{path, oldpath: .oldPath, action}) else . end
    | if .oldpath then . else del(.oldpath) end
    ;

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );


Fix jq script for building dialect in go

. as $root
| (
  [ to_entries[]
    | [
        "\t",(.key|@json),": &Dialect{\n",
        "\t\t", (.key|@json),", ", (.value.name|@json),", ", (.value.native|@json), ", map[string][]string{\n"
      ] + (
          [ .value
            | {"feature","rule","background","scenario","scenarioOutline","examples","given","when","then","and","but"}
            | to_entries[]
            | "\t\t\t"+(.key), ": {\n",
              ([ .value[] | "\t\t\t\t", @json, ",\n"  ]|add),
              "\t\t\t},\n"
          ]
      ) + [
        "\t\t},\n",
        "map[string]messages.StepKeywordType {\n"
      ] + (
        [ .value.given
          | (
              [ .[] | select(. != "* ") |
                "\t\t\t\t",
                @json,
                ": messages.StepKeywordType_CONTEXT",
                ",\n"
              ] | add
            ),
            "\t\t\t\n"
        ]
        +
        [ .value.when
          | (
              [ .[] | select(. != "* ") |
                "\t\t\t\t",
                @json,
                ": messages.StepKeywordType_ACTION",
                ",\n"
              ] | add
            ),
            "\t\t\t\n"
        ]
        +
        [ .value.then
          | (
              [ .[] | select(. != "* ") |
                "\t\t\t\t",
                @json,
                ": messages.StepKeywordType_OUTCOME",
                ",\n"
              ] | add
            ),
            "\t\t\t\n"
        ]
        +
        [ .value.and
          | (
              [ .[] | select(. != "* ") |
                "\t\t\t\t",
                @json,
                ": messages.StepKeywordType_CONJUNCTION",
                ",\n"
              ] | add
            ),
            "\t\t\t\n"
        ]
        +
        [ .value.but
          | (
              [ .[] | select(. != "* ") |
                "\t\t\t\t",
                @json,
                ": messages.StepKeywordType_CONJUNCTION",
                ",\n"
              ] | add
            ),
            "\t\t\t\n"
        ]
        + [
          "\"* \": messages.StepKeywordType_UNKNOWN,\n"
        ]
      ) + [
        "}",
        "\t},\n"
      ]
    | add
  ]
  | add
  )
| "package gherkin\n\n"
+ "import messages \"github.com/cucumber/common/messages/go/v19\"\n\n"
+ "// Builtin dialects for " + ([ $root | to_entries[] | .key+" ("+.value.name+")" ] | join(", ")) + "\n"
+ "func DialectsBuiltin() DialectProvider {\n"
+ "\treturn builtinDialects\n"
+ "}\n\n"
+ "const (\n"
+ (
  ["feature","rule","background","scenario","scenarioOutline","examples","given","when","then","and","but"]
  | [ .[] | "\t" + . + " = " + (.|@json) + "\n" ]
  | add )
+ ")\n\n"
+ "var builtinDialects = gherkinDialectMap{\n"
+ .
+ "}\n"

. as $root
| (
  [ to_entries[]
    | [
        "\t",(.key|@json),": &Dialect{\n",
        "\t\t", (.key|@json),", ", (.value.name|@json),", ", (.value.native|@json), ", map[string][]string{\n"
      ] + (
          [ .value
            | {"feature","rule","background","scenario","scenarioOutline","examples","given","when","then","and","but"}
            | to_entries[]
            | "\t\t\t"+(.key), ": {\n",
              ([ .value[] | "\t\t\t\t", @json, ",\n"  ]|add),
              "\t\t\t},\n"
          ]
      ) + [
        "\t\t},\n",
        "map[string]messages.StepKeywordType {\n"
      ] + (
        [ .value.given
          | (
              [ .[] | select(. != "* ") |
                "\t\t\t\t",
                @json,
                ": messages.StepKeywordType_CONTEXT",
                ",\n"
              ] | add
            ),
            "\t\t\t\n"
        ]
        +
        [ .value.when
          | (
              [ .[] | select(. != "* ") |
                "\t\t\t\t",
                @json,
                ": messages.StepKeywordType_ACTION",
                ",\n"
              ] | add
            ),
            "\t\t\t\n"
        ]
        +
        [ .value.then
          | (
              [ .[] | select(. != "* ") |
                "\t\t\t\t",
                @json,
                ": messages.StepKeywordType_OUTCOME",
                ",\n"
              ] | add
            ),
            "\t\t\t\n"
        ]
        +
        [ .value.and
          | (
              [ .[] | select(. != "* ") |
                "\t\t\t\t",
                @json,
                ": messages.StepKeywordType_CONJUNCTION",
                ",\n"
              ] | add
            ),
            "\t\t\t\n"
        ]
        +
        [ .value.but
          | (
              [ .[] | select(. != "* ") |
                "\t\t\t\t",
                @json,
                ": messages.StepKeywordType_CONJUNCTION",
                ",\n"
              ] | add
            ),
            "\t\t\t\n"
        ]
        + [
          "\"* \": messages.StepKeywordType_UNKNOWN,\n"
        ]
      ) + [
        "}",
        "\t},\n"
      ]
    | add
  ]
  | add
  )
| "package gherkin\n\n"
+ "import messages \"github.com/cucumber/common/messages/go/v18\"\n\n"
+ "// Builtin dialects for " + ([ $root | to_entries[] | .key+" ("+.value.name+")" ] | join(", ")) + "\n"
+ "func DialectsBuiltin() DialectProvider {\n"
+ "\treturn builtinDialects\n"
+ "}\n\n"
+ "const (\n"
+ (
  ["feature","rule","background","scenario","scenarioOutline","examples","given","when","then","and","but"]
  | [ .[] | "\t" + . + " = " + (.|@json) + "\n" ]
  | add )
+ ")\n\n"
+ "var builtinDialects = gherkinDialectMap{\n"
+ .
+ "}\n"


Beautify error message

Beautify error message

import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

declare %an:nondeterministic function local:test-concepts($expected as integer, $params as string) as atomic
{
    let $elements as object* := parse-json(http-client:get("http://" || request:server-name() || ":" || request:server-port() || "/v1/_queries/public/api/report-elements.jq?_method=POST" || $params).body.content).ReportElements[]
    let $actual as integer := count($elements)
    let $duplicates := for $element in $elements where $element.CIK instance of array return $element
    return
        if (exists($duplicates)) then
            "false [CIK " || serialize($duplicates[1].CIK, ()) || " is not unique]"
        else if ($actual eq $expected) then
            true
        else
            "false [Actual="||$actual||", Expected="||$expected ||"]"
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    all: local:test-concepts(3, "&cik=4962&name=us-gaap:Assets"),
    onlyNames: local:test-concepts(16143, "&tag=DOW30&fiscalYear=2012&fiscalPeriod=FY&onlyNames=true"),
    disclosures: local:test-concepts(1855, "&ticker=ko&fiscalYear=2012&fiscalPeriod=FY&report=Disclosure&concept=disc:ConsolidationVariableInterestEntityPolicy")
})

import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

declare %an:nondeterministic function local:test-concepts($expected as integer, $params as string) as atomic
{
    let $elements as object* := parse-json(http-client:get("http://" || request:server-name() || ":" || request:server-port() || "/v1/_queries/public/api/report-elements.jq?_method=POST" || $params).body.content).ReportElements[]
    let $actual as integer := count($elements)
    let $duplicates := for $element in $elements where $element.CIK instance of array return $element
    return
        if (exists($duplicates)) then
            "false [CIK " || $duplicates[1].CIK || " is not unique]"
        else if ($actual eq $expected) then
            true
        else
            "false [Actual="||$actual||", Expected="||$expected ||"]"
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    all: local:test-concepts(3, "&cik=4962&name=us-gaap:Assets"),
    onlyNames: local:test-concepts(16143, "&tag=DOW30&fiscalYear=2012&fiscalPeriod=FY&onlyNames=true"),
    disclosures: local:test-concepts(1855, "&ticker=ko&fiscalYear=2012&fiscalPeriod=FY&report=Disclosure&concept=disc:ConsolidationVariableInterestEntityPolicy")
})


Update facts.jq

Update facts.jq

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facts"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-empty($params as object) as item
{
    let $endpoint := "facts"
    let $res as object := test:invoke-raw($endpoint, $params)
    return if($res.status eq 200 and $res.headers."Content-Length" eq "0") then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res
    }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw($endpoint, $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "Accession Number,Concept,Entity,Period,Fiscal Period,Fiscal Year,Accepted,Legal Entity,Unit,Value,Decimals",
        "0000021344-13-000050,\"Cash and Cash Equivalents, at Carrying Value\",COCA COLA CO,2013-09-27,Q3,2013,20131024121047,Default Legal Entity,USD,11118000000,-6",
        "0000021344-13-000050,Assets,COCA COLA CO,2013-09-27,Q3,2013,20131024121047,Default Legal Entity,USD,89432000000,-6"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res
    }
};

declare %an:nondeterministic function local:test-labels-aids() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: "disc:AdvertisingCostsPolicyTextBlock",
        map: "Disclosures",
        format: "json",
        fiscalYear: 2014,
        fiscalPeriod: "FY",
        labels: true,
        aid: "0000858877-14-000029"
    }
    let $res := test:invoke($endpoint, $params)
    let $actual := [
            for $labels in $res[2].FactTable[].Labels
            return (keys($labels) ! $labels.$$)
        ]
    let $expected := [ "Advertising Costs, Policy", "Default Legal Entity", "Accession Number", "Concept", "Entity", "CISCO SYSTEMS, INC.", "Period", "Fiscal Period", "Fiscal Year", "Accepted", "Unit", "Legal Entity" ]
    let $status as integer := $res[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

local:check({
    cocacola-fy-ytd-qtd: local:test-facttable(468, {
        ticker:"ko",
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-fy: local:test-facttable(163, {
        ticker:"ko"
    }),
    cocacolaCSVLabels: local:test-labels(),
    ciscoLabelsByAid: local:test-labels-aids(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(30, {
        tag:"DOW30",
        concept:"us-gaap:Assets"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-empty({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    }),
    quartersfact: local:test-facttable(0, { (: this fact is not reported in the raw XML filing :)
        aid: "0000051143-13-000007",
        concept: "disc:AssetRetirementObligationsPolicy",
        fiscalPeriod: "Q3",
        fiscalYear: "2013",
        map: "Disclosures"
    }),
    quartersfact2: local:test-facttable(1, {
        aid: "0001193125-12-207154",
        concept: "disc:CashFlowOperatingCapitalTableTextBlock",
        fiscalPeriod: "Q1",
        fiscalYear: "2012",
        map: "Disclosures"
    }),
    generic: local:test-facttable(2, {
        "xbrl:Entity":"http://www.sec.gov/CIK%200000021344",
        "sec:Accepted":"20140227132423",
        concept:"us-gaap:Assets",
        profile-name: "generic"
    })
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facts"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-empty($params as object) as item
{
    let $endpoint := "facts"
    let $res as object := test:invoke-raw($endpoint, $params)
    return if($res.status eq 200 and $res.headers."Content-Length" eq "0") then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res
    }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw($endpoint, $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "Accession Number,Concept,Entity,Period,Fiscal Period,Fiscal Year,Accepted,Legal Entity,Unit,Value,Decimals",
        "0000021344-13-000050,\"Cash and Cash Equivalents, at Carrying Value\",COCA COLA CO,2013-09-27,Q3,2013,20131024121047,Default Legal Entity,USD,11118000000,-6",
        "0000021344-13-000050,Assets,COCA COLA CO,2013-09-27,Q3,2013,20131024121047,Default Legal Entity,USD,89432000000,-6"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res
    }
};

declare %an:nondeterministic function local:test-labels-aids() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: "disc:AdvertisingCostsPolicyTextBlock",
        map: "Disclosures",
        format: "json",
        fiscalYear: 2014,
        fiscalPeriod: "YTD4",
        labels: true,
        aid: "0000858877-14-000029"
    }
    let $res := test:invoke($endpoint, $params)
    let $actual := [
            for $labels in $res[2].FactTable[].Labels
            return (keys($labels) ! $labels.$$)
        ]
    let $expected := [ "Advertising Costs, Policy", "Default Legal Entity", "Accession Number", "Concept", "Entity", "CISCO SYSTEMS, INC.", "Period", "Fiscal Period", "Fiscal Year", "Accepted", "Unit", "Legal Entity" ]
    let $status as integer := $res[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

local:check({
    cocacola-fy-ytd-qtd: local:test-facttable(468, {
        ticker:"ko",
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-fy: local:test-facttable(163, {
        ticker:"ko"
    }),
    cocacolaCSVLabels: local:test-labels(),
    ciscoLabelsByAid: local:test-labels-aids(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(30, {
        tag:"DOW30",
        concept:"us-gaap:Assets"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-empty({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    }),
    quartersfact: local:test-facttable(0, { (: this fact is not reported in the raw XML filing :)
        aid: "0000051143-13-000007",
        concept: "disc:AssetRetirementObligationsPolicy",
        fiscalPeriod: "Q3",
        fiscalYear: "2013",
        map: "Disclosures"
    }),
    quartersfact2: local:test-facttable(1, {
        aid: "0001193125-12-207154",
        concept: "disc:CashFlowOperatingCapitalTableTextBlock",
        fiscalPeriod: "Q1",
        fiscalYear: "2012",
        map: "Disclosures"
    }),
    generic: local:test-facttable(2, {
        "xbrl:Entity":"http://www.sec.gov/CIK%200000021344",
        "sec:Accepted":"20140227132423",
        concept:"us-gaap:Assets",
        profile-name: "generic"
    })
})


Typo on $format

Typo on $format

import module namespace util = "http://secxbrl.info/modules/util";
import module namespace session = "http://apps.28.io/session";

import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";
import module namespace conversion = "http://xbrl.io/modules/bizql/conversion";

import module namespace sec = "http://xbrl.io/modules/bizql/profiles/sec/core";

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";

declare function local:to-xml($o as object*) as node()*
{
    (session:comment("xml", {
        NumFacts: count($o),
        TotalNumFacts: session:num-facts(),
        TotalNumArchives: session:num-archives(),
        TotalNumEntities: session:num-entities()
    }),
    <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
        conversion:facts-to-xml($o, { Caller: "Report"})
    }</FactTable>)
};

declare function local:contains-aspect($name as string) as boolean
{
  some $x in (request:param-names() ! starts-with($$, $name))
  satisfies $x
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube($parameters as object) as object
{
    let $shortcut-hypercube-spec :=
    {|
        let $concepts := request:param-values("concept")
        return { "xbrl:Concept" : { Domain : [ $concepts ] } }[exists($concepts)],
        util:filter-override-from-parameters($parameters, {})
    |}
    let $main-hypercube-spec :=
    {|
        {
            "dei:LegalEntityAxis" : {
                "Domain" : [ "sec:DefaultLegalEntity" ],
                "Default" : "sec:DefaultLegalEntity"
            }
        }[not local:contains-aspect("dei:LegalEntityAxis")],

        for $parameter in (
            request:param-names(),
            keys($shortcut-hypercube-spec),
            "sec:Accepted",
            "sec:FiscalPeriod",
            "sec:FiscalYear")
        where contains($parameter, ":")
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (request:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $user-defined-type as string? :=
            (request:param-values($dimension-name || "::type"), request:param-values($dimension-name || ":type"))[1]
        let $type as string? :=
            switch(true)
            case exists($user-defined-type)
                return $user-defined-type
            case exists($shortcut-hypercube-spec.$dimension-name.Type)
                return $shortcut-hypercube-spec.$dimension-name.Type
            default
                return ()
        let $values := (request:param-values($dimension-name), $shortcut-hypercube-spec.$dimension-name.Domain[])
        let $typed-values := if (exists($type)) then local:cast-sequence($values, $type) else $values
        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (request:param-values($dimension-name || "::default"), request:param-values($dimension-name || ":default"))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value
        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                { "Domain" : [ $typed-values ] }[(exists($typed-values) and not($all))],
                { "Default" : $typed-default-value }[$has-default]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($main-hypercube-spec)
};

session:audit-call();

(: Query parameters :)
let $format as string?         := request:param-values("format")
let $ciks as string*           := distinct-values(request:param-values("cik"))
let $tags as string*           := distinct-values(request:param-values("tag"))
let $tickers as string*        := distinct-values(request:param-values("ticker"))
let $sics as string*           := distinct-values(request:param-values("sic"))
let $fiscalYears as string*    := distinct-values(request:param-values("fiscalYear", "LATEST"))
let $fiscalPeriods as string*  := distinct-values(request:param-values("fiscalPeriod", "FY"))
let $aids as string*           := distinct-values(request:param-values("aid"))
let $map as string?            := request:param-values("map")
let $rule as string?            := request:param-values("rule")
let $parameters := {|
    {
        CIKs: [ $ciks ],
        Tags: [ $tags ],
        Tickers: [ $tickers ],
        SICs: [ $sics ],
        FiscalYears: [ $fiscalYears ],
        FiscalPeriods: [ $fiscalPeriods ],
        AIDs: [ $aids ]
    },
    { Format: $format }[exists($format)],
    { Map: $map }[exists($map)],
    { Rule: $rule }[exists($rule)]
|}

(: Object resolution :)
let $parameters as object := util:process-parameters($parameters)
let $archives as object* := util:filings-from-parameters($parameters, ())
let $entities := entities:entities($archives.Entity)

let $hypercube := local:hypercube($parameters)

let $facts :=
    if(empty($archives))
    then ()
    else
        for $fact in sec:facts-for(
            {|
                { Hypercube : $hypercube },
                { "ConceptMaps" : $map }[exists($map)],
                { "Rules" : $parameters.Rule }[exists($parameters.Rule)]
            |}
        )
        return {|
            $fact,
            { "EntityRegistrantName" : $entities[$$._id eq $fact.Aspects."xbrl:Entity"].Profiles.SEC.CompanyName}
        |}

let $facts := util:move-unit-out-of-aspects($facts)

let $results :=
        switch ($parameters.Format)
        case "xml" return {
            response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
            local:to-xml($facts)
        }
        case "text" case "csv" return {
            response:content-type("text/csv");
            response:header("Content-Disposition", "attachment; filename=facts.csv");
            conversion:facts-to-csv($facts, { Caller: "Report" })
        }
        case "excel" return {
            response:content-type("application/vnd.ms-excel");
            response:header("Content-Disposition", "attachment; filename=fact.csv");
            conversion:facts-to-csv($facts, { Caller: "Report" })
        }
        default return {
            response:content-type("application/json");
            response:serialization-parameters({"indent" : true});
            {|
                { NetworkIdentifier : "http://bizql.io/facts" },
                { TableName : "xbrl:Facts" },
                { FactTable : [ $facts ] },
                session:comment("json", {
                        NumFacts: count($facts),
                        TotalNumFacts: session:num-facts(),
                        TotalNumArchives: session:num-archives(),
                        TotalNumEntities: session:num-entities()
                    })
            |}
        }
return
    switch(true)
    case empty($archives) return {
        response:status-code(400);
        session:error("entities or archives not found (valid parameters: cik, ticker, tag, sic, aid)", $parameters.Format)
    }
    default return util:check-and-return-results($entities, $results, $parameters.Format)

import module namespace util = "http://secxbrl.info/modules/util";
import module namespace session = "http://apps.28.io/session";

import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";
import module namespace conversion = "http://xbrl.io/modules/bizql/conversion";

import module namespace sec = "http://xbrl.io/modules/bizql/profiles/sec/core";

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";

declare function local:to-xml($o as object*) as node()*
{
    (session:comment("xml", {
        NumFacts: count($o),
        TotalNumFacts: session:num-facts(),
        TotalNumArchives: session:num-archives(),
        TotalNumEntities: session:num-entities()
    }),
    <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
        conversion:facts-to-xml($o, { Caller: "Report"})
    }</FactTable>)
};

declare function local:contains-aspect($name as string) as boolean
{
  some $x in (request:param-names() ! starts-with($$, $name))
  satisfies $x
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube($parameters as object) as object
{
    let $shortcut-hypercube-spec :=
    {|
        let $concepts := request:param-values("concept")
        return { "xbrl:Concept" : { Domain : [ $concepts ] } }[exists($concepts)],
        util:filter-override-from-parameters($parameters, {})
    |}
    let $main-hypercube-spec :=
    {|
        {
            "dei:LegalEntityAxis" : {
                "Domain" : [ "sec:DefaultLegalEntity" ],
                "Default" : "sec:DefaultLegalEntity"
            }
        }[not local:contains-aspect("dei:LegalEntityAxis")],

        for $parameter in (
            request:param-names(),
            keys($shortcut-hypercube-spec),
            "sec:Accepted",
            "sec:FiscalPeriod",
            "sec:FiscalYear")
        where contains($parameter, ":")
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (request:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $user-defined-type as string? :=
            (request:param-values($dimension-name || "::type"), request:param-values($dimension-name || ":type"))[1]
        let $type as string? :=
            switch(true)
            case exists($user-defined-type)
                return $user-defined-type
            case exists($shortcut-hypercube-spec.$dimension-name.Type)
                return $shortcut-hypercube-spec.$dimension-name.Type
            default
                return ()
        let $values := (request:param-values($dimension-name), $shortcut-hypercube-spec.$dimension-name.Domain[])
        let $typed-values := if (exists($type)) then local:cast-sequence($values, $type) else $values
        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (request:param-values($dimension-name || "::default"), request:param-values($dimension-name || ":default"))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value
        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                { "Domain" : [ $typed-values ] }[(exists($typed-values) and not($all))],
                { "Default" : $typed-default-value }[$has-default]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($main-hypercube-spec)
};

session:audit-call();

(: Query parameters :)
let $format as string?         := request:param-values("format")
let $ciks as string*           := distinct-values(request:param-values("cik"))
let $tags as string*           := distinct-values(request:param-values("tag"))
let $tickers as string*        := distinct-values(request:param-values("ticker"))
let $sics as string*           := distinct-values(request:param-values("sic"))
let $fiscalYears as string*    := distinct-values(request:param-values("fiscalYear", "LATEST"))
let $fiscalPeriods as string*  := distinct-values(request:param-values("fiscalPeriod", "FY"))
let $aids as string*           := distinct-values(request:param-values("aid"))
let $map as string?            := request:param-values("map")
let $rule as string?            := request:param-values("rule")
let $parameters := {|
    {
        CIKs: [ $ciks ],
        Tags: [ $tags ],
        Tickers: [ $tickers ],
        SICs: [ $sics ],
        FiscalYears: [ $fiscalYears ],
        FiscalPeriods: [ $fiscalPeriods ],
        AIDs: [ $aids ]
    },
    { Format: $format }[exists($format)],
    { Map: $map }[exists($map)],
    { Rule: $rule }[exists($rule)]
|}

(: Object resolution :)
let $parameters as object := util:process-parameters($parameters)
let $archives as object* := util:filings-from-parameters($parameters, ())
let $entities := entities:entities($archives.Entity)

let $hypercube := local:hypercube($parameters)

let $facts :=
    if(empty($archives))
    then ()
    else
        for $fact in sec:facts-for(
            {|
                { Hypercube : $hypercube },
                { "ConceptMaps" : $map }[exists($map)],
                { "Rules" : $parameters.Rule }[exists($parameters.Rule)]
            |}
        )
        return {|
            $fact,
            { "EntityRegistrantName" : $entities[$$._id eq $fact.Aspects."xbrl:Entity"].Profiles.SEC.CompanyName}
        |}

let $facts := util:move-unit-out-of-aspects($facts)

let $results :=
        switch ($format)
        case "xml" return {
            response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
            local:to-xml($facts)
        }
        case "text" case "csv" return {
            response:content-type("text/csv");
            response:header("Content-Disposition", "attachment; filename=facts.csv");
            conversion:facts-to-csv($facts, { Caller: "Report" })
        }
        case "excel" return {
            response:content-type("application/vnd.ms-excel");
            response:header("Content-Disposition", "attachment; filename=fact.csv");
            conversion:facts-to-csv($facts, { Caller: "Report" })
        }
        default return {
            response:content-type("application/json");
            response:serialization-parameters({"indent" : true});
            {|
                { NetworkIdentifier : "http://bizql.io/facts" },
                { TableName : "xbrl:Facts" },
                { FactTable : [ $facts ] },
                session:comment("json", {
                        NumFacts: count($facts),
                        TotalNumFacts: session:num-facts(),
                        TotalNumArchives: session:num-archives(),
                        TotalNumEntities: session:num-entities()
                    })
            |}
        }
return
    switch(true)
    case empty($archives) return {
        response:status-code(400);
        session:error("entities or archives not found (valid parameters: cik, ticker, tag, sic, aid)", $format)
    }
    default return util:check-and-return-results($entities, $results, $parameters.Format)


:fire: Remove useless functions.

:fire: Remove useless functions.

jsoniq version "1.0";

(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for querying XBRL Models of financial reports submitted to the SEC.
 :

 :
 :
Standard $options Parameter

 :
 :
Most functions in the BizQL package allow an additional $options
 :    parameter. The options parameter is a JSON object which is defined in the documentation
 :    of the facts module.

 :
 :
In addition to the option fields defined in the
 :    facts module the following
 :    fields can be used in the options of a function in the SEC profile:

 :
 :

 :
HideAmendedFacts: if set to true (default behavior) all facts that have been
 :     amended are not included in the result.

 :
IncludeImpliedTable: set to true in order to output the implied table if
 :     there is no Table (default is false).

 :

 :
 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace networks = "http://28.io/modules/xbrl/networks";
import module namespace report-schemas = "http://28.io/modules/xbrl/report-schemas";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Scheme for the SEC entity identifiers (i.e. http://www.sec.gov/CIK)
 :)
declare variable $sec:CIK as xs:string := "http://www.sec.gov/CIK";

(:~
 : Joker value for all archives or all concepts.
 :)
declare variable $sec:ALL_OF_THEM as boolean := true;

(:~
 : All prefixes pre-declared in SEC's EDGAR.
 :)
declare variable $sec:EDGAR_PREFIXES :=
  ( "xbrli", "link", "xl", "xlink", "xbrldt", "xbrldi", "nonnum",
    "num", "ref", "us-gaap", "dei", "us-types", "invest", "country",
    "currency", "exch", "naics", "sic", "stpr", "rr", "rr-ent",
    "rr-cal", "rr-def", "rr-pre" );

(:~
 : All SEC characteristics that are always there with their defaults
 :)
declare variable $sec:DEFAULTS as object := {
    "dei:LegalEntityAxis" : {
      Default: "sec:DefaultLegalEntity",
      Domain: [ "sec:DefaultLegalEntity" ]
    },
    "sec:Accepted" : {
      Type: "xs:string"
    },
    "sec:FiscalYear" : {
      Type: "xs:integer"
    },
    "sec:FiscalPeriod" : {
      Type: "xs:string"
    }
  };

(:~
 :
Return all facts that match a given filter object optionally interpreted
 :    in the context of an optionally given hypercube.
 :    Only returns the facts that match the fiscal focus of the archive they
 :    belong to.

 :
 : @param $options standard fact retrieving options.
 :
 : @error facts:INVALID-RULE-TYPE the type of a rule is not unknown/invalid
 : @error facts:RULE-EXECUTION-ERROR a rule raised an error whilst being executed
 : @error facts:FILTER-TOO-GENERIC The filter object must have at least one of the
 :        fields Archive, Aspects.xbrl:Concept, Aspects.xbrl:Period, or
 :        Aspects.xbrl:Entity.
 : @return all facts satisfying the filter and options.
 :)
declare function sec:facts-for(
  $options as object?) as object*
{
  let $facts := facts:facts-for($options)
  return sec:hide-amended-facts($facts, $options)
};

(:~
 :
Retrieves the facts associated with the supplied concepts
 : from the supplied archive, that match the fiscal focus of the archive.

 :
 : @deprecated use sec:facts-for#1 and sec:user-defined-hypercube#1 instead.
 :
 : @param $archive_or_ids a sequence of archives or their ids or $sec:ALL_OF_THEM for no filtering.
 : @param $concepts the concepts or $sec:ALL_OF_THEM for no filtering.
 :
 : @return the latest facts associated with the supplied concept.
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function sec:facts-for-archives-and-concepts(
    $archive_or_ids as item*,
    $concepts as item*) as object*
{
  sec:facts-for-archives-and-concepts($archive_or_ids, $concepts, ())
};

(:~
 :
Retrieves the facts associated with the supplied concepts
 : from the supplied archive, that match the fiscal focus of the archive.

 :
 : @deprecated use sec:facts-for#1 and sec:user-defined-hypercube#1 instead.
 :
 : @param $archive_or_ids a sequence of archives or their ids or $sec:ALL_OF_THEM for no filtering.
 : @param $concepts the concepts or $sec:ALL_OF_THEM for no filtering.
 : @param $options standard SEC BizQL options.
 :
 : @return the latest facts associated with the supplied concept.
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function sec:facts-for-archives-and-concepts(
    $archive_or_ids as item*,
    $concepts as item*,
    $options as object?) as object*
{
  let $options := if(empty($options.Hypercube))
                  then facts:merge-objects(
                      {
                          "Hypercube" : sec:dimensionless-hypercube(())
                      },
                      $options,
                      true)
                  else $options
  let $facts :=
    facts:facts-for-archives-and-concepts(
      $archive_or_ids,
      $concepts,
      $options)
  return sec:hide-amended-facts($facts, $options)
};

(:~
 :
 :
Filters a list of facts and returns only the latest amending facts.
 :    Accordingly, all amended facts are filtered out.

 :
 : @param $facts a list of facts.
 : @param $options optional parameters:
 :

 :
HideAmendedFacts (boolean, true by default): activates or deactivates amendment hiding. If false,
 :     the function will just return the input facts.

 :

 :
 : @return a sequence of facts without amended facts.
 :)
declare function sec:hide-amended-facts(
    $facts as object*,
    $options as object?
)
as object*
{
  let $hide-amended-facts as boolean := ($options.HideAmendedFacts, true)[1]
  return
    if(not $hide-amended-facts)
    then $facts
    else
      for $grouped-facts in $facts
      group by
        $entity := $grouped-facts.Aspects."xbrl:Entity",
        $fiscal-period := $grouped-facts.Aspects."sec:FiscalPeriod",
        $fiscal-year := $grouped-facts.Aspects."sec:FiscalYear"
      return
        if (empty($fiscal-period) or empty($fiscal-year) or $fiscal-period instance of null or $fiscal-year instance of null)
        then
          (: $grouped-facts is already grouped by entity.
             Now, group by period as a fallback. :)
          for $regrouped-facts in $grouped-facts
          group by $period := $regrouped-facts.Aspects."xbrl:Period"
          return
            let $latest-accepted :=
              max(distinct-values($regrouped-facts.Aspects."sec:Accepted"))
            return
              if (empty($latest-accepted))
              then $regrouped-facts
              else
                (: hide amended facts :)
                $regrouped-facts[$$.Aspects."sec:Accepted" eq $latest-accepted]

        else
          let $latest-accepted :=
            max(distinct-values($grouped-facts.Aspects."sec:Accepted"))
          return
            if (empty($latest-accepted))
            then $grouped-facts
            else
              (: hide amended facts :)
              $grouped-facts[$$.Aspects."sec:Accepted" eq $latest-accepted]
};

(:~
 :
Return all facts for concepts that are defined by one of the common
 : edgar taxonomies. In particular, this function returns facts for concepts
 : having one of the following prefixes:

 :
 :

 :
xbrli

 :
link

 :
xl

 :
xlink

 :
xbrldt

 :
xbrldi

 :
nonnum

 :
num

 :
ref

 :
us-gaap

 :
dei

 :
us-types

 :
invest

 :
country

 :
currency

 :
exch

 :
naics

 :
sic

 :
stpr

 :
rr

 :
rr-ent

 :
rr-cal

 :
rr-def

 :
rr-pre

 :

 :
 : @param $archive_or_ids a sequence of archive or archive IDs to filter.
 : @return all facts having one of the above prefixes.
 :)
declare function sec:edgar-facts-for-archives(
  $archive_or_ids as item*) as object*
{
  sec:facts-for-archives-and-concepts(
    $archive_or_ids,
    $sec:ALL_OF_THEM,
    {
      Filter: {
        Profiles: {
          SEC: {
            IsExtension: false
          }
        }
      }
    })
};

(:~
 :
Return all facts for concepts that are NOT defined by one of the common
 : edgar taxonomies. In particular, this function returns facts for concepts
 : having none of the prefixes listed under function
 : sec:edgar-facts-for-archives.

 :
 : @param $archive_or_ids a sequence of archive or archive IDs to filter.
 : @return all facts having none of the common edgar prefixes.
 :)
declare function sec:extension-facts-for-archives(
  $archive_or_ids as item*) as object*
{
  sec:facts-for-archives-and-concepts(
    $archive_or_ids,
    $sec:ALL_OF_THEM,
    {
      Filter: {
        Profiles: {
          SEC: {
            IsExtension: true
          }
        }
      }
    })
};

(:~
 :
Returns an instantiation of a dimensionless Hypercube containing the basic
 :    characteristics (xbrl:Concept, xbrl:Period, xbrl:Entity, and xbrl:Unit).
 :    For each of those included aspects the value space is not limited, however
 :    you can restrict it using the options. If you need more
 :    XBRL dimensions, use the user-defined-hypercube function instead.

 :
In the SEC profile, there is an additional axis that is always here:
 : dei:LegalEntityAxis, with a default value sec:DefaultLegalEntity and a filtering
 : only accepting this default value.

 :
 : @return dimensionless hypercube instantiation.
 : @param $options: additional options among which:
 : - Concepts: an array of concept names to include in the hypercube.
 : - Periods: an array of periods to include in the hypercube.
 : - Entities: an array of EIDs to include in the hypercube.
 : - Units: an array of units to include in the hypercube.
 : - LegalEntities: an array of legal entities to include in the hypercube.
 :)
declare function sec:dimensionless-hypercube($options as object?) as object
{
  sec:user-defined-hypercube(
    {|
      if(exists($options.Concepts))
      then { "xbrl:Concept" : { Domain: $options.Concepts } }
      else (),
      if(exists($options.Entities))
      then { "xbrl:Entity" : { Domain: [ entities:eid($options.Entities[]) ] } }
      else (),
      if(exists($options.Periods))
      then { "xbrl:Period" : { Domain: $options.Periods } }
      else (),
      if(exists($options.Units))
      then { "xbrl:Unit" : { Domain: $options.Units } }
      else (),
      if(exists($options.LegalEntities))
      then { "dei:LegalEntityAxis" : { Domain: $options.LegalEntities } }
      else ()
    |}
  )
};

(:~
 :
Returns an instantiation of a Hypercube with no extra dimensions besides the legal entity axis.
 :    The four basic characteristics (xbrl:Concept, xbrl:Period, xbrl:Entity, and xbrl:Unit)
 :    are always added, but can be overriden. For each of those included aspects the value space is,
 :    by default, not limited, and only xbrl:Unit has a default value.

 :
The characteristic dei:LegalEntityAxis is also here. It has
 :    a default value of sec:DefaultLegalEntity and restricts the value
 :    space to this default value.

 :
 :
 : @return the hypercube instantiation.
 :)
declare function sec:user-defined-hypercube() as object
{
  sec:user-defined-hypercube(())
};

(:~
 :
Returns an instantiation of a Hypercube containing the specified dimensions.
 :    The four basic characteristics (xbrl:Concept, xbrl:Period, xbrl:Entity, and xbrl:Unit)
 :    are always added, but can be overriden. For each of those included aspects the value space is,
 :    by default, not limited, and only xbrl:Unit has a default value.

 :
The following dimensions are always present, but the values can be overriden:

 :
{
 :     "dei:LegalEntityAxis" : {
 :       Default: "sec:DefaultLegalEntity", -- overridable with HasDefault : false
 :       Domain: [ "sec:DefaultLegalEntity" ] -- overridable with Domain: null
 :     },
 :     "sec:Accepted" : {
 :       Type: "string"
 :     },
 :     "sec:FiscalYear" : {
 :       Type: "integer"
 :     },
 :     "sec:FiscalPeriod" : {
 :       Type: "string"
 :     }
 :    }

 :
 : @return the hypercube instantiation.
 : @param $dimensions
an object with pairs of (dimension name, object with dimension information). These
 : dimension information subobjects
 : may have four (optional) fields:

 :

 :
Type (string, absent by default): the name of a type (string, integer, ...). This creates a typed dimension (if absent, an explicit dimension).

 :
Default (a string, absent by default): must be either castable to Type, or null (explicit dimensions: must be a string with a member name, null not allowed).

 :
HasDefault (a boolean, true by default): if false, the dimension has no default even if Default is present (this includes the possibility to override predefined dimensions).

 :
Domain (null or an array of atomics, absent by default): restricts the domain of the dimension to the specified values, that must all be castable to Type or null (or all strings with member names for explicit dimensions). Don't forget to include the default value in there if any. If null is used instead of an array, this removes any predefined restrictions.
.
 :

 :)
declare function sec:user-defined-hypercube($dimensions as object?) as object
{
  let $dimensions := facts:merge-objects($dimensions, $sec:DEFAULTS, true)
  let $extended-dimensions := {|
    for $d in keys($dimensions)
    let $dimension-spec := $dimensions.$d

    let $dimension-default as atomic? :=
      if(not $dimension-spec.HasDefault eq false)
      then $dimension-spec.Default
      else ()

    let $dimension-domain as array? :=
        typeswitch($dimension-spec.Domain)
        case null return ()
        default return $dimension-spec.Domain

    return {
      $d : {|
        { Default : $dimension-default }[exists($dimension-default)],
        { Type : $dimension-spec.Type }[exists($dimension-spec.Type)],
        { Domain : $dimension-domain }[exists($dimension-domain)]
      |}
    }
  |}
  return hypercubes:user-defined-hypercube($extended-dimensions)
};

jsoniq version "1.0";

(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for querying XBRL Models of financial reports submitted to the SEC.
 :

 :
 :
Standard $options Parameter

 :
 :
Most functions in the BizQL package allow an additional $options
 :    parameter. The options parameter is a JSON object which is defined in the documentation
 :    of the facts module.

 :
 :
In addition to the option fields defined in the
 :    facts module the following
 :    fields can be used in the options of a function in the SEC profile:

 :
 :

 :
HideAmendedFacts: if set to true (default behavior) all facts that have been
 :     amended are not included in the result.

 :
IncludeImpliedTable: set to true in order to output the implied table if
 :     there is no Table (default is false).

 :

 :
 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace networks = "http://28.io/modules/xbrl/networks";
import module namespace report-schemas = "http://28.io/modules/xbrl/report-schemas";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Scheme for the SEC entity identifiers (i.e. http://www.sec.gov/CIK)
 :)
declare variable $sec:CIK as xs:string := "http://www.sec.gov/CIK";

(:~
 : Joker value for all archives or all concepts.
 :)
declare variable $sec:ALL_OF_THEM as boolean := true;

(:~
 : All prefixes pre-declared in SEC's EDGAR.
 :)
declare variable $sec:EDGAR_PREFIXES :=
  ( "xbrli", "link", "xl", "xlink", "xbrldt", "xbrldi", "nonnum",
    "num", "ref", "us-gaap", "dei", "us-types", "invest", "country",
    "currency", "exch", "naics", "sic", "stpr", "rr", "rr-ent",
    "rr-cal", "rr-def", "rr-pre" );

(:~
 : All SEC characteristics that are always there with their defaults
 :)
declare variable $sec:DEFAULTS as object := {
    "dei:LegalEntityAxis" : {
      Default: "sec:DefaultLegalEntity",
      Domain: [ "sec:DefaultLegalEntity" ]
    },
    "sec:Accepted" : {
      Type: "xs:string"
    },
    "sec:FiscalYear" : {
      Type: "xs:integer"
    },
    "sec:FiscalPeriod" : {
      Type: "xs:string"
    }
  };

(:~
 :
Return all facts that match a given filter object optionally interpreted
 :    in the context of an optionally given hypercube.
 :    Only returns the facts that match the fiscal focus of the archive they
 :    belong to.

 :
 : @param $options standard fact retrieving options.
 :
 : @error facts:INVALID-RULE-TYPE the type of a rule is not unknown/invalid
 : @error facts:RULE-EXECUTION-ERROR a rule raised an error whilst being executed
 : @error facts:FILTER-TOO-GENERIC The filter object must have at least one of the
 :        fields Archive, Aspects.xbrl:Concept, Aspects.xbrl:Period, or
 :        Aspects.xbrl:Entity.
 : @return all facts satisfying the filter and options.
 :)
declare function sec:facts-for(
  $options as object?) as object*
{
  let $facts := facts:facts-for($options)
  return sec:hide-amended-facts($facts, $options)
};

(:~
 :
 :
Retrieves all facts in the supplied hypercube, across all archives.

 :
Default values will be populated automatically.

 :
Only returns the facts that match the fiscal focus of the archive they
 :    belong to.

 :
 :
 : @param $hypercube a hypercube.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function sec:facts-for-hypercube(
    $hypercube as object
)
as object*
{
  sec:facts-for({ Hypercube: $hypercube })
};

(:~
 :
 :
Retrieves all facts in the supplied hypercube, across all archives.

 :
Default values will be populated automatically.

 :
Only returns the facts that match the fiscal focus of the archive they
 :    belong to.

 :
 :
 : @param $hypercube a hypercube.
 : @param $options standard fact retrieving options.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function sec:facts-for-hypercube(
    $hypercube as object,
    $options as object?
)
as object*
{
  let $options as object := facts:merge-objects(
    { Hypercube: $hypercube },
    $options,
    true
  )
  return sec:facts-for($options)
};

(:~
 :
Retrieves the facts associated with the supplied concepts
 : from the supplied archive, that match the fiscal focus of the archive.

 :
 : @deprecated use sec:facts-for#1 and sec:user-defined-hypercube#1 instead.
 :
 : @param $archive_or_ids a sequence of archives or their ids or $sec:ALL_OF_THEM for no filtering.
 : @param $concepts the concepts or $sec:ALL_OF_THEM for no filtering.
 :
 : @return the latest facts associated with the supplied concept.
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function sec:facts-for-archives-and-concepts(
    $archive_or_ids as item*,
    $concepts as item*) as object*
{
  sec:facts-for-archives-and-concepts($archive_or_ids, $concepts, ())
};

(:~
 :
Retrieves the facts associated with the supplied concepts
 : from the supplied archive, that match the fiscal focus of the archive.

 :
 : @deprecated use sec:facts-for#1 and sec:user-defined-hypercube#1 instead.
 :
 : @param $archive_or_ids a sequence of archives or their ids or $sec:ALL_OF_THEM for no filtering.
 : @param $concepts the concepts or $sec:ALL_OF_THEM for no filtering.
 : @param $options standard SEC BizQL options.
 :
 : @return the latest facts associated with the supplied concept.
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function sec:facts-for-archives-and-concepts(
    $archive_or_ids as item*,
    $concepts as item*,
    $options as object?) as object*
{
  let $options := if(empty($options.Hypercube))
                  then facts:merge-objects(
                      {
                          "Hypercube" : sec:dimensionless-hypercube(())
                      },
                      $options,
                      true)
                  else $options
  let $facts :=
    facts:facts-for-archives-and-concepts(
      $archive_or_ids,
      $concepts,
      $options)
  return sec:hide-amended-facts($facts, $options)
};

(:~
 :
 :
Filters a list of facts and returns only the latest amending facts.
 :    Accordingly, all amended facts are filtered out.

 :
 : @param $facts a list of facts.
 : @param $options optional parameters:
 :

 :
HideAmendedFacts (boolean, true by default): activates or deactivates amendment hiding. If false,
 :     the function will just return the input facts.

 :

 :
 : @return a sequence of facts without amended facts.
 :)
declare function sec:hide-amended-facts(
    $facts as object*,
    $options as object?
)
as object*
{
  let $hide-amended-facts as boolean := ($options.HideAmendedFacts, true)[1]
  return
    if(not $hide-amended-facts)
    then $facts
    else
      for $grouped-facts in $facts
      group by
        $entity := $grouped-facts.Aspects."xbrl:Entity",
        $fiscal-period := $grouped-facts.Aspects."sec:FiscalPeriod",
        $fiscal-year := $grouped-facts.Aspects."sec:FiscalYear"
      return
        if (empty($fiscal-period) or empty($fiscal-year) or $fiscal-period instance of null or $fiscal-year instance of null)
        then
          (: $grouped-facts is already grouped by entity.
             Now, group by period as a fallback. :)
          for $regrouped-facts in $grouped-facts
          group by $period := $regrouped-facts.Aspects."xbrl:Period"
          return
            let $latest-accepted :=
              max(distinct-values($regrouped-facts.Aspects."sec:Accepted"))
            return
              if (empty($latest-accepted))
              then $regrouped-facts
              else
                (: hide amended facts :)
                $regrouped-facts[$$.Aspects."sec:Accepted" eq $latest-accepted]

        else
          let $latest-accepted :=
            max(distinct-values($grouped-facts.Aspects."sec:Accepted"))
          return
            if (empty($latest-accepted))
            then $grouped-facts
            else
              (: hide amended facts :)
              $grouped-facts[$$.Aspects."sec:Accepted" eq $latest-accepted]
};

(:~
 :
Return all facts for concepts that are defined by one of the common
 : edgar taxonomies. In particular, this function returns facts for concepts
 : having one of the following prefixes:

 :
 :

 :
xbrli

 :
link

 :
xl

 :
xlink

 :
xbrldt

 :
xbrldi

 :
nonnum

 :
num

 :
ref

 :
us-gaap

 :
dei

 :
us-types

 :
invest

 :
country

 :
currency

 :
exch

 :
naics

 :
sic

 :
stpr

 :
rr

 :
rr-ent

 :
rr-cal

 :
rr-def

 :
rr-pre

 :

 :
 : @param $archive_or_ids a sequence of archive or archive IDs to filter.
 : @return all facts having one of the above prefixes.
 :)
declare function sec:edgar-facts-for-archives(
  $archive_or_ids as item*) as object*
{
  sec:facts-for-archives-and-concepts(
    $archive_or_ids,
    $sec:ALL_OF_THEM,
    {
      Filter: {
        Profiles: {
          SEC: {
            IsExtension: false
          }
        }
      }
    })
};

(:~
 :
Return all facts for concepts that are NOT defined by one of the common
 : edgar taxonomies. In particular, this function returns facts for concepts
 : having none of the prefixes listed under function
 : sec:edgar-facts-for-archives.

 :
 : @param $archive_or_ids a sequence of archive or archive IDs to filter.
 : @return all facts having none of the common edgar prefixes.
 :)
declare function sec:extension-facts-for-archives(
  $archive_or_ids as item*) as object*
{
  sec:facts-for-archives-and-concepts(
    $archive_or_ids,
    $sec:ALL_OF_THEM,
    {
      Filter: {
        Profiles: {
          SEC: {
            IsExtension: true
          }
        }
      }
    })
};

(:~
 :
Returns an instantiation of a dimensionless Hypercube containing the basic
 :    characteristics (xbrl:Concept, xbrl:Period, xbrl:Entity, and xbrl:Unit).
 :    For each of those included aspects the value space is not limited, however
 :    you can restrict it using the options. If you need more
 :    XBRL dimensions, use the user-defined-hypercube function instead.

 :
In the SEC profile, there is an additional axis that is always here:
 : dei:LegalEntityAxis, with a default value sec:DefaultLegalEntity and a filtering
 : only accepting this default value.

 :
 : @return dimensionless hypercube instantiation.
 : @param $options: additional options among which:
 : - Concepts: an array of concept names to include in the hypercube.
 : - Periods: an array of periods to include in the hypercube.
 : - Entities: an array of EIDs to include in the hypercube.
 : - Units: an array of units to include in the hypercube.
 : - LegalEntities: an array of legal entities to include in the hypercube.
 :)
declare function sec:dimensionless-hypercube($options as object?) as object
{
  sec:user-defined-hypercube(
    {|
      if(exists($options.Concepts))
      then { "xbrl:Concept" : { Domain: $options.Concepts } }
      else (),
      if(exists($options.Entities))
      then { "xbrl:Entity" : { Domain: [ entities:eid($options.Entities[]) ] } }
      else (),
      if(exists($options.Periods))
      then { "xbrl:Period" : { Domain: $options.Periods } }
      else (),
      if(exists($options.Units))
      then { "xbrl:Unit" : { Domain: $options.Units } }
      else (),
      if(exists($options.LegalEntities))
      then { "dei:LegalEntityAxis" : { Domain: $options.LegalEntities } }
      else ()
    |}
  )
};

(:~
 :
Returns an instantiation of a Hypercube with no extra dimensions besides the legal entity axis.
 :    The four basic characteristics (xbrl:Concept, xbrl:Period, xbrl:Entity, and xbrl:Unit)
 :    are always added, but can be overriden. For each of those included aspects the value space is,
 :    by default, not limited, and only xbrl:Unit has a default value.

 :
The characteristic dei:LegalEntityAxis is also here. It has
 :    a default value of sec:DefaultLegalEntity and restricts the value
 :    space to this default value.

 :
 :
 : @return the hypercube instantiation.
 :)
declare function sec:user-defined-hypercube() as object
{
  sec:user-defined-hypercube(())
};

(:~
 :
Returns an instantiation of a Hypercube containing the specified dimensions.
 :    The four basic characteristics (xbrl:Concept, xbrl:Period, xbrl:Entity, and xbrl:Unit)
 :    are always added, but can be overriden. For each of those included aspects the value space is,
 :    by default, not limited, and only xbrl:Unit has a default value.

 :
The following dimensions are always present, but the values can be overriden:

 :
{
 :     "dei:LegalEntityAxis" : {
 :       Default: "sec:DefaultLegalEntity", -- overridable with HasDefault : false
 :       Domain: [ "sec:DefaultLegalEntity" ] -- overridable with Domain: null
 :     },
 :     "sec:Accepted" : {
 :       Type: "string"
 :     },
 :     "sec:FiscalYear" : {
 :       Type: "integer"
 :     },
 :     "sec:FiscalPeriod" : {
 :       Type: "string"
 :     }
 :    }

 :
 : @return the hypercube instantiation.
 : @param $dimensions
an object with pairs of (dimension name, object with dimension information). These
 : dimension information subobjects
 : may have four (optional) fields:

 :

 :
Type (string, absent by default): the name of a type (string, integer, ...). This creates a typed dimension (if absent, an explicit dimension).

 :
Default (a string, absent by default): must be either castable to Type, or null (explicit dimensions: must be a string with a member name, null not allowed).

 :
HasDefault (a boolean, true by default): if false, the dimension has no default even if Default is present (this includes the possibility to override predefined dimensions).

 :
Domain (null or an array of atomics, absent by default): restricts the domain of the dimension to the specified values, that must all be castable to Type or null (or all strings with member names for explicit dimensions). Don't forget to include the default value in there if any. If null is used instead of an array, this removes any predefined restrictions.
.
 :

 :)
declare function sec:user-defined-hypercube($dimensions as object?) as object
{
  let $dimensions := facts:merge-objects($dimensions, $sec:DEFAULTS, true)
  let $extended-dimensions := {|
    for $d in keys($dimensions)
    let $dimension-spec := $dimensions.$d

    let $dimension-default as atomic? :=
      if(not $dimension-spec.HasDefault eq false)
      then $dimension-spec.Default
      else ()

    let $dimension-domain as array? :=
        typeswitch($dimension-spec.Domain)
        case null return ()
        default return $dimension-spec.Domain

    return {
      $d : {|
        { Default : $dimension-default }[exists($dimension-default)],
        { Type : $dimension-spec.Type }[exists($dimension-spec.Type)],
        { Domain : $dimension-domain }[exists($dimension-domain)]
      |}
    }
  |}
  return hypercubes:user-defined-hypercube($extended-dimensions)
};


:pencil: Add test.

:pencil: Add test.

import module namespace test = "http://apps.28.io/test";

test:check-all-success({
    cocacola: test:invoke-and-assert-deep-equal(
      "facttable-for-component",
      {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
      },
      function($b as item*) as item* { count($b.FactTable[]) },
      1
    ),
    tickerrole: test:invoke-and-assert-deep-equal(
      "facttable-for-component",
      {
        ticker:"ko",
        fiscalYear:"2013",
        networkIdentifier:"http://www.thecocacolacompany.com/role/ConsolidatedStatementsOfIncome"
      },
      function($b as item*) as item* { count($b.FactTable[]) },
      60
    ),
    tickerconcept: test:invoke-and-assert-deep-equal(
      "facttable-for-component",
      {
        ticker:"ko",
        concept:"us-gaap:Assets",
        fiscalYear:"2013",
        fiscalPeriod: "FY"
      },
      function($b as item*) as item* { count($b.FactTable[]) },
      76
    ),
    tickerfyfprole: test:invoke-and-assert-deep-equal(
      "facttable-for-component",
      {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        networkIdentifier:"http://www.thecoca-colacompany.com/role/CondensedConsolidatedBalanceSheets"
      },
      function($b as item*) as item* { count($b.FactTable[]) },
      70
    ),
    generic: test:invoke-and-assert-deep-equal(
      "facttable-for-component",
      {
        profile-name: "generic",
        aid: "0000021344-14-000008",
        role: "http://www.thecocacolacompany.com/role/ConsolidatedBalanceSheets"
      },
      function($b as item*) as item* { count($b.FactTable[]) },
      76
    ),
    merge: test:invoke-and-assert-deep-equal(
      "facttable-for-component",
      {
          ticker:"ko",
          fiscalYear: ["2012", "2013" ],
          fiscalPeriod: "FY",
          disclosure:"BalanceSheet",
          merge: "true"
      },
      function($b as item*) as item* { count($b.FactTable[]) },
      153
    ),
    filter: test:invoke-and-assert-deep-equal(
      "facttable-for-component",
      {
          ticker:"ko",
          fiscalYear: ["2012", "2013" ],
          fiscalPeriod: "FY",
          disclosure:"BalanceSheet",
          merge: "true",
          override: "true"
      },
      function($b as item*) as item* { count($b.FactTable[]) },
      72
    )
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facttable-for-component"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    cocacola: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tickerrole: local:test-facttable(60, {
        ticker:"ko",
        fiscalYear:"2013",
        networkIdentifier:"http://www.thecocacolacompany.com/role/ConsolidatedStatementsOfIncome"
    }),
    tickerconcept: local:test-facttable(76, {
        ticker:"ko",
        concept:"us-gaap:Assets",
        fiscalYear:"2013",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-facttable(70, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        networkIdentifier:"http://www.thecoca-colacompany.com/role/CondensedConsolidatedBalanceSheets"
    }),
    generic: local:test-facttable(76, {
        profile-name: "generic",
        aid: "0000021344-14-000008",
        role: "http://www.thecocacolacompany.com/role/ConsolidatedBalanceSheets"
    }),
    merge: local:test-facttable(153, {
        ticker:"ko",
        fiscalYear: ["2012", "2013" ],
        fiscalPeriod: "FY",
        disclosure:"BalanceSheet",
        merge: "true"
    })
})


Update spreadsheet-for-report.jq

Update spreadsheet-for-report.jq

import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";
import module namespace reports = "http://xbrl.io/modules/bizql/reports";
import module namespace components2 = "http://xbrl.io/modules/bizql/components2";

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace sec-fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";

import module namespace session = "http://apps.28.io/session";
import module namespace util = "http://secxbrl.info/modules/util";

session:audit-call();

(: Query parameters :)
let $format         := lower-case((request:param-values("format"), substring-after(request:path(), ".jq."))[1])
let $ciks           := distinct-values(companies:eid(request:param-values("cik")))
let $tags           := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers        := distinct-values(request:param-values("ticker"))
let $sics           := distinct-values(request:param-values("sic"))
let $fiscalYears    := distinct-values(
                        request:param-values("fiscalYear")
                            [$$ = ("LATEST", "ALL") or $$ castable as integer]
                    )
let $fiscalPeriods  := distinct-values(let $fp := request:param-values("fiscalPeriod")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $sec-fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids           := archives:aid(request:param-values("aid"))
let $eliminate      := request:param-values("eliminate")
let $report         := request:param-values("report")
let $validate       := request:param-values("validate", "false")

(: Object resolution :)
let $entities := util:entities($ciks, $tags, $tickers, $sics, $aids)
let $report := reports:reports($report)

(: Fact resolution :)
let $filter-override as object? :=
    util:filter-override($entities, $fiscalPeriods, $fiscalYears, $aids)
let $filtered-aspects :=
    let $report := reports:reports($report)
    let $hypercube := hypercubes:hypercubes-for-components($report, "xbrl:DefaultHypercube")
    return values($hypercube.Aspects)[exists(($$.Domains, $$.DomainRestriction))]
let $spreadsheet as object? :=
    switch(true)

    case exists($filter-override)
    return
        components2:spreadsheet(
            $report,
            {
                FilterOverride: $filter-override,
                FlattenRows: true,
                Eliminate: boolean($eliminate eq "true"),
                Validate: boolean($validate eq "true")
            }
        )

    case count($filtered-aspects) ge 2
    return components2:spreadsheet(
        $report,
        {
            FlattenRows: true,
            Eliminate: boolean($eliminate eq "true"),
            Validate: boolean($validate eq "true")
        })

    default return ()
let $results :=
        if(count($filtered-aspects) lt 2 and not exists(($filter-override)))
        then {
              response:status-code(403);
              session:error("The report filters are too weak, which leads to too big an output.", $format)
        }
        else {
            switch ($format)
            case "xml" return {} (:)
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($spreadsheet)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=facts.csv");
                conversion:facts-to-csv($spreadsheet, { Caller: "Report"})
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=fact.csv");
                conversion:facts-to-csv($spreadsheet, { Caller: "Report"})
            }:)
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                $spreadsheet
            }
        }
return
    util:check-and-return-results($entities, $results, $format)

import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";
import module namespace reports = "http://xbrl.io/modules/bizql/reports";
import module namespace components2 = "http://xbrl.io/modules/bizql/components2";

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace sec-fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";

import module namespace session = "http://apps.28.io/session";
import module namespace util = "http://secxbrl.info/modules/util";

session:audit-call();

(: Query parameters :)
let $format         := lower-case((request:param-values("format"), substring-after(request:path(), ".jq."))[1])
let $ciks           := distinct-values(companies:eid(request:param-values("cik")))
let $tags           := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers        := distinct-values(request:param-values("ticker"))
let $sics           := distinct-values(request:param-values("sic"))
let $fiscalYears    := distinct-values(
                        request:param-values("fiscalYear", "LATEST")
                            [$$ = ("LATEST", "ALL") or $$ castable as integer]
                    )
let $fiscalPeriods  := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $sec-fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids           := archives:aid(request:param-values("aid"))
let $eliminate      := request:param-values("eliminate")
let $report         := request:param-values("report")
let $validate       := request:param-values("validate", "false")

(: Object resolution :)
let $entities := util:entities($ciks, $tags, $tickers, $sics, $aids)
let $report := reports:reports($report)

(: Fact resolution :)
let $filter-override as object? :=
    util:filter-override($entities, $fiscalPeriods, $fiscalYears, $aids)
let $filtered-aspects :=
    let $report := reports:reports($report)
    let $hypercube := hypercubes:hypercubes-for-components($report, "xbrl:DefaultHypercube")
    return values($hypercube.Aspects)[exists(($$.Domains, $$.DomainRestriction))]
let $spreadsheet as object? :=
    switch(true)

    case exists($filter-override)
    return
        components2:spreadsheet(
            $report,
            {
                FilterOverride: $filter-override,
                FlattenRows: true,
                Eliminate: boolean($eliminate eq "true"),
                Validate: boolean($validate eq "true")
            }
        )

    case count($filtered-aspects) ge 2
    return components2:spreadsheet(
        $report,
        {
            FlattenRows: true,
            Eliminate: boolean($eliminate eq "true"),
            Validate: boolean($validate eq "true")
        })

    default return ()
let $results :=
        if(count($filtered-aspects) lt 2 and not exists(($filter-override)))
        then {
              response:status-code(403);
              session:error("The report filters are too weak, which leads to too big an output.", $format)
        }
        else {
            switch ($format)
            case "xml" return {} (:)
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($spreadsheet)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=facts.csv");
                conversion:facts-to-csv($spreadsheet, { Caller: "Report"})
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=fact.csv");
                conversion:facts-to-csv($spreadsheet, { Caller: "Report"})
            }:)
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                $spreadsheet
            }
        }
return
    util:check-and-return-results($entities, $results, $format)