:green_heart: Fix typo.

:green_heart: Fix typo.

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facttable-for-component"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    cocacola: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tickerrole: local:test-facttable(60, {
        ticker:"ko",
        fiscalYear:"2013",
        networkIdentifier:"http://www.thecocacolacompany.com/role/ConsolidatedStatementsOfIncome"
    }),
    tickerconcept: local:test-facttable(76, {
        ticker:"ko",
        concept:"us-gaap:Assets",
        fiscalYear:"2013",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-facttable(70, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        networkIdentifier:"http://www.thecoca-colacompany.com/role/CondensedConsolidatedBalanceSheets"
    }),
    generic: local:test-facttable(76, {
        profile-name: "generic",
        aid: "0000021344-14-000008",
        role: "http://www.thecocacolacompany.com/role/ConsolidatedBalanceSheets"
    }),
    merge: local:test-facttable(153, {
        ticker:"ko",
        fiscalYear: ["2012", "2013" ],
        fiscalPeriod: "FY",
        disclosure:"BalanceSheet",
        merge: "true"
    })
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facttable-for-component"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    cocacola: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY",
    }),
    tickerrole: local:test-facttable(60, {
        ticker:"ko",
        fiscalYear:"2013",
        networkIdentifier:"http://www.thecocacolacompany.com/role/ConsolidatedStatementsOfIncome"
    }),
    tickerconcept: local:test-facttable(76, {
        ticker:"ko",
        concept:"us-gaap:Assets",
        fiscalYear:"2013",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-facttable(70, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        networkIdentifier:"http://www.thecoca-colacompany.com/role/CondensedConsolidatedBalanceSheets"
    }),
    generic: local:test-facttable(76, {
        profile-name: "generic",
        aid: "0000021344-14-000008",
        role: "http://www.thecocacolacompany.com/role/ConsolidatedBalanceSheets"
    }),
    merge: local:test-facttable(153, {
        ticker:"ko",
        fiscalYear: ["2012", "2013" ],
        fiscalPeriod: "FY",
        disclosure:"BalanceSheet",
        merge: "true"
    })
})


:green_heart: Fix filings test.

:green_heart: Fix filings test.


Simplify removal code.

Simplify removal code.

Now that I decided to not have a generic method for removal anyway, it
doesn't really make sense to keep doing it in such a roundabout way ---
by marking for removal and then removing. This is much clearer: we just
walk through the object explicitly and remove where necessary.

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    [. == (null,false,0,{},[])] | any | not
;

# Is a given array at least length $n?
def at_least($n):
    (. | length) >= $n
;

# Generate this object and all its descendants.
def entries:
    ., recurse(.contents?[]?)
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`. We use `name` and `contents` to be compatible
# with the output of the `tree` program on the shell.
def tree($path):
    if $path | at_least(2)
    then { "contents": [ tree($path[1:]) ] }
    else .
    end | .name = $path[0]
;

# Merge an array of page objects into a single page object.
def merge:
    map(to_entries) | add | group_by(.key) | map(
        { "key": (.[0].key)
        , "value": (
            if .[0].key == "contents"
            then [ .[].value ] | add | group_by(.name) | map(merge)
            elif [ .[].value == .[0].value ] | all
            then .[0].value
            else error("can't merge incompatible values for key '\(.[0].key)'")
            end
        ) }
    ) | from_entries
;

# Add paths to each object, that is, the names of the ancestors.
def directory:
    def f($d): .dir=($d|join("/")) | .name as $n | .contents[]?|=f($d+[$n]);
    f([])
;

# Every leaf node gets a basename, eg name without extension.
def basename:
    (entries | select(has("contents") | not)) |= (
        .basename = (.name | sub(".([A-z]*)$";""))
    )
;

# Any page gets its target formats.
def formats($all_formats):
    entries |= (
        .formats = [
            (.make // empty) |
            if . == "null" then empty else . end | # fix for wrong YAML parse
            if (. | type) == "array" then .[] else . end |
            tostring | ascii_downcase |
            if [. == $all_formats[]] | any then .
            elif . == "all" then $all_formats[]
            else error("unrecognized format: \(.)")
            end
        ]
    )
;

# Any page is linked to its first target format.
def link:
    (entries | select(.formats | at_least(1))) |= (
        .link = "\(.dir)/\(.basename).\(.formats[0])"
    )
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def frontmatter:
    (entries | select(has("contents"))) |= (
        (.frontmatter = [.contents[] | select(.frontmatter | bool)][0]) |
        (.contents = .contents - [.frontmatter])
    )
;

# Any page that has no link should be excluded from the index. It would be
# nicer if we could remove in one swoop with `entries | ... |= empty`. It only
# works sometimes, presumably since we'd change the objects we'd iterate over.
def clean:
    if has("contents") then .contents |= map(clean)
    elif has("link")   then .
    else empty end
;

# Add links to the next and previous page for every linked page.
def navigation:
    def f($paths; $i; $j; $key):
        setpath($paths[$i]+[$key];
            getpath($paths[$j]) | {link,title,date}
        )
    ;
    [ path(entries | select(has("link"))) ] as $paths
    | ($paths | length) as $n
    | reduce range(0; $n-1) as $i (.; f($paths; $i; $i+1; "next"))
    | reduce range(1, $n)   as $i (.; f($paths; $i; $i-1; "prev"))
;

# Sort content first according to sort order in metadata.
def sorting:
    (entries | select(has("contents")) | .contents) |= (
        sort_by(.sort // .title // .name)
    )
;

# Add a note that tells us whether this has only children who have no more
# subchildren.
def annotate_leaves:
    (entries | select(has("contents"))) |= (
        .only_leaves = (.contents | all(has("contents") | not))
    )
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index($all_formats):
    merge
    | directory
    | basename
    | formats($all_formats)
    | link
    | clean
    | navigation
    | frontmatter
    | sorting
    | annotate_leaves
;

# Get target files and their dependencies as an enumeration of {"target":...,
# "dependencies":...} objects.
def targets($dest; $default_style):
    [ entries
        | ., (.frontmatter // empty)
        | .formats[] as $format
        | "\($dest)/\(.dir)/\(.basename).\($format)" as $doc
        | ["\($dest)/\(.dir)/\(.resources?[])"] as $external
        | ["\($dest)/\(.style // $default_style).css"] as $css
        | (if $format == "html" then ($css + $external) else [] end) as $linked
        | (if $format == "pdf"  then ($css + $external) else [] end) as $embedded
        | {"target": $format, "dependencies": ([$doc] + $linked) }
        , {"target": $doc, "dependencies": $embedded }
    ]
    | group_by(.target)
    | map(
        {"target": (.[0].target)
        , "dependencies": (map(.dependencies) | add | unique) }
    )
    | .[]?
;

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    [. == (null,false,0,{},[])] | any | not
;

# Is a given array at least length $n?
def at_least($n):
    (. | length) >= $n
;

# Generate this object and all its descendants.
def entries:
    ., recurse(.contents?[]?)
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`. We use `name` and `contents` to be compatible
# with the output of the `tree` program on the shell.
def tree($path):
    if $path | at_least(2)
    then { "contents": [ tree($path[1:]) ] }
    else .
    end | .name = $path[0]
;

# Merge an array of page objects into a single page object.
def merge:
    map(to_entries) | add | group_by(.key) | map(
        { "key": (.[0].key)
        , "value": (
            if .[0].key == "contents"
            then [ .[].value ] | add | group_by(.name) | map(merge)
            elif [ .[].value == .[0].value ] | all
            then .[0].value
            else error("can't merge incompatible values for key '\(.[0].key)'")
            end
        ) }
    ) | from_entries
;

# Add paths to each object, that is, the names of the ancestors.
def directory:
    def f($d): .dir=($d|join("/")) | .name as $n | .contents[]?|=f($d+[$n]);
    f([])
;

# Every leaf node gets a basename, eg name without extension.
def basename:
    (entries | select(has("contents") | not)) |= (
        .basename = (.name | sub(".([A-z]*)$";""))
    )
;

# Any page gets its target formats.
def formats($all_formats):
    entries |= (
        .formats = [
            (.make // empty) |
            if . == "null" then empty else . end | # fix for wrong YAML parse
            if (. | type) == "array" then .[] else . end |
            tostring | ascii_downcase |
            if [. == $all_formats[]] | any then .
            elif . == "all" then $all_formats[]
            else error("unrecognized format: \(.)")
            end
        ]
    )
;

# Any page is linked to its first target format.
def link:
    (entries | select(.formats | at_least(1))) |= (
        .link = "\(.dir)/\(.basename).\(.formats[0])"
    )
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def frontmatter:
    (entries | select(has("contents"))) |= (
        (.frontmatter = [.contents[] | select(.frontmatter | bool)][0]) |
        (.contents = .contents - [.frontmatter])
    )
;

# Anything that does not have a link, nor has any children, should be
# considered a draft and removed from the index.
# To remove an object, we first mark it for removal, then actually remove it
# later. It would be nicer if we could just do `entries ... |= empty`,
# which works in some cases but not all. I suppose it has to do with changing
# objects as we are iterating over them.
def clean:
    def mark:
        .r = true
    ;
    def remove:
        if .r == true
        then empty
        else (.contents//empty) |= map(remove)
        end
    ;
    ((entries | select((has("contents") or has("link")) | not)) |= mark)
    | remove
;

# Add links to the next and previous page for every linked page.
def navigation:
    def f($paths; $i; $j; $key):
        setpath($paths[$i]+[$key];
            getpath($paths[$j]) | {link,title,date}
        )
    ;
    [ path(entries | select(has("link"))) ] as $paths
    | ($paths | length) as $n
    | reduce range(0; $n-1) as $i (.; f($paths; $i; $i+1; "next"))
    | reduce range(1, $n)   as $i (.; f($paths; $i; $i-1; "prev"))
;

# Sort content first according to sort order in metadata.
def sorting:
    (entries | select(has("contents")) | .contents) |= (
        sort_by(.sort // .title // .name)
    )
;

# Add a note that tells us whether this has only children who have no more
# subchildren.
def annotate_leaves:
    (entries | select(has("contents"))) |= (
        .only_leaves = (.contents | all(has("contents") | not))
    )
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index($all_formats):
    merge
    | directory
    | basename
    | formats($all_formats)
    | link
    | clean
    | navigation
    | frontmatter
    | sorting
    | annotate_leaves
;

# Get target files and their dependencies as an enumeration of {"target":...,
# "dependencies":...} objects.
def targets($dest; $default_style):
    [ entries
        | ., (.frontmatter // empty)
        | .formats[] as $format
        | "\($dest)/\(.dir)/\(.basename).\($format)" as $doc
        | ["\($dest)/\(.dir)/\(.resources?[])"] as $external
        | ["\($dest)/\(.style // $default_style).css"] as $css
        | (if $format == "html" then ($css + $external) else [] end) as $linked
        | (if $format == "pdf"  then ($css + $external) else [] end) as $embedded
        | {"target": $format, "dependencies": ([$doc] + $linked) }
        , {"target": $doc, "dependencies": $embedded }
    ]
    | group_by(.target)
    | map(
        {"target": (.[0].target)
        , "dependencies": (map(.dependencies) | add | unique) }
    )
    | .[]?
;


:white_check_mark: add test to check that fsa:* concepts have labels.

import module namespace test = "http://apps.28.io/test";

test:check-all-success({
    taxonomy-label: test:invoke-and-assert-deep-equal(
      "report-elements",
      {aid:"STANDARD-TAXONOMY-2014", label:"\"Accounting policy for deferred assets\""},
      function($b as item*) as item* { $b.ReportElements },
      test:get-expected-result("edinet/report-elements-expected1.jq"),
      { NoArrayOrder: true }
    ),
    taxonomy-name: test:invoke-and-assert-deep-equal(
      "report-elements",
      {aid:"STANDARD-TAXONOMY-2014", name:"jppfs-cor:CurrentAssets"},
      function($b as item*) as item* { $b.ReportElements },
      test:get-expected-result("edinet/report-elements-expected2.jq"),
      { NoArrayOrder: true }
    ),
    company-label: test:invoke-and-assert-deep-equal(
      "report-elements",
      {cik:"E01225", label:"\"Loss of inactive facilities\""},
      function($b as item*) as item* { $b.ReportElements },
      test:get-expected-result("edinet/report-elements-expected3.jq"),
      { NoArrayOrder: true }
    ),
    company-name: test:invoke-and-assert-deep-equal(
      "report-elements",
      {cik:"E01225", name:"jpcrp-cor:NotesConsolidatedStatementOfCashFlowsHeading"},
      function($b as item*) as item* { $b.ReportElements },
      test:get-expected-result("edinet/report-elements-expected4.jq"),
      { NoArrayOrder: true }
    ),
    company-name-xml : test:invoke-and-assert-deep-equal(
      "report-elements",
      {cik:"E01225", name:"jpcrp-cor:NotesConsolidatedStatementOfCashFlowsHeading", format: "xml"},
      function($b as item*) as item* { document { $b/ReportElements } },
      test:get-expected-result-xml("edinet/report-elements-expected4-xml.jq"),
      { Format : "xml" }
    ),
    taxonomy-name-2: test:invoke-and-assert-deep-equal(
      "report-elements",
      {aid:"STANDARD-TAXONOMY-2014", name: ["fsa:FiscalYear", "fsa:FiscalPeriod", "fsa:FiscalPeriodType", "fsa:Submitted"] },
      function($b as item*) as item* { count($b.ReportElements[].Labels[]) },
      4
    )
})

import module namespace test = "http://apps.28.io/test";

test:check-all-success({
    taxonomy-label: test:invoke-and-assert-deep-equal(
      "report-elements",
      {aid:"STANDARD-TAXONOMY-2014", label:"\"Accounting policy for deferred assets\""},
      function($b as item*) as item* { $b.ReportElements },
      test:get-expected-result("edinet/report-elements-expected1.jq"),
      { NoArrayOrder: true }
    ),
    taxonomy-name: test:invoke-and-assert-deep-equal(
      "report-elements",
      {aid:"STANDARD-TAXONOMY-2014", name:"jppfs-cor:CurrentAssets"},
      function($b as item*) as item* { $b.ReportElements },
      test:get-expected-result("edinet/report-elements-expected2.jq"),
      { NoArrayOrder: true }
    ),
    company-label: test:invoke-and-assert-deep-equal(
      "report-elements",
      {cik:"E01225", label:"\"Loss of inactive facilities\""},
      function($b as item*) as item* { $b.ReportElements },
      test:get-expected-result("edinet/report-elements-expected3.jq"),
      { NoArrayOrder: true }
    ),
    company-name: test:invoke-and-assert-deep-equal(
      "report-elements",
      {cik:"E01225", name:"jpcrp-cor:NotesConsolidatedStatementOfCashFlowsHeading"},
      function($b as item*) as item* { $b.ReportElements },
      test:get-expected-result("edinet/report-elements-expected4.jq"),
      { NoArrayOrder: true }
    ),
    company-name-xml : test:invoke-and-assert-deep-equal(
      "report-elements",
      {cik:"E01225", name:"jpcrp-cor:NotesConsolidatedStatementOfCashFlowsHeading", format: "xml"},
      function($b as item*) as item* { document { $b/ReportElements } },
      test:get-expected-result-xml("edinet/report-elements-expected4-xml.jq"),
      { Format : "xml" }
    )
})


:green_heart: Fix build.

:green_heart: Fix build.

import module namespace test = "http://apps.28.io/test";

test:check-all-success({|
  {
    one: test:invoke-and-assert-deep-equal(
      "components",
      {
        aid: "S1002FC3",
        role: "http://disclosure.edinet-fsa.go.jp/role/jpcrp/rol_CoverPage"
      },
      function($res as item*) as item* { count($res.Components[]) },
      1
    ),
    aid: test:invoke-and-assert-deep-equal(
        "components",
        {
          aid: "S1002FC3"
        },
        function($res as item*) as item* { count($res.Components[]) },
        14
      )
    }
|})

import module namespace test = "http://apps.28.io/test";

let $is-japan-big := test:is-japan-big()
return
test:check-all-success({|
  {
    one: test:invoke-and-assert-deep-equal(
      "components",
      {
        aid: "S1002FC3",
        role: "http://disclosure.edinet-fsa.go.jp/role/jpcrp/rol_CoverPage"
      },
      function($res as item*) as item* { count($res.Components[]) },
      1
    ),
    aid: test:invoke-and-assert-deep-equal(
        "components",
        {
          aid: "S1002FC3"
        },
        function($res as item*) as item* { count($res.Components[]) },
        14
      )
    }
|})


:green_heart: Fix test.

:green_heart: Fix test.

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facts"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-404($params as object) as item
{
    let $endpoint := "facts"
    let $res as object := test:invoke-raw($endpoint, $params)
    return if($res.status eq 404)
           then true
           else {
              url: test:url($endpoint, $params),
              unexpectedResponse: $res
           }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw($endpoint, $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "Archive [Axis],Fiscal Period [Axis],Fiscal Year [Axis],Fiscal Period Type [Axis],Period [Axis],Reporting Entity [Axis],Concept [Axis],Unit [Axis],Legal Entity [Axis],Value,Decimals\r\n0000021344-13-000050,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),\"Cash and Cash Equivalents, at Carrying Value\",USD,Default Legal Entity [Member],11118000000,-6\r\n0000021344-13-000050,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),Assets,USD,Default Legal Entity [Member],89432000000,-6\r\n"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res,
        expected: $expectedLines
    }
};

declare %an:nondeterministic function local:test-labels-aids() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: "disc:AdvertisingCostsPolicyTextBlock",
        report: "Disclosures",
        format: "json",
        fiscalYear: 2014,
        fiscalPeriod: "FY",
        labels: true,
        aid: "0000858877-14-000029"
    }
    let $res := test:invoke($endpoint, $params)
    let $actual := [
            for $labels in $res[2].FactTable[].Labels
            return (keys($labels) ! $labels.$$)
        ]
    let $expected := [ "Archive [Axis]", "Fiscal Year [Axis]", "Non-Numeric [Member]", "Default Legal Entity [Member]", "Unit [Axis]", "Advertising Costs, Policy", "Fiscal Period [Axis]", "Concept [Axis]", "Reporting Entity [Axis]", "Legal Entity [Axis]", "Fiscal Period Type [Axis]", "Period [Axis]", "From July 28, 2013 to July 26, 2014", "CISCO SYSTEMS, INC. (858877)" ]
    let $status as integer := $res[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

local:check({
    cocacola-latest:
    (: this test will fail and needs to be updated if a newer report has been filed.
       current latest filing: 2014 :)
    local:test-facttable(462, {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    cocacola-all: local:test-facttable(468, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-all-q3: local:test-facttable(341, {
        ticker:"ko",
        fiscalYear: 2014,
        fiscalPeriod: [ "Q3" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-instant: local:test-facttable(163, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant" ]
    }),
    cocacola-ytd: local:test-facttable(305, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-qtd: local:test-facttable(90, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "Q2" ],
        fiscalPeriodType: [ "QTD" ]
    }),
    cocacolaCSVLabels: local:test-labels(),
    ciscoLabelsByAid: local:test-labels-aids(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(31, { (: Wrong stamp for AT&T, two FY 2014s:)
        tag:"DOW30",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-404({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    }),
    quartersfact: local:test-facttable(0, { (: this fact is not reported in the raw XML filing :)
        aid: "0000051143-13-000007",
        concept: "disc:AssetRetirementObligationsPolicy",
        fiscalPeriod: "Q3",
        fiscalYear: "2013",
        map: "Disclosures"
    }),
    quartersfact2: local:test-facttable(1, {
        aid: "0001193125-12-207154",
        concept: "disc:CashFlowOperatingCapitalTableTextBlock",
        fiscalPeriod: "Q1",
        fiscalYear: "2012",
        map: "Disclosures"
    }),
    generic: local:test-facttable(2, {
        "xbrl:Entity":"http://www.sec.gov/CIK 0000021344",
        "sec:Accepted":"20140227132423",
        concept:"us-gaap:Assets",
        profile-name: "generic"
    }),
    count: test:invoke-and-assert-deep-equal(
          "facts",
          {
              "ticker" : "ko",
              "fiscalYear" : ["2012", "2013", "2014"],
              "fiscalPeriod" : "FY",
              "count" : "true"
          },
          function($res as item*) as item* { $res.Statistics.NumFacts },
          1405
        )
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facts"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-404($params as object) as item
{
    let $endpoint := "facts"
    let $res as object := test:invoke-raw($endpoint, $params)
    return if($res.status eq 404)
           then true
           else {
              url: test:url($endpoint, $params),
              unexpectedResponse: $res
           }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw($endpoint, $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "Archive [Axis],Fiscal Period [Axis],Fiscal Year [Axis],Fiscal Period Type [Axis],Period [Axis],Reporting Entity [Axis],Concept [Axis],Unit [Axis],Legal Entity [Axis],Value,Decimals\r\n0000021344-13-000050,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),\"Cash and Cash Equivalents, at Carrying Value\",USD,Default Legal Entity [Member],11118000000,-6\r\n20131024121047,0000021344-13-000050,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),Assets,USD,Default Legal Entity [Member],89432000000,-6\r\n"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res,
        expected: $expectedLines
    }
};

declare %an:nondeterministic function local:test-labels-aids() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: "disc:AdvertisingCostsPolicyTextBlock",
        report: "Disclosures",
        format: "json",
        fiscalYear: 2014,
        fiscalPeriod: "FY",
        labels: true,
        aid: "0000858877-14-000029"
    }
    let $res := test:invoke($endpoint, $params)
    let $actual := [
            for $labels in $res[2].FactTable[].Labels
            return (keys($labels) ! $labels.$$)
        ]
    let $expected := [ "Archive [Axis]", "Fiscal Year [Axis]", "Non-Numeric [Member]", "Default Legal Entity [Member]", "Unit [Axis]", "Advertising Costs, Policy", "Fiscal Period [Axis]", "Concept [Axis]", "Reporting Entity [Axis]", "Legal Entity [Axis]", "Fiscal Period Type [Axis]", "Period [Axis]", "From July 28, 2013 to July 26, 2014", "CISCO SYSTEMS, INC. (858877)" ]
    let $status as integer := $res[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

local:check({
    cocacola-latest:
    (: this test will fail and needs to be updated if a newer report has been filed.
       current latest filing: 2014 :)
    local:test-facttable(462, {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    cocacola-all: local:test-facttable(468, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-all-q3: local:test-facttable(341, {
        ticker:"ko",
        fiscalYear: 2014,
        fiscalPeriod: [ "Q3" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-instant: local:test-facttable(163, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant" ]
    }),
    cocacola-ytd: local:test-facttable(305, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-qtd: local:test-facttable(90, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "Q2" ],
        fiscalPeriodType: [ "QTD" ]
    }),
    cocacolaCSVLabels: local:test-labels(),
    ciscoLabelsByAid: local:test-labels-aids(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(31, { (: Wrong stamp for AT&T, two FY 2014s:)
        tag:"DOW30",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-404({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    }),
    quartersfact: local:test-facttable(0, { (: this fact is not reported in the raw XML filing :)
        aid: "0000051143-13-000007",
        concept: "disc:AssetRetirementObligationsPolicy",
        fiscalPeriod: "Q3",
        fiscalYear: "2013",
        map: "Disclosures"
    }),
    quartersfact2: local:test-facttable(1, {
        aid: "0001193125-12-207154",
        concept: "disc:CashFlowOperatingCapitalTableTextBlock",
        fiscalPeriod: "Q1",
        fiscalYear: "2012",
        map: "Disclosures"
    }),
    generic: local:test-facttable(2, {
        "xbrl:Entity":"http://www.sec.gov/CIK 0000021344",
        "sec:Accepted":"20140227132423",
        concept:"us-gaap:Assets",
        profile-name: "generic"
    }),
    count: test:invoke-and-assert-deep-equal(
          "facts",
          {
              "ticker" : "ko",
              "fiscalYear" : ["2012", "2013", "2014"],
              "fiscalPeriod" : "FY",
              "count" : "true"
          },
          function($res as item*) as item* { $res.Statistics.NumFacts },
          1405
        )
})


Update entities.jq

Update entities.jq

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/test";

import module namespace http-request = "http://www.28msec.com/modules/http/request";
import module namespace csv = "http://zorba.io/modules/json-csv";

import module namespace entities = "http://28.io/modules/xbrl/entities";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token        as string? external;
declare  %rest:env                              variable $request-uri  as string  external;
declare  %rest:case-insensitive                 variable $format       as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $eid          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cik          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker       as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic          as string* external;
declare  %rest:case-insensitive                 variable $profile-name as string  external := $config:profile-name;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)
let $tag := if (exists(($cik, $tag, $ticker, $sic)))
             then $tag
             else "ALL"

(: Object resolution :)
let $entities :=
    switch($profile-name)
    case "sec" return
        for $entity in companies:companies(
            $cik,
            $tag,
            $ticker,
            $sic)
        order by $entity.Profiles.SEC.CompanyName
        return $entity
    default return
        for $entity in
            if(exists($eid)) then entities:entities($eid)
                                else entities:entities()
        return {
            EID: $entity._id
        }
let $comment :=
{
    NumEntities: count($entities),
    TotalNumEntities: session:num-entities()
}
let $entities :=
  switch($profile-name)
  case "sec" return
    for $entity in $entities
    return {|
      project($entity, "_id"),
      {
        Archives: backend:url("filings", {
          cik: tokenize($entity._id, " ")[2],
          fiscalYear: "ALL",
          fiscalPeriod: "ALL",
          format: $format,
          profile-name: $profile-name
        }),
      trim($entity, "_id")
    |}
  default return
    for $entity in $entities
    return {|
      $entity,
      {
        Archives: backend:url("filings", {
          eid: encode-for-uri($entity.EID),
          format: $format,
          profile-name: $profile-name
        })
      }
  |}

let $result := { "Entities" : [ $entities ] }
let $serializers := {
    to-xml : function($res as object) as node() {
        switch($profile-name)
        case "sec" return
        <Entities>{
            companies:to-xml($res.Entities[])
        }</Entities>
        default return <Entities>
            {
                for $id in $res.Entities[].EID
                return <EID>{$id}</EID>
            }
        </Entities>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec"
        return string-join(companies:to-csv($res.Entities[]))
        default return
            string-join(
                csv:serialize($res.Entities[], { serialize-null-as : "" }),
        "")
    }
}
return api:serialize($result, $comment, $serializers, $format, "entities")

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/test";

import module namespace http-request = "http://www.28msec.com/modules/http/request";
import module namespace csv = "http://zorba.io/modules/json-csv";

import module namespace entities = "http://28.io/modules/xbrl/entities";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token        as string? external;
declare  %rest:env                              variable $request-uri  as string  external;
declare  %rest:case-insensitive                 variable $format       as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $eid          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cik          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker       as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic          as string* external;
declare  %rest:case-insensitive                 variable $profile-name as string  external := $config:profile-name;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)
let $tag := if (exists(($cik, $tag, $ticker, $sic)))
             then $tag
             else "ALL"

(: Object resolution :)
let $entities :=
    switch($profile-name)
    case "sec" return
        for $entity in companies:companies(
            $cik,
            $tag,
            $ticker,
            $sic)
        order by $entity.Profiles.SEC.CompanyName
        return $entity
    default return
        for $entity in
            if(exists($eid)) then entities:entities($eid)
                                else entities:entities()
        return {
            EID: $entity._id
        }
let $comment :=
{
    NumEntities: count($entities),
    TotalNumEntities: session:num-entities()
}
let $entities :=
  switch($profile-name)
  case "sec" return
    for $entity in $entities
    return {|
      project($entity, "_id"),
      {
        Archives: backend:url({
          cik: tokenize($entity._id, " ")[2],
          fiscalYear: "ALL",
          fiscalPeriod: "ALL",
          format: $format,
          profile-name: $profile-name
        }),
      trim($entity, "_id")
    |}
  default return
    for $entity in $entities
    return {|
      $entity,
      {
        Archives: backend:url({
          eid: encode-for-uri($entity.EID),
          format: $format,
          profile-name: $profile-name
        })
      }
  |}

let $result := { "Entities" : [ $entities ] }
let $serializers := {
    to-xml : function($res as object) as node() {
        switch($profile-name)
        case "sec" return
        <Entities>{
            companies:to-xml($res.Entities[])
        }</Entities>
        default return <Entities>
            {
                for $id in $res.Entities[].EID
                return <EID>{$id}</EID>
            }
        </Entities>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec"
        return string-join(companies:to-csv($res.Entities[]))
        default return
            string-join(
                csv:serialize($res.Entities[], { serialize-null-as : "" }),
        "")
    }
}
return api:serialize($result, $comment, $serializers, $format, "entities")


Update spreadsheet-for-component.jq

Update spreadsheet-for-component.jq

import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

declare %an:nondeterministic function local:test-spreadsheet($expected as integer, $params as string) as atomic
{
    let $actual as integer := count(parse-json(http-client:get("http://" || request:server-name() || ":" || request:server-port() || "/v1/_queries/public/api/spreadsheet-for-component.jq?_method=POST" || $params).body.content).TableSet[].TableCells.Facts[][].Value)
    return if ($actual eq $expected) then true else "false [Actual="||$actual||", Expected="||$expected ||"]"
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

let $db := request:param-values("db", "all-sec-filings")
return local:check({
    cocacola: local:test-spreadsheet(1, "&ticker=ko"),
    tickerrole: local:test-spreadsheet(60, "&ticker=ko&networkIdentifier=http://www.thecocacolacompany.com/role/ConsolidatedStatementsOfIncome"),
    tickerconcept: local:test-spreadsheet(switch($db) case "all-dow30" return 93 default return 60, "&ticker=ko&fiscalYear=2014&concept=us-gaap:NetIncomeLoss"),
    tickerfyfprole: local:test-spreadsheet(70, "&ticker=ko&fiscalYear=2012&fiscalPeriod=Q1&networkIdentifier=http://www.thecoca-colacompany.com/role/CondensedConsolidatedBalanceSheets")
})

import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

declare %an:nondeterministic function local:test-spreadsheet($expected as integer*, $params as string) as atomic
{
    let $actual as integer := count(parse-json(http-client:get("http://" || request:server-name() || ":" || request:server-port() || "/v1/_queries/public/api/spreadsheet-for-component.jq?_method=POST" || $params).body.content).TableSet[].TableCells.Facts[][].Value)
    return if ($actual = $expected) then true else "false [Actual="||$actual||", Expected="||$expected ||"]"
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    cocacola: local:test-spreadsheet(1, "&ticker=ko"),
    tickerrole: local:test-spreadsheet(60, "&ticker=ko&networkIdentifier=http://www.thecocacolacompany.com/role/ConsolidatedStatementsOfIncome"),
    tickerconcept: local:test-spreadsheet((93, 60), "&ticker=ko&concept=us-gaap:NetIncomeLoss"),
    tickerfyfprole: local:test-spreadsheet(70, "&ticker=ko&fiscalYear=2012&fiscalPeriod=Q1&networkIdentifier=http://www.thecoca-colacompany.com/role/CondensedConsolidatedBalanceSheets")
})


Update patientfilter.jq

Update patientfilter.jq

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv
.[]
# start from the .results array
|.results

# remove .patient.drug array
|map(del(.patient.drug))

# remove .patient.reaction array
|map(del(.patient.reaction))

# flattens .patient object and renames child keys with patient_ prefix
|map(with_entries(select(.key != "patient")) +
       (.patient|with_entries(.key = "patient_" + .key)))

# flattens all the objects and renames child keys
|map(with_entries(select(.key != "primarysource" and
                         .key != "sender" and
                         .key != "receiver" and
                         .key != "patient_summary" and
                         .key != "reportduplicate" and
                         .key != "patient_patientdeath")) +
       (.primarysource//{primarysource:{"NA":"NA"}}|with_entries(.key = "primarysource_" + .key)) +
       (.sender//{sender:{"NA":"NA"}}|with_entries(.key = "sender_" + .key)) +
       (.receiver//{receiver:{"NA":"NA"}}|with_entries(.key = "receiver_" + .key)) +
       (.patient_summary//{patient_summary:{"NA":"NA"}}|with_entries(.key = "patient_summary_" + .key)) +
       (.report_duplicate//{report_duplicate:{"NA":"NA"}}|with_entries(.key = "report_duplicate_" + .key))+
       (.patient_patientdeath//{patient_patientdeath:{"NA":"NA"}}|with_entries(.key = "patient_patientdeath_" + .key)))

|map(del(.primarysource_primary_source))
|map(del(.sender_sender))
|map(del(.receiver_receiver))
|map(del(.report_duplicate_report_duplicate))
|map(del(.patient_summary_patient_summary))
|map(del(.patient_patientdeath_patient_patientdeath))

# grabs all unique keys and spits out CSV
|(map(keys)|add|unique) as $keys| $keys, (.[]|[.[$keys[]]])|@csv

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv
.[]
# start from the .results array
|.results

# remove .patient.drug array
|map(del(.patient.drug))

# remove .patient.reaction array
|map(del(.patient.reaction))

# flattens .patient object and renames child keys with patient_ prefix
|map(with_entries(select(.key != "patient")) +
       (.patient|with_entries(.key = "patient_" + .key)))

# flattens all the objects and renames child keys
|map(with_entries(select(.key != "primarysource" and
                         .key != "sender" and
                         .key != "receiver" and
                         .key != "patient_summary" and
                         .key != "reportduplicate" and
                         .key != "patient_patientdeath")) +
       (.primarysource//{patientsource:{"NA":"NA"}}|with_entries(.key = "primarysource_" + .key)) +
       (.sender//{sender:{"NA":"NA"}}|with_entries(.key = "sender_" + .key)) +
       (.receiver//{receiver:{"NA":"NA"}}|with_entries(.key = "receiver_" + .key)) +
       (.patient_summary//{patient_summary:{"NA":"NA"}}|with_entries(.key = "patient_summary_" + .key)) +
       (.report_duplicate//{report_duplicate:{"NA":"NA"}}|with_entries(.key = "report_duplicate_" + .key))+
       (.patient_patientdeath//{patient_patientdeath:{"NA":"NA"}}|with_entries(.key = "patient_patientdeath_" + .key)))

|map(del(.primarysource_primary_source))
|map(del(.sender_sender))
|map(del(.receiver_receiver))
|map(del(.report_duplicate_report_duplicate))
|map(del(.patient_summary_patient_summary))
|map(del(.patient_patientdeath_patient_patientdeath))

# grabs all unique keys and spits out CSV
|(map(keys)|add|unique) as $keys| $keys, (.[]|[.[$keys[]]])|@csv


Fix for wrong YAML parse of `null` in formats.

Pandoc parses `null` as a string.

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    [. == (null,false,0,{},[])] | any | not
;

# Generate this object and all its children.
def all_children:
    ., recurse(.contents?[]?)
;

# To remove an object, we first mark it for removal, then actually remove it
# later. It would be nicer if we could just do `all_children ... |= empty`,
# which works in some cases but not all. I suppose it has to do with changing
# objects as we are iterating over them.
def mark:
    .remove = true
;

# Remove all objects marked for removal.
def remove_marked:
    if .remove == true
    then empty
    else (.contents // empty) |= map(remove_marked)
    end
;

# Enumerate the listed `make` formats of a page.
def formats:
    (.meta.make // empty)
    |   if . == "null" then empty else . end # fix for wrong YAML parse
    |   if (. | type) == "array" then .[] else . end
    |   tostring
    |   ascii_downcase

;

# Enumerate the `make` formats of a page (including "all").
def target_formats($all_formats):
    formats |
    if [. == $all_formats[]] | any then .
    elif . == "all" then $all_formats[]
    else error("unrecognized format: " + .)
    end
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`.
def tree($path):
    if ($path | length) <= 1
    then .name = $path[0]
    else { "name": $path[0], "contents": [ tree($path[1:]) ] }
    end
;

# Merge an array of page objects into a single page object.
def merge:
    map(to_entries) | add | group_by(.key) | map(
        { "key": (.[0].key)
        , "value": (
            if .[0].key == "contents"
            then [ .[].value ] | add | group_by(.name) | map(merge)
            elif [ .[].value == .[0].value ] | all
            then .[0].value
            else error("can't merge incompatible values")
            end
        ) }
    ) | from_entries
;

# Add paths to each object, that is, the names of the ancestors.
def directory:
    def f($d): .directory = $d | .name as $n | .contents[]? |= f($d+[$n]);
    f([])
;

# Every leaf node gets a basename, eg name without extension.
def basename:
    (all_children | select(has("contents") | not)) |= (
        .basename = (.name | sub(".([A-z]*)$";""))
    )
;

# Any Markdown page gets a link to its HTML page (or PDF if it has no HTML
# target).
def link:
    (all_children | select(has("directory") and has("basename") and (.name | test(".(md|markdown)$";"i")))) |= (
        [target_formats(["html","pdf"])][0] as $ext |
        .link = ((.directory + [.basename + "." + $ext]) | join("/") | ltrimstr("./"))
    )
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def frontmatter:
    (all_children | select(has("contents"))) |= (
        (.frontmatter = [.contents[] | select(.meta.frontmatter | bool)][0]) |
        (.contents = .contents - [.frontmatter])
    )
;

# Drafts can be excluded from being uploaded and included in the table of
# contents. To not count as a draft, a document should be explicitly marked as
# "make" in the metadata.
def explicit_make:
    def make: has("contents") or .external or .meta.make | bool;
    (( all_children | select(make | not)) |= mark) | remove_marked
;

# Sort content first according to sort order in metadata.
def sort_content:
    (all_children | select(has("contents")) | .contents) |= (
        sort_by(.meta.sort // .meta.title // .name)
    )
;

# Add a note that tells us whether this has only children who have no more
# subchildren.
def annotate_leaves:
    (all_children | select(has("contents"))) |= (
        .only_leaves = (.contents | all(has("contents") | not))
    )
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    merge
    | directory
    | basename
    | link
    | explicit_make
    | frontmatter
    | sort_content
    | annotate_leaves
;

# Get a list of target documents and their Makefile dependencies as an array of
# {"target":..., "deps":...} objects.
def targets($dest; $all_formats; $default_style):
    def in_dir($dir; $file):
        $dir + [$file] | join("/") | ltrimstr("./") | ($dest + "/" + .)
    ;
    [ all_children
        | ., (.frontmatter // empty)
        | select(has("directory") and has("basename") and (has("external") | not))
        | target_formats($all_formats) as $format
        | in_dir(.directory; .basename + "." + $format) as $doc
        | [in_dir(.directory; .targets?[])] as $external
        | [in_dir([]; (.meta.style // $default_style) + ".css")] as $css
        | (if $format == "html" then ($css + $external) else [] end) as $linked
        | (if $format == "pdf"  then ($css + $external) else [] end) as $embedded
        | {"target": $format, "deps": ([$doc] + $linked) }
        , {"target": $doc, "deps": $embedded }
    ]
    | group_by(.target)
    | map({"target":(.[0].target), "deps": (map(.deps) | add | unique)})
    | . // []
;

# Turn a target object into makefile recipes.
def as_makefile:
    .[] | (.target + ": " + (.deps | join(" ")))
;

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    [. == (null,false,0,{},[])] | any | not
;

# Generate this object and all its children.
def all_children:
    ., recurse(.contents[]?)
;

# To remove an object, we first mark it for removal, then actually remove it
# later. It would be nicer if we could just do `all_children ... |= empty`,
# which works in some cases but not all. I suppose it has to do with changing
# objects as we are iterating over them.
def mark:
    .remove = true
;

# Remove all objects marked for removal.
def remove_marked:
    if .remove == true
    then empty
    else (.contents // empty) |= map(remove_marked)
    end
;

# Obtain all target formats ("html", "pdf"...) for a particular page.
def target_formats($all_formats):
    (.meta.make // empty)
    | ( if (. | type) == "array" then .[] else . end )
    | tostring | ascii_downcase
    | ( if ([. == $all_formats[]] | any) then .
        elif . == "all" then $all_formats[]
        else error("unrecognized format:" + .)
        end )
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`.
def tree($path):
    if ($path | length) <= 1
    then .name = $path[0]
    else { "name": $path[0], "contents": [ tree($path[1:]) ] }
    end
;

# Merge an array of page objects into a single page object.
def merge:
    map(to_entries) | add | group_by(.key) | map(
        { "key": (.[0].key)
        , "value": (
            if .[0].key == "contents"
            then [ .[].value ] | add | group_by(.name) | map(merge)
            elif [ .[].value == .[0].value ] | all
            then .[0].value
            else error("can't merge incompatible values")
            end
        ) }
    ) | from_entries
;

# Add paths to each object, that is, the names of the ancestors.
def directory:
    def f($d): .directory = $d | .name as $n | .contents[]? |= f($d+[$n]);
    f([])
;

# Every leaf node gets a basename, eg name without extension.
def basename:
    (all_children | select(has("contents") | not)) |= (
        .basename = (.name | sub(".([A-z]*)$";""))
    )
;

# Any Markdown page gets a link to its HTML page (or PDF if it has no HTML
# target).
def link:
    (all_children | select(has("directory") and has("basename") and (.name | test(".(md|markdown)$";"i")))) |= (
        [target_formats(["html","pdf"])][0] as $ext |
        .link = ((.directory + [.basename + "." + $ext]) | join("/") | ltrimstr("./"))
    )
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def frontmatter:
    (all_children | select(has("contents"))) |= (
        (.frontmatter = [.contents[] | select(.meta.frontmatter | bool)][0]) |
        (.contents = .contents - [.frontmatter])
    )
;

# Drafts can be excluded from being uploaded and included in the table of
# contents. To not count as a draft, a document should be explicitly marked as
# "make" in the metadata.
def explicit_make:
    def make: has("contents") or .external or .meta.make | bool;
    (( all_children | select(make | not)) |= mark) | remove_marked
;

# Sort content first according to sort order in metadata.
def sort_content:
    (all_children | select(has("contents")) | .contents) |= (
        sort_by(.meta.sort // .meta.title // .name)
    )
;

# Add a note that tells us whether this has only children who have no more
# subchildren.
def annotate_leaves:
    (all_children | select(has("contents"))) |= (
        .only_leaves = (.contents | all(has("contents") | not))
    )
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    merge
    | directory
    | basename
    | link
    | explicit_make
    | frontmatter
    | sort_content
    | annotate_leaves
;

# Get a list of target documents and their Makefile dependencies as an array of
# {"target":..., "deps":...} objects.
def targets($dest; $all_formats; $default_style):
    def in_dir($dir; $file):
        $dir + [$file] | join("/") | ltrimstr("./") | ($dest + "/" + .)
    ;
    [ all_children
        | ., (.frontmatter // empty)
        | select(has("directory") and has("basename") and (has("external") | not))
        | target_formats($all_formats) as $format
        | in_dir(.directory; .basename + "." + $format) as $doc
        | [in_dir(.directory; .targets?[])] as $external
        | [in_dir([]; (.meta.style // $default_style) + ".css")] as $css
        | (if $format == "html" then ($css + $external) else [] end) as $linked
        | (if $format == "pdf"  then ($css + $external) else [] end) as $embedded
        | {"target": $format, "deps": ([$doc] + $linked) }
        , {"target": $doc, "deps": $embedded }
    ]
    | group_by(.target)
    | map({"target":(.[0].target), "deps": (map(.deps) | add | unique)})
    | . // []
;

# Turn a target object into makefile recipes.
def as_makefile:
    .[] | (.target + ": " + (.deps | join(" ")))
;


Fixed filings test.

Fixed filings test.


Update rxnfilter.jq

Update rxnfilter.jq

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv
.[]
#start from the .results array
|.results

#Creates new object with .safetyreportid string and .patient.reaction array
|map(with_entries(select(.key == "safetyreportid")) + (.patient.reaction[]))

# Grabs all unique keys and spits out CSV
|(map(keys)|add|unique) as $keys| $keys, (.[]|[.[$keys[]]])|@csv

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv

#start from the .results array
.results

#Creates new object with .safetyreportid string and .patient.reaction array
|map(with_entries(select(.key == "safetyreportid")) + (.patient.reaction[]))

# Grabs all unique keys and spits out CSV
|(map(keys)|add|unique) as $keys| $keys, (.[]|[.[$keys[]]])|@csv


Change variable name for conciseness.

Change variable name for conciseness.

`all_children` was also not correct --- it generated all *descendants*,
not just all *children*.

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    [. == (null,false,0,{},[])] | any | not
;

# Is a given array at least length $n?
def at_least($n):
    (. | length) >= $n
;

# Generate this object and all its descendants.
def entries:
    ., recurse(.contents?[]?)
;

# To remove an object, we first mark it for removal, then actually remove it
# later. It would be nicer if we could just do `entries ... |= empty`,
# which works in some cases but not all. I suppose it has to do with changing
# objects as we are iterating over them.
def mark:
    .remove = true
;

# Remove all objects marked for removal.
def remove_marked:
    if .remove == true
    then empty
    else (.contents // empty) |= map(remove_marked)
    end
;

# Enumerate the `make` formats of a page.
def target_formats($all_formats):
    (.make // empty)
    |   if . == "null" then empty else . end # fix for wrong YAML parse
    |   if (. | type) == "array" then .[] else . end
    |   tostring
    |   ascii_downcase
    |   if [. == $all_formats[]] | any then .
        elif . == "all" then $all_formats[]
        else error("unrecognized format: \(.)")
        end
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`. We use `name` and `contents` to be compatible
# with the output of the `tree` program on the shell.
def tree($path):
    if $path | at_least(2)
    then { "contents": [ tree($path[1:]) ] }
    else .
    end | .name = $path[0]
;

# Merge an array of page objects into a single page object.
def merge:
    map(to_entries) | add | group_by(.key) | map(
        { "key": (.[0].key)
        , "value": (
            if .[0].key == "contents"
            then [ .[].value ] | add | group_by(.name) | map(merge)
            elif [ .[].value == .[0].value ] | all
            then .[0].value
            else error("can't merge incompatible values")
            end
        ) }
    ) | from_entries
;

# Add paths to each object, that is, the names of the ancestors.
def directory:
    def f($d):
        .dir = ($d | join("/") | ltrimstr("./"))
        | .name as $n | .contents[]? |= f($d+[$n])
    ;
    f([])
;

# Every leaf node gets a basename, eg name without extension.
def basename:
    (entries | select(has("contents") | not)) |= (
        .basename = (.name | sub(".([A-z]*)$";""))
    )
;

# Any page gets its target formats.
def formats($all_formats):
    entries |= (
        .formats = [ target_formats($all_formats) ]
    )
;

# Any page is linked to its first target format.
def link:
    (entries | select(has("dir") and has("basename") and (.formats | at_least(1)))) |= (
        .link = "\(.dir)/\(.basename).\(.formats[0])"
    )
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def frontmatter:
    (entries | select(has("contents"))) |= (
        (.frontmatter = [.contents[] | select(.frontmatter | bool)][0]) |
        (.contents = .contents - [.frontmatter])
    )
;

# Drafts can be excluded from being uploaded and included in the table of
# contents. To not count as a draft, a document should be explicitly marked as
# "make" in the metadata.
# Anything that does not have a link, nor has any children, should be removed
# from the index.
def remove_unlinked:
    def linked: has("contents") or has("link");
    (( entries | select(linked | not)) |= mark) | remove_marked
;

# Add links to the next and previous page for every linked page.
def navigation:
    def f($paths; $i; $j; $key):
        setpath($paths[$i]+[$key];
            getpath($paths[$j]) | {link,title,date}
        )
    ;
    [ path(entries | select(has("link"))) ] as $paths
    | ($paths | length) as $n
    | reduce range(0; $n-1) as $i (.; f($paths; $i; $i+1; "next"))
    | reduce range(1, $n)   as $i (.; f($paths; $i; $i-1; "prev"))
;

# Sort content first according to sort order in metadata.
def sort_content:
    (entries | select(has("contents")) | .contents) |= (
        sort_by(.sort // .title // .name)
    )
;

# Add a note that tells us whether this has only children who have no more
# subchildren.
def annotate_leaves:
    (entries | select(has("contents"))) |= (
        .only_leaves = (.contents | all(has("contents") | not))
    )
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index($all_formats):
    merge
    | directory
    | basename
    | formats($all_formats)
    | link
    | remove_unlinked
    | navigation
    | frontmatter
    | sort_content
    | annotate_leaves
;

# Get target files and their dependencies as an enumeration of {"target":...,
# "dependencies":...} objects.
def targets($dest; $default_style):
    [ entries
        | ., (.frontmatter // empty)
        | .formats[] as $format
        | "\($dest)/\(.dir)/\(.basename).\($format)" as $doc
        | ["\($dest)/\(.dir)/\(.resources?[])"] as $external
        | ["\($dest)/\(.style // $default_style).css"] as $css
        | (if $format == "html" then ($css + $external) else [] end) as $linked
        | (if $format == "pdf"  then ($css + $external) else [] end) as $embedded
        | {"target": $format, "dependencies": ([$doc] + $linked) }
        , {"target": $doc, "dependencies": $embedded }
    ]
    | group_by(.target)
    | map(
        {"target": (.[0].target)
        , "dependencies": (map(.dependencies) | add | unique) }
    )
    | .[]?
;

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    [. == (null,false,0,{},[])] | any | not
;

# Is a given array at least length $n?
def at_least($n):
    (. | length) >= $n
;

# Generate this object and all its children.
def all_children:
    ., recurse(.contents?[]?)
;

# To remove an object, we first mark it for removal, then actually remove it
# later. It would be nicer if we could just do `all_children ... |= empty`,
# which works in some cases but not all. I suppose it has to do with changing
# objects as we are iterating over them.
def mark:
    .remove = true
;

# Remove all objects marked for removal.
def remove_marked:
    if .remove == true
    then empty
    else (.contents // empty) |= map(remove_marked)
    end
;

# Enumerate the `make` formats of a page.
def target_formats($all_formats):
    (.make // empty)
    |   if . == "null" then empty else . end # fix for wrong YAML parse
    |   if (. | type) == "array" then .[] else . end
    |   tostring
    |   ascii_downcase
    |   if [. == $all_formats[]] | any then .
        elif . == "all" then $all_formats[]
        else error("unrecognized format: \(.)")
        end
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`. We use `name` and `contents` to be compatible
# with the output of the `tree` program on the shell.
def tree($path):
    if $path | at_least(2)
    then { "contents": [ tree($path[1:]) ] }
    else .
    end | .name = $path[0]
;

# Merge an array of page objects into a single page object.
def merge:
    map(to_entries) | add | group_by(.key) | map(
        { "key": (.[0].key)
        , "value": (
            if .[0].key == "contents"
            then [ .[].value ] | add | group_by(.name) | map(merge)
            elif [ .[].value == .[0].value ] | all
            then .[0].value
            else error("can't merge incompatible values")
            end
        ) }
    ) | from_entries
;

# Add paths to each object, that is, the names of the ancestors.
def directory:
    def f($d):
        .dir = ($d | join("/") | ltrimstr("./"))
        | .name as $n | .contents[]? |= f($d+[$n])
    ;
    f([])
;

# Every leaf node gets a basename, eg name without extension.
def basename:
    (all_children | select(has("contents") | not)) |= (
        .basename = (.name | sub(".([A-z]*)$";""))
    )
;

# Any page gets its target formats.
def formats($all_formats):
    all_children |= (
        .formats = [ target_formats($all_formats) ]
    )
;

# Any page is linked to its first target format.
def link:
    (all_children | select(has("dir") and has("basename") and (.formats | at_least(1)))) |= (
        .link = "\(.dir)/\(.basename).\(.formats[0])"
    )
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def frontmatter:
    (all_children | select(has("contents"))) |= (
        (.frontmatter = [.contents[] | select(.frontmatter | bool)][0]) |
        (.contents = .contents - [.frontmatter])
    )
;

# Drafts can be excluded from being uploaded and included in the table of
# contents. To not count as a draft, a document should be explicitly marked as
# "make" in the metadata.
# Anything that does not have a link, nor has any children, should be removed
# from the index.
def remove_unlinked:
    def linked: has("contents") or has("link");
    (( all_children | select(linked | not)) |= mark) | remove_marked
;

# Add links to the next and previous page for every linked page.
def navigation:
    def f($paths; $i; $j; $key):
        setpath($paths[$i]+[$key];
            getpath($paths[$j]) | {link,title,date}
        )
    ;
    [ path(all_children | select(has("link"))) ] as $paths
    | ($paths | length) as $n
    | reduce range(0; $n-1) as $i (.; f($paths; $i; $i+1; "next"))
    | reduce range(1, $n)   as $i (.; f($paths; $i; $i-1; "prev"))
;

# Sort content first according to sort order in metadata.
def sort_content:
    (all_children | select(has("contents")) | .contents) |= (
        sort_by(.sort // .title // .name)
    )
;

# Add a note that tells us whether this has only children who have no more
# subchildren.
def annotate_leaves:
    (all_children | select(has("contents"))) |= (
        .only_leaves = (.contents | all(has("contents") | not))
    )
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index($all_formats):
    merge
    | directory
    | basename
    | formats($all_formats)
    | link
    | remove_unlinked
    | navigation
    | frontmatter
    | sort_content
    | annotate_leaves
;

# Get target files and their dependencies as an enumeration of {"target":...,
# "dependencies":...} objects.
def targets($dest; $default_style):
    [ all_children
        | ., (.frontmatter // empty)
        | .formats[] as $format
        | "\($dest)/\(.dir)/\(.basename).\($format)" as $doc
        | ["\($dest)/\(.dir)/\(.resources?[])"] as $external
        | ["\($dest)/\(.style // $default_style).css"] as $css
        | (if $format == "html" then ($css + $external) else [] end) as $linked
        | (if $format == "pdf"  then ($css + $external) else [] end) as $embedded
        | {"target": $format, "dependencies": ([$doc] + $linked) }
        , {"target": $doc, "dependencies": $embedded }
    ]
    | group_by(.target)
    | map(
        {"target": (.[0].target)
        , "dependencies": (map(.dependencies) | add | unique) }
    )
    | .[]?
;


Deleted file.

Deleted file.

jsoniq version "1.0";

import module namespace components = "http://xbrl.io/modules/bizql/components";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace filings = "http://xbrl.io/modules/bizql/profiles/sec/filings";
import module namespace fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace sec = "http://xbrl.io/modules/bizql/profiles/sec/core";
import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";

import module namespace sec-networks = "http://xbrl.io/modules/bizql/profiles/sec/networks";
import module namespace networks = "http://xbrl.io/modules/bizql/networks";
import module namespace concept-maps = "http://xbrl.io/modules/bizql/concept-maps";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

declare function local:to-csv($o as object*) as string
{
    string-join(
        csv:serialize(
            for $o in $o
            let $a := $o.Aspects
            return {|
                (for $k in keys($a) return { $k : $a.$k }),
                { "Unit" :  $o.Unit },
                { "Value" : $o.Value },
                { "Type" : $o.Type },
                { "Decimals" : $o.Decimals }
            |},
            { serialize-null-as : "" }
        )
    )
};

declare function local:to-xml($o as object*) as element()*
{
    for $o in $o
    let $a := $o.Aspects
    return
        <Fact>{
            <Aspects>{
                for $k in keys($a)
                return
                    <Aspect>
                        <Name>{$k}</Name>
                        <Value>{$a.$k}</Value>
                    </Aspect>
            }</Aspects>,
            <Type>{$o.Type}</Type>,
            <Value>{$o.Value}</Value>,
            if (exists($o.Unit))
            then <Unit>{$o.Unit}</Unit>
            else(),
            if (exists($o.Decimals))
            then <Decimals>{$o.Decimals}</Decimals>
            else ()
        }</Fact>
};

declare function local:components-by-disclosures($disclosures as string*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    for $aid in $aids
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: $aid,
            "Profiles.SEC.Disclosure": { "$in" : [ $disclosures ] }
        })
};

declare function local:components-by-roles($roles as string*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: { "$in" : [ $aids ] },
            "Role": { "$in" : [ $roles ] }
        })
};

declare function local:components-by-concepts($concepts as string*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    let $ids := mongo:find($conn, "concepts",
        {|
            (
                { "Name" : { "$in" : [ $concepts ] } },
                { "Archive" : { "$in" : [ $aids ] } }
            )
        |}).Component
    return components:components($ids)
};

declare function local:filings(
    $ciks as string*,
    $tags as string*,
    $tickers as string*,
    $sics as string*,
    $fp as string*,
    $fy as string*) as object*
{
    let $entities := if ($tags = "ALL") then companies:companies()
                                        else (
                                            companies:companies($ciks),
                                            companies:companies-for-tags($tags),
                                            companies:companies-for-tickers($tickers),
                                            companies:companies-for-SIC($sics)
                                        )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                        case "ALL" return  $fiscal:ALL_FISCAL_YEARS
                    default return $fy cast as integer
                )
    for $fp in $fp
    return fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy)
};

(:
 : This function needs to go into the core. It returns all facts for a network
 : - not only the ones for the specific FY/FP
 :)
declare function local:facts($networks-or-ids as item*, $options as object?)
as object*
{
  for $component as object in components:components($networks-or-ids)
  for $table as string? allowing empty in sec-networks:tables($component).Name
  let $hypercube as object? := hypercubes:hypercubes-for-components($component, $table)
  let $hypercube as object := if (exists($hypercube))
                              then
                                if(exists($hypercube.Aspects."dei:LegalEntityAxis"))
                                then $hypercube
                                else
                                  copy $h := $hypercube
                                  modify (
                                    insert json { "dei:LegalEntityAxis" :
                                      {
                                        "Name": "dei:LegalEntityAxis",
                                        "Label": "Legal Entity Dimension",
                                        "Default" : "sec:DefaultLegalEntity"
                                      }} into $h.Aspects
                                    )
                                  return $h
                              else sec:dimensionless-hypercube({
                                  Concepts: [ sec-networks:line-items($component).Name ]
                              })
  let $facts := hypercubes:facts-for-hypercube(
    $hypercube,
    $component.Archive,
    $options
  )
  return sec:hide-amended-facts($facts, $options)
};

session:audit-call();

let $format      := lower-case((request:param-values("format"), substring-after(request:path(), ".jq."))[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids        := archives:aid(request:param-values("aid"))
let $roles       := request:param-values("networkIdentifier")
let $archives    := (
                        local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                        archives:archives($aids)
                    )
let $cid         := request:param-values("cid")
let $concepts    := distinct-values(request:param-values("concept"))
let $rollup      := distinct-values(request:param-values("rollup"))
let $map         := request:param-values("map")
let $disclosures := request:param-values("disclosure")
let $components  := (if (exists($cid))
                    then components:components($cid)
                    else (),
                    if (exists($concepts) or exists($disclosures) or exists($roles))
                    then (
                            local:components-by-concepts($concepts, $archives._id),
                            local:components-by-disclosures($disclosures, $archives._id),
                            local:components-by-roles($roles, $archives._id)
                        )
                    else components:components-for-archives($archives._id))
let $component := $components[1] (: only one for know :)
let $archive   := archives:archives($component.Archive)
let $entity    := entities:entities($archive.Entity)

return
    switch(session:check-access($entity, "data_sec"))
    case $session:ACCESS-ALLOWED return {
        let $facts := if (exists($rollup))
                     then
                         let $calc-network := networks:networks-for-components-and-short-names($component, $networks:CALCULATION_NETWORK)
                         let $hc := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube") (: sec-networks:tables($component).Name) :)
                         let $p := hypercubes:populate-networks-with-facts($calc-network, $hc, $archive)
                         let $map := concept-maps:concept-maps($map)
                         let $concepts :=
                            if (exists($map))
                            then
                                for $d in $rollup
                                return
                                    keys(descendant-objects($p)[$$.Name = keys($map.Trees($d).To)][1].To)
                            else
                                for $d in $rollup
                                return ($d, keys(descendant-objects($p)[$$.Name eq $d].To))
                         return sec:facts-for-archives-and-concepts($archive, $concepts, { Hypercube: $hc })
                     else local:facts($component, {| |})
        let $fact-table :=  for $f in $facts
                            let $a := $f.Aspects
                            return {|
                                { "Aspects" : {|
                                    { "xbrl:Entity": $a."xbrl:Entity" },
                                    if (exists($a."dei:LegalEntityAxis")) then { "dei:LegalEntityAxis": $a."dei:LegalEntityAxis" } else (),
                                    { "xbrl:Period" : $a."xbrl:Period" },
                                    { "xbrl:Concept" : $a."xbrl:Concept" },
                                    for $k in keys($f.Aspects)
                                    where $k ne "xbrl:Unit" and $k ne "xbrl:Entity" and $k ne "dei:LegalEntityAxis" and
                                          $k ne "xbrl:Period" and $k ne "xbrl:Concept"
                                    return { $k : $f.Aspects.$k }
                                |} },
                                { "Type" : $f.Type },
                                { "Value" : $f.Value },
                                if (exists($f.Decimals))
                                then { "Decimals" : $f.Decimals }
                                else (),
                                if (exists($f."Aspects"."xbrl:Unit"))
                                then { "Unit" : $f."Aspects"."xbrl:Unit" }
                                else ()
                            |}
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                (session:comment("xml", {
                            NumFacts : count($fact-table),
                            TotalNumFacts: session:num-facts(),
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        }),
                <FactTable entityRegistrantName="{$entity.Profiles.SEC.CompanyName}"
                    cik="{$entity._id}"
                    tableName="{sec-networks:tables($component, {IncludeImpliedTable: true}).Name}"
                    label="{$component.Label}"
                    accessionNumber="{$component.Archive}"
                    networkIdentifier="{$component.Role}"
                    formType="{$archive.Profiles.SEC.FormType}"
                    fiscalPeriod="{$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus}"
                    fiscalYear="{$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus}"
                    acceptanceDatetime="{filings:acceptance-dateTimes($archive)}"
                    disclosure="{$component.Profiles.SEC.Disclosure}"
                    >{
                    local:to-xml($fact-table)
                }</FactTable>)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=facttable-" || $cid || ".csv");
                local:to-csv($fact-table)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=facttable-" || $cid || ".csv");
                local:to-csv($fact-table)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { CIK : $entity._id },
                    { EntityRegistrantName : $entity.Profiles.SEC.CompanyName },
                    { TableName : sec-networks:tables($component, {IncludeImpliedTable: true}).Name },
                    { Label : $component.Label },
                    { AccessionNumber : $component.Archive },
                    { FormType : $archive.Profiles.SEC.FormType },
                    { FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus },
                    { FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus },
                    { AcceptanceDatetime : filings:acceptance-dateTimes($archive) },
                    { NetworkIdentifier: $component.Role },
                    { Disclosure : $component.Profiles.SEC.Disclosure },
                    { FactTable : [ $fact-table ] },
                    session:comment("json", {
                            NumFacts : count($fact-table),
                            TotalNumFacts: session:num-facts(),
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })
                |}
            }
       }
    case $session:ACCESS-DENIED return {
          response:status-code(403);
          session:error("accessing filings of an entity that is not in the DOW30", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          response:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()


Update facts.jq

Update facts.jq

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $request := test:invoke("facts", $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status)
};

declare %an:nondeterministic function local:test-empty($params as object) as item
{
    let $res as object := test:invoke-raw("facts", $params)
    return if($res.status eq 200 and $res.headers."Content-Length" eq "0") then true else {
        unexpectedResponse: $res
    }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw("facts", $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "Accession Number,Concept,Entity,Period,Fiscal Period,Fiscal Year,Accepted,Legal Entity,Unit,Value,Decimals",
        "0000021344-13-000050,\"Cash and Cash Equivalents, at Carrying Value\",COCA COLA CO,2013-09-27,Q3,2013,20131024121047,Default Legal Entity,USD,11118000000,-6",
        "0000021344-13-000050,Assets,COCA COLA CO,2013-09-27,Q3,2013,20131024121047,Default Legal Entity,USD,89432000000,-6"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        unexpectedResponse: $res
    }
};

local:check({
    cocacola-fy-ytd4: local:test-facttable(468, {
        ticker:"ko",
        fiscalPeriod: [ "FY", "YTD4" ]
    }),
    cocacola-fy: local:test-facttable(163, {
        ticker:"ko"
    }),
    cocacolaCSVLabels: local:test-labels(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(30, {
        tag:"DOW30",
        concept:"us-gaap:Assets"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-empty({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    })
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $request := test:invoke("facts", $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status)
};

declare %an:nondeterministic function local:test-empty($params as object) as item
{
    let $res as object := test:invoke-raw("facts", $params)
    return if($res.status eq 200 and $res.headers."Content-Length" eq "0") then true else {
        unexpectedResponse: $res
    }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw("facts", $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "Accession Number,Concept,Entity,Period,Fiscal Period,Fiscal Year,Accepted,Legal Entity,Unit,Value,Decimals",
        "0000021344-13-000050,\"Cash and Cash Equivalents, at Carrying Value\",COCA COLA CO,2013-09-27,Q3,2013,20131024121047,Default Legal Entity,USD,11118000000,-6",
        "0000021344-13-000050,Assets,COCA COLA CO,2013-09-27,Q3,2013,20131024121047,Default Legal Entity,USD,89432000000,-6"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        unexpectedResponse: $res
    }
};

local:check({
    cocacola: local:test-facttable(468, {
        ticker:"ko"
    }),
    cocacolaCSVLabels: local:test-labels(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(30, {
        tag:"DOW30",
        concept:"us-gaap:Assets"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-empty({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    })
})


:fire: Remove URIs from tests (volatile content).