erge branch 'RemoveLineItemsWorkaround' of github.com:28msec/secxbrl.info into DuplicateMetadataValidation

jsoniq version "1.0";

(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for querying SEC networks in SEC filings.

 :
SEC Networks are actually XBRL Components. In the SEC profiles, all XBRL networks
 : in an XBRL component must be consistent to each other. A way to look at it is that
 : the SEC Network can be identified with the XBRL presentation network it contains.

 :
 :
For XBRL-generic operations on XBRL components, use the
 : generic components module.

 :
 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";

import module namespace mw = "http://28.io/modules/xbrl/data-source-wrapper";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace networks = "http://28.io/modules/xbrl/networks";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Disclosure for the balance sheet.
 :)
declare variable $sec-networks:BALANCE_SHEET as xs:string := "BalanceSheet";

(:~
 : Disclosure for the parenthetical balance sheet.
 :)
declare variable $sec-networks:BALANCE_SHEET_PARENTHETICAL as xs:string := "BalanceSheetParenthetical";

(:~
 : Disclosure for the income statement.
 :)
declare variable $sec-networks:INCOME_STATEMENT as xs:string := "IncomeStatement";

(:~
 : Disclosure for the parenthetical income statement.
 :)
declare variable $sec-networks:INCOME_STATEMENT_PARENTHETICAL as xs:string := "IncomeStatementParenthetical";

(:~
 : Disclosure for the statement of comprehensive income.
 :)
declare variable $sec-networks:STATEMENT_OF_COMPREHENSIVE_INCOME as xs:string := "StatementOfComprehensiveIncome";

(:~
 : Disclosure for the cash flow statement.
 :)
declare variable $sec-networks:CASH_FLOW_STATEMENT as xs:string := "CashFlowStatement";

(:~
 : Disclosure for the parenthetical cash flow statement.
 :)
declare variable $sec-networks:CASH_FLOW_STATEMENT_PARENTHETICAL as xs:string := "CashFlowStatementParenthetical";

(:~
 : Disclosure for the default component.
 :)
declare variable $sec-networks:DEFAULT_COMPONENT as xs:string := "DefaultComponent";

(:~
 : Disclosure for the document and entity information
 :)
declare variable $sec-networks:DOCUMENT_AND_ENTITY_INFO as xs:string := "DocumentAndEntityInformation";

(:~
 :
Retrieves all SEC Networks in a given filing.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 :
 : @return all networks in the supplied filings.
 :)
declare function sec-networks:networks-for-filings(
    $archives-or-ids as item*) as object*
{
  components:components-for-archives($archives-or-ids)
};

(:~
 :
Retrieves all SEC Networks that bear the supplied disclosures.

 :
 : @param $disclosure a list of disclosures.
 :
 : @return all models that match one of the disclosures.
 :)
declare function sec-networks:networks-for-disclosures(
    $disclosures as string*) as object*
{
  for $disclosure in $disclosures
  return mw:find($components:col,{ "Profiles" : { "SEC" : { "Disclosure": $disclosure } } })
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and that match the supplied disclosures.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $disclosures a list of disclosures.
 :
 : @return all models in the archives and that match one of the disclosures.
 :)
declare function sec-networks:networks-for-filings-and-disclosures(
    $archive-or-ids as item*,
    $disclosures as string*) as object*
{
  mw:find($components:col,
  {
    $components:ARCHIVE: [ $archive-or-ids ! archives:aid($$) ],
    "Profiles" : { "SEC" : { "Disclosure": [ $disclosures ] } }
  })
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and that match the supplied category.

 :
 :
A category can be any of "Statement", "Disclosure", "Document",
 : "Schedule", or "Unknown".

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $categories a list of categories.
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-categories(
    $archive-or-ids as item*,
    $categories as string*) as object*
{
  for $aid_or_archive in $archive-or-ids
  let $aid as atomic := archives:aid($aid_or_archive)
  for $category in $categories
  return mw:find($components:col,
  {
    $components:ARCHIVE: $aid,
    "Profiles" : { "SEC" : { "Category": $category } }
  })
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and that match the supplied roles/network identifier.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $roles a list of roles.
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-roles(
    $archive-or-ids as item*,
    $roles as string*) as object*
{
  mw:find($components:col,
  {
    $components:ARCHIVE: [ $archive-or-ids ! archives:aid($$) ],
    "Role": [ $roles ]
  })
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and that contain at least one report element with a given name.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $report-elements a list of contained report element names.
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-reportElements(
    $archive-or-ids as item*,
    $report-elements as string*) as object*
{
  let $ids := mw:find($concepts:col,
      {
         $components:ARCHIVE: [ $archive-or-ids ! archives:aid($$) ],
         "Name" : [ $report-elements ]
      }).Component
  return components:components($ids)
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and whose label matches the given search string.

 :
 :
The result of the function is limited to 100.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $label-search-term a search term to search in labels
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-label(
    $archive-or-ids as item*,
    $label-search-term as string*) as object*
{
  sec-networks:networks-for-filings-and-label(
    $archive-or-ids, $label-search-term, 100)
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and whose label matches the given search string.

 :
 :
The result of the function is limited to 100.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $label-search-term a search term to search in labels
 : @param $limit limit the number of search results
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-label(
    $archive-or-ids as item*,
    $label-search-term as string*,
    $limit as integer) as object*
{
  mw:text-search($components:col,
  {
    "filter" : { "Archive" : [ $archive-or-ids ! archives:aid($$) ] },
    "search" : $label-search-term,
    "limit" : $limit,
    "score" : { "$meta" : "textScore" },
    "sort" : { score: { "$meta" : "textScore" } }
  })
};

(:~
 :
Retrieves the components with the given CID, or belonging to the given archives and corresponding to one of report
 : elements, concepts, disclosures, roles or label search.

 :
 : @param $archives-or-aids a sequence of archives or AIDs.
 : @param $components-or-ids a sequence of components or their CIDs.
 : @param $report-elements a sequence of report element names.
 : @param $concepts a sequence of concept names.
 : @param $disclosures a sequence of disclosures.
 : @param $roles a sequence of role URIs.
 : @param $search a sequence of label search strings.
 :
 : @return a sequence of components.
 :)
declare function sec-networks:components(
    $archives-or-aids as item*,
    $components-or-ids as item*,
    $report-elements as string*,
    $disclosures as string*,
    $roles as string*,
    $search as string*) as object*
{
    let $archives as object* := archives:archives($archives-or-aids)
    return(
        components:components($components-or-ids)[exists($components-or-ids)],
        if (exists(($report-elements, $disclosures, $roles, $search)))
        then (
                components:components-for-archives-and-concepts($archives, $report-elements),
                sec-networks:networks-for-filings-and-disclosures($archives, $disclosures),
                components:components-for-archives-and-roles($archives, $roles),
                sec-networks:networks-for-filings-and-label($archives, $search)
            )
        else components:components-for-archives($archives)
    )
};

(:~
 :
Builds a standard definition model out of the specified component.

 :
The concepts will be put on the y axis according to the presentation network.

 :
The other dimensions are put on the x axis, with one breakdown for each.

 :
Explicit dimensions are organized according to the dimension hierarchy from the domain-member network.

 :
Typed dimensions are organized according to the actual values.

 :
 :
Auto slicing will be performed against the fact table
 :
 : @param $component a component object.
 : @param $hypercube the name of a hypercube.
 :
 : @return a definition model
 :)
declare function sec-networks:standard-definition-models-for-component(
    $component as object,
    $hypercube as string) as object
{
    sec-networks:standard-definition-models-for-component($component, $hypercube, ())
};

(:~
 :
Builds a standard definition model out of the specified component.

 :
The concepts will be put on the y axis according to the presentation network.

 :
The other dimensions are put on the x axis, with one breakdown for each.

 :
Explicit dimensions are organized according to the dimension hierarchy from the domain-member network.

 :
Typed dimensions are organized according to the actual values.

 :
 : @param $component a component object.
 : @param $hypercube the name of a hypercube.
 : @param $options
some optional parameters, including:

 :

 :
AutoSlice: a boolean specifying whether or not slicing should be done automatically, looking at the fact table. Deactivating auto slicing will
 :  lead to better performance, but a more verbose table. If AutoSlice is active, dimensions with only one value in the fact pool will become
 : global filters rather than breakdowns on the x axis.

 :
Slicers: an object with forced slicers.

 :

 :
 : @error components:HYPERCUBE-DOES-NOT-EXIST if the specified hypercube is not found.
 : @return a definition model
 :)
declare function sec-networks:standard-definition-models-for-component(
    $component as object,
    $hypercube as string,
    $options as object?) as object
{
  if(not $hypercube = (keys($component.Hypercubes), $components:ALL-HYPERCUBES))
  then error(
    QName("components:HYPERCUBE-DOES-NOT-EXIST"),
    "Hypercube " || $hypercube || " was not found."
  )
  else
    let $table as object := hypercubes:hypercubes-for-components($component, $hypercube)

    let $auto-slice as boolean := empty($options.AutoSlice) or $options.AutoSlice
    let $facts as object*:=
        if($auto-slice)
        then hypercubes:facts($table)
        else ()
    let $dimensions as string*:= keys($table.Aspects)
    let $values-by-dimension as object := {|
        for $d in $dimensions
        return { $d : [ distinct-values($facts.Aspects.$d) ] }
    |}
    let $auto-slice-dimensions as string* :=
    (
        "xbrl:Entity"[size($values-by-dimension."xbrl:Entity") eq 1],
        keys($values-by-dimension)[
            size($values-by-dimension.$$) eq 1 and
            not $$ = ("xbrl:Entity",
                      "xbrl:Period",
                      "xbrl28:Archive",
                      "sec:Accepted",
                      "sec:FiscalYear",
                      "sec:FiscalPeriod",
                      "sec:FiscalPeriodType") ]
    )

    let $user-slice-dimensions as string* :=
        keys($options.Slicers)

    let $column-dimensions as string* := keys($values-by-dimension)[not $$ =
        ("xbrl:Concept",
        "xbrl:Period",
        "xbrl:Unit",
        "xbrl:Entity",
        "sec:Accepted",
        "xbrl28:Archive",
        "sec:FiscalYear",
        "sec:FiscalPeriod",
        "sec:FiscalPeriodType",
        $auto-slice-dimensions,
        $user-slice-dimensions)]

    let $x-breakdowns as object* := (
        components:standard-typed-dimension-breakdown($component, "xbrl:Period", $options)
          [not (($auto-slice-dimensions, $user-slice-dimensions) = "xbrl:Period")],
        for $d as string in $column-dimensions
        let $dimension-object as object := $table.Aspects.$d
        return
            if($d = ("sec:Accepted", "sec:FiscalYear", "sec:FiscalPeriod", "sec:FiscalPeriodType"))
            then components:standard-typed-dimension-breakdown(
                $component,
                $d,
                $options)
            else components:standard-explicit-dimension-breakdown(
                $component,
                $d,
                $dimension-object.Members[].Name,
                $component.Role,
                $options),
        components:standard-typed-dimension-breakdown($component, "xbrl:Entity", $options)
          [not (($auto-slice-dimensions, $user-slice-dimensions) = "xbrl:Entity")]
    )

    let $lineitems as string* := components:line-items($component)
    let $presentation-network as object? := networks:networks-for-components-and-short-names($component, "Presentation")
    let $roots as string* := distinct-values($presentation-network.Trees[].Name)
    let $lineitems as string* := if(exists($lineitems)) then $lineitems else $roots
    let $y-breakdowns as object := components:standard-concept-breakdown($lineitems, $component.Role)

    return {
        ModelKind: "DefinitionModel",
        Labels: [$component.Label],
        Parameters: {},
        Breakdowns: {
            "x" : [
                $x-breakdowns
            ],
            "y": [
                $y-breakdowns
            ]
        },
        TableFilters: {|
            for $d as string in distinct-values(($auto-slice-dimensions, $user-slice-dimensions))
            return if($d = $user-slice-dimensions)
                   then { $d : $options.Slicers.$d }
                   else { $d : $values-by-dimension.$d[] },
            if (not $auto-slice)
            then { "xbrl28:Archive" : $component.Archive }
            else ()
        |}
    }
};

(:~
 :
 :
Returns the disclosures of the suplied networks.

 :
 : @param $networks a sequence of SEC Network objects.
 : @return the disclosure names, or "UncategorizedInformation" if none.
 :
 :)
declare function sec-networks:disclosures($networks as object*) as string+
{
  for $component in $networks
  let $disclosure := $component.Profiles.SEC.Disclosure
  return if (exists($disclosure))
         then $disclosure
         else "UncategorizedInformation"
};

(:~
 :
 :
Return the categories of the supplied SEC networks (Statement, Disclosure, Document or Schedule).

 :
 : @param $networks a sequence of SEC Network objects.
 :
 : @return the category of each network.
 :)
declare function sec-networks:categories($networks as object*) as string*
{
  for $component in $networks
  return normalize-space(tokenize($component.Label, "-")[2])
};

(:~
 :
 :
Return the sub-categories of the supplied SEC networks (Detail, TextBlockLevel4, TextBLockLevel1to3).

 :
 : @param $networks a sequence of SEC Network objects.
 :
 : @return the sub-category of each network.
 :)
declare function sec-networks:sub-categories($networks as object*) as string*
{
  for $network in $networks
  let $is-text-blocks as boolean* := ($network.Concepts[])
    [not $$.SubstitutionGroup = ("xbrldt:hypercubeItem", "xbrldt:dimensionItem") and not $$.IsAbstract].IsTextBlock
  let $exists-text-blocks as boolean := exists($is-text-blocks[$$])
  let $exists-non-text-blocks as boolean := exists($is-text-blocks[not $$])
  return switch(true)
      case (: no text blocks, but still non-text-blocks :) $exists-non-text-blocks and not $exists-text-blocks return "Detail"
      case (: both :)  $exists-non-text-blocks (: and $exists-text-blocks :) return "TextBlockLevel4"
      case (: only text blocks :) $exists-text-blocks return "TextBlockLevel1To3"
      default (: exists nothing :) return "Unknown"
};

(:~
 :
 :
Return summary information for the supplied SEC networks.

 :
 : @param $networks a sequence of SEC Network objects.
 :
 : @return one object per network, containing a summary.
 :)
declare function sec-networks:summary($network as object) as object
{
  {
    Category : sec-networks:categories($network),
    SubCategory : sec-networks:sub-categories($network),
    Disclosure : sec-networks:disclosures($network)
  }
};

jsoniq version "1.0";

(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for querying SEC networks in SEC filings.

 :
SEC Networks are actually XBRL Components. In the SEC profiles, all XBRL networks
 : in an XBRL component must be consistent to each other. A way to look at it is that
 : the SEC Network can be identified with the XBRL presentation network it contains.

 :
 :
For XBRL-generic operations on XBRL components, use the
 : generic components module.

 :
 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";

import module namespace mw = "http://28.io/modules/xbrl/data-source-wrapper";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace networks = "http://28.io/modules/xbrl/networks";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Disclosure for the balance sheet.
 :)
declare variable $sec-networks:BALANCE_SHEET as xs:string := "BalanceSheet";

(:~
 : Disclosure for the parenthetical balance sheet.
 :)
declare variable $sec-networks:BALANCE_SHEET_PARENTHETICAL as xs:string := "BalanceSheetParenthetical";

(:~
 : Disclosure for the income statement.
 :)
declare variable $sec-networks:INCOME_STATEMENT as xs:string := "IncomeStatement";

(:~
 : Disclosure for the parenthetical income statement.
 :)
declare variable $sec-networks:INCOME_STATEMENT_PARENTHETICAL as xs:string := "IncomeStatementParenthetical";

(:~
 : Disclosure for the statement of comprehensive income.
 :)
declare variable $sec-networks:STATEMENT_OF_COMPREHENSIVE_INCOME as xs:string := "StatementOfComprehensiveIncome";

(:~
 : Disclosure for the cash flow statement.
 :)
declare variable $sec-networks:CASH_FLOW_STATEMENT as xs:string := "CashFlowStatement";

(:~
 : Disclosure for the parenthetical cash flow statement.
 :)
declare variable $sec-networks:CASH_FLOW_STATEMENT_PARENTHETICAL as xs:string := "CashFlowStatementParenthetical";

(:~
 : Disclosure for the default component.
 :)
declare variable $sec-networks:DEFAULT_COMPONENT as xs:string := "DefaultComponent";

(:~
 : Disclosure for the document and entity information
 :)
declare variable $sec-networks:DOCUMENT_AND_ENTITY_INFO as xs:string := "DocumentAndEntityInformation";

(:~
 :
Retrieves all SEC Networks in a given filing.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 :
 : @return all networks in the supplied filings.
 :)
declare function sec-networks:networks-for-filings(
    $archives-or-ids as item*) as object*
{
  components:components-for-archives($archives-or-ids)
};

(:~
 :
Retrieves all SEC Networks that bear the supplied disclosures.

 :
 : @param $disclosure a list of disclosures.
 :
 : @return all models that match one of the disclosures.
 :)
declare function sec-networks:networks-for-disclosures(
    $disclosures as string*) as object*
{
  for $disclosure in $disclosures
  return mw:find($components:col,{ "Profiles" : { "SEC" : { "Disclosure": $disclosure } } })
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and that match the supplied disclosures.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $disclosures a list of disclosures.
 :
 : @return all models in the archives and that match one of the disclosures.
 :)
declare function sec-networks:networks-for-filings-and-disclosures(
    $archive-or-ids as item*,
    $disclosures as string*) as object*
{
  mw:find($components:col,
  {
    $components:ARCHIVE: [ $archive-or-ids ! archives:aid($$) ],
    "Profiles" : { "SEC" : { "Disclosure": [ $disclosures ] } }
  })
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and that match the supplied category.

 :
 :
A category can be any of "Statement", "Disclosure", "Document",
 : "Schedule", or "Unknown".

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $categories a list of categories.
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-categories(
    $archive-or-ids as item*,
    $categories as string*) as object*
{
  for $aid_or_archive in $archive-or-ids
  let $aid as atomic := archives:aid($aid_or_archive)
  for $category in $categories
  return mw:find($components:col,
  {
    $components:ARCHIVE: $aid,
    "Profiles" : { "SEC" : { "Category": $category } }
  })
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and that match the supplied roles/network identifier.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $roles a list of roles.
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-roles(
    $archive-or-ids as item*,
    $roles as string*) as object*
{
  mw:find($components:col,
  {
    $components:ARCHIVE: [ $archive-or-ids ! archives:aid($$) ],
    "Role": [ $roles ]
  })
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and that contain at least one report element with a given name.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $report-elements a list of contained report element names.
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-reportElements(
    $archive-or-ids as item*,
    $report-elements as string*) as object*
{
  let $ids := mw:find($concepts:col,
      {
         $components:ARCHIVE: [ $archive-or-ids ! archives:aid($$) ],
         "Name" : [ $report-elements ]
      }).Component
  return components:components($ids)
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and whose label matches the given search string.

 :
 :
The result of the function is limited to 100.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $label-search-term a search term to search in labels
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-label(
    $archive-or-ids as item*,
    $label-search-term as string*) as object*
{
  sec-networks:networks-for-filings-and-label(
    $archive-or-ids, $label-search-term, 100)
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and whose label matches the given search string.

 :
 :
The result of the function is limited to 100.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $label-search-term a search term to search in labels
 : @param $limit limit the number of search results
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-label(
    $archive-or-ids as item*,
    $label-search-term as string*,
    $limit as integer) as object*
{
  mw:text-search($components:col,
  {
    "filter" : { "Archive" : [ $archive-or-ids ! archives:aid($$) ] },
    "search" : $label-search-term,
    "limit" : $limit,
    "score" : { "$meta" : "textScore" },
    "sort" : { score: { "$meta" : "textScore" } }
  })
};

(:~
 :
Retrieves the components with the given CID, or belonging to the given archives and corresponding to one of report
 : elements, concepts, disclosures, roles or label search.

 :
 : @param $archives-or-aids a sequence of archives or AIDs.
 : @param $components-or-ids a sequence of components or their CIDs.
 : @param $report-elements a sequence of report element names.
 : @param $concepts a sequence of concept names.
 : @param $disclosures a sequence of disclosures.
 : @param $roles a sequence of role URIs.
 : @param $search a sequence of label search strings.
 :
 : @return a sequence of components.
 :)
declare function sec-networks:components(
    $archives-or-aids as item*,
    $components-or-ids as item*,
    $report-elements as string*,
    $disclosures as string*,
    $roles as string*,
    $search as string*) as object*
{
    let $archives as object* := archives:archives($archives-or-aids)
    return(
        components:components($components-or-ids)[exists($components-or-ids)],
        if (exists(($report-elements, $disclosures, $roles, $search)))
        then (
                components:components-for-archives-and-concepts($archives, $report-elements),
                sec-networks:networks-for-filings-and-disclosures($archives, $disclosures),
                components:components-for-archives-and-roles($archives, $roles),
                sec-networks:networks-for-filings-and-label($archives, $search)
            )
        else components:components-for-archives($archives)
    )
};

(:~
 :
Builds a standard definition model out of the specified component.

 :
The concepts will be put on the y axis according to the presentation network.

 :
The other dimensions are put on the x axis, with one breakdown for each.

 :
Explicit dimensions are organized according to the dimension hierarchy from the domain-member network.

 :
Typed dimensions are organized according to the actual values.

 :
 :
Auto slicing will be performed against the fact table
 :
 : @param $component a component object.
 : @param $hypercube the name of a hypercube.
 :
 : @return a definition model
 :)
declare function sec-networks:standard-definition-models-for-component(
    $component as object,
    $hypercube as string) as object
{
    sec-networks:standard-definition-models-for-component($component, $hypercube, ())
};

(:~
 :
Builds a standard definition model out of the specified component.

 :
The concepts will be put on the y axis according to the presentation network.

 :
The other dimensions are put on the x axis, with one breakdown for each.

 :
Explicit dimensions are organized according to the dimension hierarchy from the domain-member network.

 :
Typed dimensions are organized according to the actual values.

 :
 : @param $component a component object.
 : @param $hypercube the name of a hypercube.
 : @param $options
some optional parameters, including:

 :

 :
AutoSlice: a boolean specifying whether or not slicing should be done automatically, looking at the fact table. Deactivating auto slicing will
 :  lead to better performance, but a more verbose table. If AutoSlice is active, dimensions with only one value in the fact pool will become
 : global filters rather than breakdowns on the x axis.

 :
Slicers: an object with forced slicers.

 :

 :
 : @error components:HYPERCUBE-DOES-NOT-EXIST if the specified hypercube is not found.
 : @return a definition model
 :)
declare function sec-networks:standard-definition-models-for-component(
    $component as object,
    $hypercube as string,
    $options as object?) as object
{
  if(not $hypercube = (keys($component.Hypercubes), $components:ALL-HYPERCUBES))
  then error(
    QName("components:HYPERCUBE-DOES-NOT-EXIST"),
    "Hypercube " || $hypercube || " was not found."
  )
  else
    let $table as object := hypercubes:hypercubes-for-components($component, $hypercube)

    let $auto-slice as boolean := empty($options.AutoSlice) or $options.AutoSlice
    let $facts as object*:=
        if($auto-slice)
        then hypercubes:facts($table)
        else ()
    let $dimensions as string*:= keys($table.Aspects)
    let $values-by-dimension as object := {|
        for $d in $dimensions
        return { $d : [ distinct-values($facts.Aspects.$d) ] }
    |}
    let $auto-slice-dimensions as string* :=
    (
        "xbrl:Entity"[size($values-by-dimension."xbrl:Entity") eq 1],
        keys($values-by-dimension)[
            size($values-by-dimension.$$) eq 1 and
            not $$ = ("xbrl:Entity",
                      "xbrl:Period",
                      "xbrl28:Archive",
                      "sec:Accepted",
                      "sec:FiscalYear",
                      "sec:FiscalPeriod",
                      "sec:FiscalPeriodType") ]
    )

    let $user-slice-dimensions as string* :=
        keys($options.Slicers)

    let $column-dimensions as string* := keys($values-by-dimension)[not $$ =
        ("xbrl:Concept",
        "xbrl:Period",
        "xbrl:Unit",
        "xbrl:Entity",
        "sec:Accepted",
        "xbrl28:Archive",
        "sec:FiscalYear",
        "sec:FiscalPeriod",
        "sec:FiscalPeriodType",
        $auto-slice-dimensions,
        $user-slice-dimensions)]

    let $x-breakdowns as object* := (
        components:standard-typed-dimension-breakdown($component, "xbrl:Period", $options)
          [not (($auto-slice-dimensions, $user-slice-dimensions) = "xbrl:Period")],
        for $d as string in $column-dimensions
        let $dimension-object as object := $table.Aspects.$d
        return
            if($d = ("sec:Accepted", "sec:FiscalYear", "sec:FiscalPeriod", "sec:FiscalPeriodType"))
            then components:standard-typed-dimension-breakdown(
                $component,
                $d,
                $options)
            else components:standard-explicit-dimension-breakdown(
                $component,
                $d,
                $dimension-object.Members[].Name,
                $component.Role,
                $options),
        components:standard-typed-dimension-breakdown($component, "xbrl:Entity", $options)
          [not (($auto-slice-dimensions, $user-slice-dimensions) = "xbrl:Entity")]
    )

    let $lineitems as string* := components:line-items($component, $hypercube)
    let $presentation-network as object? := networks:networks-for-components-and-short-names($component, "Presentation")
    let $roots as string* := distinct-values($presentation-network.Trees[].Name)
    let $lineitems as string* := if(exists($lineitems)) then $lineitems else $roots
    let $y-breakdowns as object := components:standard-concept-breakdown($lineitems, $component.Role)

    return {
        ModelKind: "DefinitionModel",
        Labels: [$component.Label],
        Parameters: {},
        Breakdowns: {
            "x" : [
                $x-breakdowns
            ],
            "y": [
                $y-breakdowns
            ]
        },
        TableFilters: {|
            for $d as string in distinct-values(($auto-slice-dimensions, $user-slice-dimensions))
            return if($d = $user-slice-dimensions)
                   then { $d : $options.Slicers.$d }
                   else { $d : $values-by-dimension.$d[] },
            if (not $auto-slice)
            then { "xbrl28:Archive" : $component.Archive }
            else ()
        |}
    }
};

(:~
 :
 :
Returns the disclosures of the suplied networks.

 :
 : @param $networks a sequence of SEC Network objects.
 : @return the disclosure names, or "UncategorizedInformation" if none.
 :
 :)
declare function sec-networks:disclosures($networks as object*) as string+
{
  for $component in $networks
  let $disclosure := $component.Profiles.SEC.Disclosure
  return if (exists($disclosure))
         then $disclosure
         else "UncategorizedInformation"
};

(:~
 :
 :
Return the categories of the supplied SEC networks (Statement, Disclosure, Document or Schedule).

 :
 : @param $networks a sequence of SEC Network objects.
 :
 : @return the category of each network.
 :)
declare function sec-networks:categories($networks as object*) as string*
{
  for $component in $networks
  return normalize-space(tokenize($component.Label, "-")[2])
};

(:~
 :
 :
Return the sub-categories of the supplied SEC networks (Detail, TextBlockLevel4, TextBLockLevel1to3).

 :
 : @param $networks a sequence of SEC Network objects.
 :
 : @return the sub-category of each network.
 :)
declare function sec-networks:sub-categories($networks as object*) as string*
{
  for $network in $networks
  let $is-text-blocks as boolean* := ($network.Concepts[])
    [not $$.SubstitutionGroup = ("xbrldt:hypercubeItem", "xbrldt:dimensionItem") and not $$.IsAbstract].IsTextBlock
  let $exists-text-blocks as boolean := exists($is-text-blocks[$$])
  let $exists-non-text-blocks as boolean := exists($is-text-blocks[not $$])
  return switch(true)
      case (: no text blocks, but still non-text-blocks :) $exists-non-text-blocks and not $exists-text-blocks return "Detail"
      case (: both :)  $exists-non-text-blocks (: and $exists-text-blocks :) return "TextBlockLevel4"
      case (: only text blocks :) $exists-text-blocks return "TextBlockLevel1To3"
      default (: exists nothing :) return "Unknown"
};

(:~
 :
 :
Return summary information for the supplied SEC networks.

 :
 : @param $networks a sequence of SEC Network objects.
 :
 : @return one object per network, containing a summary.
 :)
declare function sec-networks:summary($network as object) as object
{
  {
    Category : sec-networks:categories($network),
    SubCategory : sec-networks:sub-categories($network),
    Disclosure : sec-networks:disclosures($network)
  }
};


Make the c dialect transform work for both jq 1.4 and jq 1.5.

. as $root |
[(to_entries[] | .key as $lang_orig |(.key | split("-") | join("_")) as $lang | .value |
  [({"and", "background", "but", "examples", "feature", "given", "scenario", "scenarioOutline", "then", "when"} | to_entries[] |
    [(
      ["static const wchar_t* const ",$lang,"_",.key,"_KEYWORDS[] = { ",([.value[] | ["L",@json] | add] | join(", "))," };\n"] | add
     ),
     ("static const Keywords ",$lang,"_",.key,"_keywords = { ",(.value | length | tostring),", ",$lang,"_",.key,"_KEYWORDS };\n\n"
     )
    ] | add
   ),
   (["static const Dialect ",$lang,"_dialect = {\n        L\"",$lang_orig,"\",\n",
     ([{"and", "background", "but", "examples", "feature", "given", "scenario", "scenarioOutline", "then", "when"} | to_entries[] |
       ["        &",$lang,"_",.key,"_keywords"] | add
      ] | join(",\n")
     )," };\n\n"
    ] | add
   )
  ] | add
)] | add |
["#include \"dialect.h\"\n\n",
 .,
 "const Dialect* Dialect_for(const wchar_t* language) {\n",
 ([$root | to_entries[] | (.key | split("-") | join("_")) as $lang |
   "    if (wcscmp(",$lang,"_dialect.language_name, language) == 0)\n        return &",$lang,"_dialect;\n"] | add),
 "    return 0;\n}\n"
] | add

. as $root |
[(to_entries[] | .key as $lang_orig |(.key | split("-") | join("_")) as $lang | .value |
  [({"feature", "scenario", "scenarioOutline", "examples", "background", "given", "when", "then", "and", "but"} | to_entries[] |
    [(
      ["static const wchar_t* const ",$lang,"_",.key,"_KEYWORDS[] = { ",([.value[] | ["L",@json] | add] | join(", "))," };\n"] | add
     ),
     ("static const Keywords ",$lang,"_",.key,"_keywords = { ",(.value | length | tostring),", ",$lang,"_",.key,"_KEYWORDS };\n\n"
     )
    ] | add
   ),
   (["static const Dialect ",$lang,"_dialect = {\n        L\"",$lang_orig,"\",\n",
     ([{"feature", "scenario", "scenarioOutline", "examples", "background", "given", "when", "then", "and", "but"} | to_entries[] |
       ["        &",$lang,"_",.key,"_keywords"] | add
      ] | join(",\n")
     )," };\n\n"
    ] | add
   )
  ] | add
)] | add |
["#include \"dialect.h\"\n\n",
 .,
 "const Dialect* Dialect_for(const wchar_t* language) {\n",
 ([$root | to_entries[] | (.key | split("-") | join("_")) as $lang |
   "    if (wcscmp(",$lang,"_dialect.language_name, language) == 0)\n        return &",$lang,"_dialect;\n"] | add),
 "    return 0;\n}\n"
] | add


Added comment.

Added comment.

import module namespace jsv = "http://jsound.io/modules/validate";

let $jsd :=
  {
    "$namespace" : "http://www.example.com/my-schema",
    "$types" : [
      {
        "$name" : "foo",
        "$kind" : "object",
        "$content" : {
          "foo" : {
         (: "$type" missing :)
            "$optional" : true
          }
        }
      }
    ]
  }

let $instance := "foo"

return jsv:jsd-validate( $jsd, "foo", $instance )

(: vim:set syntax=xquery et sw=2 ts=2: :)

import module namespace jsv = "http://jsound.io/modules/validate";

let $jsd :=
  {
    "$namespace" : "http://www.example.com/my-schema",
    "$types" : [
      {
        "$name" : "foo",
        "$kind" : "object",
        "$content" : {
          "foo" : {
            "$optional" : true
          }
        }
      }
    ]
  }

let $instance := "foo"

return jsv:jsd-validate( $jsd, "foo", $instance )

(: vim:set syntax=xquery et sw=2 ts=2: :)


Fixed cross

Fixed cross

#!/usr/local/bin/jq -nRrf

# Output all intersections between two words

import "fadado.github.io/word/scanner" as scanner;

def chars:
    (./"")[]
;

def lpad($n):
    " "*$n + .
;

# Produces a stream of intersections between two words
def cross($word1; $word2):
    ($word1|scanner::upto($word2)) as $i |
    ($word2|scanner::upto($word1[$i:$i+1])) as $j |
    [
        ($word2[0:$j]  | chars | lpad($i)),
        $word1,
        ($word2[$j+1:] | chars | lpad($i))
    ]
;

# Entry point
def main:
    cross("computer"; "center") | (.[], "")
;

main

# vim:ai:sw=4:ts=4:et:syntax=jq

#!/usr/local/bin/jq -nRrf

# Output all intersections between two words

import "fadado.github.io/word" as word;

def chars:
    (./"")[]
;

def lpad($n):
    (" "*$n) + .
;

# Produces a stream of intersections between two words
def cross($word1; $word2):
    ($word1|word::upto($word2)) as $i |
    ($word2|word::upto($word1[$i:$i+1])) as $j |
    [
        ($word2[0:$j]  | chars | lpad($i)),
        $word1,
        ($word2[$j+1:] | chars | lpad($i))
    ]
;

# Entry point
def main:
    cross("computer"; "center") | (.[], "")
;

main

# vim:ai:sw=4:ts=4:et:syntax=jq


- removed ValidationStatisticsForReportSchema.jq

import module namespace report-schemas = "http://xbrl.io/modules/bizql/report-schemas";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace networks = "http://xbrl.io/modules/bizql/networks";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace facts = "http://xbrl.io/modules/bizql/facts";
import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";

import module namespace sec-fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace sec = "http://xbrl.io/modules/bizql/profiles/sec/core";

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";

let $format  := lower-case(request:param-values("format")[1])
let $aids     := request:param-values("aid","0000012927-14-000004")
let $report   := request:param-values("report","FundamentalAccountingConcepts")[1]
let $schema   := report-schemas:report-schemas($report)
let $fiscalYears :=     for $y in request:param-values("fiscalYear")
                        return
                            if ($y castable as integer)
                            then integer($y)
                            else ()
let $fiscalPeriods := request:param-values("fiscalPeriod")
let $archives := archives:archives(distinct-values($aids))
let $entities := entities:entities($archives.Entity)
return switch(true)
    case empty($archives) return {
        response:status-code(404);
        session:error("entities or archives not found (valid parameters: aid)", $format)
    }
    case not (session:only-dow30($entities) or session:valid()) return {
        response:status-code(401);
        session:error("accessing facts of an entity that is not in the DOW30", $format)
    }
    case empty($report) return {
        response:status-code(404);
        session:error("report does not exist", $report)
    }
    default return {

        [
            let $network as object :=
                networks:networks-for-components-and-short-names($schema, "Presentation")
            let $validation-concepts := keys(descendant-objects(descendant-objects($network)."fac:Validations").To)
            let $concepts := ("fac:PassedValidations", "fac:FailedValidations", "fac:NotApplicableValidations")
            let $options :=
                {
                    concept-maps: $report,
                    Rules: $report
                }
            let $facts := sec:facts-for-archives-and-concepts($archives, $concepts, $options)
                [if(empty($fiscalPeriods)) then true else ($$.Profiles.SEC.Fiscal.Period = $fiscalPeriods)]
                [if(empty($fiscalYears)) then true else ($$.Profiles.SEC.Fiscal.Year = $fiscalYears)]
            let $validation-details := sec:facts-for-archives-and-concepts($archives,$validation-concepts, $options)
                [if(empty($fiscalPeriods)) then true else ($$.Profiles.SEC.Fiscal.Period = $fiscalPeriods)]
                [if(empty($fiscalYears)) then true else ($$.Profiles.SEC.Fiscal.Year = $fiscalYears)]
            let $entities := companies:companies($entities)
            let $computed-archives :=
                    for $facts-by-archive in $facts
                    group by $archive := $facts-by-archive.Archive,
                             $year := $facts-by-archive.Profiles.SEC.Fiscal.Year,
                             $period := $facts-by-archive.Profiles.SEC.Fiscal.Period
                    let $entity := $entities[$$."_id" eq $facts-by-archive[1].$facts:ASPECTS.$facts:ENTITY][1]
                    let $validation-details-by-archive := $validation-details[$$.Archive eq $archive]
                    return
                        let $PassedValidations as object? := $facts-by-archive[$$.$facts:ASPECTS.$facts:CONCEPT eq "fac:PassedValidations"]
                        let $FailedValidations as object? := $facts-by-archive[$$.$facts:ASPECTS.$facts:CONCEPT eq "fac:FailedValidations"]
                        let $NotApplicableValidations as object? := $facts-by-archive[$$.$facts:ASPECTS.$facts:CONCEPT eq "fac:NotApplicableValidations"]
                        let $NotAplicabelValidationsDetails as string* :=
                          let $applied := $validation-details-by-archive.$facts:ASPECTS.$facts:CONCEPT
                          let $NotApplicableConcepts := $validation-concepts[not($$ = $applied)]
                          let $NotApplicableRules := $schema.Rules[][$$.ComputableConcepts[] = $NotApplicableConcepts]
                          return $NotApplicableRules.Label
                        return
                            {
                                Entity : $entity.Profiles.SEC.CompanyName,
                                CIK : $entity.Profiles.SEC.CIK,
                                SIC : $entity.Profiles.SEC.SIC,
                                SICDescription : $entity.Profiles.SEC.SICDescription,
                                FiscalYear: $year,
                                FiscalPeriod: $period,
                                PassedValidations : $PassedValidations.Value,
                                FailedValidations : $FailedValidations.Value,
                                NotApplicableValidations : $NotApplicableValidations.Value,
                                NotApplicableDetails: [$NotAplicabelValidationsDetails],
                                ValidationDetails: [$validation-details-by-archive]
                            }
            for $archive in $computed-archives
            order by $archive.PassedValidations descending, $archive.SICDescription ascending
            return
                $archive
        ]
    }


:pencil: Add test.

:pencil: Add test.

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";
import module namespace config = "http://apps.28.io/config";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facts"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-404($params as object) as item
{
    let $endpoint := "facts"
    let $res as object := test:invoke-raw($endpoint, $params)
    return if($res.status eq 404)
           then true
           else {
              url: test:url($endpoint, $params),
              unexpectedResponse: $res
           }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw($endpoint, $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "Archive [Axis],Fiscal Period [Axis],Fiscal Year [Axis],Fiscal Period Type [Axis],Period [Axis],Reporting Entity [Axis],Concept [Axis],Unit [Axis],Legal Entity [Axis],Value,Decimals,Balance,PeriodType,DataType\r\n0000021344-13-000050,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),Assets,USD,Default Legal Entity [Member],89432000000,-6,DEBIT,instant,xbrli:monetaryItemType\r\n0000021344-14-000036,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),\"Cash and Cash Equivalents, at Carrying Value\",USD,Default Legal Entity [Member],11118000000,-6,DEBIT,instant,xbrli:monetaryItemType\r\n"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res,
        expected: $expectedLines
    }
};

declare %an:nondeterministic function local:test-labels-aids() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: "disc:AdvertisingCostsPolicyTextBlock",
        report: "Disclosures",
        format: "json",
        fiscalYear: 2014,
        fiscalPeriod: "FY",
        labels: true,
        aid: "0000858877-14-000029"
    }
    let $res := test:invoke($endpoint, $params)
    let $actual := [
            for $labels in $res[2].FactTable[].Labels
            return (keys($labels) ! $labels.$$)
        ]
    let $expected := [ "Archive [Axis]", "Fiscal Year [Axis]", "Non Numeric", "Default Legal Entity [Member]", "Unit [Axis]", "Advertising Costs, Policy", "Fiscal Period [Axis]", "Concept [Axis]", "Reporting Entity [Axis]", "Legal Entity [Axis]", "Fiscal Period Type [Axis]", "Period [Axis]", "From July 28, 2013 to July 26, 2014", "CISCO SYSTEMS, INC. (858877)" ]
    let $status as integer := $res[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

if(not $config:content[] = ("SEC-big", "SEC-small"))
then true
else
local:check({
    cocacola-latest:
    (: this test will fail and needs to be updated if a newer report has been filed.
       current latest filing: 2014 :)
    local:test-facttable(462, {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    cocacola-all: local:test-facttable(477, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-all-q3: local:test-facttable(341, {
        ticker:"ko",
        fiscalYear: 2014,
        fiscalPeriod: [ "Q3" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-instant: local:test-facttable(170, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant" ]
    }),
    cocacola-ytd: local:test-facttable(307, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-ytd-several-years: local:test-facttable(607, {
        ticker:"ko",
        fiscalYear: [ 2013, 2014 ],
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-qtd: local:test-facttable(102, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "Q2" ],
        fiscalPeriodType: [ "QTD" ]
    }),
    cocacolaCSVLabels: local:test-labels(),
    ciscoLabelsByAid: local:test-labels-aids(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(30, { (: AT&T wrongly stamped a filing as FY, but it's now FIXED :-):)
        tag:"DOW30",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-404({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    }),
    quartersfact: local:test-facttable(0, { (: this fact is not reported in the raw XML filing :)
        aid: "0000051143-13-000007",
        concept: "disc:AssetRetirementObligationsPolicy",
        fiscalPeriod: "Q3",
        fiscalYear: "2013",
        map: "Disclosures"
    }),
    quartersfact2: local:test-facttable(1, {
        aid: "0001193125-12-207154",
        concept: "disc:CashFlowOperatingCapitalTableTextBlock",
        fiscalPeriod: "Q1",
        fiscalYear: "2012",
        map: "Disclosures"
    }),
    generic: local:test-facttable(2, {
        "xbrl:Entity":"http://www.sec.gov/CIK 0000021344",
        "sec:Accepted":"20140227132423",
        concept:"us-gaap:Assets",
        profile-name: "generic"
    }),
    labels: test:invoke-and-assert-deep-equal(
          "facts",
          {
            tag : "DOW30",
            concept:"us-gaap:Assets",
            labels: "true",
            fiscalPeriod: "FY",
            fiscalYear: "2014"
          },
          function($res as item*) as item* {
            count(distinct-values($res.FactTable[].Labels."dei:LegalEntityAxis")),
            count($res.FactTable[].Labels."dei:LegalEntityAxis")
          },
          (1, 31)
      ),
    count: test:invoke-and-assert-deep-equal(
          "facts",
          {
              "ticker" : "ko",
              "fiscalYear" : ["2012", "2013", "2014"],
              "fiscalPeriod" : "FY",
              "count" : "true"
          },
          function($res as item*) as item* { $res.Statistics.NumFacts },
          1444
        ),
    odata: test:invoke-and-assert-deep-equal(
      "facts",
      {
          "ticker" : "ko",
          "fiscalYear" : "ALL",
          "fiscalPeriod" : "FY",
          "concept" : [ "us-gaap:Assets", "us-gaap:LiabilitiesCurrent", "us-gaap:StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" ],
          "format" : "atom-xml"
      },
      function($res as item*) as item* { count($res/*:feed/*:entry) },
      18,
      { Format: "xml" }
    ),
    slicers-columns: test:invoke-and-assert-deep-equal(
          "facts",
          {
            tag : "DOW30",
            concept:"us-gaap:Assets",
            labels: "true",
            fiscalPeriod: "FY",
            fiscalYear: "2014",
            "xbrl:Period::slicer": "true",
            "xbrl28:Archive::slicer": "true",
            "dei:LegalEntityAxis::slicer": "true",
            "sec:FiscalPeriod::slicer": "true",
            "sec:FiscalYear::slicer": "true",
            "sec:FiscalPeriodType::slicer": "true",
            "xbrl:Concept::column": "1",
            "xbrl:Entity::column": "2",
            "xbrl:Unit::column": "3"
          },
          function($res as item*) as item* {
            keys($res.FactTable[[1]].Aspects)
          },
          ("xbrl:Concept", "xbrl:Entity", "xbrl:Unit")
      )
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";
import module namespace config = "http://apps.28.io/config";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facts"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-404($params as object) as item
{
    let $endpoint := "facts"
    let $res as object := test:invoke-raw($endpoint, $params)
    return if($res.status eq 404)
           then true
           else {
              url: test:url($endpoint, $params),
              unexpectedResponse: $res
           }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw($endpoint, $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "Archive [Axis],Fiscal Period [Axis],Fiscal Year [Axis],Fiscal Period Type [Axis],Period [Axis],Reporting Entity [Axis],Concept [Axis],Unit [Axis],Legal Entity [Axis],Value,Decimals,Balance,PeriodType,DataType\r\n0000021344-13-000050,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),Assets,USD,Default Legal Entity [Member],89432000000,-6,DEBIT,instant,xbrli:monetaryItemType\r\n0000021344-14-000036,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),\"Cash and Cash Equivalents, at Carrying Value\",USD,Default Legal Entity [Member],11118000000,-6,DEBIT,instant,xbrli:monetaryItemType\r\n"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res,
        expected: $expectedLines
    }
};

declare %an:nondeterministic function local:test-labels-aids() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: "disc:AdvertisingCostsPolicyTextBlock",
        report: "Disclosures",
        format: "json",
        fiscalYear: 2014,
        fiscalPeriod: "FY",
        labels: true,
        aid: "0000858877-14-000029"
    }
    let $res := test:invoke($endpoint, $params)
    let $actual := [
            for $labels in $res[2].FactTable[].Labels
            return (keys($labels) ! $labels.$$)
        ]
    let $expected := [ "Archive [Axis]", "Fiscal Year [Axis]", "Non Numeric", "Default Legal Entity [Member]", "Unit [Axis]", "Advertising Costs, Policy", "Fiscal Period [Axis]", "Concept [Axis]", "Reporting Entity [Axis]", "Legal Entity [Axis]", "Fiscal Period Type [Axis]", "Period [Axis]", "From July 28, 2013 to July 26, 2014", "CISCO SYSTEMS, INC. (858877)" ]
    let $status as integer := $res[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

if(not $config:content[] = ("SEC-big", "SEC-small"))
then true
else
local:check({
    cocacola-latest:
    (: this test will fail and needs to be updated if a newer report has been filed.
       current latest filing: 2014 :)
    local:test-facttable(462, {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    cocacola-all: local:test-facttable(477, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-all-q3: local:test-facttable(341, {
        ticker:"ko",
        fiscalYear: 2014,
        fiscalPeriod: [ "Q3" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-instant: local:test-facttable(170, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant" ]
    }),
    cocacola-ytd: local:test-facttable(307, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-ytd-several-years: local:test-facttable(607, {
        ticker:"ko",
        fiscalYear: [ 2013, 2014 ],
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-qtd: local:test-facttable(102, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "Q2" ],
        fiscalPeriodType: [ "QTD" ]
    }),
    cocacolaCSVLabels: local:test-labels(),
    ciscoLabelsByAid: local:test-labels-aids(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(30, { (: AT&T wrongly stamped a filing as FY, but it's now FIXED :-):)
        tag:"DOW30",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-404({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    }),
    quartersfact: local:test-facttable(0, { (: this fact is not reported in the raw XML filing :)
        aid: "0000051143-13-000007",
        concept: "disc:AssetRetirementObligationsPolicy",
        fiscalPeriod: "Q3",
        fiscalYear: "2013",
        map: "Disclosures"
    }),
    quartersfact2: local:test-facttable(1, {
        aid: "0001193125-12-207154",
        concept: "disc:CashFlowOperatingCapitalTableTextBlock",
        fiscalPeriod: "Q1",
        fiscalYear: "2012",
        map: "Disclosures"
    }),
    generic: local:test-facttable(2, {
        "xbrl:Entity":"http://www.sec.gov/CIK 0000021344",
        "sec:Accepted":"20140227132423",
        concept:"us-gaap:Assets",
        profile-name: "generic"
    }),
    labels: test:invoke-and-assert-deep-equal(
          "facts",
          {
            tag : "DOW30",
            concept:"us-gaap:Assets",
            labels: "true",
            fiscalPeriod: "FY",
            fiscalYear: "2014"
          },
          function($res as item*) as item* {
            count(distinct-values($res.FactTable[].Labels."dei:LegalEntityAxis")),
            count($res.FactTable[].Labels."dei:LegalEntityAxis")
          },
          (1, 31)
      ),
    count: test:invoke-and-assert-deep-equal(
          "facts",
          {
              "ticker" : "ko",
              "fiscalYear" : ["2012", "2013", "2014"],
              "fiscalPeriod" : "FY",
              "count" : "true"
          },
          function($res as item*) as item* { $res.Statistics.NumFacts },
          1444
        ),
    odata: test:invoke-and-assert-deep-equal(
      "facts",
      {
          "ticker" : "ko",
          "fiscalYear" : "ALL",
          "fiscalPeriod" : "FY",
          "concept" : [ "us-gaap:Assets", "us-gaap:LiabilitiesCurrent", "us-gaap:StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" ],
          "format" : "atom-xml"
      },
      function($res as item*) as item* { count($res/*:feed/*:entry) },
      18,
      { Format: "xml" }
    )
})


Treat JSON null as empty array

Treat JSON null as empty array

# This is a jq filter that converts the output of
# https://github.com/ecc1/medtronic/cmd/pumphistory
# to the schema that openaps expects.
# Usage:
#   pumphistory | jq -f openaps.jq > history.json

# Convert a Go duration to minutes.
def duration_to_minutes:
  if test("^(\\d+h)?(\\d+m)?0s$") then
    capture("^((?<h>\\d+)h)?((?<m>\\d+)m)?0s$") |
    60*(.h // 0 | tonumber) + (.m // 0 | tonumber)
  else
    ("unexpected duration: " + .) | error
  end;

# Output a single JSON array.
[
  # Treat null as an empty array.
  (. // []) |
  # Perform the following on each element of the input array.
  .[] |
  # Start with the timestamp field, common to all record types,
  # and the type, which is the same as the decocare type in many cases.
  {
    timestamp: .Time,
    _type: .Type
  } +
  # Add type-specific fields.
  if .Type == "TempBasalDuration" then
    {
      "duration (min)": .Info | duration_to_minutes
    }
  elif .Type == "TempBasalRate" then
    {
      _type: "TempBasal",
      temp: .Info.Type | ascii_downcase,
      rate: .Info.Value
    }
  elif .Type == "Bolus" then
    {
      amount: .Info.Amount,
      programmed: .Info.Programmed,
      unabsorbed: .Info.Unabsorbed,
      duration: .Info.Duration | duration_to_minutes
    }
  elif .Type == "Prime" then
    {
      amount: .Info.Manual,
      fixed: .Info.Fixed,
      type: "fixed"
    }
  elif .Type == "SuspendPump" then
    {
      _type: "PumpSuspend"
    }
  elif .Type == "ResumePump" then
    {
      _type: "PumpResume"
    }
  elif .Type == "BatteryChange" then
    {
      _type: "Battery"
    }
  elif .Type == "Rewind" then
    {
    }
  # Add additional cases here as needed.
  else
    # Warn about record types being skipped.
    ("skipping " + .Type + " record") | debug | empty
  end
]

# This is a jq filter that converts the output of
# https://github.com/ecc1/medtronic/cmd/pumphistory
# to the schema that openaps expects.
# Usage:
#   pumphistory | jq -f openaps.jq > history.json

# Convert a Go duration to minutes.
def duration_to_minutes:
  if test("^(\\d+h)?(\\d+m)?0s$") then
    capture("^((?<h>\\d+)h)?((?<m>\\d+)m)?0s$") |
    60*(.h // 0 | tonumber) + (.m // 0 | tonumber)
  else
    ("unexpected duration: " + .) | error
  end;

# Output a single JSON array.
[
  # Perform the following on each element of the input array.
  .[] |
  # Start with the timestamp field, common to all record types,
  # and the type, which is the same as the decocare type in many cases.
  {
    timestamp: .Time,
    _type: .Type
  } +
  # Add type-specific fields.
  if .Type == "TempBasalDuration" then
    {
      "duration (min)": .Info | duration_to_minutes
    }
  elif .Type == "TempBasalRate" then
    {
      _type: "TempBasal",
      temp: .Info.Type | ascii_downcase,
      rate: .Info.Value
    }
  elif .Type == "Bolus" then
    {
      amount: .Info.Amount,
      programmed: .Info.Programmed,
      unabsorbed: .Info.Unabsorbed,
      duration: .Info.Duration | duration_to_minutes
    }
  elif .Type == "Prime" then
    {
      amount: .Info.Manual,
      fixed: .Info.Fixed,
      type: "fixed"
    }
  elif .Type == "SuspendPump" then
    {
      _type: "PumpSuspend"
    }
  elif .Type == "ResumePump" then
    {
      _type: "PumpResume"
    }
  elif .Type == "BatteryChange" then
    {
      _type: "Battery"
    }
  elif .Type == "Rewind" then
    {
    }
  # Add additional cases here as needed.
  else
    # Warn about record types being skipped.
    ("skipping " + .Type + " record") | debug | empty
  end
]