jq: Add Merge & Sync filters [ci skip]

jq: Add Merge & Sync filters [ci skip]

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v);

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: clean | select(is_error);
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: clean | select(is_warn);
def warn_strict: clean | select(is_warn_strict);
def info_level: 30;
def is_info: .level >= info_level;
def info: clean | select(is_info);
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: clean | select(is_debug);

# Components:
def chokidar: select(.component == "Chokidar");
def LocalChange: select(.component == "local/change");
def LocalWatcher: select(.component == "LocalWatcher");
def Merge: select(.component == "Merge");
def Metadata: select(.component == "Metadata");
def Pouch: select(.component == "Pouch");
def Sync: select(.component == "Sync");

# Find conflicts:
#
#     yarn jq -c conflicts path/to/logs*
#
def is_conflict: .msg == "resolveConflictAsync";
def conflicts: clean | select(is_conflict) | {time,path};

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
def path(pattern): clean | select((.path,.oldpath,"") | strings | test(pattern));

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path};

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v);

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: clean | select(is_error);
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: clean | select(is_warn);
def warn_strict: clean | select(is_warn_strict);
def info_level: 30;
def is_info: .level >= info_level;
def info: clean | select(is_info);
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: clean | select(is_debug);

# Components:
def chokidar: select(.component == "Chokidar");
def LocalChange: select(.component == "local/change");
def LocalWatcher: select(.component == "LocalWatcher");
def Metadata: select(.component == "Metadata");
def Pouch: select(.component == "Pouch");

# Find conflicts:
#
#     yarn jq -c conflicts path/to/logs*
#
def is_conflict: .msg == "resolveConflictAsync";
def conflicts: clean | select(is_conflict) | {time,path};

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
def path(pattern): clean | select((.path,.oldpath,"") | strings | test(pattern));

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path};

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );


Modified filter-override so it doesn't always query entities.

jsoniq version "1.0";

(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for querying fiscal reports (10-K, 10-Q)
 : submitted to the SEC.

 :
 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace facts = "http://28.io/modules/xbrl/facts";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";
import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Joker for all fiscal periods.
 :)
declare variable $fiscal-core:ALL_FISCAL_PERIODS as string := "ALL";

(:~
 : Joker for all fiscal years.
 :)
declare variable $fiscal-core:ALL_FISCAL_YEARS as integer := 0;

(:~
 : Joker for the latest fiscal years.
 :)
declare variable $fiscal-core:LATEST_FISCAL_YEAR as integer := 1;

(:~
 :
Return a distinct set of companies identified by either
 :   CIKs, tags, tickers, or sics.

 :
 : @param $ciks a set of CIKs.
 : @param $tags a set of tags (ALL retrieves all companies).
 : @param $tickers a set of tickers.
 : @param $sics a set of SIC codes.
 :
 : @return the companies with the given identifiers, tags, tickers, or sic codes.
 :)
declare function fiscal-core:filings(
    $entities-or-eids as item*,
    $fiscal-periods as string*,
    $fiscal-years as integer*,
    $archives-or-aids as item*) as object*
{
    let $entities := entities:entities($entities-or-eids)
    return
        if(exists(index-of($fiscal-years, $fiscal-core:LATEST_FISCAL_YEAR)))
        then
            for $entity in $entities
            return
            fiscal-core:latest-filings($entity, $fiscal-periods)
        else
            let $fiscal-years as integer* :=
                $fiscal-years[$$ ne $fiscal-core:LATEST_FISCAL_YEAR]
            return
                fiscal-core:filings-for-entities-and-fiscal-periods-and-years($entities, $fiscal-periods, $fiscal-years),
    archives:archives($archives-or-aids)
};

declare function fiscal-core:latest-filings(
    $entities as object*,
    $fiscal-periods as string*) as object*
{
    for $entity in $entities
    for $fiscal-period in $fiscal-periods
    let $latest-fiscal-year :=
        fiscal-core:latest-reported-fiscal-period($entity, $fiscal-period).year
    return fiscal-core:filings-for-entities-and-fiscal-periods-and-years(
        $entity,
        $fiscal-period,
        $latest-fiscal-year cast as integer?
    )
};

declare function fiscal-core:filter-override(
    $entities-or-eids as item*,
    $fiscal-years as integer*,
    $fiscal-periods as string*,
    $archives-or-aids as item*
) as object?
{
    let $eids := entities:eid($entities-or-eids)
    let $entities := entities:entities($entities-or-eids)
    let $aids := archives:aid($archives-or-aids)
    let $latest-filings := fiscal-core:latest-filings($entities, $fiscal-periods)
    return
    switch(true)
    case count($aids) gt 0 return {
        "xbrl:Entity" : {
        },
        "sec:Archive" : {
            Type: "string",
            Domain : [ $aids ]
        },
        "sec:FiscalYear" : {
            Type: "integer"
        },
        "sec:FiscalPeriod" : {|
            {
                Type: "string",
                Domain: [ $fiscal-periods ]
            }[exists($fiscal-periods) and empty(index-of($fiscal-periods, $fiscal-core:ALL_FISCAL_PERIODS))]
        |}
    }
    case exists(index-of($fiscal-years, $fiscal-core:LATEST_FISCAL_YEAR))
    return {
        "sec:Archive" : {
            Type: "string",
            Domain : [archives:aid($latest-filings)]
        },
        "sec:FiscalPeriod" : {|
            {
                Type: "string",
                Domain: [ $fiscal-periods ]
            }[exists($fiscal-periods) and empty(index-of($fiscal-periods, $fiscal-core:ALL_FISCAL_PERIODS))]
        |}
    }
    case exists(($eids, $fiscal-years, $fiscal-periods))
    return {
        "xbrl:Entity" : {|
            {
                Type: "string",
                Domain: [ $eids ]
            }[exists($eids)]
        |},
        "sec:FiscalYear" :
            let $fiscal-years as integer* :=
                $fiscal-years[$$ ne $fiscal-core:LATEST_FISCAL_YEAR]
            return {|
                {
                    Type: "integer",
                    Domain: [ $fiscal-years ]
                }[exists($fiscal-years) and empty(index-of($fiscal-years, $fiscal-core:ALL_FISCAL_YEARS))]
            |},
        "sec:FiscalPeriod" : {|
            {
                Type: "string",
                Domain: [ $fiscal-periods ]
            }[exists($fiscal-periods) and empty(index-of($fiscal-periods, $fiscal-core:ALL_FISCAL_PERIODS))]
        |},
        "sec:Archive" : {}
    }
    default return ()
};

(:~
 :
Retrieves the fiscal period of a filing.

 :
 : @param $filing-or-id a filing or its id, or the empty sequence.
 :
 : @return the fiscal period (FY, YTD2, YTD3, Q1, Q2 or Q3) if any.
 :)
declare function fiscal-core:fiscal-period-for-filing(
    $filing-or-id as item?) as string?
{
  archives:archives($filing-or-id).Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
};

(:~
 :
Retrieves the fiscal period of a fact.

 :
 : @param $fact a fact object or the empty sequence.
 :
 : @return the fiscal period (FY, YTD2, YTD3, Q1, Q2 or Q3) if any.
 :)
declare function fiscal-core:fiscal-period-for-fact(
    $fact as object?) as string?
{
  $fact.Aspects."sec:FiscalPeriod"
};

(:~
 :
Retrieves the fiscal period of a filing or a fact.

 :
 : @deprecated use fiscal-period-for-fact#1 and fiscal-period-for-filing#1
 :
 : @param $filing-fact-or-id a filing, a fact, or its id.
 :
 : @return the fiscal period (FY, YTD2, YTD3, Q1, Q2 or Q3).
 :)
declare function fiscal-core:fiscal-period(
    $filing-fact-or-id as item?) as string?
{
  if(empty($filing-fact-or-id))
  then ()
  else
    typeswitch ($filing-fact-or-id)
    case object return
      switch (true)
      (: archive :)
      case exists($filing-fact-or-id.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus)
        return $filing-fact-or-id.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
      (: fact :)
      case exists($filing-fact-or-id.Aspects."sec:FiscalPeriod")
        return $filing-fact-or-id.Aspects."sec:FiscalPeriod"
      default return ()
    default return
      let $id as xs:string := string($filing-fact-or-id)
      return
        switch(true)
        (:
          "The first ten digits are the CIK of the entity that logged
           in to submit the filing. The next two digits define the
           current year. The final digits are assigned sequentially by
           CIK starting with one on the first business day of each year."

        http://www.sec.gov/info/edgar/edgarfm-vol2-v25.pdf :)
        case matches($id, "\\d{10}-\\d{2}-\\d{6}")
          return archives:archives($id).Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
        default return facts:facts($id).Aspects."sec:FiscalPeriod"
};

(:~
 :
Retrieves the fiscal year of a filing.

 :
 : @param $filing-or-id a filing or its id, or the empty sequence.
 :
 : @return the fiscal year (e.g., 2012) if any.
 :)
declare function fiscal-core:fiscal-year-for-filing(
    $filing-or-id as item?) as integer?
{
  archives:archives($filing-or-id).Profiles.SEC.Fiscal.DocumentFiscalYearFocus
};

(:~
 :
Retrieves the fiscal year of a fact.

 :
 : @param $fact a fact object or the empty sequence.
 :
 : @return the fiscal year (e.g., 2012) if any.
 :)
declare function fiscal-core:fiscal-year-for-fact(
    $fact as object?) as integer?
{
  $fact.Aspects."sec:FiscalYear"
};

(:~
 :
Retrieves the fiscal year of a filing or a fact.

 :
 : @deprecated use fiscal-core:fiscal-year-for-archive#1 or fiscal-core:fiscal-year-for-fact#1.
 :
 : @param $filing-fact-or-id a filing, a fact, or its id.
 :
 : @return the fiscal year (e.g., 2012).
 :)
declare function fiscal-core:fiscal-year(
    $filing-fact-or-id as item?) as integer?
{
  if(empty($filing-fact-or-id))
  then ()
  else
    typeswitch ($filing-fact-or-id)
    case object return
      switch (true)
      (: archive :)
      case exists($filing-fact-or-id.Profiles.SEC.Fiscal.DocumentFiscalYearFocus)
        return $filing-fact-or-id.Profiles.SEC.Fiscal.DocumentFiscalYearFocus
      (: fact :)
      case exists($filing-fact-or-id.Aspects."sec:FiscalYear")
        return $filing-fact-or-id.Aspects."sec:FiscalYear"
      default return ()
    default return
      let $id as xs:string := string($filing-fact-or-id)
      return
        switch(true)
        (:
          "The first ten digits are the CIK of the entity that logged
           in to submit the filing. The next two digits define the
           current year. The final digits are assigned sequentially by
           CIK starting with one on the first business day of each year."

          http://www.sec.gov/info/edgar/edgarfm-vol2-v25.pdf :)
        case matches($id, "\\d{10}-\\d{2}-\\d{6}")
          return archives:archives($id).Profiles.SEC.Fiscal.DocumentFiscalYearFocus
        default return facts:facts($id).Aspects."sec:FiscalYear"
};

(:~
 :
Retrieves all filings by entities and for given fiscal periods and years.

 :
 : @param $entities-or-ids entities or their ids.
 : @param $fiscal-period-focus fiscal periods (FY, YTD1, YTD2, YTD3, Q1, Q2, Q3, Q4) or $fiscal-core:ALL_FISCAL_PERIODS to accept all..
 : @param $fiscal-year-focus fiscal years or $fiscal-core:ALL_FISCAL_YEARS to accept all..
 :
 : @return all filings that match the criteria.
 :)
declare function
fiscal-core:filings-for-entities-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  let $fiscal-period-focus :=
    switch(true)
    case $fiscal-period-focus = ( "10-K", "FY") return "FY"
    case $fiscal-period-focus eq "10-Q" return ( "Q3", "Q2", "Q1" )
    case $fiscal-period-focus eq "YTD1" return "Q1"
    case $fiscal-period-focus eq "YTD2" return "Q2"
    case $fiscal-period-focus eq "YTD3" return "Q3"
    default return $fiscal-period-focus
  let $conn := fiscal-core:connection()
  for $cik-or-entity in $entities-or-ids
  let $entity-id as xs:string := companies:eid($cik-or-entity)
  let $query := {|
      { "Entity" : $entity-id },
      switch (true)
      case deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS)
        return ()
      case count($fiscal-period-focus) eq 1
        return
        {
          "Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus": $fiscal-period-focus
        }
      default return
        {
          "Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus":
            { "$in" : [ $fiscal-period-focus ] }
        },
      switch (true)
      case deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS)
        return ()
      case count($fiscal-year-focus) eq 1
        return
        {
          "Profiles.SEC.Fiscal.DocumentFiscalYearFocus": $fiscal-year-focus
        }
      default
        return
        {
          "Profiles.SEC.Fiscal.DocumentFiscalYearFocus":
            { "$in" : [ $fiscal-year-focus ] }
        }
    |}
  return mongo:find($conn, $archives:col, $query)
};

(:~
 :
Retrieves the balance sheet for the given entities and fiscal periods and years.

 :
 : @param $entity-or-ids a list of entities or their EIDs.
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 :
 : @error entities:INVALID_PARAMETER if the CIK or entity is not valid.
 :
 : @return the balance sheet at that date and for that formType.
 :)
declare function fiscal-core:balance-sheets-for-fiscal-periods-and-years(
    $entity-or-ids as item*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  for $archive in
    fiscal-core:filings-for-entities-and-fiscal-periods-and-years($entity-or-ids,
                                                                  $fiscal-period-focus,
                                                                  $fiscal-year-focus)
  return sec-networks:networks-for-filings-and-disclosures(
      $archive,
      $sec-networks:BALANCE_SHEET)
};

(:~
 :
Retrieves facts for the given aspects, fiscal periods and years.

 :
 : @param $aspects an object containing aspects to filter, among which
 :                 xbrl:Concept, xbrl:Entity and xbrl:Period (all optional).
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY).
 : @param $fiscal-year-focus the years (2012)
 :
 : @return all facts matching the given aspects, fiscal period and year.
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects as object?,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects,
    $fiscal-period-focus,
    $fiscal-year-focus,
    ())
};

(:~
 :
Retrieves facts for the given aspects, fiscal periods and years.

 :
 : @param $aspects an object containing aspects to filter, among which
 :                 xbrl:Concept, xbrl:Entity and xbrl:Period (all optional).
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 : @param $options standard SEC BizQL options.
 :
 : @return all facts matching the given aspects, fiscal period and year.
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects as object?,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*,
    $options as object?) as object*
{
  let $filter as object := {|
    {
      "Filter":
        {|
          if(exists($aspects))
          then { $facts:ASPECTS : $aspects }
          else (),
          if( deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS) and deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS))
          then { "Profiles.SEC.Fiscal" : { "$exists" : true } }
          else { "Profiles": { "SEC": { "Fiscal":
                 {|
                   if(deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS) )
                   then ()
                   else { "Period" : [ $fiscal-period-focus ] },
                   if( deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS) )
                   then ()
                   else { "Year": [ $fiscal-year-focus ] }
                 |}
               }}}
        |}
    },
    if (empty($options.Hypercube))
    then { Hypercube: sec:dimensionless-hypercube(()) }
    else ()
  |}
  let $facts :=
    facts:facts-for(
      facts:merge-objects($filter, $options, true (: giving parameters higher priority :))
    )
  return sec:hide-amended-facts($facts, $options)
};

(:~
 :
Retrieves facts for the given entities, concepts, fiscal periods and years.

 :
 : @param $entities-or-ids a list of entities or their EIDs.
 : @param $concepts a list of concept names (us-gaap:Assets)
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 :
 : @error entities:INVALID_PARAMETER if the CIK or entity is not valid.
 : @return all facts matching entities, concepts, fiscal periods and years
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $concepts as string*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids,
    $concepts,
    $fiscal-period-focus,
    $fiscal-year-focus,
    ())
};

(:~
 :
Retrieves facts for the given entities, concepts, fiscal periods and years.

 :
 : @param $entities-or-ids a list of entities or their EIDs.
 : @param $concepts a list of concept names (us-gaap:Assets)
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 : @param $options standard SEC BizQL options.
 :
 : @error entities:INVALID_PARAMETER if the CIK or entity is not valid.
 : @return all facts matching entities, concepts, fiscal periods and years
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $concepts as string*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*,
    $options as object?) as object*
{
  let $entity-ids as xs:string* := $entities-or-ids ! companies:eid($$)
  let $aspects := {|
    if(exists($entity-ids))
    then { "xbrl:Entity": [ $entity-ids ] }
    else (),
    if(exists($concepts))
    then { "xbrl:Concept": [ $concepts ] }
    else ()
  |}
  return
    fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
      $aspects,
      $fiscal-period-focus,
      $fiscal-year-focus,
      $options)
};

(:~
 :
Retrieves the latest reported fiscal period and year for the
 : supplied entity.

 :
 : @param $entity-or-id an entity or its EID.
 :
 : @return an object with two fields: period and year.
 :)
declare function fiscal-core:latest-reported-fiscal-period(
    $entity-or-id as item) as object?
{
  (
  let $entity-or-id := companies:eid($entity-or-id)
  for $archive in archives:archives-for-entities($entity-or-id)
  let $archive-fiscal-period := fiscal-core:fiscal-period-for-filing($archive)
  let $archive-fiscal-year := fiscal-core:fiscal-year-for-filing($archive)
  group by $archive-fiscal-year
  order by $archive-fiscal-year descending empty least
  return {
    year: $archive-fiscal-year,
    period: switch(true)
            case $archive-fiscal-period = "FY" return "FY"
            case $archive-fiscal-period = ("Q3", "YTD3") return "Q3"
            case $archive-fiscal-period = ("Q2", "YTD2") return "Q2"
            case $archive-fiscal-period = ("Q1", "YTD1") return "Q1"
            default return () }
  )[1]
};

(:~
 :
Retrieves the latest fiscal period and year for the
 : supplied entity.

 :
 : @param $entity-or-id an entity or its EID.
 : @param $type 10-K or 10-Q or FY, YTD1, YTD2, YTD3, Q1, Q2, or Q3, or $fiscal-core:ALL_FISCAL_PERIODS
 : to accept all.
 :
 : @return an object with two fields: period and year.
 :)
declare function fiscal-core:latest-reported-fiscal-period(
    $entity-or-id as item,
    $type as string) as object?
{
  if(deep-equal($type, $fiscal-core:ALL_FISCAL_PERIODS))
  then fiscal-core:latest-reported-fiscal-period($entity-or-id)
  else
  (
  let $type-filter :=
    switch(true)
    case $type = ( "10-K", "FY") return "FY"
    case $type eq "10-Q" return ( "Q3", "Q2", "Q1" )
    case $type eq "YTD1" return "Q1"
    case $type eq "YTD2" return "Q2"
    case $type eq "YTD3" return "Q3"
    default return $type
  let $entity-or-id := companies:eid($entity-or-id)
  for $archive in archives:archives-for-entities($entity-or-id)
  let $archive-fiscal-period := fiscal-core:fiscal-period-for-filing($archive)
  let $archive-fiscal-year := fiscal-core:fiscal-year-for-filing($archive)
  where $archive-fiscal-period = $type-filter
  group by $archive-fiscal-year
  order by $archive-fiscal-year descending empty least
  return {
    year: $archive-fiscal-year,
    period:
      switch(true)
      case $archive-fiscal-period = "FY" and ($type eq "10-K" or $type eq "FY") return "FY"
      case $archive-fiscal-period = "Q3" and ($type eq "10-Q" or $type eq "Q3") return "Q3"
      case $archive-fiscal-period = "Q2" and ($type eq "10-Q" or $type eq "Q2") return "Q2"
      case $archive-fiscal-period = "Q1" and ($type eq "10-Q" or $type eq "Q1") return "Q1"
      default return () }
  )[1]
};

(:~
 :)
declare %private %an:strictlydeterministic function fiscal-core:connection() as anyURI
{
  let $credentials :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return if (empty($credentials))
             then error(QName("fiscal-core:CONNECTION-FAILED"), "no xbrl MongoDB configured")
             else $credentials
  return
    try {
      mongo:connect($credentials)
    } catch mongo:* {
      error(QName("fiscal-core:CONNECTION-FAILED"), $err:description)
    }
};

jsoniq version "1.0";

(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for querying fiscal reports (10-K, 10-Q)
 : submitted to the SEC.

 :
 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace facts = "http://28.io/modules/xbrl/facts";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";
import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Joker for all fiscal periods.
 :)
declare variable $fiscal-core:ALL_FISCAL_PERIODS as string := "ALL";

(:~
 : Joker for all fiscal years.
 :)
declare variable $fiscal-core:ALL_FISCAL_YEARS as integer := 0;

(:~
 : Joker for the latest fiscal years.
 :)
declare variable $fiscal-core:LATEST_FISCAL_YEAR as integer := 1;

(:~
 :
Return a distinct set of companies identified by either
 :   CIKs, tags, tickers, or sics.

 :
 : @param $ciks a set of CIKs.
 : @param $tags a set of tags (ALL retrieves all companies).
 : @param $tickers a set of tickers.
 : @param $sics a set of SIC codes.
 :
 : @return the companies with the given identifiers, tags, tickers, or sic codes.
 :)
declare function fiscal-core:filings(
    $entities-or-eids as item*,
    $fiscal-periods as string*,
    $fiscal-years as integer*,
    $archives-or-aids as item*) as object*
{
    let $entities := entities:entities($entities-or-eids)
    return
        if(exists(index-of($fiscal-years, $fiscal-core:LATEST_FISCAL_YEAR)))
        then
            for $entity in $entities
            return
            fiscal-core:latest-filings($entity, $fiscal-periods)
        else
            let $fiscal-years as integer* :=
                $fiscal-years[$$ ne $fiscal-core:LATEST_FISCAL_YEAR]
            return
                fiscal-core:filings-for-entities-and-fiscal-periods-and-years($entities, $fiscal-periods, $fiscal-years),
    archives:archives($archives-or-aids)
};

declare function fiscal-core:latest-filings(
    $entities as object*,
    $fiscal-periods as string*) as object*
{
    for $entity in $entities
    for $fiscal-period in $fiscal-periods
    let $latest-fiscal-year :=
        fiscal-core:latest-reported-fiscal-period($entity, $fiscal-period).year
    return fiscal-core:filings-for-entities-and-fiscal-periods-and-years(
        $entity,
        $fiscal-period,
        $latest-fiscal-year cast as integer?
    )
};

declare function fiscal-core:filter-override(
    $entities-or-eids as item*,
    $fiscal-years as integer*,
    $fiscal-periods as string*,
    $archives-or-aids as item*
) as object?
{
    let $entities := entities:entities($entities-or-eids)
    let $aids := archives:aid($archives-or-aids)
    let $latest-filings := fiscal-core:latest-filings($entities, $fiscal-periods)
    return
    switch(true)
    case count($aids) gt 0 return {
        "xbrl:Entity" : {
        },
        "sec:Archive" : {
            Type: "string",
            Domain : [ $aids ]
        },
        "sec:FiscalYear" : {
            Type: "integer"
        },
        "sec:FiscalPeriod" : {|
            {
                Type: "string",
                Domain: [ $fiscal-periods ]
            }[exists($fiscal-periods) and empty(index-of($fiscal-periods, $fiscal-core:ALL_FISCAL_PERIODS))]
        |}
    }
    case exists(index-of($fiscal-years, $fiscal-core:LATEST_FISCAL_YEAR))
    return {
        "sec:Archive" : {
            Type: "string",
            Domain : [archives:aid($latest-filings)]
        },
        "sec:FiscalPeriod" : {|
            {
                Type: "string",
                Domain: [ $fiscal-periods ]
            }[exists($fiscal-periods) and empty(index-of($fiscal-periods, $fiscal-core:ALL_FISCAL_PERIODS))]
        |}
    }
    case exists(($entities, $fiscal-years, $fiscal-periods))
    return {
        "xbrl:Entity" : {|
            {
                Type: "string",
                Domain: [ $entities._id ]
            }[exists($entities)]
        |},
        "sec:FiscalYear" :
            let $fiscal-years as integer* :=
                $fiscal-years[$$ ne $fiscal-core:LATEST_FISCAL_YEAR]
            return {|
                {
                    Type: "integer",
                    Domain: [ $fiscal-years ]
                }[exists($fiscal-years) and empty(index-of($fiscal-years, $fiscal-core:ALL_FISCAL_YEARS))]
            |},
        "sec:FiscalPeriod" : {|
            {
                Type: "string",
                Domain: [ $fiscal-periods ]
            }[exists($fiscal-periods) and empty(index-of($fiscal-periods, $fiscal-core:ALL_FISCAL_PERIODS))]
        |},
        "sec:Archive" : {}
    }
    default return ()
};

(:~
 :
Retrieves the fiscal period of a filing.

 :
 : @param $filing-or-id a filing or its id, or the empty sequence.
 :
 : @return the fiscal period (FY, YTD2, YTD3, Q1, Q2 or Q3) if any.
 :)
declare function fiscal-core:fiscal-period-for-filing(
    $filing-or-id as item?) as string?
{
  archives:archives($filing-or-id).Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
};

(:~
 :
Retrieves the fiscal period of a fact.

 :
 : @param $fact a fact object or the empty sequence.
 :
 : @return the fiscal period (FY, YTD2, YTD3, Q1, Q2 or Q3) if any.
 :)
declare function fiscal-core:fiscal-period-for-fact(
    $fact as object?) as string?
{
  $fact.Aspects."sec:FiscalPeriod"
};

(:~
 :
Retrieves the fiscal period of a filing or a fact.

 :
 : @deprecated use fiscal-period-for-fact#1 and fiscal-period-for-filing#1
 :
 : @param $filing-fact-or-id a filing, a fact, or its id.
 :
 : @return the fiscal period (FY, YTD2, YTD3, Q1, Q2 or Q3).
 :)
declare function fiscal-core:fiscal-period(
    $filing-fact-or-id as item?) as string?
{
  if(empty($filing-fact-or-id))
  then ()
  else
    typeswitch ($filing-fact-or-id)
    case object return
      switch (true)
      (: archive :)
      case exists($filing-fact-or-id.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus)
        return $filing-fact-or-id.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
      (: fact :)
      case exists($filing-fact-or-id.Aspects."sec:FiscalPeriod")
        return $filing-fact-or-id.Aspects."sec:FiscalPeriod"
      default return ()
    default return
      let $id as xs:string := string($filing-fact-or-id)
      return
        switch(true)
        (:
          "The first ten digits are the CIK of the entity that logged
           in to submit the filing. The next two digits define the
           current year. The final digits are assigned sequentially by
           CIK starting with one on the first business day of each year."

        http://www.sec.gov/info/edgar/edgarfm-vol2-v25.pdf :)
        case matches($id, "\\d{10}-\\d{2}-\\d{6}")
          return archives:archives($id).Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
        default return facts:facts($id).Aspects."sec:FiscalPeriod"
};

(:~
 :
Retrieves the fiscal year of a filing.

 :
 : @param $filing-or-id a filing or its id, or the empty sequence.
 :
 : @return the fiscal year (e.g., 2012) if any.
 :)
declare function fiscal-core:fiscal-year-for-filing(
    $filing-or-id as item?) as integer?
{
  archives:archives($filing-or-id).Profiles.SEC.Fiscal.DocumentFiscalYearFocus
};

(:~
 :
Retrieves the fiscal year of a fact.

 :
 : @param $fact a fact object or the empty sequence.
 :
 : @return the fiscal year (e.g., 2012) if any.
 :)
declare function fiscal-core:fiscal-year-for-fact(
    $fact as object?) as integer?
{
  $fact.Aspects."sec:FiscalYear"
};

(:~
 :
Retrieves the fiscal year of a filing or a fact.

 :
 : @deprecated use fiscal-core:fiscal-year-for-archive#1 or fiscal-core:fiscal-year-for-fact#1.
 :
 : @param $filing-fact-or-id a filing, a fact, or its id.
 :
 : @return the fiscal year (e.g., 2012).
 :)
declare function fiscal-core:fiscal-year(
    $filing-fact-or-id as item?) as integer?
{
  if(empty($filing-fact-or-id))
  then ()
  else
    typeswitch ($filing-fact-or-id)
    case object return
      switch (true)
      (: archive :)
      case exists($filing-fact-or-id.Profiles.SEC.Fiscal.DocumentFiscalYearFocus)
        return $filing-fact-or-id.Profiles.SEC.Fiscal.DocumentFiscalYearFocus
      (: fact :)
      case exists($filing-fact-or-id.Aspects."sec:FiscalYear")
        return $filing-fact-or-id.Aspects."sec:FiscalYear"
      default return ()
    default return
      let $id as xs:string := string($filing-fact-or-id)
      return
        switch(true)
        (:
          "The first ten digits are the CIK of the entity that logged
           in to submit the filing. The next two digits define the
           current year. The final digits are assigned sequentially by
           CIK starting with one on the first business day of each year."

          http://www.sec.gov/info/edgar/edgarfm-vol2-v25.pdf :)
        case matches($id, "\\d{10}-\\d{2}-\\d{6}")
          return archives:archives($id).Profiles.SEC.Fiscal.DocumentFiscalYearFocus
        default return facts:facts($id).Aspects."sec:FiscalYear"
};

(:~
 :
Retrieves all filings by entities and for given fiscal periods and years.

 :
 : @param $entities-or-ids entities or their ids.
 : @param $fiscal-period-focus fiscal periods (FY, YTD1, YTD2, YTD3, Q1, Q2, Q3, Q4) or $fiscal-core:ALL_FISCAL_PERIODS to accept all..
 : @param $fiscal-year-focus fiscal years or $fiscal-core:ALL_FISCAL_YEARS to accept all..
 :
 : @return all filings that match the criteria.
 :)
declare function
fiscal-core:filings-for-entities-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  let $fiscal-period-focus :=
    switch(true)
    case $fiscal-period-focus = ( "10-K", "FY") return "FY"
    case $fiscal-period-focus eq "10-Q" return ( "Q3", "Q2", "Q1" )
    case $fiscal-period-focus eq "YTD1" return "Q1"
    case $fiscal-period-focus eq "YTD2" return "Q2"
    case $fiscal-period-focus eq "YTD3" return "Q3"
    default return $fiscal-period-focus
  let $conn := fiscal-core:connection()
  for $cik-or-entity in $entities-or-ids
  let $entity-id as xs:string := companies:eid($cik-or-entity)
  let $query := {|
      { "Entity" : $entity-id },
      switch (true)
      case deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS)
        return ()
      case count($fiscal-period-focus) eq 1
        return
        {
          "Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus": $fiscal-period-focus
        }
      default return
        {
          "Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus":
            { "$in" : [ $fiscal-period-focus ] }
        },
      switch (true)
      case deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS)
        return ()
      case count($fiscal-year-focus) eq 1
        return
        {
          "Profiles.SEC.Fiscal.DocumentFiscalYearFocus": $fiscal-year-focus
        }
      default
        return
        {
          "Profiles.SEC.Fiscal.DocumentFiscalYearFocus":
            { "$in" : [ $fiscal-year-focus ] }
        }
    |}
  return mongo:find($conn, $archives:col, $query)
};

(:~
 :
Retrieves the balance sheet for the given entities and fiscal periods and years.

 :
 : @param $entity-or-ids a list of entities or their EIDs.
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 :
 : @error entities:INVALID_PARAMETER if the CIK or entity is not valid.
 :
 : @return the balance sheet at that date and for that formType.
 :)
declare function fiscal-core:balance-sheets-for-fiscal-periods-and-years(
    $entity-or-ids as item*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  for $archive in
    fiscal-core:filings-for-entities-and-fiscal-periods-and-years($entity-or-ids,
                                                                  $fiscal-period-focus,
                                                                  $fiscal-year-focus)
  return sec-networks:networks-for-filings-and-disclosures(
      $archive,
      $sec-networks:BALANCE_SHEET)
};

(:~
 :
Retrieves facts for the given aspects, fiscal periods and years.

 :
 : @param $aspects an object containing aspects to filter, among which
 :                 xbrl:Concept, xbrl:Entity and xbrl:Period (all optional).
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY).
 : @param $fiscal-year-focus the years (2012)
 :
 : @return all facts matching the given aspects, fiscal period and year.
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects as object?,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects,
    $fiscal-period-focus,
    $fiscal-year-focus,
    ())
};

(:~
 :
Retrieves facts for the given aspects, fiscal periods and years.

 :
 : @param $aspects an object containing aspects to filter, among which
 :                 xbrl:Concept, xbrl:Entity and xbrl:Period (all optional).
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 : @param $options standard SEC BizQL options.
 :
 : @return all facts matching the given aspects, fiscal period and year.
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
    $aspects as object?,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*,
    $options as object?) as object*
{
  let $filter as object := {|
    {
      "Filter":
        {|
          if(exists($aspects))
          then { $facts:ASPECTS : $aspects }
          else (),
          if( deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS) and deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS))
          then { "Profiles.SEC.Fiscal" : { "$exists" : true } }
          else { "Profiles": { "SEC": { "Fiscal":
                 {|
                   if(deep-equal($fiscal-period-focus, $fiscal-core:ALL_FISCAL_PERIODS) )
                   then ()
                   else { "Period" : [ $fiscal-period-focus ] },
                   if( deep-equal($fiscal-year-focus, $fiscal-core:ALL_FISCAL_YEARS) )
                   then ()
                   else { "Year": [ $fiscal-year-focus ] }
                 |}
               }}}
        |}
    },
    if (empty($options.Hypercube))
    then { Hypercube: sec:dimensionless-hypercube(()) }
    else ()
  |}
  let $facts :=
    facts:facts-for(
      facts:merge-objects($filter, $options, true (: giving parameters higher priority :))
    )
  return sec:hide-amended-facts($facts, $options)
};

(:~
 :
Retrieves facts for the given entities, concepts, fiscal periods and years.

 :
 : @param $entities-or-ids a list of entities or their EIDs.
 : @param $concepts a list of concept names (us-gaap:Assets)
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 :
 : @error entities:INVALID_PARAMETER if the CIK or entity is not valid.
 : @return all facts matching entities, concepts, fiscal periods and years
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $concepts as string*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*) as object*
{
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids,
    $concepts,
    $fiscal-period-focus,
    $fiscal-year-focus,
    ())
};

(:~
 :
Retrieves facts for the given entities, concepts, fiscal periods and years.

 :
 : @param $entities-or-ids a list of entities or their EIDs.
 : @param $concepts a list of concept names (us-gaap:Assets)
 : @param $fiscal-period-focus the periods (YTD1, YTD2, YTD3, Q1, Q2, Q3 or FY) or $fiscal-core:ALL_FISCAL_PERIODS to accept all.
 : @param $fiscal-year-focus the years (2012) or $fiscal-core:ALL_FISCAL_YEARS to accept all.
 : @param $options standard SEC BizQL options.
 :
 : @error entities:INVALID_PARAMETER if the CIK or entity is not valid.
 : @return all facts matching entities, concepts, fiscal periods and years
 : @deprecated This function has been deprecated in favor of more generic
 :   functions like facts:facts-for and hypercubes:facts.
 :)
declare function
  fiscal-core:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
    $entities-or-ids as item*,
    $concepts as string*,
    $fiscal-period-focus as string*,
    $fiscal-year-focus as integer*,
    $options as object?) as object*
{
  let $entity-ids as xs:string* := $entities-or-ids ! companies:eid($$)
  let $aspects := {|
    if(exists($entity-ids))
    then { "xbrl:Entity": [ $entity-ids ] }
    else (),
    if(exists($concepts))
    then { "xbrl:Concept": [ $concepts ] }
    else ()
  |}
  return
    fiscal-core:facts-for-aspects-and-fiscal-periods-and-years(
      $aspects,
      $fiscal-period-focus,
      $fiscal-year-focus,
      $options)
};

(:~
 :
Retrieves the latest reported fiscal period and year for the
 : supplied entity.

 :
 : @param $entity-or-id an entity or its EID.
 :
 : @return an object with two fields: period and year.
 :)
declare function fiscal-core:latest-reported-fiscal-period(
    $entity-or-id as item) as object?
{
  (
  let $entity-or-id := companies:eid($entity-or-id)
  for $archive in archives:archives-for-entities($entity-or-id)
  let $archive-fiscal-period := fiscal-core:fiscal-period-for-filing($archive)
  let $archive-fiscal-year := fiscal-core:fiscal-year-for-filing($archive)
  group by $archive-fiscal-year
  order by $archive-fiscal-year descending empty least
  return {
    year: $archive-fiscal-year,
    period: switch(true)
            case $archive-fiscal-period = "FY" return "FY"
            case $archive-fiscal-period = ("Q3", "YTD3") return "Q3"
            case $archive-fiscal-period = ("Q2", "YTD2") return "Q2"
            case $archive-fiscal-period = ("Q1", "YTD1") return "Q1"
            default return () }
  )[1]
};

(:~
 :
Retrieves the latest fiscal period and year for the
 : supplied entity.

 :
 : @param $entity-or-id an entity or its EID.
 : @param $type 10-K or 10-Q or FY, YTD1, YTD2, YTD3, Q1, Q2, or Q3, or $fiscal-core:ALL_FISCAL_PERIODS
 : to accept all.
 :
 : @return an object with two fields: period and year.
 :)
declare function fiscal-core:latest-reported-fiscal-period(
    $entity-or-id as item,
    $type as string) as object?
{
  if(deep-equal($type, $fiscal-core:ALL_FISCAL_PERIODS))
  then fiscal-core:latest-reported-fiscal-period($entity-or-id)
  else
  (
  let $type-filter :=
    switch(true)
    case $type = ( "10-K", "FY") return "FY"
    case $type eq "10-Q" return ( "Q3", "Q2", "Q1" )
    case $type eq "YTD1" return "Q1"
    case $type eq "YTD2" return "Q2"
    case $type eq "YTD3" return "Q3"
    default return $type
  let $entity-or-id := companies:eid($entity-or-id)
  for $archive in archives:archives-for-entities($entity-or-id)
  let $archive-fiscal-period := fiscal-core:fiscal-period-for-filing($archive)
  let $archive-fiscal-year := fiscal-core:fiscal-year-for-filing($archive)
  where $archive-fiscal-period = $type-filter
  group by $archive-fiscal-year
  order by $archive-fiscal-year descending empty least
  return {
    year: $archive-fiscal-year,
    period:
      switch(true)
      case $archive-fiscal-period = "FY" and ($type eq "10-K" or $type eq "FY") return "FY"
      case $archive-fiscal-period = "Q3" and ($type eq "10-Q" or $type eq "Q3") return "Q3"
      case $archive-fiscal-period = "Q2" and ($type eq "10-Q" or $type eq "Q2") return "Q2"
      case $archive-fiscal-period = "Q1" and ($type eq "10-Q" or $type eq "Q1") return "Q1"
      default return () }
  )[1]
};

(:~
 :)
declare %private %an:strictlydeterministic function fiscal-core:connection() as anyURI
{
  let $credentials :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return if (empty($credentials))
             then error(QName("fiscal-core:CONNECTION-FAILED"), "no xbrl MongoDB configured")
             else $credentials
  return
    try {
      mongo:connect($credentials)
    } catch mongo:* {
      error(QName("fiscal-core:CONNECTION-FAILED"), $err:description)
    }
};


Fixed failing test -- adapted it to the new JSONiq grammar

declare namespace an = "http://www.zorba-xquery.com/annotations";

declare default function namespace "foo.com";

declare variable $input:=
  for $i in 1 to 1000
  return ({"a":3, "b" : "foo"}, {"a": { "d" : 4}, "c":[1,2,3]});

declare %an:sequential function collapse-general($x)
{
   variable $val := [], $obj := [], $arr := [];

   $x ! (typeswitch ($$)
         case object() return  { append json $$ into $obj; }
         case array() return { append json $$ into $arr; }
         default return { append json $$ into $val; });

   if (size($val) eq 0)
   then ()
   else ["_VAL", members($val)],

   if (size($obj) eq 0)
   then ()
   else ["_OBJ", collapse-objects(members($obj))],

   if (size($arr) eq 0)
   then ()
   else  ["_ARR", collapse-general(members($arr) ! members($$)) ]
};

declare %an:sequential function collapse-objects($x as object()*)
{
  {| for $y in distinct-values($x ! (keys($$)))
         let $z := $x($y)
         return {$y : collapse-general($z)} |}
};

collapse-objects($input),
"
"

declare namespace an = "http://www.zorba-xquery.com/annotations";

declare default function namespace "foo.com";

declare variable $input:=
  for $i in 1 to 1000
  return ({"a":3, "b" : "foo"}, {"a": { "d" : 4}, "c":[1,2,3]});

declare %an:sequential function collapse-general($x)
{
   variable $val := [], $obj := [], $arr := [];

   $x ! (typeswitch (.)
         case object() return  { append json . into $obj; }
         case array() return { append json . into $arr; }
         default return { append json . into $val; });

   if (size($val) eq 0)
   then ()
   else ["_VAL", members($val)],

   if (size($obj) eq 0)
   then ()
   else ["_OBJ", collapse-objects(members($obj))],

   if (size($arr) eq 0)
   then ()
   else  ["_ARR", collapse-general(members($arr) ! members(.)) ]
};

declare %an:sequential function collapse-objects($x as object()*)
{
  {| for $y in distinct-values($x ! (keys(.)))
         let $z := $x($y)
         return {$y : collapse-general($z)} |}
};

collapse-objects($input),
"
"


:pencil: Add test.

:pencil: Add test.

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facts"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-404($params as object) as item
{
    let $endpoint := "facts"
    let $res as object := test:invoke-raw($endpoint, $params)
    return if($res.status eq 404)
           then true
           else {
              url: test:url($endpoint, $params),
              unexpectedResponse: $res
           }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw($endpoint, $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "Archive [Axis],Fiscal Period [Axis],Fiscal Year [Axis],Fiscal Period Type [Axis],Period [Axis],Reporting Entity [Axis],Concept [Axis],Unit [Axis],Legal Entity [Axis],Value,Decimals\r\n0000021344-14-000036,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),\"Cash and Cash Equivalents, at Carrying Value\",USD,Default Legal Entity [Member],11118000000,-6\r\n0000021344-13-000050,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),Assets,USD,Default Legal Entity [Member],89432000000,-6\r\n"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res,
        expected: $expectedLines
    }
};

declare %an:nondeterministic function local:test-labels-aids() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: "disc:AdvertisingCostsPolicyTextBlock",
        report: "Disclosures",
        format: "json",
        fiscalYear: 2014,
        fiscalPeriod: "FY",
        labels: true,
        aid: "0000858877-14-000029"
    }
    let $res := test:invoke($endpoint, $params)
    let $actual := [
            for $labels in $res[2].FactTable[].Labels
            return (keys($labels) ! $labels.$$)
        ]
    let $expected := [ "Archive [Axis]", "Fiscal Year [Axis]", "Non-Numeric [Member]", "Default Legal Entity [Member]", "Unit [Axis]", "Advertising Costs, Policy", "Fiscal Period [Axis]", "Concept [Axis]", "Reporting Entity [Axis]", "Legal Entity [Axis]", "Fiscal Period Type [Axis]", "Period [Axis]", "From July 28, 2013 to July 26, 2014", "CISCO SYSTEMS, INC. (858877)" ]
    let $status as integer := $res[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

local:check({
    cocacola-latest:
    (: this test will fail and needs to be updated if a newer report has been filed.
       current latest filing: 2014 :)
    local:test-facttable(462, {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    cocacola-all: local:test-facttable(477, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-all-q3: local:test-facttable(341, {
        ticker:"ko",
        fiscalYear: 2014,
        fiscalPeriod: [ "Q3" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-instant: local:test-facttable(170, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant" ]
    }),
    cocacola-ytd: local:test-facttable(307, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-qtd: local:test-facttable(102, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "Q2" ],
        fiscalPeriodType: [ "QTD" ]
    }),
    cocacolaCSVLabels: local:test-labels(),
    ciscoLabelsByAid: local:test-labels-aids(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(30, { (: AT&T wrongly stamped a filing as FY, but it's now FIXED :-):)
        tag:"DOW30",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-404({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    }),
    quartersfact: local:test-facttable(0, { (: this fact is not reported in the raw XML filing :)
        aid: "0000051143-13-000007",
        concept: "disc:AssetRetirementObligationsPolicy",
        fiscalPeriod: "Q3",
        fiscalYear: "2013",
        map: "Disclosures"
    }),
    quartersfact2: local:test-facttable(1, {
        aid: "0001193125-12-207154",
        concept: "disc:CashFlowOperatingCapitalTableTextBlock",
        fiscalPeriod: "Q1",
        fiscalYear: "2012",
        map: "Disclosures"
    }),
    generic: local:test-facttable(2, {
        "xbrl:Entity":"http://www.sec.gov/CIK 0000021344",
        "sec:Accepted":"20140227132423",
        concept:"us-gaap:Assets",
        profile-name: "generic"
    }),
    count: test:invoke-and-assert-deep-equal(
          "facts",
          {
              "ticker" : "ko",
              "fiscalYear" : ["2012", "2013", "2014"],
              "fiscalPeriod" : "FY",
              "count" : "true"
          },
          function($res as item*) as item* { $res.Statistics.NumFacts },
          1444
        ),
    odata: test:invoke-and-assert-deep-equal(
      "facts",
      {
          "ticker" : "ko",
          "fiscalYear" : "ALL",
          "fiscalPeriod" : "FY",
          "concept" : [ "us-gaap:Assets", "us-gaap:LiabilitiesCurrent", "us-gaap:StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" ],
          "format" : "odata-atom"
      },
      function($res as item*) as item* { count($res/*:feed/*:entry) },
      6
    )
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facts"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-404($params as object) as item
{
    let $endpoint := "facts"
    let $res as object := test:invoke-raw($endpoint, $params)
    return if($res.status eq 404)
           then true
           else {
              url: test:url($endpoint, $params),
              unexpectedResponse: $res
           }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw($endpoint, $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "Archive [Axis],Fiscal Period [Axis],Fiscal Year [Axis],Fiscal Period Type [Axis],Period [Axis],Reporting Entity [Axis],Concept [Axis],Unit [Axis],Legal Entity [Axis],Value,Decimals\r\n0000021344-14-000036,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),\"Cash and Cash Equivalents, at Carrying Value\",USD,Default Legal Entity [Member],11118000000,-6\r\n0000021344-13-000050,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),Assets,USD,Default Legal Entity [Member],89432000000,-6\r\n"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res,
        expected: $expectedLines
    }
};

declare %an:nondeterministic function local:test-labels-aids() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: "disc:AdvertisingCostsPolicyTextBlock",
        report: "Disclosures",
        format: "json",
        fiscalYear: 2014,
        fiscalPeriod: "FY",
        labels: true,
        aid: "0000858877-14-000029"
    }
    let $res := test:invoke($endpoint, $params)
    let $actual := [
            for $labels in $res[2].FactTable[].Labels
            return (keys($labels) ! $labels.$$)
        ]
    let $expected := [ "Archive [Axis]", "Fiscal Year [Axis]", "Non-Numeric [Member]", "Default Legal Entity [Member]", "Unit [Axis]", "Advertising Costs, Policy", "Fiscal Period [Axis]", "Concept [Axis]", "Reporting Entity [Axis]", "Legal Entity [Axis]", "Fiscal Period Type [Axis]", "Period [Axis]", "From July 28, 2013 to July 26, 2014", "CISCO SYSTEMS, INC. (858877)" ]
    let $status as integer := $res[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

local:check({
    cocacola-latest:
    (: this test will fail and needs to be updated if a newer report has been filed.
       current latest filing: 2014 :)
    local:test-facttable(462, {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    cocacola-all: local:test-facttable(477, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-all-q3: local:test-facttable(341, {
        ticker:"ko",
        fiscalYear: 2014,
        fiscalPeriod: [ "Q3" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-instant: local:test-facttable(170, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant" ]
    }),
    cocacola-ytd: local:test-facttable(307, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-qtd: local:test-facttable(102, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "Q2" ],
        fiscalPeriodType: [ "QTD" ]
    }),
    cocacolaCSVLabels: local:test-labels(),
    ciscoLabelsByAid: local:test-labels-aids(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(30, { (: AT&T wrongly stamped a filing as FY, but it's now FIXED :-):)
        tag:"DOW30",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-404({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    }),
    quartersfact: local:test-facttable(0, { (: this fact is not reported in the raw XML filing :)
        aid: "0000051143-13-000007",
        concept: "disc:AssetRetirementObligationsPolicy",
        fiscalPeriod: "Q3",
        fiscalYear: "2013",
        map: "Disclosures"
    }),
    quartersfact2: local:test-facttable(1, {
        aid: "0001193125-12-207154",
        concept: "disc:CashFlowOperatingCapitalTableTextBlock",
        fiscalPeriod: "Q1",
        fiscalYear: "2012",
        map: "Disclosures"
    }),
    generic: local:test-facttable(2, {
        "xbrl:Entity":"http://www.sec.gov/CIK 0000021344",
        "sec:Accepted":"20140227132423",
        concept:"us-gaap:Assets",
        profile-name: "generic"
    }),
    count: test:invoke-and-assert-deep-equal(
          "facts",
          {
              "ticker" : "ko",
              "fiscalYear" : ["2012", "2013", "2014"],
              "fiscalPeriod" : "FY",
              "count" : "true"
          },
          function($res as item*) as item* { $res.Statistics.NumFacts },
          1444
        )
})


Also sort in the new way.

Also sort in the new way.

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    (. == {} or . == [] or . == false or . == null or . == 0) | not
;

# Generate this object and all its children.
def all_children:
    ., ((.contents // empty) | .[] | all_children)
;

# Format a date into its wordy equivalent.
def format_date:
    try
    (
        (   try (. | strptime("%Y-%m-%d"))
            catch (
                try (. | strptime("%Y/%m/%d"))
                catch (
                    try (. | strptime("%Y-%m"))
                )
            )
        ) | mktime | strftime("%B %-d, %Y")
    )
;

# Group an array of objects into an object of arrays, such that the key-value
# pairs of the new object are an accumulation of those of the old object. For
# example, `[{"a":1, "b":2}, {"a":3}]` turns into `{"a":[1, 3], "b":[3]}`. This
# can be used to partition an array.
# Try: `[1,"2",3] | map ({(.|type):.}) | group`
def group:
    reduce ([.[] | to_entries[]] | group_by(.key))[] as $group
    ( {}
    ; .[ $group[0].key ] |= (. // []) + [ $group[].value ]
    )
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`.
def tree($path):
    def tree_aux($future):
        { "name": $future[0] } +
        if ($future | length) > 1 then
            { "contents": [ tree_aux($future[1:]) ] }
        else
            .
        end
    ;
    tree_aux($path)
;

# Merge two values. If the values are both objects, the values at equal indices
# will be recursively merged; if the values are both arrays, the arrays are
# concatenated; if the values are some other type, we only check if the values
# do not conflict.
def merge(other):
    if (. | type) == "object" and (other | type) == "object" then
        (. | to_entries) + (other | to_entries)
        | group_by(.key)
        | map({ "key": (.[0].key), "value" : (reduce .[1:][].value as $x (.[0].value; merge($x)))})
        | from_entries
    elif (. | type) == "array" and (other | type) == "array" then
        . + other
    elif . == other then
        .
    else
        error("trying to merge incompatible objects " + (.|type) + " and " + (other|type))
    end
;

# Merge an array of content by grouping it by name and then merging all the
# pages with the same name. This is necessary because simply merging content
# will only concatenate pages, even if they are the same page.
def merge_content:
    reduce .[] as $x ({}; merge($x)) |
    if has("contents") then
        .contents |= ( group_by(.name) | map(merge_content) )
    else
        .
    end
;

# Add paths to each object, that is, the names of the ancestors.
def add_directories:
    def add_directories_aux($history):
        ($history + [.name]) as $present |
        .directory = $history |
        (.contents[]? |= add_directories_aux($present))
    ;
    add_directories_aux([])
;

# Any page gets a link to its target.
def add_links(target_extension):
    if (has("directory") and (.name | endswith(".md"))) then
        .link = ((.directory + [.name]) | join("/") | rtrimstr(".md") | . + "." + target_extension)
    else
        .contents[]? |= add_links(target_extension)
    end
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def add_frontmatter:
    if has("contents") then
        ( .contents | map({(.name=="index.md" or .meta.frontmatter|tostring):.}) | group) as {$true, $false}
        | .frontmatter = $true[0]
        | .contents = (($false // []) + $true[1:])
        | (.contents[] |= add_frontmatter)
    else
        .
    end
;

# If marked as such, drafts can be excluded from being uploaded and included in
# the table of contents. To not count as a draft, a document should either be
# marked as `publish`, or be *not* marked as `draft` and have children that
# *are* marked as `publish`.
def add_drafts:
    def is_publication:
        ( .publish or .meta.publish | bool)
        or
        ( .external | bool )
        or
        ( (.meta.draft | bool | not) and ((.contents // []) | any(is_publication)) )
    ;
    if has("contents") then
        ( .contents | map({(is_publication|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .drafts = ($false // [])
        | (.contents[] |= add_drafts)
    else
        .
    end
;

# A resource is anything that is neither a directory nor a page with metadata
# nor an external link.
def add_resources:
    if has("contents") then
        ( .contents | map({(has("contents") or has("meta") or (has("external") and (.external | bool))|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .resources = ($false // [])
        | (.contents[]? |= add_resources)
    else
        .
    end
;

def add_formatted_dates:
    if has("contents") then
        .contents[]? |= add_formatted_dates
    elif has("meta") then
        .meta.formatted_date = (.meta.date | format_date)
    else
        .
    end
;

# Sort content first according to sort order in metadata.
def sort_content:
    (all_children | select(has("contents")) | .contents) |= (
        sort_by(.meta.sort // .meta.title // .name)
    )
;

# Add a note that tells us whether this has only children who have no more
# subchildren.
def annotate_leaves:
    (all_children | select(has("contents"))) |= (
        .only_leaves = (.contents | all(has("contents") | not))
    )
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index(target_extension):
    merge_content
    | add_directories
    | add_links(target_extension)
    | add_drafts
    | add_frontmatter
    | add_resources
    | sort_content
    | annotate_leaves
;

# Get publishable targets from an index
def targets:
    recurse(.contents[]?)
    | ., (.frontmatter // empty)
    | select(has("link") and (has("external") | not))
    | .directory as $dir
    | [.link] + [.targets?[] | $dir + [.] | join("/") ]
    | .[]
;

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    (. == {} or . == [] or . == false or . == null or . == 0) | not
;

# Format a date into its wordy equivalent.
def format_date:
    try
    (
        (   try (. | strptime("%Y-%m-%d"))
            catch (
                try (. | strptime("%Y/%m/%d"))
                catch (
                    try (. | strptime("%Y-%m"))
                )
            )
        ) | mktime | strftime("%B %-d, %Y")
    )
;

# Group an array of objects into an object of arrays, such that the key-value
# pairs of the new object are an accumulation of those of the old object. For
# example, `[{"a":1, "b":2}, {"a":3}]` turns into `{"a":[1, 3], "b":[3]}`. This
# can be used to partition an array.
# Try: `[1,"2",3] | map ({(.|type):.}) | group`
def group:
    reduce ([.[] | to_entries[]] | group_by(.key))[] as $group
    ( {}
    ; .[ $group[0].key ] |= (. // []) + [ $group[].value ]
    )
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`.
def tree($path):
    def tree_aux($future):
        { "name": $future[0] } +
        if ($future | length) > 1 then
            { "contents": [ tree_aux($future[1:]) ] }
        else
            .
        end
    ;
    tree_aux($path)
;

# Merge two values. If the values are both objects, the values at equal indices
# will be recursively merged; if the values are both arrays, the arrays are
# concatenated; if the values are some other type, we only check if the values
# do not conflict.
def merge(other):
    if (. | type) == "object" and (other | type) == "object" then
        (. | to_entries) + (other | to_entries)
        | group_by(.key)
        | map({ "key": (.[0].key), "value" : (reduce .[1:][].value as $x (.[0].value; merge($x)))})
        | from_entries
    elif (. | type) == "array" and (other | type) == "array" then
        . + other
    elif . == other then
        .
    else
        error("trying to merge incompatible objects " + (.|type) + " and " + (other|type))
    end
;

# Merge an array of content by grouping it by name and then merging all the
# pages with the same name. This is necessary because simply merging content
# will only concatenate pages, even if they are the same page.
def merge_content:
    reduce .[] as $x ({}; merge($x)) |
    if has("contents") then
        .contents |= ( group_by(.name) | map(merge_content) )
    else
        .
    end
;

# Add paths to each object, that is, the names of the ancestors.
def add_directories:
    def add_directories_aux($history):
        ($history + [.name]) as $present |
        .directory = $history |
        (.contents[]? |= add_directories_aux($present))
    ;
    add_directories_aux([])
;

# Any page gets a link to its target.
def add_links(target_extension):
    if (has("directory") and (.name | endswith(".md"))) then
        .link = ((.directory + [.name]) | join("/") | rtrimstr(".md") | . + "." + target_extension)
    else
        .contents[]? |= add_links(target_extension)
    end
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def add_frontmatter:
    if has("contents") then
        ( .contents | map({(.name=="index.md" or .meta.frontmatter|tostring):.}) | group) as {$true, $false}
        | .frontmatter = $true[0]
        | .contents = (($false // []) + $true[1:])
        | (.contents[] |= add_frontmatter)
    else
        .
    end
;

# If marked as such, drafts can be excluded from being uploaded and included in
# the table of contents. To not count as a draft, a document should either be
# marked as `publish`, or be *not* marked as `draft` and have children that
# *are* marked as `publish`.
def add_drafts:
    def is_publication:
        ( .publish or .meta.publish | bool)
        or
        ( .external | bool )
        or
        ( (.meta.draft | bool | not) and ((.contents // []) | any(is_publication)) )
    ;
    if has("contents") then
        ( .contents | map({(is_publication|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .drafts = ($false // [])
        | (.contents[] |= add_drafts)
    else
        .
    end
;

# A resource is anything that is neither a directory nor a page with metadata
# nor an external link.
def add_resources:
    if has("contents") then
        ( .contents | map({(has("contents") or has("meta") or (has("external") and (.external | bool))|tostring):.}) | group) as {$true, $false}
        | .contents = ($true // [])
        | .resources = ($false // [])
        | (.contents[]? |= add_resources)
    else
        .
    end
;

def add_formatted_dates:
    if has("contents") then
        .contents[]? |= add_formatted_dates
    elif has("meta") then
        .meta.formatted_date = (.meta.date | format_date)
    else
        .
    end
;

# Sort content first according to sort order in metadata.
def sort_content:
    if has("contents") then
        .contents |= (sort_by(.meta.sort // .meta.title // .name) | map(sort_content))
    else
        .
    end
;

# Generate this object and all its children.
def all_children:
    ., ((.contents // empty) | .[] | all_children)
;

# Add a note that tells us whether this has only children who have no more
# subchildren.
def annotate_leaves:
    all_children |= (
        .only_leaves = (has("contents") | not) or (.contents | all(has("contents") | not))
    )
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index(target_extension):
    merge_content
    | add_directories
    | add_links(target_extension)
    | add_drafts
    | add_frontmatter
    | add_resources
    | sort_content
    | annotate_leaves
;

# Get publishable targets from an index
def targets:
    recurse(.contents[]?)
    | ., (.frontmatter // empty)
    | select(has("link") and (has("external") | not))
    | .directory as $dir
    | [.link] + [.targets?[] | $dir + [.] | join("/") ]
    | .[]
;


fixed buggy signatures (happened during warning fixes)

jsoniq version "1.0";

module namespace session = "http://apps.28.io/session";

import module namespace random = "http://zorba.io/modules/random";
import module namespace user = "http://apps.28.io/user";
import module namespace api = "http://apps.28.io/api";
import module namespace req = "http://www.28msec.com/modules/http-request";
import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace cred = "http://www.28msec.com/modules/credentials";

declare variable $session:tokens := "Tokens";
declare variable $session:VALID-TOKEN := "[a-z0-9\\-]+";
declare variable $session:ACCESS-ALLOWED := 0;
declare variable $session:ACCESS-DENIED := 1;
declare variable $session:ACCESS-AUTH-REQUIRED := 2;

declare variable $session:comment := "Generated by 28.io, for more information see http://www.28.io/xbrl (" || fn:current-dateTime() || ")";

declare function session:comment($format as string) as item*
{
    switch ($format)
    case "json" return { "Comment" : $session:comment }
    case "xml" return
        (
            comment { $session:comment },
            comment { req:server-name() || req:uri() }
        )
    default return error()
};

declare function session:comment($format as string, $stats as object)
  as item*
{
    switch ($format)
    case "json" return
        {|
            session:comment($format),
            { "Statistics" : $stats }
        |}
    case "xml" return
        (
            session:comment($format),
            comment { "Statistics: " || string-join(for $k in keys($stats) return concat ($k, ": ", $stats($k)), ", ") }
        )
    default return error()
};

declare function session:connect() as xs:anyURI
{
    let $cred := cred:credentials("MongoDB", "xbrl")
    return mongo:connect($cred)
};

declare function session:num-facts() as xs:integer
{
    let $conn := session:connect()
    return mongo:count($conn, "facts")
};

declare function session:num-archives() as xs:integer
{
    let $conn := session:connect()
    return mongo:count($conn, "archives")
};

declare function session:num-components() as xs:integer
{
    let $conn := session:connect()
    return mongo:count($conn, "components")
};

declare function session:num-concepts() as xs:integer
{
    let $conn := session:connect()
    return mongo:count($conn, "concepts")
};

declare function session:num-entities() as xs:integer
{
    let $conn := session:connect()
    return mongo:count($conn, "entities")
};

declare %an:sequential function session:start($user-id as xs:string, $expiration-date as xs:dateTime)
as string
{
    session:cleanup();

    variable $token := random:uuid();
    variable $session :=
      {
        "_id"           : $token,
        user-id         : $user-id,
        expiration-date : $expiration-date
      };
    db:insert($session:tokens, $session);

    $token
};

declare function session:get($token as string)
as string
{
    variable $session := find($session:tokens, { "_id" : $token });

    if (exists($session))
    then
        if ($session.expiration-date gt current-dateTime())
        then $session.user-id
        else fn:error(xs:QName("session:expired"), "Your session has expired at " || $session.expiration-date )
    else fn:error(xs:QName("session:expired"), "Your session is not valid")
};

declare %an:sequential function session:terminate($token as string)
as empty-sequence()
{
    variable $session := find($session:tokens, { "_id" : $token });

    if (exists($session))
    then db:delete($session);
    else ();
};

declare %an:sequential function session:cleanup()
as empty-sequence()
{
    for $session in collection($session:tokens)
    where $session.expiration-date le current-dateTime()
    return db:delete($session);
};

declare function session:validate()
as string
{
    variable $token := api:required-parameter("token", $session:VALID-TOKEN);
    session:get($token)
};

declare function session:valid()
as boolean
{
    let $token := api:parameter("token", $session:VALID-TOKEN, ())
    return
      if (exists($token))
      then try {{ session:get($token); true }} catch * {{ false }}
      else false
};

declare function session:validate($right-id as string)
as string
{
    variable $user-id := session:validate();
    if (user:is-authorized($user-id, $right-id))
    then $user-id
    else fn:error(xs:QName("session:missing-authorization"), "User does not have required right " || $right-id)
};

declare function session:valid($right-id as string)
as boolean
{
    variable $user-id := session:validate();
    if (user:is-authorized($user-id, $right-id))
    then true
    else false
};

declare function session:check-access($entities as object*, $right-id as string)
as integer
{
    if (session:only-dow30($entities))
    then $session:ACCESS-ALLOWED
    else
        try {{
            variable $user-id := session:validate();
            if (user:is-authorized($user-id, $right-id))
            then $session:ACCESS-ALLOWED
            else $session:ACCESS-DENIED
        }}
        catch * {{
            $session:ACCESS-AUTH-REQUIRED
        }}
};

declare %private function session:only-dow30($entities as object*)
as boolean
{
    count(
        for $e in $entities
        where count($e.Profiles.SEC.Tags[]) gt 0 and $e.Profiles.SEC.Tags[] = "DOW30"
        return $e) eq count($entities)
};

declare function session:error($msg as string, $format as string?) as item
{
	switch ($format)
    case "xml" return <error><message>{$msg}</message></error>
	case "text" or "csv" or "excel" return $msg
    default return
	{
		"success" : false,
		"description" : $msg
	}
};

declare %an:sequential function session:terminate()
as empty-sequence()
{
   session:terminate(api:required-parameter("token", $session:VALID-TOKEN))
};

declare %an:sequential function session:audit-call() as empty-sequence()
{
    let $dist-aspects := [ "xbrl:Concept", "xbrl:Entity", "xbrl:Period" ]
    let $facts := {
        KeyAspects : $dist-aspects,
        Aspects : {
            "xbrl:Concept" : "secxbrl:ClientIP",
            "xbrl:Entity" : if (session:valid()) then user:get-by-id(session:validate()).email else "Anonymous",
            "xbrl:Period" : string(fn:current-dateTime()),
            "xbrl:Unit" : "xbrl:NonNumeric",
            "secxbrl:Query" : req:path()
        },
        Type : "NonNumericValue",
        Value : (req:header-value("X-FORWARDED-FOR"), req:remote-addr())[1]
    }
    return db:insert("audit", $facts);
};

jsoniq version "1.0";

module namespace session = "http://apps.28.io/session";

import module namespace random = "http://zorba.io/modules/random";
import module namespace user = "http://apps.28.io/user";
import module namespace api = "http://apps.28.io/api";
import module namespace req = "http://www.28msec.com/modules/http-request";
import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace cred = "http://www.28msec.com/modules/credentials";

declare variable $session:tokens := "Tokens";
declare variable $session:VALID-TOKEN := "[a-z0-9\\-]+";
declare variable $session:ACCESS-ALLOWED := 0;
declare variable $session:ACCESS-DENIED := 1;
declare variable $session:ACCESS-AUTH-REQUIRED := 2;

declare variable $session:comment := "Generated by 28.io, for more information see http://www.28.io/xbrl (" || fn:current-dateTime() || ")";

declare function session:comment($format as string) as item*
{
    switch ($format)
    case "json" return { "Comment" : $session:comment }
    case "xml" return
        (
            comment { $session:comment },
            comment { req:server-name() || req:uri() }
        )
    default return error()
};

declare function session:comment($format as string, $stats as object)
  as item*
{
    switch ($format)
    case "json" return
        {|
            session:comment($format),
            { "Statistics" : $stats }
        |}
    case "xml" return
        (
            session:comment($format),
            comment { "Statistics: " || string-join(for $k in keys($stats) return concat ($k, ": ", $stats($k)), ", ") }
        )
    default return error()
};

declare function session:connect() as object
{
    let $cred := cred:credentials("MongoDB", "xbrl")
    return mongo:connect($cred)
};

declare function session:num-facts() as object
{
    let $conn := session:connect()
    return mongo:count($conn, "facts")
};

declare function session:num-archives() as object
{
    let $conn := session:connect()
    return mongo:count($conn, "archives")
};

declare function session:num-components() as object
{
    let $conn := session:connect()
    return mongo:count($conn, "components")
};

declare function session:num-concepts() as object
{
    let $conn := session:connect()
    return mongo:count($conn, "concepts")
};

declare function session:num-entities() as object
{
    let $conn := session:connect()
    return mongo:count($conn, "entities")
};

declare %an:sequential function session:start($user-id as xs:string, $expiration-date as xs:dateTime)
as string
{
    session:cleanup();

    variable $token := random:uuid();
    variable $session :=
      {
        "_id"           : $token,
        user-id         : $user-id,
        expiration-date : $expiration-date
      };
    db:insert($session:tokens, $session);

    $token
};

declare function session:get($token as string)
as string
{
    variable $session := find($session:tokens, { "_id" : $token });

    if (exists($session))
    then
        if ($session.expiration-date gt current-dateTime())
        then $session.user-id
        else fn:error(xs:QName("session:expired"), "Your session has expired at " || $session.expiration-date )
    else fn:error(xs:QName("session:expired"), "Your session is not valid")
};

declare %an:sequential function session:terminate($token as string)
as empty-sequence()
{
    variable $session := find($session:tokens, { "_id" : $token });

    if (exists($session))
    then db:delete($session);
    else ();
};

declare %an:sequential function session:cleanup()
as empty-sequence()
{
    for $session in collection($session:tokens)
    where $session.expiration-date le current-dateTime()
    return db:delete($session);
};

declare function session:validate()
as string
{
    variable $token := api:required-parameter("token", $session:VALID-TOKEN);
    session:get($token)
};

declare function session:valid()
as boolean
{
    let $token := api:parameter("token", $session:VALID-TOKEN, ())
    return
      if (exists($token))
      then try {{ session:get($token); true }} catch * {{ false }}
      else false
};

declare function session:validate($right-id as string)
as string
{
    variable $user-id := session:validate();
    if (user:is-authorized($user-id, $right-id))
    then $user-id
    else fn:error(xs:QName("session:missing-authorization"), "User does not have required right " || $right-id)
};

declare function session:valid($right-id as string)
as boolean
{
    variable $user-id := session:validate();
    if (user:is-authorized($user-id, $right-id))
    then true
    else false
};

declare function session:check-access($entities as object*, $right-id as string)
as integer
{
    if (session:only-dow30($entities))
    then $session:ACCESS-ALLOWED
    else
        try {{
            variable $user-id := session:validate();
            if (user:is-authorized($user-id, $right-id))
            then $session:ACCESS-ALLOWED
            else $session:ACCESS-DENIED
        }}
        catch * {{
            $session:ACCESS-AUTH-REQUIRED
        }}
};

declare %private function session:only-dow30($entities as object*)
as boolean
{
    count(
        for $e in $entities
        where count($e.Profiles.SEC.Tags[]) gt 0 and $e.Profiles.SEC.Tags[] = "DOW30"
        return $e) eq count($entities)
};

declare function session:error($msg as string, $format as string?) as item
{
	switch ($format)
    case "xml" return <error><message>{$msg}</message></error>
	case "text" or "csv" or "excel" return $msg
    default return
	{
		"success" : false,
		"description" : $msg
	}
};

declare %an:sequential function session:terminate()
as empty-sequence()
{
   session:terminate(api:required-parameter("token", $session:VALID-TOKEN))
};

declare %an:sequential function session:audit-call() as empty-sequence()
{
    let $dist-aspects := [ "xbrl:Concept", "xbrl:Entity", "xbrl:Period" ]
    let $facts := {
        KeyAspects : $dist-aspects,
        Aspects : {
            "xbrl:Concept" : "secxbrl:ClientIP",
            "xbrl:Entity" : if (session:valid()) then user:get-by-id(session:validate()).email else "Anonymous",
            "xbrl:Period" : string(fn:current-dateTime()),
            "xbrl:Unit" : "xbrl:NonNumeric",
            "secxbrl:Query" : req:path()
        },
        Type : "NonNumericValue",
        Value : (req:header-value("X-FORWARDED-FOR"), req:remote-addr())[1]
    }
    return db:insert("audit", $facts);
};