jq: Redact notes in errors and add shortPath

  Redacting notes in errors prevents us from reading the content of
  notes in log files and has the benefit of reducing the amount of text
  displayed when searching through log files.

  The `shortPath` helper keeps only the name of files and directories in
  the `path` field of log lines to increase the readability of logs when
  we don't care about the whole hierarchy.

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove notes content
def redactNote(obj):
  obj |=
    (.
    | if isempty(.metadata.content | objects) | not then .metadata.content |= "[redacted]" else . end
    | if isempty(.metadata.schema | objects) | not then .metadata.schema |= "[redacted]" else . end
    | if isempty(.remote | objects) | not then redactNote(.remote) else . end
    | if isempty(.moveFrom | objects) | not then redactNote(.moveFrom) else . end
    | if isempty(.overwrite | objects) | not then redactNote(.overwrite) else . end
    )
;

def cleanNote:
  .
  | if isempty(.doc | objects) | not then redactNote(.doc) else . end
  | if isempty(.was | objects) | not then redactNote(.was) else . end
  | if isempty(.remoteDoc | objects) | not then redactNote(.remoteDoc) else . end
  | if isempty(.note | objects) | not then redactNote(.note) else . end
  | if isempty(.file | objects) | not then redactNote(.file) else . end
  | if isempty(.err | objects) | not then .
    | .err |= (. | redactNote(.doc))
    else . end
  | if isempty(.change | objects) | not then .
    | .change |= (. | redactNote(.doc))
    | .change |= (. | redactNote(.was))
    else . end
;

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v) |
  del(.level) |
  del(.local) |
  del(.remote) |
  cleanNote |
  {time,component,msg,path}+(del(.time)|del(.component)|del(.msg)|del(.path));

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: select(is_error) | clean;
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: select(is_warn) | clean;
def warn_strict: select(is_warn_strict) | clean;
def info_level: 30;
def is_info: .level >= info_level;
def info: select(is_info) | clean;
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: select(is_debug) | clean;

# Components:
def component(pattern): select(.component | test(pattern));
def App: component("^App$");
def Chokidar: component("^Chokidar|chokidar$");
def Fs: component("^Fs|FS$");
def LocalAnalysis: component("^LocalAnalysis|local/analysis$");
def LocalChange: component("^LocalChange|local/change$");
def LocalWatcher: component("^LocalWatcher$");
def LocalWriter: component("^LocalWriter$");
def Merge: component("^Merge$");
def Metadata: component("^Metadata$");
def Perfs: component("^Perfs|PerfReports$");
def Pouch: component("^Pouch$");
def Prep: component("^Prep$");
def RemoteCozy: component("^RemoteCozy$");
def RemoteUserActionRequired: component("^RemoteUserActionRequired|remote/user_action_required$");
def RemoteWarningPoller: component("^RemoteWarningPoller$");
def RemoteWatcher: component("^RemoteWatcher$");
def RemoteWriter: component("^RemoteWriter$");
def Sync: component("^Sync$");

def Local: select(.component | test("^local|chokidar"; "i"));
def Remote: select(.component | test("^remote"; "i"));

# Exclude "up-to-date" messages
#
#     yarn jq nouptodate path/to/logs*
#
def nouptodate: select(.msg | test("up[- ]to[- ]date") | not);

# Exclude initial scan messages
#
#     yarn jq noscan path/to/logs*
#
def noscan: select(.action != "scan");

# Exclude Proxy stuff:
#
#     yarn jq noproxy path/to/logs*
#
def noproxy: select(.component | test("GUI:proxy") | not);

# Find conflicts:
#
#     yarn jq -c conflict path/to/logs*
#
def is_conflict: .msg | test("resolveConflictAsync|Resolving conflict");
def conflict: clean | select(is_conflict);

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
# FIXME: Should not output the same line twice when both path & oldpath match.
def path(pattern):
  clean | select(
    (
      try (.path // (.doc | .path) // (.change | .doc | .path) // (.idConflict | .newDoc | .path) // (.event | .path)),
      try (.oldpath // (.was | .path) // (.idConflict | .existingDoc | .path) // (.event | .oldPath)),
      ""
    )
      | strings
      | test(pattern)
  );

def shortPath:
  (
    .
    | if isempty(.path | strings) | not then .path |= (. | split("\\") | last) else . end
    | if isempty(.path | strings) | not then .path |= (. | split("/") | last) else . end
    | if isempty(.oldpath | strings) | not then .oldpath |= (. | split("\\") | last) else . end
    | if isempty(.oldpath | strings) | not then .oldpath |= (. | split("/") | last) else . end
  );

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#    yarn jq 'before("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#    yarn jq 'after("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#
def time(pattern):
  select(.time | test(pattern));
def before(time): select(.time <= time);
def after(time): select(.time >= time);

# Drop time properties
def notime: del(.time);

# Filter inode number(s):
#
#    yarn jq 'ino(7852458)' path/to/logs*
#    yarn jq 'ino(7852458; 464672)' path/to/logs*
#
def ino(n): select((.ino // (.doc|.ino) // (.change|.doc|.ino)) as $ino | [n] | flatten | map(. == $ino) | any);

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path,oldpath,event}
    | if (.event | not) then del(.event) else . end
    | if (.event | type == "object") then del(.event) + (.event|{path, oldpath: .oldPath, action}) else . end
    | if .oldpath then . else del(.oldpath) end
    ;

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove notes content
def redactNote(obj):
  obj |=
    (.
    | if isempty(.metadata.content | objects) | not then .metadata.content |= "[redacted]" else . end
    | if isempty(.metadata.schema | objects) | not then .metadata.schema |= "[redacted]" else . end
    | if isempty(.remote | objects) | not then redactNote(.remote) else . end
    | if isempty(.moveFrom | objects) | not then redactNote(.moveFrom) else . end
    | if isempty(.overwrite | objects) | not then redactNote(.overwrite) else . end
    )
;

def cleanNote:
  .
  | if isempty(.doc | objects) |not then redactNote(.doc) else . end
  | if isempty(.was | objects) |not then redactNote(.was) else . end
  | if isempty(.remoteDoc | objects) |not then redactNote(.remoteDoc) else . end
  | if isempty(.note | objects) |not then redactNote(.note) else . end
  | if isempty(.change | objects) | not then .
    | .change |= (. | redactNote(.doc))
    | .change |= (. | redactNote(.was))
    else . end
;

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v) |
  del(.level) |
  del(.local) |
  del(.remote) |
  cleanNote |
  {time,component,msg,path}+(del(.time)|del(.component)|del(.msg)|del(.path));

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: select(is_error) | clean;
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: select(is_warn) | clean;
def warn_strict: select(is_warn_strict) | clean;
def info_level: 30;
def is_info: .level >= info_level;
def info: select(is_info) | clean;
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: select(is_debug) | clean;

# Components:
def component(pattern): select(.component | test(pattern));
def App: component("^App$");
def Chokidar: component("^Chokidar|chokidar$");
def Fs: component("^Fs|FS$");
def LocalAnalysis: component("^LocalAnalysis|local/analysis$");
def LocalChange: component("^LocalChange|local/change$");
def LocalWatcher: component("^LocalWatcher$");
def LocalWriter: component("^LocalWriter$");
def Merge: component("^Merge$");
def Metadata: component("^Metadata$");
def Perfs: component("^Perfs|PerfReports$");
def Pouch: component("^Pouch$");
def Prep: component("^Prep$");
def RemoteCozy: component("^RemoteCozy$");
def RemoteUserActionRequired: component("^RemoteUserActionRequired|remote/user_action_required$");
def RemoteWarningPoller: component("^RemoteWarningPoller$");
def RemoteWatcher: component("^RemoteWatcher$");
def RemoteWriter: component("^RemoteWriter$");
def Sync: component("^Sync$");

def Local: select(.component | test("^local|chokidar"; "i"));
def Remote: select(.component | test("^remote"; "i"));

# Exclude "up-to-date" messages
#
#     yarn jq nouptodate path/to/logs*
#
def nouptodate: select(.msg | test("up[- ]to[- ]date") | not);

# Exclude initial scan messages
#
#     yarn jq noscan path/to/logs*
#
def noscan: select(.action != "scan");

# Exclude Proxy stuff:
#
#     yarn jq noproxy path/to/logs*
#
def noproxy: select(.component | test("GUI:proxy") | not);

# Find conflicts:
#
#     yarn jq -c conflict path/to/logs*
#
def is_conflict: .msg | test("resolveConflictAsync|Resolving conflict");
def conflict: clean | select(is_conflict);

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
# FIXME: Should not output the same line twice when both path & oldpath match.
def path(pattern):
  clean | select(
    (
      try (.path // (.doc | .path) // (.change | .doc | .path) // (.idConflict | .newDoc | .path) // (.event | .path)),
      try (.oldpath // (.was | .path) // (.idConflict | .existingDoc | .path) // (.event | .oldPath)),
      ""
    )
      | strings
      | test(pattern)
  );

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#    yarn jq 'before("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#    yarn jq 'after("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#
def time(pattern):
  select(.time | test(pattern));
def before(time): select(.time <= time);
def after(time): select(.time >= time);

# Drop time properties
def notime: del(.time);

# Filter inode number(s):
#
#    yarn jq 'ino(7852458)' path/to/logs*
#    yarn jq 'ino(7852458; 464672)' path/to/logs*
#
def ino(n): select((.ino // (.doc|.ino) // (.change|.doc|.ino)) as $ino | [n] | flatten | map(. == $ino) | any);

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path,oldpath,event}
    | if (.event | not) then del(.event) else . end
    | if (.event | type == "object") then del(.event) + (.event|{path, oldpath: .oldPath, action}) else . end
    | if .oldpath then . else del(.oldpath) end
    ;

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );


Fix doc for map.jq utility file.

Fix doc for map.jq utility file.

jsoniq version "1.0";

(:
 : Copyright 2006-2013 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

(:~
 : Map utility
 : @author Chris Hillery, Cezar Andrei
 :)

module namespace map = "http://jsound.io/modules/validate/map";

declare namespace an = "http://zorba.io/annotations";

(:~
 : Inserts new key and value into the map or replaces value under the exiting key.
 : Returns true all the time.
 :
 : @param $map the map JSON object
 : @param $key the key
 : @param $value the value
 : @return true
 :
 : Example:

 variable $foo := { "foo" : 5 };
 map:set($foo, "foo", 6);
 map:set($foo, "bar", 7);
 $foo

 :)
declare %an:sequential function
map:set($map as object, $key as xs:string, $value as item) as boolean
{
  if( keys($map) = $key ) then
    insert json { $key : $value } into $map;
  else
    replace value of json $map.$key with $value;

  true
};

(:~
 : Only if key doesn't exist, inserts new key and value into the map and
 : returns true. Otherwise returns false.
 :
 : @param $map the map JSON object
 : @param $key the key
 : @param $value the value
 : @return true if $key is empty, false otherwise
 :
 : Example:

 variable $foo := { "foo" : 5 };
 map:set($foo, "foo", 6);
 map:set-if-empty($foo, "foo", 7);
 $foo

 :)
declare %an:sequential function
map:set-if-empty($map as object, $key as xs:string, $value as item) as boolean
{
  if( keys($map) = $key )
  then
    false
  else
  {
    insert json { $key : $value } into $map;
    true
  }
};

(:~
 : Returns the value under the key.
 :
 : @param $map the map JSON object
 : @param $key the key
 : @return the value under the $key, () if empty
 :
 : Example:

 variable $foo := { "foo" : 5 };
 map:set($foo, "foo", 6);
 map:set($foo, "bar", 7);
 map:get($foo, "foo")

 :)
declare function
map:get($map as object, $key as string) as item
{
  $map.$key
};

(:~
 : Returns true if $key exists in the map.
 :
 : @param $map the map JSON object
 : @param $key the key
 : @return true if $key exists, false otherwise
 :
 : Example:

 variable $foo := { "foo" : 5 };
 map:set($foo, "foo", 6);
 map:has-key($foo, "foo")

 :)
declare function
map:has-key($map as object, $key as string) as boolean
{
  keys($map) = $key
};

(:
variable $foo := { "foo" : 5 };
map:set($foo, "foo", 6);
map:set($foo, "bar", 7);
$foo
:)

jsoniq version "1.0";

(:
 : Copyright 2006-2013 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

(:~
 : Map utility
 :
 :)

module namespace map = "http://jsound.io/modules/validate/map";

declare namespace an = "http://zorba.io/annotations";

(:~
 : Inserts new key and value into the map or replaces value under the exiting key.
 : Returns true all the time.
 : Example:

 variable $foo := { "foo" : 5 };
 map:set($foo, "foo", 6);
 map:set($foo, "bar", 7);
 $foo

 :)
declare %an:sequential function
map:set($map as object, $key as xs:string, $value as item) as boolean
{
  if( keys($map) = $key ) then
    insert json { $key : $value } into $map;
  else
    replace value of json $map.$key with $value;

  true
};

(:~
 : Only if key doesn't exist, inserts new key and value into the map and
 : returns true. Otherwise returns false.

 variable $foo := { "foo" : 5 };
 map:set($foo, "foo", 6);
 map:set-if-empty($foo, "foo", 7);
 $foo

 :)
declare %an:sequential function
map:set-if-empty($map as object, $key as xs:string, $value as item) as boolean
{
  if( keys($map) = $key )
  then
    false
  else
  {
    insert json { $key : $value } into $map;
    true
  }
};

(:~
 : Returns the value under the key.

 variable $foo := { "foo" : 5 };
 map:set($foo, "foo", 6);
 map:set($foo, "bar", 7);
 map:get($foo, "foo")

 :)
declare function
map:get($map as object, $key as string) as item
{
  $map.$key
};

(:~
 : Returns true if $key exists in the map.

 variable $foo := { "foo" : 5 };
 map:set($foo, "foo", 6);
 map:has-key($foo, "foo")

 :)
declare function
map:has-key($map as object, $key as string) as boolean
{
  keys($map) = $key
};

(:
variable $foo := { "foo" : 5 };
map:set($foo, "foo", 6);
map:set($foo, "bar", 7);
$foo
:)


:bug: Adding on-the-fly concept presentation if the presentation network is missing in the infoset.

jsoniq version "1.0";
(:
 : Copyright 2012-2014 28msec Inc.
 :)

(:~
 :
This module provides functionality for retrieving components.

 :
 :
Components help organizing the facts contained in archives
 : in smaller parts that "make sense" together. A component is identified
 : with a CID (component ID).

 :
 :
A component is made of networks (see the networks module) and hypercubes
 : (see the hypercubes module).

 :
 :
With this module, you can retrieve all components, all components belonging
 : to one or several archives. You can retrieve the CID of a component or retrieve
 : the components associated with a couple of CIDs.

 :
 :)
module namespace components = "http://28.io/modules/xbrl/components";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace networks = "http://28.io/modules/xbrl/networks";
import module namespace resolution = "http://28.io/modules/xbrl/resolution";
import module namespace layout = "http://28.io/modules/xbrl/layout";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";

import module namespace accountant = "http://28.io/modules/xbrl/profiles/accountant/converter";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Name of the collection the components are stored in.
 :)
declare variable $components:col as xs:string := "components";

(:~
 : Name of the field pointing to the archive.
 :)
declare variable $components:ARCHIVE as xs:string := "Archive";

(:~
 : Name of the field pointing to the role.
 :)
declare variable $components:ROLE as xs:string := "Role";

(:~
 : Name of the field pointing to the default language.
 :)
declare variable $components:DEFAULT-LANGUAGE as xs:string := "Language";

(:~
 :
Retrieves all components.

 :
 : @return all components.
 :)
declare function components:components() as object*
{
  mw:find($components:col,{})
};

(:~
 :
Retrieves the components with the given CIDs.

 :
 : @param $component-or-ids the CIDs or the components themselves.
 :
 : @return the components whose _id field matches one of these CIDs.
 :)
declare function components:components($component-or-ids as item*) as object*
{
  let $ids as string* :=
    for $s in $component-or-ids
    where $s instance of string
    return $s
  let $components as object* :=
    for $s in $component-or-ids
    where $s instance of object
    return $s
  return
    (
      $components,
      if (exists($ids))
      then mw:find($components:col,{ "_id" : { "$in" : [ $ids ! components:cid($$) ] } })
      else ()
    )
};

(:~
 :
Retrieves all components that belong to the supplied archives.

 :
 : @param $archive-or-id an archive or its AID.
 :
 : @return all components in the archive with this AID.
 :)
declare function components:components-for-archives($archive-or-ids as item*) as object*
{
  for $archive-or-id in $archive-or-ids
  return mw:find($components:col,{ $components:ARCHIVE: archives:aid($archive-or-id) })
};

(:~
 : Retrieves all components for the supplied filings and roles.
 :
 : @param $archives-or-ids a sequence of archives or AIDs.
 : @param $roles a sequence of role URIs.
 :
 : @return the components.
 :)
declare function components:components-for-archives-and-roles(
    $archives-or-ids as item*,
    $roles as string*) as object*
{
    let $aids := archives:aid($archives-or-ids)
    return
        mw:find($components:col,
        {
            $components:ARCHIVE: { "$in" : [ $aids ] },
            "Role": { "$in" : [ $roles ] }
        })
};

(:~
 : Retrieves all components for the supplied filings and that use the supplied concepts.
 :
 : @param $archives-or-ids a sequence of archives or AIDs.
 : @param $concepts a sequence of concept names.
 :
 : @return the components.
 :)
declare function components:components-for-archives-and-concepts(
    $archives-or-ids as item*,
    $concepts as string*) as object*
{
    let $aids as string* := archives:aid($archives-or-ids)
    let $concepts := mw:find($concepts:col,
        {|
            (
                { "Name" : { "$in" : [ $concepts ] } },
                { "Archive" : { "$in" : [ $aids ] } }
            )
        |})
    let $roles := $concepts.Role
    for $component in components:components-for-archives-and-roles($aids, $roles)
    where (some $concept in $concepts satisfies $concept.Archive eq $component.Archive and $concept.Role eq $component.Role)
    return $component
};

(:~
 :
Retrieves all definition models in a component.

 :
 : @param $component a component object.
 :
 : @return all definition models
 :)
declare function components:definition-models-for-components(
    $components as object*) as object*
{
    $components.DefinitionModels[]
};

(:~
 :
Retrieves definition models in a component
 : that have the supplied names.

 :
 : @param $component a component object.
 : @param $names a sequence of names.
 :
 : @return the definition models.
 :)
declare function components:definition-models-for-components(
    $components as object*,
    $names as string*) as object
{
    $components.DefinitionModels[][$$.Name = $names]
};

(:~
 :
Selects a table from the component.

 :
 : @param $component a component object.
 : @param $options
some optional parameters, including:

 :

 :
HypercubeName: a string specifying which hypercube to use. By default, one of the non-default hypercubes will be arbitrarily chosen. If none
 :  is available, the default hypercube will be picked.

 :

 :
 : @error components:HYPERCUBE-DOES-NOT-EXIST if the specified hypercube is not found.
 : @return a definition model
 :)
declare function components:select-table(
    $component as object,
    $options as object?) as object
{
    let $user-chosen-table as object? := hypercubes:hypercubes-for-components($component, $options.HypercubeName)
    let $non-implicit-table as object? := hypercubes:hypercubes-for-components($component)[1]
    return
        switch(true)
        case exists($options.HypercubeName) and empty($user-chosen-table)
            return error(QName("components:HYPERCUBE-DOES-NOT-EXIST"), $options.HypercubeName || " : Hypercube does not exist.")
        case exists($user-chosen-table)
            return $user-chosen-table
        case exists($non-implicit-table)
            return $non-implicit-table
        default return error(QName("components:HYPERCUBE-DOES-NOT-EXIST"), "Components does not contain any hypercube.")
};

(:~
 :
 :
Retrieves all facts that are relevant to the
 : supplied component, and populates them with the default dimension values
 : when missing.

 :
The default hypercube will be taken, and the concept maps and rules will
 : be used if any.

 :
 : @param $component a component object.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:facts(
    $component as object
)
as object*
{
  components:facts($component, {})
};

(:~
 :
 :
Retrieves all facts that are relevant to the
 : supplied component, and populates them with the default dimension values
 : when missing.

 :
 :
 : @param $report-or-id a report or its RID,
 : @param $options standard fact retrieving options as well as
 :

 :
FilterOverride: hypercube dimension specs (as defined as hypercubes:user-defined-hypercube)
 :   to override filters in the report's hypercube.

 :
HypercubeName: picks a different hypercube than the default one.

 :

 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:facts(
    $component as object,
    $options as object?
)
as object*
{
  let $concept-map := networks:networks-for-components-and-short-names(
      $component,
      "ConceptMap")
  let $rules := (flatten($component.Rules), flatten($options.Rules))
  let $overriden-hypercube as object? :=
        if(exists($options.Hypercube))
        then $options.Hypercube
        else components:select-table($component, $options)
    let $hypercube :=
    if(exists($options.FilterOverride))
    then hypercubes:modify-hypercube(
      $overriden-hypercube,
      $options.FilterOverride
    )
    else $overriden-hypercube
  let $concepts as object* := $component.Concepts[]
  return hypercubes:facts(
          $hypercube,
          {|
            trim($options, "Rules"),
            { "ConceptMaps": $concept-map }[exists($concept-map)],
            { "Rules": $rules }[exists($rules)],
            { "Concepts": [ $concepts ] }[exists($concepts)]
          |}
         )
};

(:~
 :
 :
Retrieves the spreadsheet-like rendering for the component.

 :
The default definition model will be taken.

 :
The default hypercube will be taken, and the concept maps and rules will
 : be used if any.

 :
 : @param $component a component object.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:spreadsheet(
    $component as object
)
as object*
{
  components:spreadsheet($component, {})
};

(:~
 :
 :
Retrieves the spreadsheet-like rendering for the component.

 :
 : @param $report-or-id a report or its RID,
 : @param $options standard fact retrieving options as well as
 :

 :
FilterOverride: hypercube dimension specs (as defined as hypercubes:user-defined-hypercube)
 :   to override filters in the report's hypercube.

 :
HypercubeName: picks a different hypercube than the default one.

 :

 :
 :
 : @error err:XPTY0004 if zero, or more than one definition models are found.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:spreadsheet(
    $component as object,
    $options as object?
)
as object*
{
    let $definition-model as object :=
        switch(true)
        case exists($options.DefinitionModel)
            return $options.DefinitionModel
        case (exists($options.DefinitionModelName))
            return components:definition-models-for-components($component, $options.DefinitionModelName)
        default
            return components:definition-models-for-components($component)
    let $overriden-hypercube as object? :=
        if(exists($options.Hypercube))
        then $options.Hypercube
        else components:select-table($component, $options)
    let $concept-map as object?:= networks:networks-for-components-and-short-names(
      $component,
      "ConceptMap")
    let $rules as array? := $component.Rules
    let $concepts as object* := $component.Concepts[]
    let $new-options as object :=
        {|
            trim($options, "Rules"),
            {"ConceptMaps": $concept-map }[exists($concept-map)],
            { "Rules": [ ( flatten($rules[]), flatten($options.Rules) ) ] }[exists(($rules, $options.Rules))],
            {
                Hypercube:
                    if(exists($options.FilterOverride))
                    then hypercubes:modify-hypercube(
                        $overriden-hypercube,
                        $options.FilterOverride
                    )
                    else $overriden-hypercube
            }[exists($overriden-hypercube)],
            { "Concepts": [ $concepts ] }[exists($concepts)]
        |}
    let $structural-model as object := resolution:resolve(
        $definition-model,
        $component,
        $new-options)
    let $layout-model as object := layout:layout($structural-model, $new-options)
    let $accountant-model as object := accountant:flatten-row-headers($layout-model)
    let $result := if($options.FlattenRows) then $accountant-model else $layout-model
    return
        if($options.Debug)
        then copy $r := $result
             modify (
                 insert json { Definition: $definition-model } into $r,
                 insert json { Structural: $structural-model } into $r
             )
             return $r
        else $result
};

(:~
 : Return the number of hypercubes in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of hypercubes
 :)
declare function components:num-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumHypercubes
};

(:~
 : Return the number of networks in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of networks
 :)
declare function components:num-networks($components as object*) as integer*
{
  $components ! $$.Statistics.NumNetworks
};

(:~
 : Return the number of (distinct) explicit dimensions in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of dimensions
 :)
declare function components:num-explicit-dimensions($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctExplicitDimensions
};

(:~
 : Return the number of (distinct) domains in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of domains
 :)
declare function components:num-domains($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctDomains
};

(:~
 : Return the number of (distinct) members in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of members
 :)
declare function components:num-members($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctMembers
};

(:~
 : Return the number of (distinct) concrete primary items being
 : in a hypercube for each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of primary items
 :)
declare function components:num-concrete-primary-items-in-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctConcretePrimaryItemsInHypercubes
};

(:~
 : Return the number of (distinct) abstract primary items being
 : in a hypercube for each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of primary items
 :)
declare function components:num-abstract-primary-items-in-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctAbstractPrimaryItemsInHypercubes
};

(:~
 : Return the number of (distinct) concrete primary items not being
 : in a hypercube for each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of primary items
 :)
declare function components:num-concrete-primary-items-not-in-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctConcretePrimaryItemsNotInHypercubes
};

(:~
 : Return the number of (distinct) abstract primary items not being
 : in a hypercube for each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of primary items
 :)
declare function components:num-distinct-abstract-primary-items-not-in-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctAbstractPrimaryItemsNotInHypercubes
};

(:~
 :
 :
Converts the input to a normalized component id (CID). The input
 : can be either an CID, or an component object which contains an _id.

 :
 : @param $component-or-id an component object or an CID.
 :
 : @error components:INVALID_PARAMETER if the CID or component is not valid.
 :
 : @return the normalized CID.
 :)
declare function components:cid($component-or-id as item) as atomic
{
  typeswitch ($component-or-id)
  case object return
    let $id := $component-or-id._id
    return if(exists($id))
           then $id
           else error(QName("components:INVALID_PARAMETER"),
                      "Invalid component provided (no _id field)")
  case $id as atomic return $id
  default return error(
      QName("components:INVALID_PARAMETER"),
      "Invalid component or id (must be an object or an atomic): "
      || serialize($component-or-id))
};

(:~
 :
Returns the standard period breakdown.

 :
 : @return the period breakdown.
 :)
declare function components:standard-period-breakdown() as object
{
    {
        BreakdownLabels: [ "Period breakdown" ],
        BreakdownTrees: [
            {
                Kind: "Rule",
                Abstract: true,
                Labels: [ "Period [Axis]" ],
                Children: [ {
                    Kind: "Aspect",
                    Aspect: "xbrl:Period"
                } ]
            }
        ]
    }
};

declare function components:standard-typed-dimension-breakdown($dimension-name as string, $dimension-values as atomic*) as object
{
    {
        BreakdownLabels: [ $dimension-name || " breakdown" ],
        BreakdownTrees: [
            {
                Kind: "Rule",
                Abstract: true,
                Labels: [ $dimension-name || " [Axis]" ],
                Children: [
                    for $value in $dimension-values
                    return {
                        Kind: "Rule",
                        Labels: [ $value ],
                        AspectRulesSet: { "" : { $dimension-name : $value } }
                    }
                ]
            }
        ]
    }
};

declare function components:standard-explicit-dimension-breakdown(
    $dimension-name as string,
    $dimension-label as string?,
    $domain-names as string*,
    $role as string) as object
{
    let $dimension-label as string := ($dimension-label, $dimension-name)[1]
    return
    {
        BreakdownLabels: [ $dimension-label || " breakdown" ],
        BreakdownTrees: [
            {
                Kind: "Rule",
                Abstract: true,
                Labels: [ $dimension-label ],
                Children: [
                    for $domain as string in $domain-names
                    return {
                        Kind: "DimensionRelationship",
                        LinkRole: $role,
                        Dimension: $dimension-name,
                        RelationshipSource: $domain,
                        FormulaAxis: "descendant",
                        Generations: 0
                    }
                ]
            }
        ]
    }
};

declare function components:standard-entity-breakdown() as object
{
    {
        BreakdownLabels: [ "Entity breakdown" ],
        BreakdownTrees: [
            {
                Kind: "Rule",
                Abstract: true,
                Labels: [ "Entity [Axis]" ],
                ConstraintSets: { "" : {} },
                Children: [ {
                    Kind: "Aspect",
                    Aspect: "xbrl:Entity"
                } ]
            }
        ]
    }
};

declare function components:standard-concept-breakdown(
    $line-items-elements as string*,
    $role as string) as object
{
    {
        BreakdownLabels: [ "Breakdown on concepts" ],
        BreakdownTrees: [
            for $lineitems as string in $line-items-elements
            return {
                Kind: "ConceptRelationship",
                LinkName: "link:presentationLink",
                LinkRole: $role,
                ArcName: "link:presentationArc",
                ArcRole: "http://www.xbrl.org/2003/arcrole/parent-child",
                RelationshipSource: $lineitems,
                FormulaAxis: "descendant",
                Generations: 0,
                RollUpAgainstCalculationNetwork: false
            }
        ]
    }
};

(:~
 :
Builds a standard definition model out of the specified component.

 :
The concepts will be put on the y axis according to the presentation network.

 :
The other dimensions are put on the x axis, with one breakdown for each.

 :
Explicit dimensions are organized according to the dimension hierarchy from the domain-member network.

 :
Typed dimensions are organized according to the actual values.

 :
 :
One of the non-default hypercubes will be arbitrarily chosen. If none is available, the default hypercube will be picked.

 :
Auto slicing will be performed against the fact table
 :
 : @param $component a component object.
 :
 : @return a definition model
 :)
declare function components:standard-definition-models-for-components($components as object*) as object
{
    components:standard-definition-models-for-components($components, ())
};

(:~
 :
Builds a standard definition model out of the specified component.

 :
The concepts will be put on the y axis according to the presentation network.

 :
The other dimensions are put on the x axis, with one breakdown for each.

 :
Explicit dimensions are organized according to the dimension hierarchy from the domain-member network.

 :
Typed dimensions are organized according to the actual values.

 :
 : @param $component a component object.
 : @param $options
some optional parameters, including:

 :

 :
HypercubeName: a string specifying which hypercube to use. By default, one of the non-default hypercubes will be arbitrarily chosen. If none
 :  is available, the default hypercube will be picked.

 :
AutoSlice: a boolean specifying whether or not slicing should be done automatically, looking at the fact table. Deactivating auto slicing will
 :  lead to better performance, but a more verbose table. If AutoSlice is active, dimensions with only one value in the fact pool will become
 : global filters rather than breakdowns on the x axis.

 :
Slicers: an object with forced slicers.

 :

 :
 : @error components:HYPERCUBE-DOES-NOT-EXIST if the specified hypercube is not found.
 : @return a definition model
 :)
declare function components:standard-definition-models-for-components($components as object*, $options as object?) as object
{
    for $component in $components
    let $table as object := components:select-table($component, $options)

    let $auto-slice as boolean := empty($options.AutoSlice) or $options.AutoSlice
    let $facts as object*:=
        if($auto-slice)
        then hypercubes:facts($table)
        else ()
    let $dimensions as string*:= keys($table.Aspects)
    let $values-by-dimension as object := {|
        for $d in $dimensions
        return { $d : [ distinct-values($facts.Aspects.$d) ] }
    |}
    let $auto-slice-dimensions as string* :=
        keys($values-by-dimension)[size($values-by-dimension.$$) eq 1 and not ($$ = ("xbrl:Period", "sec:FiscalYear",  "sec:FiscalPeriod") ) ]
    let $user-slice-dimensions as string* :=
        keys($options.Slicers)

    let $column-dimensions as string* := keys($values-by-dimension)[not $$ =
        ("xbrl:Concept",
        "xbrl:Period",
        "xbrl:Unit",
        "xbrl:Entity",
        "xbrl28:Archive",
        $auto-slice-dimensions,
        $user-slice-dimensions)]

    let $x-breakdowns as object* := (
        components:standard-period-breakdown()[not (($auto-slice-dimensions, $user-slice-dimensions) = "xbrl:Period")],
        for $d as string in $column-dimensions
        let $metadata as object? := ($component.Concepts[])[$$.Name eq $d]
        let $label as string? :=
          concepts:labels(
            $d,
            $concepts:VERBOSE_LABEL_ROLE,
            ($options.Language, "en")[1],
            $components.Concepts[],
            $options
          )[1]
        return
            components:standard-explicit-dimension-breakdown(
                $d,
                ($label, $metadata.Label)[1],
                $table.Aspects.$d.Members[].Name,
                $component.Role),
        components:standard-entity-breakdown()[not (($auto-slice-dimensions, $user-slice-dimensions) = "xbrl:Entity")]
    )

    let $lineitems as string* := components:line-items($component)
    let $presentation-network as object? := networks:networks-for-components-and-short-names($component, "Presentation")
    let $roots as string* := $presentation-network.Trees[].Name
    let $lineitems as string* := if(exists($lineitems)) then $lineitems else $roots
    let $y-breakdowns as object :=
      let $concept-label := (concepts:labels(
        "xbrl:Concept",
        $concepts:VERBOSE_LABEL_ROLE,
        ($options.Language, "en")[1],
        $components.Concepts[],
        $options
      ), concepts:labels(
        "xbrl:Concept",
        $concepts:STANDARD_LABEL_ROLE,
        ($options.Language, "en")[1],
        $components.Concepts[],
        $options
      ))[1]
      return if(exists($presentation-network))
              then components:standard-concept-breakdown($lineitems, $component.Role)
              else components:standard-explicit-dimension-breakdown(
                "xbrl:Concept",
                $concept-label,
                $table.Aspects."xbrl:Concept".Members[].Name,
                $component.Role)

    return {
        ModelKind: "DefinitionModel",
        Labels: [$component.Label],
        Parameters: {},
        Breakdowns: {
            "x" : [
                $x-breakdowns
            ],
            "y": [
                $y-breakdowns
            ]
        },
        TableFilters: {|
            for $d as string in distinct-values(($auto-slice-dimensions, $user-slice-dimensions))
            return if($d = $user-slice-dimensions)
                   then { $d : $options.Slicers.$d }
                   else { $d : $values-by-dimension.$d[] },
            if (not $auto-slice)
            then { "xbrl28:Archive" : $component.Archive }
            else ()
        |}
    }
};

declare function components:filter-override(
    $entities-or-eids as item*,
    $archives-or-aids as item*
) as object?
{
    let $override := exists(($entities-or-eids, $archives-or-aids))
    where $override
    let $eids := entities:eid($entities-or-eids)
    let $aids := archives:aid($archives-or-aids)
    return
     {
        "xbrl:Entity" : {|
          { Type: "string" },
          { Domain: [ $eids ] }[exists($eids)]
        |},

        "xbrl28:Archive" : {|
          { Type: "string" },
          { Domain : [ $aids ] }[exists($aids)]
        |}
    }
};

(:~
:
Merges the supplied components, grouping networks by arc/link name/role
: and merging all hypercubes into a single one.

:
: @param $components the input components.
:
: @return the merged component.
:)
declare function components:merge($components as object*) as object
{
    {
        Archive: null,
        Role: $components.Role[1],
        Label: $components.Label[1],
        Networks: [
            networks:merge($components.Networks[])
        ],
        Hypercubes:
            let $merged := hypercubes:merge(values($components.Hypercubes))
            return { $merged.Name: $merged },
        Concepts: [
            for $concept in $components.Concepts[]
            group by $name := $concept.Name
            return $concept[1]
        ]
    }
};

(:~
 :
Returns the line items, that is the top-level abstracts or concepts in the
 : presentation network.

 :
 : @param $components the input components.
 :
 : @return the line items report elements.
 :)
declare function components:line-items($components as object*) as string*
{
    let $presentation-networks as object* := networks:networks-for-components-and-short-names($components, "Presentation")
    return components:line-items-recursive($components, $presentation-networks.Trees[])
};

declare %private function components:line-items-recursive($components as object*, $networks as object*) as string*
{
  let $concepts as object* := $components.Concepts[]
  for $network in $networks
  let $concept-name := $network.Name
  let $substitution-group as string := $concepts[$$.Name eq $concept-name].SubstitutionGroup[1]
  let $has-table-child as boolean := $concepts[$$.Name = $network.To[].Name].SubstitutionGroup = "xbrldt:hypercubeItem"
  return switch(true)
          case $has-table-child return components:line-items-recursive($components, $network.To[])
          case $substitution-group eq "xbrldt:dimensionItem" return ()
          case $substitution-group eq "xbrldt:hypercubeItem" return ()
          default return $concept-name
};

jsoniq version "1.0";
(:
 : Copyright 2012-2014 28msec Inc.
 :)

(:~
 :
This module provides functionality for retrieving components.

 :
 :
Components help organizing the facts contained in archives
 : in smaller parts that "make sense" together. A component is identified
 : with a CID (component ID).

 :
 :
A component is made of networks (see the networks module) and hypercubes
 : (see the hypercubes module).

 :
 :
With this module, you can retrieve all components, all components belonging
 : to one or several archives. You can retrieve the CID of a component or retrieve
 : the components associated with a couple of CIDs.

 :
 :)
module namespace components = "http://28.io/modules/xbrl/components";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace networks = "http://28.io/modules/xbrl/networks";
import module namespace resolution = "http://28.io/modules/xbrl/resolution";
import module namespace layout = "http://28.io/modules/xbrl/layout";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";

import module namespace accountant = "http://28.io/modules/xbrl/profiles/accountant/converter";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Name of the collection the components are stored in.
 :)
declare variable $components:col as xs:string := "components";

(:~
 : Name of the field pointing to the archive.
 :)
declare variable $components:ARCHIVE as xs:string := "Archive";

(:~
 : Name of the field pointing to the role.
 :)
declare variable $components:ROLE as xs:string := "Role";

(:~
 : Name of the field pointing to the default language.
 :)
declare variable $components:DEFAULT-LANGUAGE as xs:string := "Language";

(:~
 :
Retrieves all components.

 :
 : @return all components.
 :)
declare function components:components() as object*
{
  mw:find($components:col,{})
};

(:~
 :
Retrieves the components with the given CIDs.

 :
 : @param $component-or-ids the CIDs or the components themselves.
 :
 : @return the components whose _id field matches one of these CIDs.
 :)
declare function components:components($component-or-ids as item*) as object*
{
  let $ids as string* :=
    for $s in $component-or-ids
    where $s instance of string
    return $s
  let $components as object* :=
    for $s in $component-or-ids
    where $s instance of object
    return $s
  return
    (
      $components,
      if (exists($ids))
      then mw:find($components:col,{ "_id" : { "$in" : [ $ids ! components:cid($$) ] } })
      else ()
    )
};

(:~
 :
Retrieves all components that belong to the supplied archives.

 :
 : @param $archive-or-id an archive or its AID.
 :
 : @return all components in the archive with this AID.
 :)
declare function components:components-for-archives($archive-or-ids as item*) as object*
{
  for $archive-or-id in $archive-or-ids
  return mw:find($components:col,{ $components:ARCHIVE: archives:aid($archive-or-id) })
};

(:~
 : Retrieves all components for the supplied filings and roles.
 :
 : @param $archives-or-ids a sequence of archives or AIDs.
 : @param $roles a sequence of role URIs.
 :
 : @return the components.
 :)
declare function components:components-for-archives-and-roles(
    $archives-or-ids as item*,
    $roles as string*) as object*
{
    let $aids := archives:aid($archives-or-ids)
    return
        mw:find($components:col,
        {
            $components:ARCHIVE: { "$in" : [ $aids ] },
            "Role": { "$in" : [ $roles ] }
        })
};

(:~
 : Retrieves all components for the supplied filings and that use the supplied concepts.
 :
 : @param $archives-or-ids a sequence of archives or AIDs.
 : @param $concepts a sequence of concept names.
 :
 : @return the components.
 :)
declare function components:components-for-archives-and-concepts(
    $archives-or-ids as item*,
    $concepts as string*) as object*
{
    let $aids as string* := archives:aid($archives-or-ids)
    let $concepts := mw:find($concepts:col,
        {|
            (
                { "Name" : { "$in" : [ $concepts ] } },
                { "Archive" : { "$in" : [ $aids ] } }
            )
        |})
    let $roles := $concepts.Role
    for $component in components:components-for-archives-and-roles($aids, $roles)
    where (some $concept in $concepts satisfies $concept.Archive eq $component.Archive and $concept.Role eq $component.Role)
    return $component
};

(:~
 :
Retrieves all definition models in a component.

 :
 : @param $component a component object.
 :
 : @return all definition models
 :)
declare function components:definition-models-for-components(
    $components as object*) as object*
{
    $components.DefinitionModels[]
};

(:~
 :
Retrieves definition models in a component
 : that have the supplied names.

 :
 : @param $component a component object.
 : @param $names a sequence of names.
 :
 : @return the definition models.
 :)
declare function components:definition-models-for-components(
    $components as object*,
    $names as string*) as object
{
    $components.DefinitionModels[][$$.Name = $names]
};

(:~
 :
Selects a table from the component.

 :
 : @param $component a component object.
 : @param $options
some optional parameters, including:

 :

 :
HypercubeName: a string specifying which hypercube to use. By default, one of the non-default hypercubes will be arbitrarily chosen. If none
 :  is available, the default hypercube will be picked.

 :

 :
 : @error components:HYPERCUBE-DOES-NOT-EXIST if the specified hypercube is not found.
 : @return a definition model
 :)
declare function components:select-table(
    $component as object,
    $options as object?) as object
{
    let $user-chosen-table as object? := hypercubes:hypercubes-for-components($component, $options.HypercubeName)
    let $non-implicit-table as object? := hypercubes:hypercubes-for-components($component)[1]
    return
        switch(true)
        case exists($options.HypercubeName) and empty($user-chosen-table)
            return error(QName("components:HYPERCUBE-DOES-NOT-EXIST"), $options.HypercubeName || " : Hypercube does not exist.")
        case exists($user-chosen-table)
            return $user-chosen-table
        case exists($non-implicit-table)
            return $non-implicit-table
        default return error(QName("components:HYPERCUBE-DOES-NOT-EXIST"), "Components does not contain any hypercube.")
};

(:~
 :
 :
Retrieves all facts that are relevant to the
 : supplied component, and populates them with the default dimension values
 : when missing.

 :
The default hypercube will be taken, and the concept maps and rules will
 : be used if any.

 :
 : @param $component a component object.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:facts(
    $component as object
)
as object*
{
  components:facts($component, {})
};

(:~
 :
 :
Retrieves all facts that are relevant to the
 : supplied component, and populates them with the default dimension values
 : when missing.

 :
 :
 : @param $report-or-id a report or its RID,
 : @param $options standard fact retrieving options as well as
 :

 :
FilterOverride: hypercube dimension specs (as defined as hypercubes:user-defined-hypercube)
 :   to override filters in the report's hypercube.

 :
HypercubeName: picks a different hypercube than the default one.

 :

 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:facts(
    $component as object,
    $options as object?
)
as object*
{
  let $concept-map := networks:networks-for-components-and-short-names(
      $component,
      "ConceptMap")
  let $rules := (flatten($component.Rules), flatten($options.Rules))
  let $overriden-hypercube as object? :=
        if(exists($options.Hypercube))
        then $options.Hypercube
        else components:select-table($component, $options)
    let $hypercube :=
    if(exists($options.FilterOverride))
    then hypercubes:modify-hypercube(
      $overriden-hypercube,
      $options.FilterOverride
    )
    else $overriden-hypercube
  let $concepts as object* := $component.Concepts[]
  return hypercubes:facts(
          $hypercube,
          {|
            trim($options, "Rules"),
            { "ConceptMaps": $concept-map }[exists($concept-map)],
            { "Rules": $rules }[exists($rules)],
            { "Concepts": [ $concepts ] }[exists($concepts)]
          |}
         )
};

(:~
 :
 :
Retrieves the spreadsheet-like rendering for the component.

 :
The default definition model will be taken.

 :
The default hypercube will be taken, and the concept maps and rules will
 : be used if any.

 :
 : @param $component a component object.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:spreadsheet(
    $component as object
)
as object*
{
  components:spreadsheet($component, {})
};

(:~
 :
 :
Retrieves the spreadsheet-like rendering for the component.

 :
 : @param $report-or-id a report or its RID,
 : @param $options standard fact retrieving options as well as
 :

 :
FilterOverride: hypercube dimension specs (as defined as hypercubes:user-defined-hypercube)
 :   to override filters in the report's hypercube.

 :
HypercubeName: picks a different hypercube than the default one.

 :

 :
 :
 : @error err:XPTY0004 if zero, or more than one definition models are found.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:spreadsheet(
    $component as object,
    $options as object?
)
as object*
{
    let $definition-model as object :=
        switch(true)
        case exists($options.DefinitionModel)
            return $options.DefinitionModel
        case (exists($options.DefinitionModelName))
            return components:definition-models-for-components($component, $options.DefinitionModelName)
        default
            return components:definition-models-for-components($component)
    let $overriden-hypercube as object? :=
        if(exists($options.Hypercube))
        then $options.Hypercube
        else components:select-table($component, $options)
    let $concept-map as object?:= networks:networks-for-components-and-short-names(
      $component,
      "ConceptMap")
    let $rules as array? := $component.Rules
    let $concepts as object* := $component.Concepts[]
    let $new-options as object :=
        {|
            trim($options, "Rules"),
            {"ConceptMaps": $concept-map }[exists($concept-map)],
            { "Rules": [ ( flatten($rules[]), flatten($options.Rules) ) ] }[exists(($rules, $options.Rules))],
            {
                Hypercube:
                    if(exists($options.FilterOverride))
                    then hypercubes:modify-hypercube(
                        $overriden-hypercube,
                        $options.FilterOverride
                    )
                    else $overriden-hypercube
            }[exists($overriden-hypercube)],
            { "Concepts": [ $concepts ] }[exists($concepts)]
        |}
    let $structural-model as object := resolution:resolve(
        $definition-model,
        $component,
        $new-options)
    let $layout-model as object := layout:layout($structural-model, $new-options)
    let $accountant-model as object := accountant:flatten-row-headers($layout-model)
    let $result := if($options.FlattenRows) then $accountant-model else $layout-model
    return
        if($options.Debug)
        then copy $r := $result
             modify (
                 insert json { Definition: $definition-model } into $r,
                 insert json { Structural: $structural-model } into $r
             )
             return $r
        else $result
};

(:~
 : Return the number of hypercubes in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of hypercubes
 :)
declare function components:num-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumHypercubes
};

(:~
 : Return the number of networks in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of networks
 :)
declare function components:num-networks($components as object*) as integer*
{
  $components ! $$.Statistics.NumNetworks
};

(:~
 : Return the number of (distinct) explicit dimensions in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of dimensions
 :)
declare function components:num-explicit-dimensions($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctExplicitDimensions
};

(:~
 : Return the number of (distinct) domains in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of domains
 :)
declare function components:num-domains($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctDomains
};

(:~
 : Return the number of (distinct) members in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of members
 :)
declare function components:num-members($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctMembers
};

(:~
 : Return the number of (distinct) concrete primary items being
 : in a hypercube for each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of primary items
 :)
declare function components:num-concrete-primary-items-in-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctConcretePrimaryItemsInHypercubes
};

(:~
 : Return the number of (distinct) abstract primary items being
 : in a hypercube for each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of primary items
 :)
declare function components:num-abstract-primary-items-in-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctAbstractPrimaryItemsInHypercubes
};

(:~
 : Return the number of (distinct) concrete primary items not being
 : in a hypercube for each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of primary items
 :)
declare function components:num-concrete-primary-items-not-in-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctConcretePrimaryItemsNotInHypercubes
};

(:~
 : Return the number of (distinct) abstract primary items not being
 : in a hypercube for each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of primary items
 :)
declare function components:num-distinct-abstract-primary-items-not-in-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctAbstractPrimaryItemsNotInHypercubes
};

(:~
 :
 :
Converts the input to a normalized component id (CID). The input
 : can be either an CID, or an component object which contains an _id.

 :
 : @param $component-or-id an component object or an CID.
 :
 : @error components:INVALID_PARAMETER if the CID or component is not valid.
 :
 : @return the normalized CID.
 :)
declare function components:cid($component-or-id as item) as atomic
{
  typeswitch ($component-or-id)
  case object return
    let $id := $component-or-id._id
    return if(exists($id))
           then $id
           else error(QName("components:INVALID_PARAMETER"),
                      "Invalid component provided (no _id field)")
  case $id as atomic return $id
  default return error(
      QName("components:INVALID_PARAMETER"),
      "Invalid component or id (must be an object or an atomic): "
      || serialize($component-or-id))
};

(:~
 :
Returns the standard period breakdown.

 :
 : @return the period breakdown.
 :)
declare function components:standard-period-breakdown() as object
{
    {
        BreakdownLabels: [ "Period breakdown" ],
        BreakdownTrees: [
            {
                Kind: "Rule",
                Abstract: true,
                Labels: [ "Period [Axis]" ],
                Children: [ {
                    Kind: "Aspect",
                    Aspect: "xbrl:Period"
                } ]
            }
        ]
    }
};

declare function components:standard-typed-dimension-breakdown($dimension-name as string, $dimension-values as atomic*) as object
{
    {
        BreakdownLabels: [ $dimension-name || " breakdown" ],
        BreakdownTrees: [
            {
                Kind: "Rule",
                Abstract: true,
                Labels: [ $dimension-name || " [Axis]" ],
                Children: [
                    for $value in $dimension-values
                    return {
                        Kind: "Rule",
                        Labels: [ $value ],
                        AspectRulesSet: { "" : { $dimension-name : $value } }
                    }
                ]
            }
        ]
    }
};

declare function components:standard-explicit-dimension-breakdown(
    $dimension-name as string,
    $dimension-label as string?,
    $domain-names as string*,
    $role as string) as object
{
    let $dimension-label as string := ($dimension-label, $dimension-name)[1]
    return
    {
        BreakdownLabels: [ $dimension-label || " breakdown" ],
        BreakdownTrees: [
            {
                Kind: "Rule",
                Abstract: true,
                Labels: [ $dimension-label ],
                Children: [
                    for $domain as string in $domain-names
                    return {
                        Kind: "DimensionRelationship",
                        LinkRole: $role,
                        Dimension: $dimension-name,
                        RelationshipSource: $domain,
                        FormulaAxis: "descendant",
                        Generations: 0
                    }
                ]
            }
        ]
    }
};

declare function components:standard-entity-breakdown() as object
{
    {
        BreakdownLabels: [ "Entity breakdown" ],
        BreakdownTrees: [
            {
                Kind: "Rule",
                Abstract: true,
                Labels: [ "Entity [Axis]" ],
                ConstraintSets: { "" : {} },
                Children: [ {
                    Kind: "Aspect",
                    Aspect: "xbrl:Entity"
                } ]
            }
        ]
    }
};

declare function components:standard-concept-breakdown(
    $line-items-elements as string*,
    $role as string) as object
{
    {
        BreakdownLabels: [ "Breakdown on concepts" ],
        BreakdownTrees: [
            for $lineitems as string in $line-items-elements
            return {
                Kind: "ConceptRelationship",
                LinkName: "link:presentationLink",
                LinkRole: $role,
                ArcName: "link:presentationArc",
                ArcRole: "http://www.xbrl.org/2003/arcrole/parent-child",
                RelationshipSource: $lineitems,
                FormulaAxis: "descendant",
                Generations: 0,
                RollUpAgainstCalculationNetwork: false
            }
        ]
    }
};

(:~
 :
Builds a standard definition model out of the specified component.

 :
The concepts will be put on the y axis according to the presentation network.

 :
The other dimensions are put on the x axis, with one breakdown for each.

 :
Explicit dimensions are organized according to the dimension hierarchy from the domain-member network.

 :
Typed dimensions are organized according to the actual values.

 :
 :
One of the non-default hypercubes will be arbitrarily chosen. If none is available, the default hypercube will be picked.

 :
Auto slicing will be performed against the fact table
 :
 : @param $component a component object.
 :
 : @return a definition model
 :)
declare function components:standard-definition-models-for-components($components as object*) as object
{
    components:standard-definition-models-for-components($components, ())
};

(:~
 :
Builds a standard definition model out of the specified component.

 :
The concepts will be put on the y axis according to the presentation network.

 :
The other dimensions are put on the x axis, with one breakdown for each.

 :
Explicit dimensions are organized according to the dimension hierarchy from the domain-member network.

 :
Typed dimensions are organized according to the actual values.

 :
 : @param $component a component object.
 : @param $options
some optional parameters, including:

 :

 :
HypercubeName: a string specifying which hypercube to use. By default, one of the non-default hypercubes will be arbitrarily chosen. If none
 :  is available, the default hypercube will be picked.

 :
AutoSlice: a boolean specifying whether or not slicing should be done automatically, looking at the fact table. Deactivating auto slicing will
 :  lead to better performance, but a more verbose table. If AutoSlice is active, dimensions with only one value in the fact pool will become
 : global filters rather than breakdowns on the x axis.

 :
Slicers: an object with forced slicers.

 :

 :
 : @error components:HYPERCUBE-DOES-NOT-EXIST if the specified hypercube is not found.
 : @return a definition model
 :)
declare function components:standard-definition-models-for-components($components as object*, $options as object?) as object
{
    for $component in $components
    let $table as object := components:select-table($component, $options)

    let $auto-slice as boolean := empty($options.AutoSlice) or $options.AutoSlice
    let $facts as object*:=
        if($auto-slice)
        then hypercubes:facts($table)
        else ()
    let $dimensions as string*:= keys($table.Aspects)
    let $values-by-dimension as object := {|
        for $d in $dimensions
        return { $d : [ distinct-values($facts.Aspects.$d) ] }
    |}
    let $auto-slice-dimensions as string* :=
        keys($values-by-dimension)[size($values-by-dimension.$$) eq 1 and not ($$ = ("xbrl:Period", "sec:FiscalYear",  "sec:FiscalPeriod") ) ]
    let $user-slice-dimensions as string* :=
        keys($options.Slicers)

    let $column-dimensions as string* := keys($values-by-dimension)[not $$ =
        ("xbrl:Concept",
        "xbrl:Period",
        "xbrl:Unit",
        "xbrl:Entity",
        "xbrl28:Archive",
        $auto-slice-dimensions,
        $user-slice-dimensions)]

    let $x-breakdowns as object* := (
        components:standard-period-breakdown()[not (($auto-slice-dimensions, $user-slice-dimensions) = "xbrl:Period")],
        for $d as string in $column-dimensions
        let $metadata as object? := ($component.Concepts[])[$$.Name eq $d]
        let $label as string? :=
          concepts:labels(
            $d,
            $concepts:VERBOSE_LABEL_ROLE,
            ($options.Language, "en")[1],
            $components.Concepts[],
            $options
          )[1]
        return
            components:standard-explicit-dimension-breakdown(
                $d,
                ($label, $metadata.Label)[1],
                $table.Aspects.$d.Members[].Name,
                $component.Role),
        components:standard-entity-breakdown()[not (($auto-slice-dimensions, $user-slice-dimensions) = "xbrl:Entity")]
    )

    let $lineitems as string* := components:line-items($component)
    let $presentation-network as object? := networks:networks-for-components-and-short-names($component, "Presentation")
    let $roots as string* := $presentation-network.Trees[].Name
    let $lineitems as string* := if(exists($lineitems)) then $lineitems else $roots
    let $y-breakdowns as object := components:standard-concept-breakdown($lineitems, $component.Role)

    return {
        ModelKind: "DefinitionModel",
        Labels: [$component.Label],
        Parameters: {},
        Breakdowns: {
            "x" : [
                $x-breakdowns
            ],
            "y": [
                $y-breakdowns
            ]
        },
        TableFilters: {|
            for $d as string in distinct-values(($auto-slice-dimensions, $user-slice-dimensions))
            return if($d = $user-slice-dimensions)
                   then { $d : $options.Slicers.$d }
                   else { $d : $values-by-dimension.$d[] },
            if (not $auto-slice)
            then { "xbrl28:Archive" : $component.Archive }
            else ()
        |}
    }
};

declare function components:filter-override(
    $entities-or-eids as item*,
    $archives-or-aids as item*
) as object?
{
    let $override := exists(($entities-or-eids, $archives-or-aids))
    where $override
    let $eids := entities:eid($entities-or-eids)
    let $aids := archives:aid($archives-or-aids)
    return
     {
        "xbrl:Entity" : {|
          { Type: "string" },
          { Domain: [ $eids ] }[exists($eids)]
        |},

        "xbrl28:Archive" : {|
          { Type: "string" },
          { Domain : [ $aids ] }[exists($aids)]
        |}
    }
};

(:~
:
Merges the supplied components, grouping networks by arc/link name/role
: and merging all hypercubes into a single one.

:
: @param $components the input components.
:
: @return the merged component.
:)
declare function components:merge($components as object*) as object
{
    {
        Archive: null,
        Role: $components.Role[1],
        Label: $components.Label[1],
        Networks: [
            networks:merge($components.Networks[])
        ],
        Hypercubes:
            let $merged := hypercubes:merge(values($components.Hypercubes))
            return { $merged.Name: $merged },
        Concepts: [
            for $concept in $components.Concepts[]
            group by $name := $concept.Name
            return $concept[1]
        ]
    }
};

(:~
 :
Returns the line items, that is the top-level abstracts or concepts in the
 : presentation network.

 :
 : @param $components the input components.
 :
 : @return the line items report elements.
 :)
declare function components:line-items($components as object*) as string*
{
    let $presentation-networks as object* := networks:networks-for-components-and-short-names($components, "Presentation")
    return components:line-items-recursive($components, $presentation-networks.Trees[])
};

declare %private function components:line-items-recursive($components as object*, $networks as object*) as string*
{
  let $concepts as object* := $components.Concepts[]
  for $network in $networks
  let $concept-name := $network.Name
  let $substitution-group as string := $concepts[$$.Name eq $concept-name].SubstitutionGroup[1]
  let $has-table-child as boolean := $concepts[$$.Name = $network.To[].Name].SubstitutionGroup = "xbrldt:hypercubeItem"
  return switch(true)
          case $has-table-child return components:line-items-recursive($components, $network.To[])
          case $substitution-group eq "xbrldt:dimensionItem" return ()
          case $substitution-group eq "xbrldt:hypercubeItem" return ()
          default return $concept-name
};


Fixed broken test.

Fixed broken test.

jsoniq version "1.0";

module namespace conversion = "http://28.io/modules/xbrl/conversion";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare %private variable $conversion:STANDARD_LABELS :=
    {
        "xbrl28:Archive" : "Accession Number",
        "xbrl:Concept" : "Concept",
        "xbrl:Entity" : "Entity",
        "xbrl:Period" : "Period",
        "sec:FiscalPeriod" : "Fiscal Period",
        "sec:FiscalYear" : "Fiscal Year",
        "sec:FiscalPeriodType" : "Fiscal Period Type",
        "sec:Accepted" : "Accepted",
        "xbrl:Unit": "Unit",
        "dei:LegalEntityAxis": "Legal Entity"
    };

declare %private function conversion:aspect-label(
    $aspect as string,
    $fact as object) as string
{
    ($fact.Labels.$aspect, $conversion:STANDARD_LABELS.$aspect, $aspect)[1]
};

declare %private function conversion:aspect-value-or-label(
    $aspect as string,
    $fact as object) as atomic
{
    let $aspect-value as atomic := $fact.Aspects.($aspect)
    let $labels as object? := $fact.Labels
    return
        switch(true)
        case ($aspect-value instance of string and exists($labels.($aspect-value)))
          return $labels.($aspect-value)
        default return $aspect-value
};

declare function conversion:get-standard-labels($fact as object, $entityName as string?) as object {
    {|
        for $aspect as string in keys($fact.Aspects)
        let $aspect-label as string := conversion:aspect-label($aspect, $fact)
        let $aspect-value as atomic := conversion:aspect-value-or-label($aspect, $fact)
        return
            (
                {
                    $aspect : $aspect-label
                },
                switch(true)
                case ($aspect eq "xbrl:Entity" and starts-with($aspect-value, "http://www.sec.gov/CIK "))
                  return {
                      $aspect-value : ($entityName, substring-after($aspect-value, "http://www.sec.gov/CIK "))[1]
                  }
                case ($aspect eq "xbrl:Unit" and starts-with($aspect-value, "iso4217:"))
                  return {
                      $aspect-value : substring-after($aspect-value, "iso4217:")
                  }
                default return ()
            ),
        if(exists($fact.Unit) and starts-with($fact.Unit, "iso4217:"))
        then
            {
                $fact.Unit : substring-after($fact.Unit, "iso4217:")
            }
        else ()
    |}
};

declare function conversion:facts-to-csv(
    $facts as object*,
    $options as object?) as string?
{
    let $projection :=
        if($options.Caller eq "Report")
        then ("Value",
              "Decimals",
              "EntityRegistrantName",
              "ReportedConcept")
        else ("Value",
              "Type",
              "Decimals")
    let $use-labels as boolean := exists($facts.Labels)
    return if (exists($facts)) (: bug in csv:serialize :)
           then string-join(
                csv:serialize(
                    for $fact in $facts
                    return
                        {|
                            if($use-labels)
                            then
                                for $aspect as string in keys($fact.Aspects)[$$ ne "sec:Archive"]
                                let $aspect-label as string := conversion:aspect-label($aspect, $fact)
                                let $aspect-value as atomic := conversion:aspect-value-or-label($aspect, $fact)
                                return
                                    {
                                        $aspect-label : $aspect-value
                                    }
                            else
                                $fact.Aspects,
                            {
                                "Unit":
                                    if(starts-with($fact.Unit, "iso4217:"))
                                    then substring-after($fact.Unit, "iso4217:")
                                    else $fact.Unit
                            }[exists($fact.Unit)],
                            project($fact, $projection)
                        |},
                    { serialize-null-as : "" }
                )
            )
            else ()
};

declare function conversion:facts-to-xml(
    $facts as object*,
    $options as object?) as element()*
{
    for $fact in $facts
    let $aspects := $fact.Aspects
    let $use-labels as boolean := exists($facts.Labels)
    return
        <Fact>{
            <Aspects>{
                for $aspect in keys($aspects)
                return
                    <Aspect>
                        <Name>{
                            if($use-labels)
                            then attribute { "label" } { conversion:aspect-label($aspect, $fact) }
                            else (),
                            $aspect
                        }</Name>
                        <Value>{
                            if($use-labels)
                            then attribute { "label" } { conversion:aspect-value-or-label($aspect, $fact) }
                            else (),
                            $aspects.$aspect
                        }</Value>
                    </Aspect>
            }</Aspects>,
            if($options.Caller eq "Report")
            then (
                <Value>{$fact.Value}</Value>,
                <Type>{$fact.Type}</Type>,
                <Unit>{$fact.Unit}</Unit>[exists($fact.Unit)],
                <Decimals>{$fact.Decimals}</Decimals>[exists($fact.Decimals)],
                <EntityRegistrantName>{$fact.EntityRegistrantName}</EntityRegistrantName>,
                conversion:audittrail-to-xml($fact.AuditTrails)[exists($fact.AuditTrails)]
            ) else (
                <Type>{$fact.Type}</Type>,
                <Value>{$fact.Value}</Value>,
                <Unit>{$fact.Unit}</Unit>[exists($fact.Unit)],
                <Decimals>{$fact.Decimals}</Decimals>[exists($fact.Decimals)]
            )
        }</Fact>
};

declare function conversion:audittrail-to-xml($audit as item) as element()
{
    <AuditTrails>{
        for $a in flatten($audit)
        return (
            <Type>{$a.Type}</Type>,
            <Label>{$a.Label}</Label>,
            <Message>{$a.Message}</Message>,
            <Data>{
                if (exists($a.Data.OriginalConcept))
                then <OriginalConcept>{$a.Data.OriginalConcept}</OriginalConcept>
                else (),
                if (exists($a.Data.OutputConcept))
                then <OutputConcept>{$a.Data.OutputConcept}</OutputConcept>
                else (),
                if (exists($a.Data.AuditTrails))
                then conversion:audittrail-to-xml($a.Data.AuditTrails)
                else ()
            }</Data>
        )
    }</AuditTrails>
};

jsoniq version "1.0";

module namespace conversion = "http://28.io/modules/xbrl/conversion";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare %private function conversion:aspect-label(
    $aspect as string,
    $fact as object) as string
{
    ($fact.Labels.$aspect, $aspect)[1]
};

declare %private function conversion:aspect-value-or-label(
    $aspect as string,
    $fact as object) as atomic
{
    let $aspect-value as atomic := $fact.Aspects.($aspect)
    let $labels as object? := $fact.Labels
    return
        switch(true)
        case ($aspect-value instance of string and exists($labels.($aspect-value)))
          return $labels.($aspect-value)
        default return $aspect-value
};

declare function conversion:get-standard-labels($fact as object, $entityName as string?) as object {
    {|
        for $aspect as string in keys($fact.Aspects)
        let $aspect-label as string := conversion:aspect-label($aspect, $fact)
        let $aspect-value as atomic := conversion:aspect-value-or-label($aspect, $fact)
        return
            (
                {
                    $aspect : $aspect-label
                },
                switch(true)
                case ($aspect eq "xbrl:Entity" and starts-with($aspect-value, "http://www.sec.gov/CIK "))
                  return {
                      $aspect-value : ($entityName, substring-after($aspect-value, "http://www.sec.gov/CIK "))[1]
                  }
                case ($aspect eq "xbrl:Unit" and starts-with($aspect-value, "iso4217:"))
                  return {
                      $aspect-value : substring-after($aspect-value, "iso4217:")
                  }
                default return ()
            ),
        if(exists($fact.Unit) and starts-with($fact.Unit, "iso4217:"))
        then
            {
                $fact.Unit : substring-after($fact.Unit, "iso4217:")
            }
        else ()
    |}
};

declare function conversion:facts-to-csv(
    $facts as object*,
    $options as object?) as string?
{
    let $projection :=
        if($options.Caller eq "Report")
        then ("Value",
              "Decimals",
              "EntityRegistrantName",
              "ReportedConcept")
        else ("Value",
              "Type",
              "Decimals")
    let $use-labels as boolean := exists($facts.Labels)
    return if (exists($facts)) (: bug in csv:serialize :)
           then string-join(
                csv:serialize(
                    for $fact in $facts
                    return
                        {|
                            if($use-labels)
                            then
                                for $aspect as string in keys($fact.Aspects)[$$ ne "sec:Archive"]
                                let $aspect-label as string := conversion:aspect-label($aspect, $fact)
                                let $aspect-value as atomic := conversion:aspect-value-or-label($aspect, $fact)
                                return
                                    {
                                        $aspect-label : $aspect-value
                                    }
                            else
                                $fact.Aspects,
                            {
                                "Unit":
                                    if(starts-with($fact.Unit, "iso4217:"))
                                    then substring-after($fact.Unit, "iso4217:")
                                    else $fact.Unit
                            }[exists($fact.Unit)],
                            project($fact, $projection)
                        |},
                    { serialize-null-as : "" }
                )
            )
            else ()
};

declare function conversion:facts-to-xml(
    $facts as object*,
    $options as object?) as element()*
{
    for $fact in $facts
    let $aspects := $fact.Aspects
    let $use-labels as boolean := exists($facts.Labels)
    return
        <Fact>{
            <Aspects>{
                for $aspect in keys($aspects)
                return
                    <Aspect>
                        <Name>{
                            if($use-labels)
                            then attribute { "label" } { conversion:aspect-label($aspect, $fact) }
                            else (),
                            $aspect
                        }</Name>
                        <Value>{
                            if($use-labels)
                            then attribute { "label" } { conversion:aspect-value-or-label($aspect, $fact) }
                            else (),
                            $aspects.$aspect
                        }</Value>
                    </Aspect>
            }</Aspects>,
            if($options.Caller eq "Report")
            then (
                <Value>{$fact.Value}</Value>,
                <Type>{$fact.Type}</Type>,
                <Unit>{$fact.Unit}</Unit>[exists($fact.Unit)],
                <Decimals>{$fact.Decimals}</Decimals>[exists($fact.Decimals)],
                <EntityRegistrantName>{$fact.EntityRegistrantName}</EntityRegistrantName>,
                conversion:audittrail-to-xml($fact.AuditTrails)[exists($fact.AuditTrails)]
            ) else (
                <Type>{$fact.Type}</Type>,
                <Value>{$fact.Value}</Value>,
                <Unit>{$fact.Unit}</Unit>[exists($fact.Unit)],
                <Decimals>{$fact.Decimals}</Decimals>[exists($fact.Decimals)]
            )
        }</Fact>
};

declare function conversion:audittrail-to-xml($audit as item) as element()
{
    <AuditTrails>{
        for $a in flatten($audit)
        return (
            <Type>{$a.Type}</Type>,
            <Label>{$a.Label}</Label>,
            <Message>{$a.Message}</Message>,
            <Data>{
                if (exists($a.Data.OriginalConcept))
                then <OriginalConcept>{$a.Data.OriginalConcept}</OriginalConcept>
                else (),
                if (exists($a.Data.OutputConcept))
                then <OutputConcept>{$a.Data.OutputConcept}</OutputConcept>
                else (),
                if (exists($a.Data.AuditTrails))
                then conversion:audittrail-to-xml($a.Data.AuditTrails)
                else ()
            }</Data>
        )
    }</AuditTrails>
};