:lipstick: Add signature to fix warning.

jsoniq version "1.0";
(:
 : Copyright 2012-2014 28msec Inc.
 :)

(:~
 :
This module provides functionality for retrieving components.

 :
 :
Components help organizing the facts contained in archives
 : in smaller parts that "make sense" together. A component is identified
 : with a CID (component ID).

 :
 :
A component is made of networks (see the networks module) and hypercubes
 : (see the hypercubes module).

 :
 :
With this module, you can retrieve all components, all components belonging
 : to one or several archives. You can retrieve the CID of a component or retrieve
 : the components associated with a couple of CIDs.

 :
 :)
module namespace components = "http://28.io/modules/xbrl/components";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace networks = "http://28.io/modules/xbrl/networks";
import module namespace resolution = "http://28.io/modules/xbrl/resolution";
import module namespace layout = "http://28.io/modules/xbrl/layout";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";

import module namespace accountant = "http://28.io/modules/xbrl/profiles/accountant/converter";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Name of the collection the components are stored in.
 :)
declare variable $components:col as xs:string := "components";

(:~
 : Name of the field pointing to the archive.
 :)
declare variable $components:ARCHIVE as xs:string := "Archive";

(:~
 : Name of the field pointing to the role.
 :)
declare variable $components:ROLE as xs:string := "Role";

(:~
 : Name of the field pointing to the default language.
 :)
declare variable $components:DEFAULT-LANGUAGE as xs:string := "Language";

(:~
 : Placeholder for all archive IDs..
 :)
declare variable $components:ALL-ARCHIVES as xs:string := "";

(:~
 : Placeholder for all component roles..
 :)
declare variable $components:ALL-ROLES as xs:string := "";

(:~
 : Placeholder for all conceptss..
 :)
declare variable $components:ALL-CONCEPTS as xs:string := "";

(:~
 : Placeholder for all labels..
 :)
declare variable $components:ALL-LABELS as xs:string := "";

(:~
 :
Retrieves all components.

 :
 : @return all components.
 :)
declare function components:components() as object*
{
  mw:find($components:col,{})
};

(:~
 :
Retrieves the components with the given CIDs.

 :
 : @param $component-or-ids the CIDs or the components themselves.
 :
 : @return the components whose _id field matches one of these CIDs.
 :)
declare function components:components($component-or-ids as item*) as object*
{
  let $ids as string* :=
    for $s in $component-or-ids
    where $s instance of string
    return $s
  let $components as object* :=
    for $s in $component-or-ids
    where $s instance of object
    return $s
  return
    (
      $components,
      if (exists($ids))
      then mw:find($components:col,{ "_id" : { "$in" : [ $ids ! components:cid($$) ] } })
      else ()
    )
};

declare function components:components-for(
  $archives-or-ids as item*,
  $roles as string*,
  $concepts as string*,
  $exact-labels as string*,
  $options as object*
) as object*
{
  let $aids := if(deep-equal($archives-or-ids, $components:ALL-ARCHIVES))
               then $components:ALL-ARCHIVES
               else archives:aid($archives-or-ids)
  let $query := {|
    { "Archive" : { "$in" : [ $aids ] } }[not $aids = $components:ALL-ARCHIVES],
    { "Role" : { "$in" : [ $roles ] } }[not $roles = $components:ALL-ROLES],
    { "Concepts.Name" : { "$in" : [ $concepts ] } }[not $concepts = $components:ALL-CONCEPTS],
    { "Concepts.Labels.Value" : { "$in" : [ $exact-labels ] } }[not $exact-labels = $components:ALL-LABELS]
  |}
  return if($options.LabelsOnly)
         then mw:find($components:col, $query, {
           "Archive" : 1,
           "Role" : 1,
           "Concepts.Labels" : 1,
           "Concepts.Name" : 1 }
         )
         else mw:find($components:col, $query)
};

(:~
 :
Retrieves all components that belong to the supplied archives.

 :
 : @param $archive-or-id an archive or its AID.
 :
 : @return all components in the archive with this AID.
 :)
declare function components:components-for-archives($archive-or-ids as item*) as object*
{
  for $archive-or-id in $archive-or-ids
  return mw:find($components:col,{ $components:ARCHIVE: archives:aid($archive-or-id) })
};

(:~
 : Retrieves all components for the supplied filings and roles.
 :
 : @param $archives-or-ids a sequence of archives or AIDs.
 : @param $roles a sequence of role URIs.
 :
 : @return the components.
 :)
declare function components:components-for-archives-and-roles(
    $archives-or-ids as item*,
    $roles as string*) as object*
{
    let $aids := archives:aid($archives-or-ids)
    return
        mw:find($components:col,
        {
            $components:ARCHIVE: { "$in" : [ $aids ] },
            "Role": { "$in" : [ $roles ] }
        })
};

(:~
 : Retrieves all components for the supplied filings and that use the supplied concepts.
 :
 : @param $archives-or-ids a sequence of archives or AIDs.
 : @param $concepts a sequence of concept names.
 :
 : @return the components.
 :)
declare function components:components-for-archives-and-concepts(
    $archives-or-ids as item*,
    $concepts as string*) as object*
{
    let $aids as string* := archives:aid($archives-or-ids)
    let $concepts := mw:find($concepts:col,
        {|
            (
                { "Name" : { "$in" : [ $concepts ] } },
                { "Archive" : { "$in" : [ $aids ] } }
            )
        |})
    let $roles := $concepts.Role
    for $component in components:components-for-archives-and-roles($aids, $roles)
    where (some $concept in $concepts satisfies $concept.Archive eq $component.Archive and $concept.Role eq $component.Role)
    return $component
};

(:~
 :
Retrieves all definition models in a component.

 :
 : @param $component a component object.
 :
 : @return all definition models
 :)
declare function components:definition-models-for-components(
    $components as object*) as object*
{
    $components.DefinitionModels[]
};

(:~
 :
Retrieves definition models in a component
 : that have the supplied names.

 :
 : @param $component a component object.
 : @param $names a sequence of names.
 :
 : @return the definition models.
 :)
declare function components:definition-models-for-components(
    $components as object*,
    $names as string*) as object
{
    $components.DefinitionModels[][$$.Name = $names]
};

(:~
 :
Selects a table from the component.

 :
 : @param $component a component object.
 : @param $options
some optional parameters, including:

 :

 :
HypercubeName: a string specifying which hypercube to use. By default, one of the non-default hypercubes will be arbitrarily chosen. If none
 :  is available, the default hypercube will be picked.

 :

 :
 : @error components:HYPERCUBE-DOES-NOT-EXIST if the specified hypercube is not found.
 : @return a definition model
 :)
declare function components:select-table(
    $component as object,
    $options as object?) as object
{
    let $user-chosen-table as object? := hypercubes:hypercubes-for-components($component, $options.HypercubeName)
    let $non-implicit-table as object? := hypercubes:hypercubes-for-components($component)[1]
    return
        switch(true)
        case exists($options.HypercubeName) and empty($user-chosen-table)
            return error(QName("components:HYPERCUBE-DOES-NOT-EXIST"), $options.HypercubeName || " : Hypercube does not exist.")
        case exists($user-chosen-table)
            return $user-chosen-table
        case exists($non-implicit-table)
            return $non-implicit-table
        default return error(QName("components:HYPERCUBE-DOES-NOT-EXIST"), "Components does not contain any hypercube.")
};

(:~
 :
 :
Retrieves all facts that are relevant to the
 : supplied component, and populates them with the default dimension values
 : when missing.

 :
The default hypercube will be taken, and the concept maps and rules will
 : be used if any.

 :
 : @param $component a component object.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:facts(
    $component as object
)
as object*
{
  components:facts($component, {})
};

(:~
 :
 :
Retrieves all facts that are relevant to the
 : supplied component, and populates them with the default dimension values
 : when missing.

 :
 :
 : @param $report-or-id a report or its RID,
 : @param $options standard fact retrieving options as well as
 :

 :
FilterOverride: hypercube dimension specs (as defined as hypercubes:user-defined-hypercube)
 :   to override filters in the report's hypercube.

 :
HypercubeName: picks a different hypercube than the default one.

 :

 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:facts(
    $component as object,
    $options as object?
)
as object*
{
  let $concept-map := networks:networks-for-components-and-short-names(
      $component,
      "ConceptMap")
  let $rules := (flatten($component.Rules), flatten($options.Rules))
  let $overriden-hypercube as object? :=
        if(exists($options.Hypercube))
        then $options.Hypercube
        else components:select-table($component, $options)
    let $hypercube :=
    if(exists($options.FilterOverride))
    then hypercubes:modify-hypercube(
      $overriden-hypercube,
      $options.FilterOverride
    )
    else $overriden-hypercube
  let $concepts as object* := $component.Concepts[]
  return hypercubes:facts(
          $hypercube,
          {|
            trim($options, "Rules"),
            { "ConceptMaps": $concept-map }[exists($concept-map)],
            { "Rules": $rules }[exists($rules)],
            { "Concepts": [ $concepts ] }[exists($concepts)]
          |}
         )
};

(:~
 :
 :
Retrieves the spreadsheet-like rendering for the component.

 :
The default definition model will be taken.

 :
The default hypercube will be taken, and the concept maps and rules will
 : be used if any.

 :
 : @param $component a component object.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:spreadsheet(
    $component as object
)
as object*
{
  components:spreadsheet($component, {})
};

(:~
 :
 :
Retrieves the spreadsheet-like rendering for the component.

 :
 : @param $report-or-id a report or its RID,
 : @param $options standard fact retrieving options as well as
 :

 :
FilterOverride: hypercube dimension specs (as defined as hypercubes:user-defined-hypercube)
 :   to override filters in the report's hypercube.

 :
HypercubeName: picks a different hypercube than the default one.

 :

 :
 :
 : @error err:XPTY0004 if zero, or more than one definition models are found.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:spreadsheet(
    $component as object,
    $options as object?
)
as object*
{
    let $definition-model as object :=
        switch(true)
        case exists($options.DefinitionModel)
            return $options.DefinitionModel
        case (exists($options.DefinitionModelName))
            return components:definition-models-for-components($component, $options.DefinitionModelName)
        default
            return components:definition-models-for-components($component)
    let $overriden-hypercube as object? :=
        if(exists($options.Hypercube))
        then $options.Hypercube
        else components:select-table($component, $options)
    let $concept-map as object?:= networks:networks-for-components-and-short-names(
      $component,
      "ConceptMap")
    let $rules as array? := $component.Rules
    let $concepts as object* := $component.Concepts[]
    let $new-options as object :=
        {|
            trim($options, "Rules"),
            {"ConceptMaps": $concept-map }[exists($concept-map)],
            { "Rules": [ ( flatten($rules[]), flatten($options.Rules) ) ] }[exists(($rules, $options.Rules))],
            {
                Hypercube:
                    if(exists($options.FilterOverride))
                    then hypercubes:modify-hypercube(
                        $overriden-hypercube,
                        $options.FilterOverride
                    )
                    else $overriden-hypercube
            }[exists($overriden-hypercube)],
            { "Concepts": [ $concepts ] }[exists($concepts)]
        |}
    let $structural-model as object := resolution:resolve(
        $definition-model,
        $component,
        $new-options)
    let $layout-model as object := layout:layout($structural-model, $new-options)
    let $accountant-model as object := accountant:flatten-row-headers($layout-model)
    let $result := if($options.FlattenRows) then $accountant-model else $layout-model
    return
        if($options.Debug)
        then copy $r := $result
             modify (
                 insert json { Definition: $definition-model } into $r,
                 insert json { Structural: $structural-model } into $r
             )
             return $r
        else $result
};

(:~
 : Return the number of hypercubes in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of hypercubes
 :)
declare function components:num-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumHypercubes
};

(:~
 : Return the number of networks in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of networks
 :)
declare function components:num-networks($components as object*) as integer*
{
  $components ! $$.Statistics.NumNetworks
};

(:~
 : Return the number of (distinct) explicit dimensions in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of dimensions
 :)
declare function components:num-explicit-dimensions($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctExplicitDimensions
};

(:~
 : Return the number of (distinct) domains in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of domains
 :)
declare function components:num-domains($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctDomains
};

(:~
 : Return the number of (distinct) members in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of members
 :)
declare function components:num-members($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctMembers
};

(:~
 : Return the number of (distinct) concrete primary items being
 : in a hypercube for each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of primary items
 :)
declare function components:num-concrete-primary-items-in-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctConcretePrimaryItemsInHypercubes
};

(:~
 : Return the number of (distinct) abstract primary items being
 : in a hypercube for each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of primary items
 :)
declare function components:num-abstract-primary-items-in-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctAbstractPrimaryItemsInHypercubes
};

(:~
 : Return the number of (distinct) concrete primary items not being
 : in a hypercube for each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of primary items
 :)
declare function components:num-concrete-primary-items-not-in-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctConcretePrimaryItemsNotInHypercubes
};

(:~
 : Return the number of (distinct) abstract primary items not being
 : in a hypercube for each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of primary items
 :)
declare function components:num-distinct-abstract-primary-items-not-in-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctAbstractPrimaryItemsNotInHypercubes
};

(:~
 :
 :
Converts the input to a normalized component id (CID). The input
 : can be either an CID, or an component object which contains an _id.

 :
 : @param $component-or-id an component object or an CID.
 :
 : @error components:INVALID_PARAMETER if the CID or component is not valid.
 :
 : @return the normalized CID.
 :)
declare function components:cid($component-or-id as item) as atomic
{
  typeswitch ($component-or-id)
  case object return
    let $id := $component-or-id._id
    return if(exists($id))
           then $id
           else error(QName("components:INVALID_PARAMETER"),
                      "Invalid component provided (no _id field)")
  case $id as atomic return $id
  default return error(
      QName("components:INVALID_PARAMETER"),
      "Invalid component or id (must be an object or an atomic): "
      || serialize($component-or-id))
};

declare function components:standard-typed-dimension-breakdown(
    $components as object*,
    $dimension-name as string,
    $options as object?) as object
{
    let $label as string := (
      concepts:labels(
        $dimension-name,
        $concepts:VERBOSE_LABEL_ROLE,
        ($options.Language, $components.$components:DEFAULT-LANGUAGE)[1],
        $components.Concepts[],
        $options
      ),
      concepts:labels(
        $dimension-name,
        $concepts:STANDARD_LABEL_ROLE,
        ($options.Language, $components.$components:DEFAULT-LANGUAGE)[1],
        $components.Concepts[],
        $options
      ),
      $dimension-name)[1]
    return
    {
      BreakdownLabels: [ $label || " breakdown" ],
      BreakdownTrees: [
        {
            Kind: "Rule",
            Abstract: true,
            Labels: [ $label ],
            Children: [
              {
                Kind: "Aspect",
                Aspect: $dimension-name
              }
            ]
        }
      ]
    }
};

declare function components:standard-explicit-dimension-breakdown(
    $components as object*,
    $dimension-name as string,
    $domain-names as string*,
    $role as string,
    $options as object?) as object
{
    let $label as string := (
      concepts:labels(
        $dimension-name,
        $concepts:VERBOSE_LABEL_ROLE,
        ($options.Language, $components.$components:DEFAULT-LANGUAGE)[1],
        $components.Concepts[],
        $options
      ),
      concepts:labels(
        $dimension-name,
        $concepts:STANDARD_LABEL_ROLE,
        ($options.Language, $components.$components:DEFAULT-LANGUAGE)[1],
        $components.Concepts[],
        $options
      ),
      $dimension-name)[1]
    return
    {
        BreakdownLabels: [ $label || " breakdown" ],
        BreakdownTrees: [
            {
                Kind: "Rule",
                Abstract: true,
                Labels: [ $label ],
                Children: [
                    for $domain as string in $domain-names
                    return {
                        Kind: "DimensionRelationship",
                        LinkRole: $role,
                        Dimension: $dimension-name,
                        RelationshipSource: $domain,
                        FormulaAxis: "descendant",
                        Generations: 0
                    }
                ]
            }
        ]
    }
};

declare function components:standard-concept-breakdown(
    $line-items-elements as string*,
    $role as string) as object
{
    {
        BreakdownLabels: [ "Breakdown on concepts" ],
        BreakdownTrees: [
            for $lineitems as string in $line-items-elements
            return {
                Kind: "ConceptRelationship",
                LinkName: "link:presentationLink",
                LinkRole: $role,
                ArcName: "link:presentationArc",
                ArcRole: "http://www.xbrl.org/2003/arcrole/parent-child",
                RelationshipSource: $lineitems,
                FormulaAxis: "descendant",
                Generations: 0,
                RollUpAgainstCalculationNetwork: false
            }
        ]
    }
};

(:~
 :
Builds a standard definition model out of the specified component.

 :
The concepts will be put on the y axis according to the presentation network.

 :
The other dimensions are put on the x axis, with one breakdown for each.

 :
Explicit dimensions are organized according to the dimension hierarchy from the domain-member network.

 :
Typed dimensions are organized according to the actual values.

 :
 :
One of the non-default hypercubes will be arbitrarily chosen. If none is available, the default hypercube will be picked.

 :
Auto slicing will be performed against the fact table
 :
 : @param $component a component object.
 :
 : @return a definition model
 :)
declare function components:standard-definition-models-for-components($components as object*) as object
{
    components:standard-definition-models-for-components($components, ())
};

(:~
 :
Builds a standard definition model out of the specified component.

 :
The concepts will be put on the y axis according to the presentation network.

 :
The other dimensions are put on the x axis, with one breakdown for each.

 :
Explicit dimensions are organized according to the dimension hierarchy from the domain-member network.

 :
Typed dimensions are organized according to the actual values.

 :
 : @param $component a component object.
 : @param $options
some optional parameters, including:

 :

 :
HypercubeName: a string specifying which hypercube to use. By default, one of the non-default hypercubes will be arbitrarily chosen. If none
 :  is available, the default hypercube will be picked.

 :
AutoSlice: a boolean specifying whether or not slicing should be done automatically, looking at the fact table. Deactivating auto slicing will
 :  lead to better performance, but a more verbose table. If AutoSlice is active, dimensions with only one value in the fact pool will become
 : global filters rather than breakdowns on the x axis.

 :
Slicers: an object with forced slicers.

 :

 :
 : @error components:HYPERCUBE-DOES-NOT-EXIST if the specified hypercube is not found.
 : @return a definition model
 :)
declare function components:standard-definition-models-for-components($components as object*, $options as object?) as object
{
    for $component in $components
    let $table as object := components:select-table($component, $options)

    let $auto-slice as boolean := empty($options.AutoSlice) or $options.AutoSlice
    let $facts as object*:=
        if($auto-slice)
        then hypercubes:facts($table)
        else ()
    let $dimensions as string*:= keys($table.Aspects)
    let $values-by-dimension as object := {|
        for $d in $dimensions
        return { $d : [ distinct-values($facts.Aspects.$d) ] }
    |}
    let $auto-slice-dimensions as string* :=
        keys($values-by-dimension)[size($values-by-dimension.$$) eq 1 and not ($$ = ("xbrl:Period", "sec:FiscalYear",  "sec:FiscalPeriod") ) ]
    let $user-slice-dimensions as string* :=
        keys($options.Slicers)

    let $column-dimensions as string* := keys($values-by-dimension)[not $$ =
        ("xbrl:Concept",
        "xbrl:Period",
        "xbrl:Unit",
        "xbrl:Entity",
        "xbrl28:Archive",
        $options.HideDimensions[],
        $auto-slice-dimensions,
        $user-slice-dimensions)]

    let $x-breakdowns as object* := (
        components:standard-typed-dimension-breakdown($components, "xbrl:Period", $options)
          [not (($auto-slice-dimensions, $user-slice-dimensions) = "xbrl:Period")],
        for $d as string in $column-dimensions
        let $dimension-object as object := $table.Aspects.$d
        let $is-typed as boolean := boolean($dimension-object.Kind eq "TypedDimension")
        return
            if($is-typed)
            then
              components:standard-typed-dimension-breakdown(
                $components,
                $d,
                $options)
            else
              components:standard-explicit-dimension-breakdown(
                $components,
                $d,
                $dimension-object.Members[].Name,
                $component.Role,
                $options),
        components:standard-typed-dimension-breakdown($components, "xbrl:Entity", $options)
          [not (($auto-slice-dimensions, $user-slice-dimensions) = "xbrl:Entity")]
    )

    let $lineitems as string* := components:line-items($component)
    let $presentation-network as object? := networks:networks-for-components-and-short-names($component, "Presentation")
    let $roots as string* := $presentation-network.Trees[].Name
    let $lineitems as string* := if(exists($lineitems)) then $lineitems else $roots
    let $y-breakdowns as object :=
      if(exists($presentation-network))
        then components:standard-concept-breakdown($lineitems, $component.Role)
        else components:standard-explicit-dimension-breakdown(
            $components,
            "xbrl:Concept",
            $table.Aspects."xbrl:Concept".Members[].Name,
            $component.Role,
            $options)

    return {
        ModelKind: "DefinitionModel",
        Labels: [$component.Label],
        Parameters: {},
        Breakdowns: {
            "x" : [
                $x-breakdowns
            ],
            "y": [
                $y-breakdowns
            ]
        },
        TableFilters: {|
            for $d as string in distinct-values(($auto-slice-dimensions, $user-slice-dimensions))
            return if($d = $user-slice-dimensions)
                   then { $d : $options.Slicers.$d }
                   else { $d : $values-by-dimension.$d[] },
            if (not $auto-slice)
            then { "xbrl28:Archive" : $component.Archive }
            else ()
        |}
    }
};

declare function components:filter-override(
    $entities-or-eids as item*,
    $archives-or-aids as item*
) as object?
{
    let $override := exists(($entities-or-eids, $archives-or-aids))
    where $override
    let $eids := entities:eid($entities-or-eids)
    let $aids := archives:aid($archives-or-aids)
    return
     {
        "xbrl:Entity" : {|
          { Type: "string" },
          { Domain: [ $eids ] }[exists($eids)]
        |},

        "xbrl28:Archive" : {|
          { Type: "string" },
          { Domain : [ $aids ] }[exists($aids)]
        |}
    }
};

(:~
:
Merges the supplied components, grouping networks by arc/link name/role
: and merging all hypercubes into a single one.

:
: @param $components the input components.
:
: @return the merged component.
:)
declare function components:merge($components as object*) as object
{
    {
        Archive: null,
        Role: $components.Role[1],
        Label: $components.Label[1],
        Networks: [
            networks:merge($components.Networks[])
        ],
        Hypercubes:
            let $merged := hypercubes:merge(values($components.Hypercubes))
            return { $merged.Name: $merged },
        Concepts: [
            for $concept in $components.Concepts[]
            group by $name := $concept.Name
            return $concept[1]
        ]
    }
};

(:~
 :
Returns the line items, that is the top-level abstracts or concepts in the
 : presentation network.

 :
 : @param $components the input components.
 :
 : @return the line items report elements.
 :)
declare function components:line-items($components as object*) as string*
{
    let $presentation-networks as object* := networks:networks-for-components-and-short-names($components, "Presentation")
    return components:line-items-recursive($components, $presentation-networks.Trees[])
};

declare %private function components:line-items-recursive($components as object*, $networks as object*) as string*
{
  let $concepts as object* := $components.Concepts[]
  for $network in $networks
  let $concept-name := $network.Name
  let $substitution-group as string := $concepts[$$.Name eq $concept-name].SubstitutionGroup[1]
  let $has-table-child as boolean := $concepts[$$.Name = $network.To[].Name].SubstitutionGroup = "xbrldt:hypercubeItem"
  return switch(true)
          case $has-table-child return components:line-items-recursive($components, $network.To[])
          case $substitution-group eq "xbrldt:dimensionItem" return ()
          case $substitution-group eq "xbrldt:hypercubeItem" return ()
          default return $concept-name
};

jsoniq version "1.0";
(:
 : Copyright 2012-2014 28msec Inc.
 :)

(:~
 :
This module provides functionality for retrieving components.

 :
 :
Components help organizing the facts contained in archives
 : in smaller parts that "make sense" together. A component is identified
 : with a CID (component ID).

 :
 :
A component is made of networks (see the networks module) and hypercubes
 : (see the hypercubes module).

 :
 :
With this module, you can retrieve all components, all components belonging
 : to one or several archives. You can retrieve the CID of a component or retrieve
 : the components associated with a couple of CIDs.

 :
 :)
module namespace components = "http://28.io/modules/xbrl/components";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace networks = "http://28.io/modules/xbrl/networks";
import module namespace resolution = "http://28.io/modules/xbrl/resolution";
import module namespace layout = "http://28.io/modules/xbrl/layout";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";

import module namespace accountant = "http://28.io/modules/xbrl/profiles/accountant/converter";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Name of the collection the components are stored in.
 :)
declare variable $components:col as xs:string := "components";

(:~
 : Name of the field pointing to the archive.
 :)
declare variable $components:ARCHIVE as xs:string := "Archive";

(:~
 : Name of the field pointing to the role.
 :)
declare variable $components:ROLE as xs:string := "Role";

(:~
 : Name of the field pointing to the default language.
 :)
declare variable $components:DEFAULT-LANGUAGE as xs:string := "Language";

(:~
 : Placeholder for all archive IDs..
 :)
declare variable $components:ALL-ARCHIVES as xs:string := "";

(:~
 : Placeholder for all component roles..
 :)
declare variable $components:ALL-ROLES as xs:string := "";

(:~
 : Placeholder for all conceptss..
 :)
declare variable $components:ALL-CONCEPTS as xs:string := "";

(:~
 : Placeholder for all labels..
 :)
declare variable $components:ALL-LABELS as xs:string := "";

(:~
 :
Retrieves all components.

 :
 : @return all components.
 :)
declare function components:components() as object*
{
  mw:find($components:col,{})
};

(:~
 :
Retrieves the components with the given CIDs.

 :
 : @param $component-or-ids the CIDs or the components themselves.
 :
 : @return the components whose _id field matches one of these CIDs.
 :)
declare function components:components($component-or-ids as item*) as object*
{
  let $ids as string* :=
    for $s in $component-or-ids
    where $s instance of string
    return $s
  let $components as object* :=
    for $s in $component-or-ids
    where $s instance of object
    return $s
  return
    (
      $components,
      if (exists($ids))
      then mw:find($components:col,{ "_id" : { "$in" : [ $ids ! components:cid($$) ] } })
      else ()
    )
};

declare function components:components-for(
  $archives-or-ids as item*,
  $roles as string*,
  $concepts as string*,
  $exact-labels as string*,
  $options as object*
)
{
  let $aids := if(deep-equal($archives-or-ids, $components:ALL-ARCHIVES))
               then $components:ALL-ARCHIVES
               else archives:aid($archives-or-ids)
  let $query := {|
    { "Archive" : { "$in" : [ $aids ] } }[not $aids = $components:ALL-ARCHIVES],
    { "Role" : { "$in" : [ $roles ] } }[not $roles = $components:ALL-ROLES],
    { "Concepts.Name" : { "$in" : [ $concepts ] } }[not $concepts = $components:ALL-CONCEPTS],
    { "Concepts.Labels.Value" : { "$in" : [ $exact-labels ] } }[not $exact-labels = $components:ALL-LABELS]
  |}
  return if($options.LabelsOnly)
         then mw:find($components:col, $query, {
           "Archive" : 1,
           "Role" : 1,
           "Concepts.Labels" : 1,
           "Concepts.Name" : 1 }
         )
         else mw:find($components:col, $query)
};

(:~
 :
Retrieves all components that belong to the supplied archives.

 :
 : @param $archive-or-id an archive or its AID.
 :
 : @return all components in the archive with this AID.
 :)
declare function components:components-for-archives($archive-or-ids as item*) as object*
{
  for $archive-or-id in $archive-or-ids
  return mw:find($components:col,{ $components:ARCHIVE: archives:aid($archive-or-id) })
};

(:~
 : Retrieves all components for the supplied filings and roles.
 :
 : @param $archives-or-ids a sequence of archives or AIDs.
 : @param $roles a sequence of role URIs.
 :
 : @return the components.
 :)
declare function components:components-for-archives-and-roles(
    $archives-or-ids as item*,
    $roles as string*) as object*
{
    let $aids := archives:aid($archives-or-ids)
    return
        mw:find($components:col,
        {
            $components:ARCHIVE: { "$in" : [ $aids ] },
            "Role": { "$in" : [ $roles ] }
        })
};

(:~
 : Retrieves all components for the supplied filings and that use the supplied concepts.
 :
 : @param $archives-or-ids a sequence of archives or AIDs.
 : @param $concepts a sequence of concept names.
 :
 : @return the components.
 :)
declare function components:components-for-archives-and-concepts(
    $archives-or-ids as item*,
    $concepts as string*) as object*
{
    let $aids as string* := archives:aid($archives-or-ids)
    let $concepts := mw:find($concepts:col,
        {|
            (
                { "Name" : { "$in" : [ $concepts ] } },
                { "Archive" : { "$in" : [ $aids ] } }
            )
        |})
    let $roles := $concepts.Role
    for $component in components:components-for-archives-and-roles($aids, $roles)
    where (some $concept in $concepts satisfies $concept.Archive eq $component.Archive and $concept.Role eq $component.Role)
    return $component
};

(:~
 :
Retrieves all definition models in a component.

 :
 : @param $component a component object.
 :
 : @return all definition models
 :)
declare function components:definition-models-for-components(
    $components as object*) as object*
{
    $components.DefinitionModels[]
};

(:~
 :
Retrieves definition models in a component
 : that have the supplied names.

 :
 : @param $component a component object.
 : @param $names a sequence of names.
 :
 : @return the definition models.
 :)
declare function components:definition-models-for-components(
    $components as object*,
    $names as string*) as object
{
    $components.DefinitionModels[][$$.Name = $names]
};

(:~
 :
Selects a table from the component.

 :
 : @param $component a component object.
 : @param $options
some optional parameters, including:

 :

 :
HypercubeName: a string specifying which hypercube to use. By default, one of the non-default hypercubes will be arbitrarily chosen. If none
 :  is available, the default hypercube will be picked.

 :

 :
 : @error components:HYPERCUBE-DOES-NOT-EXIST if the specified hypercube is not found.
 : @return a definition model
 :)
declare function components:select-table(
    $component as object,
    $options as object?) as object
{
    let $user-chosen-table as object? := hypercubes:hypercubes-for-components($component, $options.HypercubeName)
    let $non-implicit-table as object? := hypercubes:hypercubes-for-components($component)[1]
    return
        switch(true)
        case exists($options.HypercubeName) and empty($user-chosen-table)
            return error(QName("components:HYPERCUBE-DOES-NOT-EXIST"), $options.HypercubeName || " : Hypercube does not exist.")
        case exists($user-chosen-table)
            return $user-chosen-table
        case exists($non-implicit-table)
            return $non-implicit-table
        default return error(QName("components:HYPERCUBE-DOES-NOT-EXIST"), "Components does not contain any hypercube.")
};

(:~
 :
 :
Retrieves all facts that are relevant to the
 : supplied component, and populates them with the default dimension values
 : when missing.

 :
The default hypercube will be taken, and the concept maps and rules will
 : be used if any.

 :
 : @param $component a component object.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:facts(
    $component as object
)
as object*
{
  components:facts($component, {})
};

(:~
 :
 :
Retrieves all facts that are relevant to the
 : supplied component, and populates them with the default dimension values
 : when missing.

 :
 :
 : @param $report-or-id a report or its RID,
 : @param $options standard fact retrieving options as well as
 :

 :
FilterOverride: hypercube dimension specs (as defined as hypercubes:user-defined-hypercube)
 :   to override filters in the report's hypercube.

 :
HypercubeName: picks a different hypercube than the default one.

 :

 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:facts(
    $component as object,
    $options as object?
)
as object*
{
  let $concept-map := networks:networks-for-components-and-short-names(
      $component,
      "ConceptMap")
  let $rules := (flatten($component.Rules), flatten($options.Rules))
  let $overriden-hypercube as object? :=
        if(exists($options.Hypercube))
        then $options.Hypercube
        else components:select-table($component, $options)
    let $hypercube :=
    if(exists($options.FilterOverride))
    then hypercubes:modify-hypercube(
      $overriden-hypercube,
      $options.FilterOverride
    )
    else $overriden-hypercube
  let $concepts as object* := $component.Concepts[]
  return hypercubes:facts(
          $hypercube,
          {|
            trim($options, "Rules"),
            { "ConceptMaps": $concept-map }[exists($concept-map)],
            { "Rules": $rules }[exists($rules)],
            { "Concepts": [ $concepts ] }[exists($concepts)]
          |}
         )
};

(:~
 :
 :
Retrieves the spreadsheet-like rendering for the component.

 :
The default definition model will be taken.

 :
The default hypercube will be taken, and the concept maps and rules will
 : be used if any.

 :
 : @param $component a component object.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:spreadsheet(
    $component as object
)
as object*
{
  components:spreadsheet($component, {})
};

(:~
 :
 :
Retrieves the spreadsheet-like rendering for the component.

 :
 : @param $report-or-id a report or its RID,
 : @param $options standard fact retrieving options as well as
 :

 :
FilterOverride: hypercube dimension specs (as defined as hypercubes:user-defined-hypercube)
 :   to override filters in the report's hypercube.

 :
HypercubeName: picks a different hypercube than the default one.

 :

 :
 :
 : @error err:XPTY0004 if zero, or more than one definition models are found.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:spreadsheet(
    $component as object,
    $options as object?
)
as object*
{
    let $definition-model as object :=
        switch(true)
        case exists($options.DefinitionModel)
            return $options.DefinitionModel
        case (exists($options.DefinitionModelName))
            return components:definition-models-for-components($component, $options.DefinitionModelName)
        default
            return components:definition-models-for-components($component)
    let $overriden-hypercube as object? :=
        if(exists($options.Hypercube))
        then $options.Hypercube
        else components:select-table($component, $options)
    let $concept-map as object?:= networks:networks-for-components-and-short-names(
      $component,
      "ConceptMap")
    let $rules as array? := $component.Rules
    let $concepts as object* := $component.Concepts[]
    let $new-options as object :=
        {|
            trim($options, "Rules"),
            {"ConceptMaps": $concept-map }[exists($concept-map)],
            { "Rules": [ ( flatten($rules[]), flatten($options.Rules) ) ] }[exists(($rules, $options.Rules))],
            {
                Hypercube:
                    if(exists($options.FilterOverride))
                    then hypercubes:modify-hypercube(
                        $overriden-hypercube,
                        $options.FilterOverride
                    )
                    else $overriden-hypercube
            }[exists($overriden-hypercube)],
            { "Concepts": [ $concepts ] }[exists($concepts)]
        |}
    let $structural-model as object := resolution:resolve(
        $definition-model,
        $component,
        $new-options)
    let $layout-model as object := layout:layout($structural-model, $new-options)
    let $accountant-model as object := accountant:flatten-row-headers($layout-model)
    let $result := if($options.FlattenRows) then $accountant-model else $layout-model
    return
        if($options.Debug)
        then copy $r := $result
             modify (
                 insert json { Definition: $definition-model } into $r,
                 insert json { Structural: $structural-model } into $r
             )
             return $r
        else $result
};

(:~
 : Return the number of hypercubes in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of hypercubes
 :)
declare function components:num-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumHypercubes
};

(:~
 : Return the number of networks in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of networks
 :)
declare function components:num-networks($components as object*) as integer*
{
  $components ! $$.Statistics.NumNetworks
};

(:~
 : Return the number of (distinct) explicit dimensions in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of dimensions
 :)
declare function components:num-explicit-dimensions($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctExplicitDimensions
};

(:~
 : Return the number of (distinct) domains in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of domains
 :)
declare function components:num-domains($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctDomains
};

(:~
 : Return the number of (distinct) members in each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of members
 :)
declare function components:num-members($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctMembers
};

(:~
 : Return the number of (distinct) concrete primary items being
 : in a hypercube for each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of primary items
 :)
declare function components:num-concrete-primary-items-in-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctConcretePrimaryItemsInHypercubes
};

(:~
 : Return the number of (distinct) abstract primary items being
 : in a hypercube for each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of primary items
 :)
declare function components:num-abstract-primary-items-in-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctAbstractPrimaryItemsInHypercubes
};

(:~
 : Return the number of (distinct) concrete primary items not being
 : in a hypercube for each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of primary items
 :)
declare function components:num-concrete-primary-items-not-in-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctConcretePrimaryItemsNotInHypercubes
};

(:~
 : Return the number of (distinct) abstract primary items not being
 : in a hypercube for each of the given components.
 :
 : @param $components sequence of components.
 :
 : @return the said number of primary items
 :)
declare function components:num-distinct-abstract-primary-items-not-in-hypercubes($components as object*) as integer*
{
  $components ! $$.Statistics.NumDistinctAbstractPrimaryItemsNotInHypercubes
};

(:~
 :
 :
Converts the input to a normalized component id (CID). The input
 : can be either an CID, or an component object which contains an _id.

 :
 : @param $component-or-id an component object or an CID.
 :
 : @error components:INVALID_PARAMETER if the CID or component is not valid.
 :
 : @return the normalized CID.
 :)
declare function components:cid($component-or-id as item) as atomic
{
  typeswitch ($component-or-id)
  case object return
    let $id := $component-or-id._id
    return if(exists($id))
           then $id
           else error(QName("components:INVALID_PARAMETER"),
                      "Invalid component provided (no _id field)")
  case $id as atomic return $id
  default return error(
      QName("components:INVALID_PARAMETER"),
      "Invalid component or id (must be an object or an atomic): "
      || serialize($component-or-id))
};

declare function components:standard-typed-dimension-breakdown(
    $components as object*,
    $dimension-name as string,
    $options as object?) as object
{
    let $label as string := (
      concepts:labels(
        $dimension-name,
        $concepts:VERBOSE_LABEL_ROLE,
        ($options.Language, $components.$components:DEFAULT-LANGUAGE)[1],
        $components.Concepts[],
        $options
      ),
      concepts:labels(
        $dimension-name,
        $concepts:STANDARD_LABEL_ROLE,
        ($options.Language, $components.$components:DEFAULT-LANGUAGE)[1],
        $components.Concepts[],
        $options
      ),
      $dimension-name)[1]
    return
    {
      BreakdownLabels: [ $label || " breakdown" ],
      BreakdownTrees: [
        {
            Kind: "Rule",
            Abstract: true,
            Labels: [ $label ],
            Children: [
              {
                Kind: "Aspect",
                Aspect: $dimension-name
              }
            ]
        }
      ]
    }
};

declare function components:standard-explicit-dimension-breakdown(
    $components as object*,
    $dimension-name as string,
    $domain-names as string*,
    $role as string,
    $options as object?) as object
{
    let $label as string := (
      concepts:labels(
        $dimension-name,
        $concepts:VERBOSE_LABEL_ROLE,
        ($options.Language, $components.$components:DEFAULT-LANGUAGE)[1],
        $components.Concepts[],
        $options
      ),
      concepts:labels(
        $dimension-name,
        $concepts:STANDARD_LABEL_ROLE,
        ($options.Language, $components.$components:DEFAULT-LANGUAGE)[1],
        $components.Concepts[],
        $options
      ),
      $dimension-name)[1]
    return
    {
        BreakdownLabels: [ $label || " breakdown" ],
        BreakdownTrees: [
            {
                Kind: "Rule",
                Abstract: true,
                Labels: [ $label ],
                Children: [
                    for $domain as string in $domain-names
                    return {
                        Kind: "DimensionRelationship",
                        LinkRole: $role,
                        Dimension: $dimension-name,
                        RelationshipSource: $domain,
                        FormulaAxis: "descendant",
                        Generations: 0
                    }
                ]
            }
        ]
    }
};

declare function components:standard-concept-breakdown(
    $line-items-elements as string*,
    $role as string) as object
{
    {
        BreakdownLabels: [ "Breakdown on concepts" ],
        BreakdownTrees: [
            for $lineitems as string in $line-items-elements
            return {
                Kind: "ConceptRelationship",
                LinkName: "link:presentationLink",
                LinkRole: $role,
                ArcName: "link:presentationArc",
                ArcRole: "http://www.xbrl.org/2003/arcrole/parent-child",
                RelationshipSource: $lineitems,
                FormulaAxis: "descendant",
                Generations: 0,
                RollUpAgainstCalculationNetwork: false
            }
        ]
    }
};

(:~
 :
Builds a standard definition model out of the specified component.

 :
The concepts will be put on the y axis according to the presentation network.

 :
The other dimensions are put on the x axis, with one breakdown for each.

 :
Explicit dimensions are organized according to the dimension hierarchy from the domain-member network.

 :
Typed dimensions are organized according to the actual values.

 :
 :
One of the non-default hypercubes will be arbitrarily chosen. If none is available, the default hypercube will be picked.

 :
Auto slicing will be performed against the fact table
 :
 : @param $component a component object.
 :
 : @return a definition model
 :)
declare function components:standard-definition-models-for-components($components as object*) as object
{
    components:standard-definition-models-for-components($components, ())
};

(:~
 :
Builds a standard definition model out of the specified component.

 :
The concepts will be put on the y axis according to the presentation network.

 :
The other dimensions are put on the x axis, with one breakdown for each.

 :
Explicit dimensions are organized according to the dimension hierarchy from the domain-member network.

 :
Typed dimensions are organized according to the actual values.

 :
 : @param $component a component object.
 : @param $options
some optional parameters, including:

 :

 :
HypercubeName: a string specifying which hypercube to use. By default, one of the non-default hypercubes will be arbitrarily chosen. If none
 :  is available, the default hypercube will be picked.

 :
AutoSlice: a boolean specifying whether or not slicing should be done automatically, looking at the fact table. Deactivating auto slicing will
 :  lead to better performance, but a more verbose table. If AutoSlice is active, dimensions with only one value in the fact pool will become
 : global filters rather than breakdowns on the x axis.

 :
Slicers: an object with forced slicers.

 :

 :
 : @error components:HYPERCUBE-DOES-NOT-EXIST if the specified hypercube is not found.
 : @return a definition model
 :)
declare function components:standard-definition-models-for-components($components as object*, $options as object?) as object
{
    for $component in $components
    let $table as object := components:select-table($component, $options)

    let $auto-slice as boolean := empty($options.AutoSlice) or $options.AutoSlice
    let $facts as object*:=
        if($auto-slice)
        then hypercubes:facts($table)
        else ()
    let $dimensions as string*:= keys($table.Aspects)
    let $values-by-dimension as object := {|
        for $d in $dimensions
        return { $d : [ distinct-values($facts.Aspects.$d) ] }
    |}
    let $auto-slice-dimensions as string* :=
        keys($values-by-dimension)[size($values-by-dimension.$$) eq 1 and not ($$ = ("xbrl:Period", "sec:FiscalYear",  "sec:FiscalPeriod") ) ]
    let $user-slice-dimensions as string* :=
        keys($options.Slicers)

    let $column-dimensions as string* := keys($values-by-dimension)[not $$ =
        ("xbrl:Concept",
        "xbrl:Period",
        "xbrl:Unit",
        "xbrl:Entity",
        "xbrl28:Archive",
        $options.HideDimensions[],
        $auto-slice-dimensions,
        $user-slice-dimensions)]

    let $x-breakdowns as object* := (
        components:standard-typed-dimension-breakdown($components, "xbrl:Period", $options)
          [not (($auto-slice-dimensions, $user-slice-dimensions) = "xbrl:Period")],
        for $d as string in $column-dimensions
        let $dimension-object as object := $table.Aspects.$d
        let $is-typed as boolean := boolean($dimension-object.Kind eq "TypedDimension")
        return
            if($is-typed)
            then
              components:standard-typed-dimension-breakdown(
                $components,
                $d,
                $options)
            else
              components:standard-explicit-dimension-breakdown(
                $components,
                $d,
                $dimension-object.Members[].Name,
                $component.Role,
                $options),
        components:standard-typed-dimension-breakdown($components, "xbrl:Entity", $options)
          [not (($auto-slice-dimensions, $user-slice-dimensions) = "xbrl:Entity")]
    )

    let $lineitems as string* := components:line-items($component)
    let $presentation-network as object? := networks:networks-for-components-and-short-names($component, "Presentation")
    let $roots as string* := $presentation-network.Trees[].Name
    let $lineitems as string* := if(exists($lineitems)) then $lineitems else $roots
    let $y-breakdowns as object :=
      if(exists($presentation-network))
        then components:standard-concept-breakdown($lineitems, $component.Role)
        else components:standard-explicit-dimension-breakdown(
            $components,
            "xbrl:Concept",
            $table.Aspects."xbrl:Concept".Members[].Name,
            $component.Role,
            $options)

    return {
        ModelKind: "DefinitionModel",
        Labels: [$component.Label],
        Parameters: {},
        Breakdowns: {
            "x" : [
                $x-breakdowns
            ],
            "y": [
                $y-breakdowns
            ]
        },
        TableFilters: {|
            for $d as string in distinct-values(($auto-slice-dimensions, $user-slice-dimensions))
            return if($d = $user-slice-dimensions)
                   then { $d : $options.Slicers.$d }
                   else { $d : $values-by-dimension.$d[] },
            if (not $auto-slice)
            then { "xbrl28:Archive" : $component.Archive }
            else ()
        |}
    }
};

declare function components:filter-override(
    $entities-or-eids as item*,
    $archives-or-aids as item*
) as object?
{
    let $override := exists(($entities-or-eids, $archives-or-aids))
    where $override
    let $eids := entities:eid($entities-or-eids)
    let $aids := archives:aid($archives-or-aids)
    return
     {
        "xbrl:Entity" : {|
          { Type: "string" },
          { Domain: [ $eids ] }[exists($eids)]
        |},

        "xbrl28:Archive" : {|
          { Type: "string" },
          { Domain : [ $aids ] }[exists($aids)]
        |}
    }
};

(:~
:
Merges the supplied components, grouping networks by arc/link name/role
: and merging all hypercubes into a single one.

:
: @param $components the input components.
:
: @return the merged component.
:)
declare function components:merge($components as object*) as object
{
    {
        Archive: null,
        Role: $components.Role[1],
        Label: $components.Label[1],
        Networks: [
            networks:merge($components.Networks[])
        ],
        Hypercubes:
            let $merged := hypercubes:merge(values($components.Hypercubes))
            return { $merged.Name: $merged },
        Concepts: [
            for $concept in $components.Concepts[]
            group by $name := $concept.Name
            return $concept[1]
        ]
    }
};

(:~
 :
Returns the line items, that is the top-level abstracts or concepts in the
 : presentation network.

 :
 : @param $components the input components.
 :
 : @return the line items report elements.
 :)
declare function components:line-items($components as object*) as string*
{
    let $presentation-networks as object* := networks:networks-for-components-and-short-names($components, "Presentation")
    return components:line-items-recursive($components, $presentation-networks.Trees[])
};

declare %private function components:line-items-recursive($components as object*, $networks as object*) as string*
{
  let $concepts as object* := $components.Concepts[]
  for $network in $networks
  let $concept-name := $network.Name
  let $substitution-group as string := $concepts[$$.Name eq $concept-name].SubstitutionGroup[1]
  let $has-table-child as boolean := $concepts[$$.Name = $network.To[].Name].SubstitutionGroup = "xbrldt:hypercubeItem"
  return switch(true)
          case $has-table-child return components:line-items-recursive($components, $network.To[])
          case $substitution-group eq "xbrldt:dimensionItem" return ()
          case $substitution-group eq "xbrldt:hypercubeItem" return ()
          default return $concept-name
};


:lipstick: Implement Dennis' comments.

:lipstick: Implement Dennis' comments.


Implement review

Implement review

jsoniq version "1.0";

module namespace api = "http://apps.28.io/api";

import module namespace session    = "http://apps.28.io/session";
import module namespace sec-fiscal =
    "http://28.io/modules/xbrl/profiles/sec/fiscal/core";
import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace request = "http://www.28msec.com/modules/http-request";

declare function api:json-to-csv($objects as object*) as string
{
  let $flattened as object* := api:flatten-json-object($objects, ())
  return string-join(
    csv:serialize($flattened, { field-names: [ keys($flattened) ], serialize-null-as : "" }),
    "")
  };

declare function api:json-to-xml($objects as object*, $root-name as string) as element()*
{
  for $flattened-object as object in api:flatten-json-object($objects, { KeepArray: true })
  return element { $root-name } {
    for $key in keys($flattened-object)
    let $value := $flattened-object.$key
    return typeswitch($value)
           case array return
             let $key-with-s := if(substring($key, string-length($key), 1) eq "s")
                                then $key
                                else $key || "s"
             let $key-no-s := if(substring($key, string-length($key), 1) eq "s")
                                then substring($key, 1, string-length($key) - 1)
                                else $key
             return element {$key-with-s} { for $v in $value[] return element { $key-no-s } { $v } }
           default return element { $key } { $value }
  }
};

declare %private function api:flatten-json-object(
  $items as item*,
  $options as object?) as item*
{
  for $item in $items
  return typeswitch($item)
         case atomic return $item
         case array return let $atomic-items := flatten($item)[$$ instance of atomic]
                           return if ($options.KeepArray)
                                 then [ $atomic-items ]
                                  else string-join($atomic-items, ",\r")
         case object return {|
             for $key in keys($item)
             return typeswitch($item.$key)
                    case object return api:flatten-json-object($item.$key, $options)
                    default return { $key: api:flatten-json-object($item.$key, $options) }
         |}
         default return ()
};

declare function api:validate-regexp($name as string, $value as string?, $regexp as string)
as ()
{
  if (exists($value))
  then
    if (matches($value, "^" || $regexp || "$"))
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " does not match reg. expression " || $regexp || ".");
  else ();
};

declare function api:validate-enum($name as string, $value as string?, $enum as string*)
as ()
{
  if (exists($value))
  then
    if ($value = $enum)
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " is not one of these: " || string-join($enum,", ") || ".");
  else ();
};

declare %private function api:sgpl($count as integer, $singular as string, $plural as string)
as string
{
    if ($count eq 1)
    then $count || " " || $singular
    else $count || " " || $plural
};

declare function api:passed($timestamp as xs:dateTime?)
as string
{
    if (empty($timestamp) or $timestamp eq null)
    then "Never"
    else
        let $now := current-dateTime()
        let $difference := $now - $timestamp
        let $years := years-from-duration($difference)
        let $month := months-from-duration($difference)
        let $days := days-from-duration($difference)
        let $hours := hours-from-duration($difference)
        let $minutes := minutes-from-duration($difference)
        return
            if ($years gt 0) then api:sgpl($years,"year ago","years ago")
            else if ($month gt 0) then api:sgpl($month,"month ago","month ago")
            else if ($days gt 0) then api:sgpl($days, "day ago", "days ago")
            else if ($hours gt 0) then api:sgpl($hours, "hour ago", "hours ago")
            else if ($minutes gt 0) then api:sgpl($minutes, "minute ago", "minutes ago")
            else if ($difference div xs:dayTimeDuration("PT1S") lt 0) then "future"
            else "Just Now"
};

declare function api:success() as object
{
  { "success" : true }
};

declare function api:success($data as object()) as object
{
  {|
     {"success" : true },
     $data
  |}
};

declare function api:check-and-return-results(
    $token as string?,
    $results as item*,
    $format as string?
) as item*
{
    switch(session:has-access($token, "data_sec"))
    case $session:ACCESS-ALLOWED return
        $results
    case $session:ACCESS-DENIED return {
          { status: 403 },
          session:error("access denied", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          { status: 401 },
          session:error("authentication required or session expired", $format)
       }
    default return error()
};

declare function api:normalize-facts(
    $facts as object*) as object*
{
  trim($facts, "_id")
};

declare function api:serialize(
    $result as json-item,
    $comment as object,
    $serializers as object,
    $format as string?,
    $file-name as string) as item*
{
    switch ($format)
    case "xml" return {
        {
            serialization: { method: "xml", "omit-xml-declaration" : false, indent : true }
        },
        session:comment("xml", $comment),
        $serializers.to-xml($result)
    }
    case "text" case "csv" return {
        {
            "content-type": "text/csv",
            headers: { "Content-Disposition": "attachment; filename=" || $file-name || ".csv" }
        },
        $serializers.to-csv($result)
    }
    case "html" return {
        let $csv as string := $serializers.to-csv($result)
        return
        {
            { "content-type": "text/html" },
            api:csv-to-html($csv)
        }
    }
    case "excel" return {
        {
            "content-type": "application/vnd.ms-excel",
            headers: { "Content-Disposition": "attachment; filename=" || $file-name || ".csv" }
        },
        $serializers.to-csv($result)
    }
    default return {
        { serialization: { method: "json", indent : true } },
        {|
            session:comment("json", $comment),
            $result
        |}
    }
};

declare function api:csv-to-html(
    $csv as string) as item*
{
    let $csv as string* := $csv
    let $rows :=
      let $newline-tokens :=
        (: We go through all quotes. Outside of quotes, we mark newlines. Inside
           quotes, we don't interpret newlines :)
        for tumbling window $token as string* in tokenize($csv, "\"")
        start when true
        end $u at $j next $v
            (: If we've seen an even number of quotes so far, and the current
               token is not empty, then the current token (outside of quotes)
               is completed.
               If it's empty, then it means that the last closing quote was
               actually an escaped quote "" and we cannot close the window yet:)
            when ($j mod 2 eq 1 and $u ne "")
            (: If we've seen an odd number of quotes so far, and the next
               token is not empty, then the current token (inside quotes)
               is completed.
               If it's empty, then it means that the upcoming closing quote is
               actually an escaped quote "" and we cannot close the window yet:)
            or ($j mod 2 eq 0 and $v ne "")
        count $k
        return if($k mod 2 eq 1)
               (: We interpret \n as a csv newline for odd windows :)
               then $token ! replace($$, "\n", "______28newline______")
               (: We don't interpret anything for even windows, but paste
                  tokens back together (this is the "" escaping) :)
               else "\"" || string-join($token, "\"") || "\""
      return tokenize(string-join($newline-tokens), "______28newline______")
    let $header := $rows[1]
    let $body := subsequence($rows, 2)
    let $display-cell := function($row as string?, $cell-tag-name as string) {
      for $cell in
        (: We go through all quotes. Outside of quotes, we mark commas. Inside
           quotes, we don't interpret commas :)
        let $comma-tokens :=
          for tumbling window $token as string* in tokenize($row, "\"")
          start when true
          end $u at $j next $v
            (: If we've seen an even number of quotes so far, and the current
               token is not empty, then the current token (outside of quotes)
               is completed.
               If it's empty, then it means that the last closing quote was
               actually an escaped quote "" and we cannot close the window yet:)
              when ($j mod 2 eq 1 and $u ne "")
            (: If we've seen an odd number of quotes so far, and the next
               token is not empty, then the current token (inside quotes)
               is completed.
               If it's empty, then it means that the upcoming closing quote is
               actually an escaped quote "" and we cannot close the window yet:)
              or ($j mod 2 eq 0 and $v ne "")
          count $k
          return if($k mod 2 eq 1)
               (: We interpret the comma as a csv comma for odd windows :)
                 then $token ! replace($$, ",", "______28comma______")
               (: We don't interpret anything for even windows, but paste
                  tokens back together (this is the "" escaping) :)
                 else string-join($token, "\"")
        return tokenize(string-join($comma-tokens), "______28comma______")
      return switch(true)
             case contains($cell, "http://") and
                  (contains($cell, request:server-name()) or
                   contains($cell, "rendering.secxbrl.info"))
             return element { $cell-tag-name } { Link }
             default return element { $cell-tag-name } {tokenize($cell, "\r") ! ($$,
)}
    }
    return

        <title>Cell Store REST API</title>

{$display-cell($header, "th")}

          {
            for $row as string in $body
            return
{$display-cell($row, "td")}

          }

};

declare function api:preprocess-fiscal-years($fiscal-years as string*) as integer*
{
  distinct-values(
    for $fy in $fiscal-years ! upper-case($$)
    return switch($fy)
           case "LATEST" return $sec-fiscal:LATEST_FISCAL_YEAR
           case "ALL" return $sec-fiscal:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
  )
};

declare function api:preprocess-fiscal-periods($fiscal-periods as string*) as string*
{
  distinct-values(
    for $fp in $fiscal-periods ! upper-case($$)
    return switch($fp)
           case "Q1"
           case "Q2"
           case "Q3"
           case "FY"
             return $fp
           case "Q4"
             return "FY"
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIODS
           default
             return error(xs:QName("local:INVALID-PERIOD"),
               $fp || ": fiscalPeriod values must be one or more of Q1, Q2, Q3, Q4, FY, ALL")
  )
};

declare function api:preprocess-fiscal-period-types($fiscal-period-types as string*) as string*
{
  distinct-values(
    for $fpt in $fiscal-period-types
    return switch($fpt)
           case "instant"
           case "QTD"
           case "YTD"
             return $fpt
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIOD_TYPES
           default
             return error(xs:QName("local:INVALID-PERIOD-TYPE"),
               $fpt || ": fiscalPeriodType values must be one or more of instant, YTD, QTD")
  )
};

declare function api:preprocess-format($format as string?, $request-uri as string) as string?
{
  let $request-path := tokenize($request-uri, "\\?")[1]
  return lower-case(($format, substring-after($request-path, ".jq."))[1])
};

declare function api:preprocess-tags($tags as string*) as string*
{
  distinct-values($tags ! upper-case($$))
};

declare function api:preprocess-concept-kind($kind as string*) as string*
{
  distinct-values(
    for $kind in $kind
    return switch(lower-case($kind))
           case "hypercube" return "Hypercube"
           case "dimension" return "Dimension"
           case "member" return "Member"
           case "lineitems" return "LineItems"
           case "abstract" return "Abstract"
           case "concept" return "Concept"
           default return ()
  )
};

declare function api:preprocess-boolean($name as string, $value as string)
as boolean
{
  if ($value eq "")
  then true
  else
  {
      api:validate-enum($name, lower-case($value), ("true", "false"));
      boolean($value)
  }
};

declare function api:not-found($name as string) as item*
{
  { status: 404, content-type: "application/json" },
  session:error("No such " || $name || " could be found.", "json")
};


added filing stats to query

added filing stats to query

import module namespace facts = "http://xbrl.io/modules/bizql/facts";

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace filings = "http://xbrl.io/modules/bizql/profiles/sec/filings";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace session = "http://apps.28.io/session";

declare function local:to-xml($filings as object*) as node()*
{
    ( session:comment("xml"),
    <Filings>{
        for $f in $filings
        return
        <Filing>
            <CIK>{$f.CIK}</CIK>
            <EntityRegistrantName>{$f.EntityRegistrantName}</EntityRegistrantName>
            <FormType>{$f.FormType}</FormType>
            <FiscalYear>{$f.FiscalYear}</FiscalYear>
            <FiscalPeriod>{$f.FiscalPeriod}</FiscalPeriod>
            <Accepted>{$f.Accepted}</Accepted>
            <Generator>{$f.Generator}</Generator>
            <AccessionNumber>{$f.AccessionNumber}</AccessionNumber>
            <SECFilingPage>{$f.SECFilingPage}</SECFilingPage>
            <XBRLInstanceURL>{$f.XBRLInstanceURL}</XBRLInstanceURL>
            <Networks>{$f.Networks}</Networks>
            <ReportElements>{$f.ReportElements}</ReportElements>
            <Tables>{$f.Tables}</Tables>
            <Axis>{$f.Axis}</Axis>
            <Members>{$f.Members}</Members>
            <LineItems>{$f.LineItems}</LineItems>
            <Concepts>{$f.Concepts}</Concepts>
            <Abstracts>{$f.Abstracts}</Abstracts>
            <Footnotes>{$f.Footnotes}</Footnotes>
            <Facts>{$f.Facts}</Facts>
            <ExtensionFacts>{$f.ExtensionFacts}</ExtensionFacts>
            <ExtensionConcepts>{$f.ExtensionConcepts}</ExtensionConcepts>
            <ExtensionAbstracts>{$f.ExtensionAbstracts}</ExtensionAbstracts>
        </Filing>
    }</Filings>)
};

declare function local:to-csv($filings as object*) as string*
{
    csv:serialize($filings, { serialize-null-as : "" })
};

declare function local:summary($a)
{
    {
        CIK : $a.Entity,
        EntityRegistrantName :
            facts:facts-for-archives-and-concepts($a, "dei:EntityRegistrantName").Value,
        FormType : filings:document-types($a),
        FiscalYear : fiscal:fiscal-year($a),
        FiscalPeriod : fiscal:fiscal-period($a),
        Accepted : filings:acceptance-dateTimes($a),
        Generator : filings:generators($a),
        AccessionNumber: $a._id,
        SECFilingPage : $a.Profiles.SEC.SECFilingPage,
        XBRLInstanceURL : $a.InstanceURL,
        Networks : filings:num-networks($a),
        ReportElements : filings:num-report-elements($a),
        Tables : filings:num-tables($a),
        Axis : filings:num-axes($a),
        Members : filings:num-members($a),
        LineItems : $a.Statistics.Profiles.SEC.NumDistinctReportElementNamesEndingWithLineItems,
        Concepts : filings:num-concepts($a),
        Abstracts : filings:num-abstracts($a),
        Footnotes : filings:num-footnotes($a),
        Facts: filings:num-facts($a),
        ExtensionFacts:  filings:num-extension-facts($a),
        ExtensionConcepts : filings:num-extension-concepts($a),
        ExtensionAbstracts : filings:num-extension-abstracts($a)
    }
};

let $format  := lower-case(request:param-values("format")[1])
let $ciks     := request:param-values("cik")
let $aids     := request:param-values("aid")
let $archives := (archives:archives($aids), filings:filings-for-companies($ciks))
let $entities := companies:companies($archives.Entities)
return
    if (session:only-dow30($entities) or session:valid())
    then {
        let $archives :=
                for $a in $archives
                order by filings:acceptance-dateTimes($a) descending
                return local:summary($a)
        return
            switch ($format)
            case "xml"  return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($archives)
            }
            case "csv" case "text" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=filings.csv");
                string-join(local:to-csv($archives), "")
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=filings.csv");
                string-join(local:to-csv($archives))
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { "Archives" : [ $archives ] },
                    session:comment("json")
                |}
            }
    } else {
        response:status-code(401);
        session:error("accessing filings of an entity that is not in the DOW30", $format)
    }

import module namespace facts = "http://xbrl.io/modules/bizql/facts";

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace filings = "http://xbrl.io/modules/bizql/profiles/sec/filings";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace session = "http://apps.28.io/session";

declare function local:to-xml($filings as object*) as node()*
{
    ( session:comment("xml"),
    <Filings>{
        for $f in $filings
        return
        <Filing>
            <CIK>{$f.CIK}</CIK>
            <EntityRegistrantName>{$f.EntityRegistrantName}</EntityRegistrantName>
            <FormType>{$f.FormType}</FormType>
            <FiscalYear>{$f.FiscalYear}</FiscalYear>
            <FiscalPeriod>{$f.FiscalPeriod}</FiscalPeriod>
            <Accepted>{$f.Accepted}</Accepted>
            <Generator>{$f.Generator}</Generator>
            <AccessionNumber>{$f.AccessionNumber}</AccessionNumber>
            <SECFilingPage>{$f.SECFilingPage}</SECFilingPage>
            <XBRLInstanceURL>{$f.XBRLInstanceURL}</XBRLInstanceURL>
        </Filing>
    }</Filings>)
};

declare function local:to-csv($filings as object*) as string*
{
    csv:serialize($filings, { serialize-null-as : "" })
};

declare function local:summary($a)
{
    {
        CIK : $a.Entity,
        EntityRegistrantName :
            facts:facts-for-archives-and-concepts($a, "dei:EntityRegistrantName").Value,
        FormType : filings:document-types($a),
        FiscalYear : fiscal:fiscal-year($a),
        FiscalPeriod : fiscal:fiscal-period($a),
        Accepted : filings:acceptance-dateTimes($a),
        Generator : filings:generators($a),
        AccessionNumber: $a._id,
        SECFilingPage : $a.Profiles.SEC.SECFilingPage,
        XBRLInstanceURL : $a.InstanceURL
    }
};

let $format  := lower-case(request:param-values("format")[1])
let $ciks     := request:param-values("cik")
let $fids     := request:param-values("fid")
let $archives := (archives:archives($fids), filings:filings-for-companies($ciks))
let $entities := companies:companies($archives.Entities)
return
    if (session:only-dow30($entities) or session:valid())
    then {
        let $archives :=
                for $a in $archives
                order by filings:acceptance-dateTimes($a) descending
                return local:summary($a)
        return
            switch ($format)
            case "xml"  return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($archives)
            }
            case "csv" case "text" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=filings.csv");
                string-join(local:to-csv($archives), "")
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=filings.csv");
                string-join(local:to-csv($archives))
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { "Archives" : [ $archives ] },
                    session:comment("json")
                |}
            }
    } else {
        response:status-code(401);
        session:error("accessing filings of an entity that is not in the DOW30", $format)
    }