:bug: Only output balance for monetary types.

jsoniq version "1.0";
(:
 : Copyright 2014 28msec Inc.
 :)

(:~
 :
This module provides functionality for retrieving concepts
 : and labels.

 :
 : @author Federico Cavalieri
 :)
module namespace concepts = "http://28.io/modules/xbrl/concepts";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

declare namespace xbrli = "http://www.xbrl.org/2003/instance";
declare namespace xlink = "http://www.w3.org/1999/xlink";
declare namespace link = "http://www.xbrl.org/2003/linkbase";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Name of the collection the concepts are stored in.
 :)
declare variable $concepts:col as xs:string := "concepts";

(:~
 : Name of the field pointing to the archive.
 :)
declare variable $concepts:ARCHIVE as xs:string := "Archive";

(:~
 : Name of the field pointing to the role.
 :)
declare variable $concepts:ROLE as xs:string := "Role";

(:~
 : Name of the field pointing to the name.
 :)
declare variable $concepts:NAME as xs:string := "Name";

(:~
 : Name of the field pointing to the labels.
 :)
declare variable $concepts:LABELS as xs:string := "Labels";

(:~
 : The default component link role.
 :)
declare variable $concepts:DEFAULT_COMPONENT_LINK_ROLE as xs:string :=
    "http://www.xbrl.org/2003/role/link";

(:~
 : Allow any component link role
 :)
declare variable $concepts:ANY_COMPONENT_LINK_ROLE as xs:string :=
    ":any component:";

(:~
 : A joker to ask for all concept names.
 :)
declare variable $concepts:ALL_CONCEPT_NAMES as xs:string := ":any concept:";

(:~
 : A joker to ask for all concept labels.
 :)
declare variable $concepts:ALL_CONCEPT_LABELS as xs:string := ":any concept label:";

(:~
 : The concept report element
 :)
declare variable $concepts:CONCEPT_REPORT_ELEMENT as xs:string := "xbrl:Concept";

(:~
 : The period report element
 :)
declare variable $concepts:PERIOD_REPORT_ELEMENT as xs:string := "xbrl:Period";

(:~
 : The entity report element
 :)
declare variable $concepts:ENTITY_REPORT_ELEMENT as xs:string := "xbrl:Entity";

(:~
 : The unit report element
 :)
declare variable $concepts:UNIT_REPORT_ELEMENT as xs:string := "xbrl:Unit";

(:~
 : The archive report element
 :)
declare variable $concepts:ARCHIVE_REPORT_ELEMENT as xs:string := "xbrl28:Archive";

(:~
 : The implied table report element
 :)
declare variable $concepts:IMPLIED_TABLE_REPORT_ELEMENT as xs:string := "xbrl28:ImpliedTable";

(:~
 : The non-numeric table report element
 :)
declare variable $concepts:NON_NUMERIC_REPORT_ELEMENT as xs:string := "xbrl:NonNumeric";

(:~
 :
Retrieves all concepts.

 :
 : @return all concepts.
 :)
declare function concepts:concepts() as object*
{
  mw:find($concepts:col,{})
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives-or-ids the archives or their IDs.
 : @param $component-roles the component roles.
 : @param $labels labels for exact matches.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives-or-ids as item*,
    $component-roles as string*,
    $labels as string*
  ) as object*
{
  concepts:concepts($concept-names, $archives-or-ids, $component-roles, $labels, ())
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives-or-ids the archives or their IDs.
 : @param $component-roles the component roles.
 : @param $labels labels for exact matches.
 : @param $options one option OnlyNames to only output concepts with the Name field.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives-or-ids as item*,
    $component-roles as string*,
    $labels as string*,
    $options as object?
  ) as object*
{
  let $projection as object := if($options.OnlyNames eq true) then
  {
    Name: 1,
    IsAbstract: 1,
    IsTextBlock: 1
  } else {}
  let $archives := archives:aid($archives-or-ids)
  return
  if (exists($archives))
  then mw:find($concepts:col,
    {|
      {
        $concepts:ARCHIVE : { "$in" : [ $archives ] }
      },
      {
        $concepts:ROLE : { "$in" :
          [ $component-roles ]
        }
      }[not $component-roles = $concepts:ANY_COMPONENT_LINK_ROLE],
      {
        $concepts:NAME : { "$in" : [ $concept-names ] }
      }[not $concept-names = $concepts:ALL_CONCEPT_NAMES],
      {
        "Labels.Value" : { "$in" : [ $labels ] }
      }[not $labels = $concepts:ALL_CONCEPT_LABELS]
    |},
    $projection
  )
  else ()
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives-or-ids the archives or their IDs.
 : @param $component-roles the component roles.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives-or-ids as item*,
    $component-roles as string*
  ) as object*
{
  concepts:concepts(
    $concept-names,
    $archives-or-ids,
    $component-roles,
    $concepts:ALL_CONCEPT_LABELS
  )
};

(:~
 :
Retrieves the concepts which:
 :  - concept name matches a given one;
 :  - archive number matches that of a given component;
 :  - component role matches that of a given component or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $component-or-ids the CIDs or the components themselves.
 : @param $labels labels for exact matches.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts-for-components(
    $concept-names as string*,
    $labels as string*,
    $component-or-ids as item*,
    $options as object?) as object*
{
  for $component in components:components($component-or-ids)
  return concepts:concepts($concept-names,
                           $component.Archive[not $$ instance of null],
                           $component.Role,
                           $labels,
                           $options)
};

(:~
 :
Transforms a concept tree into XBRL arcs and locators.
 :    Those can then be added to an extended link.

 :
 : @param $trees the trees to transform.
 :
 : @return the locator and arcs elements
 :)
declare function concepts:tree-to-xbrl(
    $tree as object,
    $arcRole as string,
    $taxonomyName as string,
    $level as integer,
    $pos as integer
) as element()*
{
  let $children as object* := ($tree.To[], $tree.Members[])
  let $fromID as string := replace($tree.Name, ":", "_")
  let $fromLabel as string := $fromID || "_" || $level || "_" || $pos
  let $fromLocator as element() := <link:loc xlink:type="locator" xlink:href="{$taxonomyName}.xsd#{$fromID}" xlink:label="{$fromLabel}" xlink:title="{normalize-space($tree.Label)}" />

  return
  (
    $fromLocator,

    for $child at $pos in $children
    order by $child.Order ascending
    let $toID as string := replace($child.Name, ":", "_")
    let $toLabel as string := $toID || "_" || ($level + 1) || "_" || $pos
    let $order as decimal :=
      if(empty($child.Order) or $child.Order eq null)
      then 1
      else $child.Order
    let $arc as element() :=
      element {
        switch ($arcRole)
        case "http://www.xbrl.org/2003/arcrole/concept-label" return xs:QName("link:labelArc")
        case "http://www.xbrl.org/2003/arcrole/parent-child" return xs:QName("link:presentationArc")
        case "http://www.xbrl.org/2003/arcrole/summation-item" return xs:QName("link:calculationArc")
        case "http://xbrl.org/int/dim/arcrole/all"
        case "http://xbrl.org/int/dim/arcrole/hypercube-dimension"
        case "http://xbrl.org/int/dim/arcrole/dimension-domain"
        case "http://xbrl.org/int/dim/arcrole/dimension-default"
        case "http://xbrl.org/int/dim/arcrole/domain-member"
        return xs:QName("link:definitionArc")
        default return error(xs:QName("err:UnknownArcRole"), "arcrole " || $arcRole || " unknown or not supported.")
      }
      {
        attribute { "xlink:type" } { "arc" },
        attribute { "xlink:arcrole" } { $arcRole },
        attribute { "xlink:from" } { $fromLabel },
        attribute { "xlink:to" } { $toLabel },
        attribute { "use" } { "optional" },
        attribute { "order" } { $order },
        (attribute { "weight" } { $child.Weight } )[$arcRole eq "http://www.xbrl.org/2003/arcrole/summation-item"]
      }
    return
      (
        $arc,
        concepts:tree-to-xbrl($child, $arcRole, $taxonomyName, $level+1, $pos)
      )
  )
};

(:~
 :
Transforms concepts into XBRL definitions (XML Schema element definition).

 :
 : @param $concepts the concepts to transform.
 :
 : @return the XML Schema element definitions.
 :)
declare function concepts:to-xbrl(
    $concepts as object*) as element()*
{
  for $concept as object in $concepts
  where not $concept.Name = ($concepts:CONCEPT_REPORT_ELEMENT,
                             $concepts:ENTITY_REPORT_ELEMENT,
                             $concepts:PERIOD_REPORT_ELEMENT,
                             $concepts:UNIT_REPORT_ELEMENT,
                             $concepts:ARCHIVE_REPORT_ELEMENT,
                             $concepts:IMPLIED_TABLE_REPORT_ELEMENT,
                             $concepts:NON_NUMERIC_REPORT_ELEMENT)
  let $name as string := substring-after($concept.Name, ":")
  let $id as string := replace($concept.Name, ":", "_")
  let $substitutionGroup as string := $concept.SubstitutionGroup
  let $nillable as boolean := (boolean($concept.IsNillable), false)[1]
  let $abstract as boolean := (boolean($concept.IsAbstract), false)[1]
  let $periodType as attribute()? := if(exists($concept.PeriodType)) then attribute { "xbrli:periodType" } {$concept.PeriodType } else ()
  let $balance as attribute()? :=
      (attribute {"xbrli:balance"} { $concept.Balance })[
          $concept.DataType eq "xbrli:monetaryItemType" and
          exists($concept.Balance)]
  let $type as string := $concept.DataType
  let $textBlock as boolean := boolean($concept.IsTextBlock)
  return
      <element xmlns="http://www.w3.org/2001/XMLSchema"
          id="{$id}"
          name="{$name}"
          nillable="{$nillable}"
          substitutionGroup="{$substitutionGroup}"
          abstract="{$abstract}"
          type="{$type}">{
              (
                  $periodType,
                  $balance
              )
      }</element>
};

jsoniq version "1.0";
(:
 : Copyright 2014 28msec Inc.
 :)

(:~
 :
This module provides functionality for retrieving concepts
 : and labels.

 :
 : @author Federico Cavalieri
 :)
module namespace concepts = "http://28.io/modules/xbrl/concepts";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

declare namespace xbrli = "http://www.xbrl.org/2003/instance";
declare namespace xlink = "http://www.w3.org/1999/xlink";
declare namespace link = "http://www.xbrl.org/2003/linkbase";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Name of the collection the concepts are stored in.
 :)
declare variable $concepts:col as xs:string := "concepts";

(:~
 : Name of the field pointing to the archive.
 :)
declare variable $concepts:ARCHIVE as xs:string := "Archive";

(:~
 : Name of the field pointing to the role.
 :)
declare variable $concepts:ROLE as xs:string := "Role";

(:~
 : Name of the field pointing to the name.
 :)
declare variable $concepts:NAME as xs:string := "Name";

(:~
 : Name of the field pointing to the labels.
 :)
declare variable $concepts:LABELS as xs:string := "Labels";

(:~
 : The default component link role.
 :)
declare variable $concepts:DEFAULT_COMPONENT_LINK_ROLE as xs:string :=
    "http://www.xbrl.org/2003/role/link";

(:~
 : Allow any component link role
 :)
declare variable $concepts:ANY_COMPONENT_LINK_ROLE as xs:string :=
    ":any component:";

(:~
 : A joker to ask for all concept names.
 :)
declare variable $concepts:ALL_CONCEPT_NAMES as xs:string := ":any concept:";

(:~
 : A joker to ask for all concept labels.
 :)
declare variable $concepts:ALL_CONCEPT_LABELS as xs:string := ":any concept label:";

(:~
 : The concept report element
 :)
declare variable $concepts:CONCEPT_REPORT_ELEMENT as xs:string := "xbrl:Concept";

(:~
 : The period report element
 :)
declare variable $concepts:PERIOD_REPORT_ELEMENT as xs:string := "xbrl:Period";

(:~
 : The entity report element
 :)
declare variable $concepts:ENTITY_REPORT_ELEMENT as xs:string := "xbrl:Entity";

(:~
 : The unit report element
 :)
declare variable $concepts:UNIT_REPORT_ELEMENT as xs:string := "xbrl:Unit";

(:~
 : The archive report element
 :)
declare variable $concepts:ARCHIVE_REPORT_ELEMENT as xs:string := "xbrl28:Archive";

(:~
 : The implied table report element
 :)
declare variable $concepts:IMPLIED_TABLE_REPORT_ELEMENT as xs:string := "xbrl28:ImpliedTable";

(:~
 : The non-numeric table report element
 :)
declare variable $concepts:NON_NUMERIC_REPORT_ELEMENT as xs:string := "xbrl:NonNumeric";

(:~
 :
Retrieves all concepts.

 :
 : @return all concepts.
 :)
declare function concepts:concepts() as object*
{
  mw:find($concepts:col,{})
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives-or-ids the archives or their IDs.
 : @param $component-roles the component roles.
 : @param $labels labels for exact matches.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives-or-ids as item*,
    $component-roles as string*,
    $labels as string*
  ) as object*
{
  concepts:concepts($concept-names, $archives-or-ids, $component-roles, $labels, ())
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives-or-ids the archives or their IDs.
 : @param $component-roles the component roles.
 : @param $labels labels for exact matches.
 : @param $options one option OnlyNames to only output concepts with the Name field.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives-or-ids as item*,
    $component-roles as string*,
    $labels as string*,
    $options as object?
  ) as object*
{
  let $projection as object := if($options.OnlyNames eq true) then
  {
    Name: 1,
    IsAbstract: 1,
    IsTextBlock: 1
  } else {}
  let $archives := archives:aid($archives-or-ids)
  return
  if (exists($archives))
  then mw:find($concepts:col,
    {|
      {
        $concepts:ARCHIVE : { "$in" : [ $archives ] }
      },
      {
        $concepts:ROLE : { "$in" :
          [ $component-roles ]
        }
      }[not $component-roles = $concepts:ANY_COMPONENT_LINK_ROLE],
      {
        $concepts:NAME : { "$in" : [ $concept-names ] }
      }[not $concept-names = $concepts:ALL_CONCEPT_NAMES],
      {
        "Labels.Value" : { "$in" : [ $labels ] }
      }[not $labels = $concepts:ALL_CONCEPT_LABELS]
    |},
    $projection
  )
  else ()
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives-or-ids the archives or their IDs.
 : @param $component-roles the component roles.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives-or-ids as item*,
    $component-roles as string*
  ) as object*
{
  concepts:concepts(
    $concept-names,
    $archives-or-ids,
    $component-roles,
    $concepts:ALL_CONCEPT_LABELS
  )
};

(:~
 :
Retrieves the concepts which:
 :  - concept name matches a given one;
 :  - archive number matches that of a given component;
 :  - component role matches that of a given component or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $component-or-ids the CIDs or the components themselves.
 : @param $labels labels for exact matches.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts-for-components(
    $concept-names as string*,
    $labels as string*,
    $component-or-ids as item*,
    $options as object?) as object*
{
  for $component in components:components($component-or-ids)
  return concepts:concepts($concept-names,
                           $component.Archive[not $$ instance of null],
                           $component.Role,
                           $labels,
                           $options)
};

(:~
 :
Transforms a concept tree into XBRL arcs and locators.
 :    Those can then be added to an extended link.

 :
 : @param $trees the trees to transform.
 :
 : @return the locator and arcs elements
 :)
declare function concepts:tree-to-xbrl(
    $tree as object,
    $arcRole as string,
    $taxonomyName as string,
    $level as integer,
    $pos as integer
) as element()*
{
  let $children as object* := ($tree.To[], $tree.Members[])
  let $fromID as string := replace($tree.Name, ":", "_")
  let $fromLabel as string := $fromID || "_" || $level || "_" || $pos
  let $fromLocator as element() := <link:loc xlink:type="locator" xlink:href="{$taxonomyName}.xsd#{$fromID}" xlink:label="{$fromLabel}" xlink:title="{normalize-space($tree.Label)}" />

  return
  (
    $fromLocator,

    for $child at $pos in $children
    order by $child.Order ascending
    let $toID as string := replace($child.Name, ":", "_")
    let $toLabel as string := $toID || "_" || ($level + 1) || "_" || $pos
    let $order as decimal :=
      if(empty($child.Order) or $child.Order eq null)
      then 1
      else $child.Order
    let $arc as element() :=
      element {
        switch ($arcRole)
        case "http://www.xbrl.org/2003/arcrole/concept-label" return xs:QName("link:labelArc")
        case "http://www.xbrl.org/2003/arcrole/parent-child" return xs:QName("link:presentationArc")
        case "http://www.xbrl.org/2003/arcrole/summation-item" return xs:QName("link:calculationArc")
        case "http://xbrl.org/int/dim/arcrole/all"
        case "http://xbrl.org/int/dim/arcrole/hypercube-dimension"
        case "http://xbrl.org/int/dim/arcrole/dimension-domain"
        case "http://xbrl.org/int/dim/arcrole/dimension-default"
        case "http://xbrl.org/int/dim/arcrole/domain-member"
        return xs:QName("link:definitionArc")
        default return error(xs:QName("err:UnknownArcRole"), "arcrole " || $arcRole || " unknown or not supported.")
      }
      {
        attribute { "xlink:type" } { "arc" },
        attribute { "xlink:arcrole" } { $arcRole },
        attribute { "xlink:from" } { $fromLabel },
        attribute { "xlink:to" } { $toLabel },
        attribute { "use" } { "optional" },
        attribute { "order" } { $order },
        (attribute { "weight" } { $child.Weight } )[$arcRole eq "http://www.xbrl.org/2003/arcrole/summation-item"]
      }
    return
      (
        $arc,
        concepts:tree-to-xbrl($child, $arcRole, $taxonomyName, $level+1, $pos)
      )
  )
};

(:~
 :
Transforms concepts into XBRL definitions (XML Schema element definition).

 :
 : @param $concepts the concepts to transform.
 :
 : @return the XML Schema element definitions.
 :)
declare function concepts:to-xbrl(
    $concepts as object*) as element()*
{
  for $concept as object in $concepts
  where not $concept.Name = ($concepts:CONCEPT_REPORT_ELEMENT,
                             $concepts:ENTITY_REPORT_ELEMENT,
                             $concepts:PERIOD_REPORT_ELEMENT,
                             $concepts:UNIT_REPORT_ELEMENT,
                             $concepts:ARCHIVE_REPORT_ELEMENT,
                             $concepts:IMPLIED_TABLE_REPORT_ELEMENT,
                             $concepts:NON_NUMERIC_REPORT_ELEMENT)
  let $name as string := substring-after($concept.Name, ":")
  let $id as string := replace($concept.Name, ":", "_")
  let $substitutionGroup as string := $concept.SubstitutionGroup
  let $nillable as boolean := (boolean($concept.IsNillable), false)[1]
  let $abstract as boolean := (boolean($concept.IsAbstract), false)[1]
  let $periodType as attribute()? := if($concept.PeriodType) then attribute { "xbrli:periodType" } {$concept.PeriodType } else ()
  let $balance as attribute()? := if($concept.Balance) then attribute { "xbrli:balance" } {$concept.Balance } else ()
  let $type as string := $concept.DataType
  let $textBlock := $concept.IsTextBlock
  return
      <element xmlns="http://www.w3.org/2001/XMLSchema"
          id="{$id}"
          name="{$name}"
          nillable="{$nillable}"
          substitutionGroup="{$substitutionGroup}"
          abstract="{$abstract}"
          type="{$type}">{
              (
                  $periodType,
                  $balance
              )
      }</element>
};


:lipstick: Split lines.

:lipstick: Split lines.

jsoniq version "1.0";

module namespace search = "http://apps.28.io/search";

import module namespace config = "http://apps.28.io/config";
import module namespace es = "http://28.io/modules/elasticsearch";
import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";
import module namespace mongo = "http://www.28msec.com/modules/mongodb";

declare variable $search:es-conn := es:connection("xbrl");
declare variable $search:mongo-conn := mw:connection();

declare variable $search:index := $config:db-name;
declare variable $search:mapping := $config:elastic-search-mapping;

declare %an:sequential function search:sync-entities() as string+
{
  variable $entities as object* := mongo:find($search:mongo-conn, "entities");
  variable $es-entities as object* := es:list(
      $search:es-conn,
      $search:index,
      "entities").hits.hits[]._source;
  variable $to-be-inserted as object* :=
    for $entity in $entities
    where not(exists($es-entities[$$._id eq $entity._id]))
    return $entity;
  variable $to-be-updated as object* :=
    for $entity in $entities
    let $existing as object? := $es-entities[$$._id eq $entity._id]
    where exists($existing)
    let $changed as boolean := not(deep-equal($existing, $entity))
    where $changed
    return $entity;

  (: insert new entities :)
  for $insert in $to-be-inserted
  return es:insert($search:es-conn, $search:index, "entities", $insert);

  (: update entities if changed :)
  for $update in $to-be-updated
  return es:insert($search:es-conn, $search:index, "entities", $update);

  (
    count($to-be-inserted) ||
        " entities have been inserted into elastic search",
    count($to-be-updated) ||
        " entities have been updated in elastic search"
  )
};

declare %an:sequential function search:init() as string
{
  let $index-created as boolean :=
    if(not(es:exists($search:es-conn, $search:index)))
    then {
      es:create($search:es-conn, $search:index, $search:mapping);
      true
    }
    else false
  return
    if($index-created)
    then "Index " || $search:index || " created."
    else "Index " || $search:index || " already exists."
};

jsoniq version "1.0";

module namespace search = "http://apps.28.io/search";

import module namespace config = "http://apps.28.io/config";
import module namespace es = "http://28.io/modules/elasticsearch";
import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";
import module namespace mongo = "http://www.28msec.com/modules/mongodb";

declare variable $search:es-conn := es:connection("xbrl");
declare variable $search:mongo-conn := mw:connection();

declare variable $search:index := $config:db-name;
declare variable $search:mapping := $config:elastic-search-mapping;

declare %an:sequential function search:sync-entities() as string+
{
  variable $entities as object* := mongo:find($search:mongo-conn, "entities");
  variable $es-entities as object* := es:list($search:es-conn, $search:index, "entities").hits.hits[]._source;
  variable $to-be-inserted as object* :=
    for $entity in $entities
    where not(exists($es-entities[$$._id eq $entity._id]))
    return $entity;
  variable $to-be-updated as object* :=
    for $entity in $entities
    let $existing as object? := $es-entities[$$._id eq $entity._id]
    where exists($existing)
    let $changed as boolean := not(deep-equal($existing, $entity))
    where $changed
    return $entity;

  (: insert new entities :)
  for $insert in $to-be-inserted
  return es:insert($search:es-conn, $search:index, "entities", $insert);

  (: update entities if changed :)
  for $update in $to-be-updated
  return es:insert($search:es-conn, $search:index, "entities", $update);

  (
    count($to-be-inserted) || " entities have been inserted into elastic search",
    count($to-be-updated) || " entities have been updated in elastic search"
  )
};

declare %an:sequential function search:init() as string
{
  let $index-created as boolean :=
    if(not(es:exists($search:es-conn, $search:index)))
    then {
      es:create($search:es-conn, $search:index, $search:mapping);
      true
    }
    else false
  return
    if($index-created)
    then "Index " || $search:index || " created."
    else "Index " || $search:index || " already exists."
};


:green_heart: Fix an SEC test as well.

:green_heart: Fix an SEC test as well.

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace test = "http://apps.28.io/test";
import module namespace config = "http://apps.28.io/config";

declare variable $local:expected as object :=
    {
      "dow30" : [ "http://www.sec.gov/CIK 0000066740", "http://www.sec.gov/CIK 0000004962", "http://www.sec.gov/CIK 0000732717", "http://www.sec.gov/CIK 0000012927", "http://www.sec.gov/CIK 0000018230", "http://www.sec.gov/CIK 0000093410", "http://www.sec.gov/CIK 0000858877", "http://www.sec.gov/CIK 0000021344", "http://www.sec.gov/CIK 0000030554", "http://www.sec.gov/CIK 0000034088", "http://www.sec.gov/CIK 0000040545", "http://www.sec.gov/CIK 0000886982", "http://www.sec.gov/CIK 0000354950", "http://www.sec.gov/CIK 0000050863", "http://www.sec.gov/CIK 0000051143", "http://www.sec.gov/CIK 0000200406", "http://www.sec.gov/CIK 0000019617", "http://www.sec.gov/CIK 0000063908", "http://www.sec.gov/CIK 0000789019", "http://www.sec.gov/CIK 0000310158", "http://www.sec.gov/CIK 0000320187", "http://www.sec.gov/CIK 0000078003", "http://www.sec.gov/CIK 0000080424", "http://www.sec.gov/CIK 0000086312", "http://www.sec.gov/CIK 0000101829", "http://www.sec.gov/CIK 0000731766", "http://www.sec.gov/CIK 0000732712", "http://www.sec.gov/CIK 0001403161", "http://www.sec.gov/CIK 0000104169", "http://www.sec.gov/CIK 0001001039" ],
      "generic" : [ "http://www.sec.gov/CIK 0000004962" ],
      "cik" : [ "http://www.sec.gov/CIK 0000004962" ],
      "ticker" : [ "http://www.sec.gov/CIK 0000104169" ],
      "ticker2" : [ "http://www.sec.gov/CIK 0000021344", "http://www.sec.gov/CIK 0000104169" ],
      "sic" : [ "http://www.sec.gov/CIK 0001585644", "http://www.sec.gov/CIK 0001608249", "http://www.sec.gov/CIK 0001023731", "http://www.sec.gov/CIK 0001089511", "http://www.sec.gov/CIK 0000104777", "http://www.sec.gov/CIK 0001047919", "http://www.sec.gov/CIK 0000732717", "http://www.sec.gov/CIK 0000879585", "http://www.sec.gov/CIK 0001122063", "http://www.sec.gov/CIK 0001104358", "http://www.sec.gov/CIK 0001205727", "http://www.sec.gov/CIK 0000018926", "http://www.sec.gov/CIK 0000716133", "http://www.sec.gov/CIK 0001161706", "http://www.sec.gov/CIK 0001457737", "http://www.sec.gov/CIK 0001304421", "http://www.sec.gov/CIK 0001086740", "http://www.sec.gov/CIK 0001352819", "http://www.sec.gov/CIK 0001084384", "http://www.sec.gov/CIK 0001101239", "http://www.sec.gov/CIK 0000766561", "http://www.sec.gov/CIK 0001062613", "http://www.sec.gov/CIK 0001504678", "http://www.sec.gov/CIK 0000020520", "http://www.sec.gov/CIK 0001071411", "http://www.sec.gov/CIK 0000075679", "http://www.sec.gov/CIK 0000808461", "http://www.sec.gov/CIK 0001061322", "http://www.sec.gov/CIK 0000746210", "http://www.sec.gov/CIK 0001384929", "http://www.sec.gov/CIK 0001315255", "http://www.sec.gov/CIK 0001006837", "http://www.sec.gov/CIK 0001005731", "http://www.sec.gov/CIK 0001041954", "http://www.sec.gov/CIK 0001292653", "http://www.sec.gov/CIK 0001160142", "http://www.sec.gov/CIK 0001084752", "http://www.sec.gov/CIK 0000794323", "http://www.sec.gov/CIK 0001130888", "http://www.sec.gov/CIK 0001520744", "http://www.sec.gov/CIK 0001005699", "http://www.sec.gov/CIK 0000732412", "http://www.sec.gov/CIK 0000071557", "http://www.sec.gov/CIK 0001328571", "http://www.sec.gov/CIK 0001288359", "http://www.sec.gov/CIK 0001372041", "http://www.sec.gov/CIK 0000090721", "http://www.sec.gov/CIK 0001558294", "http://www.sec.gov/CIK 0001405660", "http://www.sec.gov/CIK 0001458704", "http://www.sec.gov/CIK 0001143238", "http://www.sec.gov/CIK 0001037949", "http://www.sec.gov/CIK 0000068622", "http://www.sec.gov/CIK 0001341319", "http://www.sec.gov/CIK 0000354963", "http://www.sec.gov/CIK 0000101830", "http://www.sec.gov/CIK 0000943117", "http://www.sec.gov/CIK 0001560158", "http://www.sec.gov/CIK 0001051512", "http://www.sec.gov/CIK 0001101688", "http://www.sec.gov/CIK 0001499355", "http://www.sec.gov/CIK 0000732712", "http://www.sec.gov/CIK 0001270400", "http://www.sec.gov/CIK 0001322791", "http://www.sec.gov/CIK 0001353505", "http://www.sec.gov/CIK 0001272830", "http://www.sec.gov/CIK 0001562636", "http://www.sec.gov/CIK 0001282266", "http://www.sec.gov/CIK 0000919742", "http://www.sec.gov/CIK 0001111634", "http://www.sec.gov/CIK 0001448962", "http://www.sec.gov/CIK 0001502756", "http://www.sec.gov/CIK 0001373444", "http://www.sec.gov/CIK 0001057758" ],
      "sic-dow30": [ "http://www.sec.gov/CIK 0000732717", "http://www.sec.gov/CIK 0000732712" ],
      "mixed" : [ "http://www.sec.gov/CIK 0001585644", "http://www.sec.gov/CIK 0001608249", "http://www.sec.gov/CIK 0001023731", "http://www.sec.gov/CIK 0001089511", "http://www.sec.gov/CIK 0000104777", "http://www.sec.gov/CIK 0001047919", "http://www.sec.gov/CIK 0000004962", "http://www.sec.gov/CIK 0000732717", "http://www.sec.gov/CIK 0000879585", "http://www.sec.gov/CIK 0001122063", "http://www.sec.gov/CIK 0001104358", "http://www.sec.gov/CIK 0001205727", "http://www.sec.gov/CIK 0000018926", "http://www.sec.gov/CIK 0000716133", "http://www.sec.gov/CIK 0000021344", "http://www.sec.gov/CIK 0001161706", "http://www.sec.gov/CIK 0001457737", "http://www.sec.gov/CIK 0001304421", "http://www.sec.gov/CIK 0001086740", "http://www.sec.gov/CIK 0001352819", "http://www.sec.gov/CIK 0001084384", "http://www.sec.gov/CIK 0001101239", "http://www.sec.gov/CIK 0000766561", "http://www.sec.gov/CIK 0001062613", "http://www.sec.gov/CIK 0001504678", "http://www.sec.gov/CIK 0000020520", "http://www.sec.gov/CIK 0001071411", "http://www.sec.gov/CIK 0000075679", "http://www.sec.gov/CIK 0000808461", "http://www.sec.gov/CIK 0001061322", "http://www.sec.gov/CIK 0000746210", "http://www.sec.gov/CIK 0001384929", "http://www.sec.gov/CIK 0001315255", "http://www.sec.gov/CIK 0001006837", "http://www.sec.gov/CIK 0001005731", "http://www.sec.gov/CIK 0001041954", "http://www.sec.gov/CIK 0001292653", "http://www.sec.gov/CIK 0001160142", "http://www.sec.gov/CIK 0001084752", "http://www.sec.gov/CIK 0000794323", "http://www.sec.gov/CIK 0001130888", "http://www.sec.gov/CIK 0001520744", "http://www.sec.gov/CIK 0001005699", "http://www.sec.gov/CIK 0000732412", "http://www.sec.gov/CIK 0000071557", "http://www.sec.gov/CIK 0001328571", "http://www.sec.gov/CIK 0001288359", "http://www.sec.gov/CIK 0001372041", "http://www.sec.gov/CIK 0000090721", "http://www.sec.gov/CIK 0001558294", "http://www.sec.gov/CIK 0001405660", "http://www.sec.gov/CIK 0001458704", "http://www.sec.gov/CIK 0001143238", "http://www.sec.gov/CIK 0001037949", "http://www.sec.gov/CIK 0000068622", "http://www.sec.gov/CIK 0001341319", "http://www.sec.gov/CIK 0000354963", "http://www.sec.gov/CIK 0000101830", "http://www.sec.gov/CIK 0000943117", "http://www.sec.gov/CIK 0001560158", "http://www.sec.gov/CIK 0001051512", "http://www.sec.gov/CIK 0001101688", "http://www.sec.gov/CIK 0001499355", "http://www.sec.gov/CIK 0000732712", "http://www.sec.gov/CIK 0001270400", "http://www.sec.gov/CIK 0001322791", "http://www.sec.gov/CIK 0001353505", "http://www.sec.gov/CIK 0001272830", "http://www.sec.gov/CIK 0001562636", "http://www.sec.gov/CIK 0000104169", "http://www.sec.gov/CIK 0001282266", "http://www.sec.gov/CIK 0000919742", "http://www.sec.gov/CIK 0001111634", "http://www.sec.gov/CIK 0001448962", "http://www.sec.gov/CIK 0001502756", "http://www.sec.gov/CIK 0001373444", "http://www.sec.gov/CIK 0001057758" ],
      "mixed-dow30": [ "http://www.sec.gov/CIK 0000004962", "http://www.sec.gov/CIK 0000732717", "http://www.sec.gov/CIK 0000021344", "http://www.sec.gov/CIK 0000732712", "http://www.sec.gov/CIK 0000104169" ]
    };

declare %an:nondeterministic function local:test-example1() as item
{
  let $expected :=
  [ {
    "_id" : "http://www.sec.gov/CIK 0000021344",
    "Archives" : "http://" || request:server-name() || ":" || request:server-port() ||
      "/v1/_queries/public/api/filings.jq?_method=POST&eid=http%3A%2F%2Fwww.sec.gov%2FCIK%200000021344&format=&profile-name=sec&token=" || $config:test-token || "&fiscalYear=ALL&fiscalPeriod=ALL",
    "Profiles" : {
      "SEC" : {
        "Name" : "SEC",
        "CompanyName" : "COCA COLA CO",
        "CompanyType" : "Corporation",
        "CIK" : "0000021344",
        "SIC" : "2080",
        "SICDescription" : "BEVERAGES",
        "SICGroup" : 9,
        "Taxonomy" : "ci",
        "Sector" : "Food/Kindred Products",
        "Tickers" : [ "ko" ],
        "Tags" : [ "FORTUNE100", "DOW30", "RUSSELL1000", "SP500" ],
        "IsTrust" : false
      }
    }
  } ]
  let $endpoint := "entities"
  let $params := {cik:"21344"}
  let $request := test:invoke($endpoint, $params)
  let $status as integer := $request[1]
  let $actual as array? := $request[2].Entities
  return test:assert-deep-equal($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-entities(
    $expected as array,
    $params as object) as item
{
    let $endpoint := "entities"
    let $request := test:invoke($endpoint, $params)
    let $actual as array := [ $request[2].Entities[]._id ]
    let $status as integer := $request[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

let $dow30 := test:is-dow30()
return local:check({
    dow30: local:test-entities($local:expected.dow30, {tag:"DOW30"}),
    generic: local:test-entities($local:expected.generic, {eid:"http://www.sec.gov/CIK 0000004962", profile-name:"generic"}),
    cik: local:test-entities($local:expected.cik, {cik:"4962"}),
    ticker: local:test-entities($local:expected.ticker, {ticker:"wmt"}),
    ticker2: local:test-entities($local:expected.ticker2, {ticker:["wmt","ko"]}),
    sic: local:test-entities(if($dow30) then $local:expected.sic-dow30 else $local:expected.sic, {sic:"4813"}),
    mixed: local:test-entities(if($dow30) then $local:expected.mixed-dow30 else $local:expected.mixed, {cik:"4962", sic:"4813", ticker:[ "wmt", "ko" ]}),
    example1: local:test-example1()
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace test = "http://apps.28.io/test";
import module namespace config = "http://apps.28.io/config";

declare variable $local:expected as object :=
    {
      "dow30" : [ "http://www.sec.gov/CIK 0000066740", "http://www.sec.gov/CIK 0000004962", "http://www.sec.gov/CIK 0000732717", "http://www.sec.gov/CIK 0000012927", "http://www.sec.gov/CIK 0000018230", "http://www.sec.gov/CIK 0000093410", "http://www.sec.gov/CIK 0000858877", "http://www.sec.gov/CIK 0000021344", "http://www.sec.gov/CIK 0000030554", "http://www.sec.gov/CIK 0000034088", "http://www.sec.gov/CIK 0000040545", "http://www.sec.gov/CIK 0000886982", "http://www.sec.gov/CIK 0000354950", "http://www.sec.gov/CIK 0000050863", "http://www.sec.gov/CIK 0000051143", "http://www.sec.gov/CIK 0000200406", "http://www.sec.gov/CIK 0000019617", "http://www.sec.gov/CIK 0000063908", "http://www.sec.gov/CIK 0000789019", "http://www.sec.gov/CIK 0000310158", "http://www.sec.gov/CIK 0000320187", "http://www.sec.gov/CIK 0000078003", "http://www.sec.gov/CIK 0000080424", "http://www.sec.gov/CIK 0000086312", "http://www.sec.gov/CIK 0000101829", "http://www.sec.gov/CIK 0000731766", "http://www.sec.gov/CIK 0000732712", "http://www.sec.gov/CIK 0001403161", "http://www.sec.gov/CIK 0000104169", "http://www.sec.gov/CIK 0001001039" ],
      "generic" : [ "http://www.sec.gov/CIK 0000004962" ],
      "cik" : [ "http://www.sec.gov/CIK 0000004962" ],
      "ticker" : [ "http://www.sec.gov/CIK 0000104169" ],
      "ticker2" : [ "http://www.sec.gov/CIK 0000021344", "http://www.sec.gov/CIK 0000104169" ],
      "sic" : [ "http://www.sec.gov/CIK 0001585644", "http://www.sec.gov/CIK 0001608249", "http://www.sec.gov/CIK 0001023731", "http://www.sec.gov/CIK 0001089511", "http://www.sec.gov/CIK 0000104777", "http://www.sec.gov/CIK 0001047919", "http://www.sec.gov/CIK 0000732717", "http://www.sec.gov/CIK 0000879585", "http://www.sec.gov/CIK 0001122063", "http://www.sec.gov/CIK 0001104358", "http://www.sec.gov/CIK 0001205727", "http://www.sec.gov/CIK 0000018926", "http://www.sec.gov/CIK 0000716133", "http://www.sec.gov/CIK 0001161706", "http://www.sec.gov/CIK 0001457737", "http://www.sec.gov/CIK 0001304421", "http://www.sec.gov/CIK 0001086740", "http://www.sec.gov/CIK 0001352819", "http://www.sec.gov/CIK 0001084384", "http://www.sec.gov/CIK 0001101239", "http://www.sec.gov/CIK 0000766561", "http://www.sec.gov/CIK 0001062613", "http://www.sec.gov/CIK 0001504678", "http://www.sec.gov/CIK 0000020520", "http://www.sec.gov/CIK 0001071411", "http://www.sec.gov/CIK 0000075679", "http://www.sec.gov/CIK 0000808461", "http://www.sec.gov/CIK 0001061322", "http://www.sec.gov/CIK 0000746210", "http://www.sec.gov/CIK 0001384929", "http://www.sec.gov/CIK 0001315255", "http://www.sec.gov/CIK 0001006837", "http://www.sec.gov/CIK 0001005731", "http://www.sec.gov/CIK 0001041954", "http://www.sec.gov/CIK 0001292653", "http://www.sec.gov/CIK 0001160142", "http://www.sec.gov/CIK 0001084752", "http://www.sec.gov/CIK 0000794323", "http://www.sec.gov/CIK 0001130888", "http://www.sec.gov/CIK 0001520744", "http://www.sec.gov/CIK 0001005699", "http://www.sec.gov/CIK 0000732412", "http://www.sec.gov/CIK 0000071557", "http://www.sec.gov/CIK 0001328571", "http://www.sec.gov/CIK 0001288359", "http://www.sec.gov/CIK 0001372041", "http://www.sec.gov/CIK 0000090721", "http://www.sec.gov/CIK 0001558294", "http://www.sec.gov/CIK 0001405660", "http://www.sec.gov/CIK 0001458704", "http://www.sec.gov/CIK 0001143238", "http://www.sec.gov/CIK 0001037949", "http://www.sec.gov/CIK 0000068622", "http://www.sec.gov/CIK 0001341319", "http://www.sec.gov/CIK 0000354963", "http://www.sec.gov/CIK 0000101830", "http://www.sec.gov/CIK 0000943117", "http://www.sec.gov/CIK 0001560158", "http://www.sec.gov/CIK 0001051512", "http://www.sec.gov/CIK 0001101688", "http://www.sec.gov/CIK 0001499355", "http://www.sec.gov/CIK 0000732712", "http://www.sec.gov/CIK 0001270400", "http://www.sec.gov/CIK 0001322791", "http://www.sec.gov/CIK 0001353505", "http://www.sec.gov/CIK 0001272830", "http://www.sec.gov/CIK 0001562636", "http://www.sec.gov/CIK 0001282266", "http://www.sec.gov/CIK 0000919742", "http://www.sec.gov/CIK 0001111634", "http://www.sec.gov/CIK 0001448962", "http://www.sec.gov/CIK 0001502756", "http://www.sec.gov/CIK 0001373444", "http://www.sec.gov/CIK 0001057758" ],
      "sic-dow30": [ "http://www.sec.gov/CIK 0000732717", "http://www.sec.gov/CIK 0000732712" ],
      "mixed" : [ "http://www.sec.gov/CIK 0001585644", "http://www.sec.gov/CIK 0001608249", "http://www.sec.gov/CIK 0001023731", "http://www.sec.gov/CIK 0001089511", "http://www.sec.gov/CIK 0000104777", "http://www.sec.gov/CIK 0001047919", "http://www.sec.gov/CIK 0000004962", "http://www.sec.gov/CIK 0000732717", "http://www.sec.gov/CIK 0000879585", "http://www.sec.gov/CIK 0001122063", "http://www.sec.gov/CIK 0001104358", "http://www.sec.gov/CIK 0001205727", "http://www.sec.gov/CIK 0000018926", "http://www.sec.gov/CIK 0000716133", "http://www.sec.gov/CIK 0000021344", "http://www.sec.gov/CIK 0001161706", "http://www.sec.gov/CIK 0001457737", "http://www.sec.gov/CIK 0001304421", "http://www.sec.gov/CIK 0001086740", "http://www.sec.gov/CIK 0001352819", "http://www.sec.gov/CIK 0001084384", "http://www.sec.gov/CIK 0001101239", "http://www.sec.gov/CIK 0000766561", "http://www.sec.gov/CIK 0001062613", "http://www.sec.gov/CIK 0001504678", "http://www.sec.gov/CIK 0000020520", "http://www.sec.gov/CIK 0001071411", "http://www.sec.gov/CIK 0000075679", "http://www.sec.gov/CIK 0000808461", "http://www.sec.gov/CIK 0001061322", "http://www.sec.gov/CIK 0000746210", "http://www.sec.gov/CIK 0001384929", "http://www.sec.gov/CIK 0001315255", "http://www.sec.gov/CIK 0001006837", "http://www.sec.gov/CIK 0001005731", "http://www.sec.gov/CIK 0001041954", "http://www.sec.gov/CIK 0001292653", "http://www.sec.gov/CIK 0001160142", "http://www.sec.gov/CIK 0001084752", "http://www.sec.gov/CIK 0000794323", "http://www.sec.gov/CIK 0001130888", "http://www.sec.gov/CIK 0001520744", "http://www.sec.gov/CIK 0001005699", "http://www.sec.gov/CIK 0000732412", "http://www.sec.gov/CIK 0000071557", "http://www.sec.gov/CIK 0001328571", "http://www.sec.gov/CIK 0001288359", "http://www.sec.gov/CIK 0001372041", "http://www.sec.gov/CIK 0000090721", "http://www.sec.gov/CIK 0001558294", "http://www.sec.gov/CIK 0001405660", "http://www.sec.gov/CIK 0001458704", "http://www.sec.gov/CIK 0001143238", "http://www.sec.gov/CIK 0001037949", "http://www.sec.gov/CIK 0000068622", "http://www.sec.gov/CIK 0001341319", "http://www.sec.gov/CIK 0000354963", "http://www.sec.gov/CIK 0000101830", "http://www.sec.gov/CIK 0000943117", "http://www.sec.gov/CIK 0001560158", "http://www.sec.gov/CIK 0001051512", "http://www.sec.gov/CIK 0001101688", "http://www.sec.gov/CIK 0001499355", "http://www.sec.gov/CIK 0000732712", "http://www.sec.gov/CIK 0001270400", "http://www.sec.gov/CIK 0001322791", "http://www.sec.gov/CIK 0001353505", "http://www.sec.gov/CIK 0001272830", "http://www.sec.gov/CIK 0001562636", "http://www.sec.gov/CIK 0000104169", "http://www.sec.gov/CIK 0001282266", "http://www.sec.gov/CIK 0000919742", "http://www.sec.gov/CIK 0001111634", "http://www.sec.gov/CIK 0001448962", "http://www.sec.gov/CIK 0001502756", "http://www.sec.gov/CIK 0001373444", "http://www.sec.gov/CIK 0001057758" ],
      "mixed-dow30": [ "http://www.sec.gov/CIK 0000004962", "http://www.sec.gov/CIK 0000732717", "http://www.sec.gov/CIK 0000021344", "http://www.sec.gov/CIK 0000732712", "http://www.sec.gov/CIK 0000104169" ]
    };

declare %an:nondeterministic function local:test-example1() as item
{
  let $expected :=
  [ {
    "_id" : "http://www.sec.gov/CIK 0000021344",
    "Archives" : "http://" || request:server-name() || ":" || request:server-port() ||
      "/v1/_queries/public/api/filings.jq?_method=POST&token=" || $config:test-token || "&eid=http%3A%2F%2Fwww.sec.gov%2FCIK%200000021344&format=&profile-name=sec&fiscalYear=ALL&fiscalPeriod=ALL",
    "Profiles" : {
      "SEC" : {
        "Name" : "SEC",
        "CompanyName" : "COCA COLA CO",
        "CompanyType" : "Corporation",
        "CIK" : "0000021344",
        "SIC" : "2080",
        "SICDescription" : "BEVERAGES",
        "SICGroup" : 9,
        "Taxonomy" : "ci",
        "Sector" : "Food/Kindred Products",
        "Tickers" : [ "ko" ],
        "Tags" : [ "FORTUNE100", "DOW30", "RUSSELL1000", "SP500" ],
        "IsTrust" : false
      }
    }
  } ]
  let $endpoint := "entities"
  let $params := {cik:"21344"}
  let $request := test:invoke($endpoint, $params)
  let $status as integer := $request[1]
  let $actual as array? := $request[2].Entities
  return test:assert-deep-equal($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-entities(
    $expected as array,
    $params as object) as item
{
    let $endpoint := "entities"
    let $request := test:invoke($endpoint, $params)
    let $actual as array := [ $request[2].Entities[]._id ]
    let $status as integer := $request[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

let $dow30 := test:is-dow30()
return local:check({
    dow30: local:test-entities($local:expected.dow30, {tag:"DOW30"}),
    generic: local:test-entities($local:expected.generic, {eid:"http://www.sec.gov/CIK 0000004962", profile-name:"generic"}),
    cik: local:test-entities($local:expected.cik, {cik:"4962"}),
    ticker: local:test-entities($local:expected.ticker, {ticker:"wmt"}),
    ticker2: local:test-entities($local:expected.ticker2, {ticker:["wmt","ko"]}),
    sic: local:test-entities(if($dow30) then $local:expected.sic-dow30 else $local:expected.sic, {sic:"4813"}),
    mixed: local:test-entities(if($dow30) then $local:expected.mixed-dow30 else $local:expected.mixed, {cik:"4962", sic:"4813", ticker:[ "wmt", "ko" ]}),
    example1: local:test-example1()
})


:green_heart: Fix again.

:green_heart: Fix again.

jsoniq version "1.0";

(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for querying SEC networks in SEC filings.

 :
SEC Networks are actually XBRL Components. In the SEC profiles, all XBRL networks
 : in an XBRL component must be consistent to each other. A way to look at it is that
 : the SEC Network can be identified with the XBRL presentation network it contains.

 :
 :
For XBRL-generic operations on XBRL components, use the
 : generic components module.

 :
 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace networks = "http://28.io/modules/xbrl/networks";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Disclosure for the balance sheet.
 :)
declare variable $sec-networks:BALANCE_SHEET as xs:string := "BalanceSheet";

(:~
 : Disclosure for the parenthetical balance sheet.
 :)
declare variable $sec-networks:BALANCE_SHEET_PARENTHETICAL as xs:string := "BalanceSheetParenthetical";

(:~
 : Disclosure for the income statement.
 :)
declare variable $sec-networks:INCOME_STATEMENT as xs:string := "IncomeStatement";

(:~
 : Disclosure for the parenthetical income statement.
 :)
declare variable $sec-networks:INCOME_STATEMENT_PARENTHETICAL as xs:string := "IncomeStatementParenthetical";

(:~
 : Disclosure for the statement of comprehensive income.
 :)
declare variable $sec-networks:STATEMENT_OF_COMPREHENSIVE_INCOME as xs:string := "StatementOfComprehensiveIncome";

(:~
 : Disclosure for the cash flow statement.
 :)
declare variable $sec-networks:CASH_FLOW_STATEMENT as xs:string := "CashFlowStatement";

(:~
 : Disclosure for the parenthetical cash flow statement.
 :)
declare variable $sec-networks:CASH_FLOW_STATEMENT_PARENTHETICAL as xs:string := "CashFlowStatementParenthetical";

(:~
 : Disclosure for the default component.
 :)
declare variable $sec-networks:DEFAULT_COMPONENT as xs:string := "DefaultComponent";

(:~
 : Disclosure for the document and entity information
 :)
declare variable $sec-networks:DOCUMENT_AND_ENTITY_INFO as xs:string := "DocumentAndEntityInformation";

(:~
 :
Retrieves all SEC Networks in a given filing.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 :
 : @return all networks in the supplied filings.
 :)
declare function sec-networks:networks-for-filings(
    $archives-or-ids as item*) as object*
{
  components:components-for-archives($archives-or-ids)
};

(:~
 :
Retrieves all SEC Networks that bear the supplied disclosures.

 :
 : @param $disclosure a list of disclosures.
 :
 : @return all models that match one of the disclosures.
 :)
declare function sec-networks:networks-for-disclosures(
    $disclosures as string*) as object*
{
  for $disclosure in $disclosures
  return mw:find($components:col,{ "Profiles" : { "SEC" : { "Disclosure": $disclosure } } })
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and that match the supplied disclosures.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $disclosures a list of disclosures.
 :
 : @return all models in the archives and that match one of the disclosures.
 :)
declare function sec-networks:networks-for-filings-and-disclosures(
    $archive-or-ids as item*,
    $disclosures as string*) as object*
{
  mw:find($components:col,
  {
    $components:ARCHIVE: [ $archive-or-ids ! archives:aid($$) ],
    "Profiles" : { "SEC" : { "Disclosure": [ $disclosures ] } }
  })
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and that match the supplied category.

 :
 :
A category can be any of "Statement", "Disclosure", "Document",
 : "Schedule", or "Unknown".

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $categories a list of categories.
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-categories(
    $archive-or-ids as item*,
    $categories as string*) as object*
{
  for $aid_or_archive in $archive-or-ids
  let $aid as atomic := archives:aid($aid_or_archive)
  for $category in $categories
  return mw:find($components:col,
  {
    $components:ARCHIVE: $aid,
    "Profiles" : { "SEC" : { "Category": $category } }
  })
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and that match the supplied roles/network identifier.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $roles a list of roles.
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-roles(
    $archive-or-ids as item*,
    $roles as string*) as object*
{
  mw:find($components:col,
  {
    $components:ARCHIVE: [ $archive-or-ids ! archives:aid($$) ],
    "Role": [ $roles ]
  })
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and that contain at least one report element with a given name.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $report-elements a list of contained report element names.
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-reportElements(
    $archive-or-ids as item*,
    $report-elements as string*) as object*
{
  let $ids := mw:find($concepts:col,
      {
         $components:ARCHIVE: [ $archive-or-ids ! archives:aid($$) ],
         "Name" : [ $report-elements ]
      }).Component
  return components:components($ids)
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and whose label matches the given search string.

 :
 :
The result of the function is limited to 100.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $label-search-term a search term to search in labels
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-label(
    $archive-or-ids as item*,
    $label-search-term as string*) as object*
{
  sec-networks:networks-for-filings-and-label(
    $archive-or-ids, $label-search-term, 100)
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and whose label matches the given search string.

 :
 :
The result of the function is limited to 100.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $label-search-term a search term to search in labels
 : @param $limit limit the number of search results
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-label(
    $archive-or-ids as item*,
    $label-search-term as string*,
    $limit as integer) as object*
{
  mw:run-cmd-deterministic(
  {
    "text" : $components:col,
    "filter" : { "Archive" : [ $archive-or-ids ! archives:aid($$) ] },
    "search" : $label-search-term,
    "limit" : $limit,
    "score" : { "$meta" : "textScore" },
    "sort" : { score: { "$meta" : "textScore" } }
  }).results[].obj
};

(:~
 :
Retrieves the components with the given CID, or belonging to the given archives and corresponding to one of report
 : elements, concepts, disclosures, roles or label search.

 :
 : @param $archives-or-aids a sequence of archives or AIDs.
 : @param $components-or-ids a sequence of components or their CIDs.
 : @param $report-elements a sequence of report element names.
 : @param $concepts a sequence of concept names.
 : @param $disclosures a sequence of disclosures.
 : @param $roles a sequence of role URIs.
 : @param $search a sequence of label search strings.
 :
 : @return a sequence of components.
 :)
declare function sec-networks:components(
    $archives-or-aids as item*,
    $components-or-ids as item*,
    $report-elements as string*,
    $disclosures as string*,
    $roles as string*,
    $search as string*) as object*
{
    let $archives as object* := archives:archives($archives-or-aids)
    return(
        components:components($components-or-ids)[exists($components-or-ids)],
        if (exists(($report-elements, $disclosures, $roles, $search)))
        then (
                components:components-for-archives-and-concepts($archives, $report-elements),
                sec-networks:networks-for-filings-and-disclosures($archives, $disclosures),
                components:components-for-archives-and-roles($archives, $roles),
                sec-networks:networks-for-filings-and-label($archives, $search)
            )
        else components:components-for-archives($archives)
    )
};

(:~
 :
Builds a standard definition model out of the specified component.

 :
The concepts will be put on the y axis according to the presentation network.

 :
The other dimensions are put on the x axis, with one breakdown for each.

 :
Explicit dimensions are organized according to the dimension hierarchy from the domain-member network.

 :
Typed dimensions are organized according to the actual values.

 :
 :
Auto slicing will be performed against the fact table
 :
 : @param $component a component object.
 : @param $hypercube the name of a hypercube.
 :
 : @return a definition model
 :)
declare function sec-networks:standard-definition-models-for-component(
    $component as object,
    $hypercube as string) as object
{
    sec-networks:standard-definition-models-for-component($component, $hypercube, ())
};

(:~
 :
Builds a standard definition model out of the specified component.

 :
The concepts will be put on the y axis according to the presentation network.

 :
The other dimensions are put on the x axis, with one breakdown for each.

 :
Explicit dimensions are organized according to the dimension hierarchy from the domain-member network.

 :
Typed dimensions are organized according to the actual values.

 :
 : @param $component a component object.
 : @param $hypercube the name of a hypercube.
 : @param $options
some optional parameters, including:

 :

 :
AutoSlice: a boolean specifying whether or not slicing should be done automatically, looking at the fact table. Deactivating auto slicing will
 :  lead to better performance, but a more verbose table. If AutoSlice is active, dimensions with only one value in the fact pool will become
 : global filters rather than breakdowns on the x axis.

 :
Slicers: an object with forced slicers.

 :

 :
 : @error components:HYPERCUBE-DOES-NOT-EXIST if the specified hypercube is not found.
 : @return a definition model
 :)
declare function sec-networks:standard-definition-models-for-component(
    $component as object,
    $hypercube as string,
    $options as object?) as object
{
  if(not $hypercube = (keys($component.Hypercubes), $components:ALL-HYPERCUBES))
  then error(
    QName("components:HYPERCUBE-DOES-NOT-EXIST"),
    "Hypercube " || $hypercube || " was not found."
  )
  else
    let $table as object := hypercubes:hypercubes-for-components($component, $hypercube)

    let $auto-slice as boolean := empty($options.AutoSlice) or $options.AutoSlice
    let $facts as object*:=
        if($auto-slice)
        then hypercubes:facts($table)
        else ()
    let $dimensions as string*:= keys($table.Aspects)
    let $values-by-dimension as object := {|
        for $d in $dimensions
        return { $d : [ distinct-values($facts.Aspects.$d) ] }
    |}
    let $auto-slice-dimensions as string* :=
    (
        "xbrl:Entity"[size($values-by-dimension."xbrl:Entity") eq 1],
        keys($values-by-dimension)[
            size($values-by-dimension.$$) eq 1 and
            not $$ = ("xbrl:Entity",
                      "xbrl:Period",
                      "xbrl28:Archive",
                      "sec:Accepted",
                      "sec:FiscalYear",
                      "sec:FiscalPeriod",
                      "sec:FiscalPeriodType") ]
    )

    let $user-slice-dimensions as string* :=
        keys($options.Slicers)

    let $column-dimensions as string* := keys($values-by-dimension)[not $$ =
        ("xbrl:Concept",
        "xbrl:Period",
        "xbrl:Unit",
        "xbrl:Entity",
        "sec:Accepted",
        "xbrl28:Archive",
        "sec:FiscalYear",
        "sec:FiscalPeriod",
        "sec:FiscalPeriodType",
        $auto-slice-dimensions,
        $user-slice-dimensions)]

    let $x-breakdowns as object* := (
        components:standard-typed-dimension-breakdown($component, "xbrl:Period", $options)
          [not (($auto-slice-dimensions, $user-slice-dimensions) = "xbrl:Period")],
        for $d as string in $column-dimensions
        let $dimension-object as object := $table.Aspects.$d
        return
            if($d = ("sec:Accepted", "sec:FiscalYear", "sec:FiscalPeriod", "sec:FiscalPeriodType"))
            then components:standard-typed-dimension-breakdown(
                $component,
                $d,
                $options)
            else components:standard-explicit-dimension-breakdown(
                $component,
                $d,
                $dimension-object.Members[].Name,
                $component.Role,
                $options),
        components:standard-typed-dimension-breakdown($component, "xbrl:Entity", $options)
          [not (($auto-slice-dimensions, $user-slice-dimensions) = "xbrl:Entity")]
    )

    let $lineitems as string* := components:line-items($component, $hypercube)
    let $presentation-network as object? := networks:networks-for-components-and-short-names($component, "Presentation")
    let $roots as string* := distinct-values($presentation-network.Trees[].Name)
    let $lineitems as string* := if(exists($lineitems)) then $lineitems else $roots
    let $y-breakdowns as object := components:standard-concept-breakdown($lineitems, $component.Role)

    return {
        ModelKind: "DefinitionModel",
        Labels: [$component.Label],
        Parameters: {},
        Breakdowns: {
            "x" : [
                $x-breakdowns
            ],
            "y": [
                $y-breakdowns
            ]
        },
        TableFilters: {|
            for $d as string in distinct-values(($auto-slice-dimensions, $user-slice-dimensions))
            return if($d = $user-slice-dimensions)
                   then { $d : $options.Slicers.$d }
                   else { $d : $values-by-dimension.$d[] },
            if (not $auto-slice)
            then { "xbrl28:Archive" : $component.Archive }
            else ()
        |}
    }
};

(:~
 :
 :
Retrieves all facts belonging to the SEC Network.

 :
 : @param $networks-or-ids a sequence of SEC Network objects, or their XBRL Component IDs.
 : @return a sequence of facts.
 :)
declare function sec-networks:facts(
    $networks-or-ids as item*
)
as object*
{
  sec-networks:facts($networks-or-ids, ())
};

(:~
 :
 :
Retrieves all facts belonging to the SEC Network.

 :
 : @param $networks a sequence of SEC Network objects.
 : @param $options standard SEC BizQL options.
 :
 : @return a sequence of facts.
 :)
declare function sec-networks:facts(
    $networks as object*,
    $options as object?
)
as object*
{
  for $component as object in $networks
  for $table as string? allowing empty in components:hypercubes($component)[$$ ne "xbrl28:ImpliedTable"]
  let $hypercube as object? := hypercubes:hypercubes-for-components($component, $table)
  let $hypercube as object := if (exists($hypercube))
                              then $hypercube
                              else sec:dimensionless-hypercube({
                                  Concepts: [ components:concepts($component) ]
                              })
  let $facts := hypercubes:facts-for-hypercube(
    $hypercube,
    $component.Archive,
    if (exists($options.Filter.Profiles.SEC.Fiscal))
    then $options
    else facts:merge-objects(
        { Filter: { "Profiles" : { "SEC" : { "Fiscal" : { "$exists" : true } } } } },
        $options,
        true
    )
  )
  return sec:hide-amended-facts($facts, $options)
};

(:~
 :
 :
Returns the disclosures of the suplied networks.

 :
 : @param $networks a sequence of SEC Network objects.
 : @return the disclosure names, or "UncategorizedInformation" if none.
 :
 :)
declare function sec-networks:disclosures($networks as object*) as string+
{
  for $component in $networks
  let $disclosure := $component.Profiles.SEC.Disclosure
  return if (exists($disclosure))
         then $disclosure
         else "UncategorizedInformation"
};

(:~
 :
 :
Return the categories of the supplied SEC networks (Statement, Disclosure, Document or Schedule).

 :
 : @param $networks a sequence of SEC Network objects.
 :
 : @return the category of each network.
 :)
declare function sec-networks:categories($networks as object*) as string*
{
  for $component in $networks
  return normalize-space(tokenize($component.Label, "-")[2])
};

(:~
 :
 :
Return the sub-categories of the supplied SEC networks (Detail, TextBlockLevel4, TextBLockLevel1to3).

 :
 : @param $networks a sequence of SEC Network objects.
 :
 : @return the sub-category of each network.
 :)
declare function sec-networks:sub-categories($networks as object*) as string*
{
  for $network in $networks
  let $is-text-blocks as boolean* := ($network.Concepts[])
    [not $$.SubstitutionGroup = ("xbrldt:hypercubeItem", "xbrldt:dimensionItem") and not $$.IsAbstract].IsTextBlock
  let $exists-text-blocks as boolean := exists($is-text-blocks[$$])
  let $exists-non-text-blocks as boolean := exists($is-text-blocks[not $$])
  return switch(true)
      case (: no text blocks, but still non-text-blocks :) $exists-non-text-blocks and not $exists-text-blocks return "Detail"
      case (: both :)  $exists-non-text-blocks (: and $exists-text-blocks :) return "TextBlockLevel4"
      case (: only text blocks :) $exists-text-blocks return "TextBlockLevel1To3"
      default (: exists nothing :) return "Unknown"
};

(:~
 :
 :
Return summary information for the supplied SEC networks.

 :
 : @param $networks a sequence of SEC Network objects.
 :
 : @return one object per network, containing a summary.
 :)
declare function sec-networks:summary($network as object) as object
{
  {
    Category : sec-networks:categories($network),
    SubCategory : sec-networks:sub-categories($network),
    Disclosure : sec-networks:disclosures($network)
  }
};

jsoniq version "1.0";

(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for querying SEC networks in SEC filings.

 :
SEC Networks are actually XBRL Components. In the SEC profiles, all XBRL networks
 : in an XBRL component must be consistent to each other. A way to look at it is that
 : the SEC Network can be identified with the XBRL presentation network it contains.

 :
 :
For XBRL-generic operations on XBRL components, use the
 : generic components module.

 :
 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace networks = "http://28.io/modules/xbrl/networks";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Disclosure for the balance sheet.
 :)
declare variable $sec-networks:BALANCE_SHEET as xs:string := "BalanceSheet";

(:~
 : Disclosure for the parenthetical balance sheet.
 :)
declare variable $sec-networks:BALANCE_SHEET_PARENTHETICAL as xs:string := "BalanceSheetParenthetical";

(:~
 : Disclosure for the income statement.
 :)
declare variable $sec-networks:INCOME_STATEMENT as xs:string := "IncomeStatement";

(:~
 : Disclosure for the parenthetical income statement.
 :)
declare variable $sec-networks:INCOME_STATEMENT_PARENTHETICAL as xs:string := "IncomeStatementParenthetical";

(:~
 : Disclosure for the statement of comprehensive income.
 :)
declare variable $sec-networks:STATEMENT_OF_COMPREHENSIVE_INCOME as xs:string := "StatementOfComprehensiveIncome";

(:~
 : Disclosure for the cash flow statement.
 :)
declare variable $sec-networks:CASH_FLOW_STATEMENT as xs:string := "CashFlowStatement";

(:~
 : Disclosure for the parenthetical cash flow statement.
 :)
declare variable $sec-networks:CASH_FLOW_STATEMENT_PARENTHETICAL as xs:string := "CashFlowStatementParenthetical";

(:~
 : Disclosure for the default component.
 :)
declare variable $sec-networks:DEFAULT_COMPONENT as xs:string := "DefaultComponent";

(:~
 : Disclosure for the document and entity information
 :)
declare variable $sec-networks:DOCUMENT_AND_ENTITY_INFO as xs:string := "DocumentAndEntityInformation";

(:~
 :
Retrieves all SEC Networks in a given filing.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 :
 : @return all networks in the supplied filings.
 :)
declare function sec-networks:networks-for-filings(
    $archives-or-ids as item*) as object*
{
  components:components-for-archives($archives-or-ids)
};

(:~
 :
Retrieves all SEC Networks that bear the supplied disclosures.

 :
 : @param $disclosure a list of disclosures.
 :
 : @return all models that match one of the disclosures.
 :)
declare function sec-networks:networks-for-disclosures(
    $disclosures as string*) as object*
{
  for $disclosure in $disclosures
  return mw:find($components:col,{ "Profiles" : { "FSA" : { "SEC" : { "Disclosure": $disclosure } } } })
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and that match the supplied disclosures.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $disclosures a list of disclosures.
 :
 : @return all models in the archives and that match one of the disclosures.
 :)
declare function sec-networks:networks-for-filings-and-disclosures(
    $archive-or-ids as item*,
    $disclosures as string*) as object*
{
  mw:find($components:col,
  {
    $components:ARCHIVE: [ $archive-or-ids ! archives:aid($$) ],
    "Profiles" : { "FSA" : { "SEC" : { "Disclosure": [ $disclosures ] } } }
  })
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and that match the supplied category.

 :
 :
A category can be any of "Statement", "Disclosure", "Document",
 : "Schedule", or "Unknown".

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $categories a list of categories.
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-categories(
    $archive-or-ids as item*,
    $categories as string*) as object*
{
  for $aid_or_archive in $archive-or-ids
  let $aid as atomic := archives:aid($aid_or_archive)
  for $category in $categories
  return mw:find($components:col,
  {
    $components:ARCHIVE: $aid,
    "Profiles" : { "FSA" : { "SEC" : { "Category": $category } } }
  })
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and that match the supplied roles/network identifier.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $roles a list of roles.
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-roles(
    $archive-or-ids as item*,
    $roles as string*) as object*
{
  mw:find($components:col,
  {
    $components:ARCHIVE: [ $archive-or-ids ! archives:aid($$) ],
    "Role": [ $roles ]
  })
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and that contain at least one report element with a given name.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $report-elements a list of contained report element names.
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-reportElements(
    $archive-or-ids as item*,
    $report-elements as string*) as object*
{
  let $ids := mw:find($concepts:col,
      {
         $components:ARCHIVE: [ $archive-or-ids ! archives:aid($$) ],
         "Name" : [ $report-elements ]
      }).Component
  return components:components($ids)
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and whose label matches the given search string.

 :
 :
The result of the function is limited to 100.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $label-search-term a search term to search in labels
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-label(
    $archive-or-ids as item*,
    $label-search-term as string*) as object*
{
  sec-networks:networks-for-filings-and-label(
    $archive-or-ids, $label-search-term, 100)
};

(:~
 :
Retrieves all models that belong to the archives given as first
 : parameter and whose label matches the given search string.

 :
 :
The result of the function is limited to 100.

 :
 : @param $archive-or-ids an arbitrary number of archive identifiers (AIDs)
 :                          or archive objects.
 : @param $label-search-term a search term to search in labels
 : @param $limit limit the number of search results
 :
 : @return all said models
 :)
declare function sec-networks:networks-for-filings-and-label(
    $archive-or-ids as item*,
    $label-search-term as string*,
    $limit as integer) as object*
{
  mw:run-cmd-deterministic(
  {
    "text" : $components:col,
    "filter" : { "Archive" : [ $archive-or-ids ! archives:aid($$) ] },
    "search" : $label-search-term,
    "limit" : $limit,
    "score" : { "$meta" : "textScore" },
    "sort" : { score: { "$meta" : "textScore" } }
  }).results[].obj
};

(:~
 :
Retrieves the components with the given CID, or belonging to the given archives and corresponding to one of report
 : elements, concepts, disclosures, roles or label search.

 :
 : @param $archives-or-aids a sequence of archives or AIDs.
 : @param $components-or-ids a sequence of components or their CIDs.
 : @param $report-elements a sequence of report element names.
 : @param $concepts a sequence of concept names.
 : @param $disclosures a sequence of disclosures.
 : @param $roles a sequence of role URIs.
 : @param $search a sequence of label search strings.
 :
 : @return a sequence of components.
 :)
declare function sec-networks:components(
    $archives-or-aids as item*,
    $components-or-ids as item*,
    $report-elements as string*,
    $disclosures as string*,
    $roles as string*,
    $search as string*) as object*
{
    let $archives as object* := archives:archives($archives-or-aids)
    return(
        components:components($components-or-ids)[exists($components-or-ids)],
        if (exists(($report-elements, $disclosures, $roles, $search)))
        then (
                components:components-for-archives-and-concepts($archives, $report-elements),
                sec-networks:networks-for-filings-and-disclosures($archives, $disclosures),
                components:components-for-archives-and-roles($archives, $roles),
                sec-networks:networks-for-filings-and-label($archives, $search)
            )
        else components:components-for-archives($archives)
    )
};

(:~
 :
Builds a standard definition model out of the specified component.

 :
The concepts will be put on the y axis according to the presentation network.

 :
The other dimensions are put on the x axis, with one breakdown for each.

 :
Explicit dimensions are organized according to the dimension hierarchy from the domain-member network.

 :
Typed dimensions are organized according to the actual values.

 :
 :
Auto slicing will be performed against the fact table
 :
 : @param $component a component object.
 : @param $hypercube the name of a hypercube.
 :
 : @return a definition model
 :)
declare function sec-networks:standard-definition-models-for-component(
    $component as object,
    $hypercube as string) as object
{
    sec-networks:standard-definition-models-for-component($component, $hypercube, ())
};

(:~
 :
Builds a standard definition model out of the specified component.

 :
The concepts will be put on the y axis according to the presentation network.

 :
The other dimensions are put on the x axis, with one breakdown for each.

 :
Explicit dimensions are organized according to the dimension hierarchy from the domain-member network.

 :
Typed dimensions are organized according to the actual values.

 :
 : @param $component a component object.
 : @param $hypercube the name of a hypercube.
 : @param $options
some optional parameters, including:

 :

 :
AutoSlice: a boolean specifying whether or not slicing should be done automatically, looking at the fact table. Deactivating auto slicing will
 :  lead to better performance, but a more verbose table. If AutoSlice is active, dimensions with only one value in the fact pool will become
 : global filters rather than breakdowns on the x axis.

 :
Slicers: an object with forced slicers.

 :

 :
 : @error components:HYPERCUBE-DOES-NOT-EXIST if the specified hypercube is not found.
 : @return a definition model
 :)
declare function sec-networks:standard-definition-models-for-component(
    $component as object,
    $hypercube as string,
    $options as object?) as object
{
  if(not $hypercube = (keys($component.Hypercubes), $components:ALL-HYPERCUBES))
  then error(
    QName("components:HYPERCUBE-DOES-NOT-EXIST"),
    "Hypercube " || $hypercube || " was not found."
  )
  else
    let $table as object := hypercubes:hypercubes-for-components($component, $hypercube)

    let $auto-slice as boolean := empty($options.AutoSlice) or $options.AutoSlice
    let $facts as object*:=
        if($auto-slice)
        then hypercubes:facts($table)
        else ()
    let $dimensions as string*:= keys($table.Aspects)
    let $values-by-dimension as object := {|
        for $d in $dimensions
        return { $d : [ distinct-values($facts.Aspects.$d) ] }
    |}
    let $auto-slice-dimensions as string* :=
    (
        "xbrl:Entity"[size($values-by-dimension."xbrl:Entity") eq 1],
        keys($values-by-dimension)[
            size($values-by-dimension.$$) eq 1 and
            not $$ = ("xbrl:Entity",
                      "xbrl:Period",
                      "xbrl28:Archive",
                      "sec:Accepted",
                      "sec:FiscalYear",
                      "sec:FiscalPeriod",
                      "sec:FiscalPeriodType") ]
    )

    let $user-slice-dimensions as string* :=
        keys($options.Slicers)

    let $column-dimensions as string* := keys($values-by-dimension)[not $$ =
        ("xbrl:Concept",
        "xbrl:Period",
        "xbrl:Unit",
        "xbrl:Entity",
        "sec:Accepted",
        "xbrl28:Archive",
        "sec:FiscalYear",
        "sec:FiscalPeriod",
        "sec:FiscalPeriodType",
        $auto-slice-dimensions,
        $user-slice-dimensions)]

    let $x-breakdowns as object* := (
        components:standard-typed-dimension-breakdown($component, "xbrl:Period", $options)
          [not (($auto-slice-dimensions, $user-slice-dimensions) = "xbrl:Period")],
        for $d as string in $column-dimensions
        let $dimension-object as object := $table.Aspects.$d
        return
            if($d = ("sec:Accepted", "sec:FiscalYear", "sec:FiscalPeriod", "sec:FiscalPeriodType"))
            then components:standard-typed-dimension-breakdown(
                $component,
                $d,
                $options)
            else components:standard-explicit-dimension-breakdown(
                $component,
                $d,
                $dimension-object.Members[].Name,
                $component.Role,
                $options),
        components:standard-typed-dimension-breakdown($component, "xbrl:Entity", $options)
          [not (($auto-slice-dimensions, $user-slice-dimensions) = "xbrl:Entity")]
    )

    let $lineitems as string* := components:line-items($component, $hypercube)
    let $presentation-network as object? := networks:networks-for-components-and-short-names($component, "Presentation")
    let $roots as string* := distinct-values($presentation-network.Trees[].Name)
    let $lineitems as string* := if(exists($lineitems)) then $lineitems else $roots
    let $y-breakdowns as object := components:standard-concept-breakdown($lineitems, $component.Role)

    return {
        ModelKind: "DefinitionModel",
        Labels: [$component.Label],
        Parameters: {},
        Breakdowns: {
            "x" : [
                $x-breakdowns
            ],
            "y": [
                $y-breakdowns
            ]
        },
        TableFilters: {|
            for $d as string in distinct-values(($auto-slice-dimensions, $user-slice-dimensions))
            return if($d = $user-slice-dimensions)
                   then { $d : $options.Slicers.$d }
                   else { $d : $values-by-dimension.$d[] },
            if (not $auto-slice)
            then { "xbrl28:Archive" : $component.Archive }
            else ()
        |}
    }
};

(:~
 :
 :
Retrieves all facts belonging to the SEC Network.

 :
 : @param $networks-or-ids a sequence of SEC Network objects, or their XBRL Component IDs.
 : @return a sequence of facts.
 :)
declare function sec-networks:facts(
    $networks-or-ids as item*
)
as object*
{
  sec-networks:facts($networks-or-ids, ())
};

(:~
 :
 :
Retrieves all facts belonging to the SEC Network.

 :
 : @param $networks a sequence of SEC Network objects.
 : @param $options standard SEC BizQL options.
 :
 : @return a sequence of facts.
 :)
declare function sec-networks:facts(
    $networks as object*,
    $options as object?
)
as object*
{
  for $component as object in $networks
  for $table as string? allowing empty in components:hypercubes($component)[$$ ne "xbrl28:ImpliedTable"]
  let $hypercube as object? := hypercubes:hypercubes-for-components($component, $table)
  let $hypercube as object := if (exists($hypercube))
                              then $hypercube
                              else sec:dimensionless-hypercube({
                                  Concepts: [ components:concepts($component) ]
                              })
  let $facts := hypercubes:facts-for-hypercube(
    $hypercube,
    $component.Archive,
    if (exists($options.Filter.Profiles.SEC.Fiscal))
    then $options
    else facts:merge-objects(
        { Filter: { "Profiles" : { "FSA" : { "SEC" : { "Fiscal" : { "$exists" : true } } } } } },
        $options,
        true
    )
  )
  return sec:hide-amended-facts($facts, $options)
};

(:~
 :
 :
Returns the disclosures of the suplied networks.

 :
 : @param $networks a sequence of SEC Network objects.
 : @return the disclosure names, or "UncategorizedInformation" if none.
 :
 :)
declare function sec-networks:disclosures($networks as object*) as string+
{
  for $component in $networks
  let $disclosure := $component.Profiles.SEC.Disclosure
  return if (exists($disclosure))
         then $disclosure
         else "UncategorizedInformation"
};

(:~
 :
 :
Return the categories of the supplied SEC networks (Statement, Disclosure, Document or Schedule).

 :
 : @param $networks a sequence of SEC Network objects.
 :
 : @return the category of each network.
 :)
declare function sec-networks:categories($networks as object*) as string*
{
  for $component in $networks
  return normalize-space(tokenize($component.Label, "-")[2])
};

(:~
 :
 :
Return the sub-categories of the supplied SEC networks (Detail, TextBlockLevel4, TextBLockLevel1to3).

 :
 : @param $networks a sequence of SEC Network objects.
 :
 : @return the sub-category of each network.
 :)
declare function sec-networks:sub-categories($networks as object*) as string*
{
  for $network in $networks
  let $is-text-blocks as boolean* := ($network.Concepts[])
    [not $$.SubstitutionGroup = ("xbrldt:hypercubeItem", "xbrldt:dimensionItem") and not $$.IsAbstract].IsTextBlock
  let $exists-text-blocks as boolean := exists($is-text-blocks[$$])
  let $exists-non-text-blocks as boolean := exists($is-text-blocks[not $$])
  return switch(true)
      case (: no text blocks, but still non-text-blocks :) $exists-non-text-blocks and not $exists-text-blocks return "Detail"
      case (: both :)  $exists-non-text-blocks (: and $exists-text-blocks :) return "TextBlockLevel4"
      case (: only text blocks :) $exists-text-blocks return "TextBlockLevel1To3"
      default (: exists nothing :) return "Unknown"
};

(:~
 :
 :
Return summary information for the supplied SEC networks.

 :
 : @param $networks a sequence of SEC Network objects.
 :
 : @return one object per network, containing a summary.
 :)
declare function sec-networks:summary($network as object) as object
{
  {
    Category : sec-networks:categories($network),
    SubCategory : sec-networks:sub-categories($network),
    Disclosure : sec-networks:disclosures($network)
  }
};