Convert output from JSON to Unix

Convert output from JSON to Unix

#!/usr/bin/jq -rMf

# List modules in order of the gap between their timestamps.
# May show misleading outliers due to recently-frozen modules
# and in-progress bot passes.

# Usage: wget -q -O - http://status.ksp-ckan.space/status/netkan.json | slow-modules.jq

(now - 3*60*60) as $cutoff
| to_entries
| map(select(.value.frozen != true)
    | {id:.key, dt:(.value.last_checked[0:19]+"Z"|fromdate)}
    | select(.dt >= $cutoff))
| sort_by(.dt)
| [range(1;length) as $i
   | .[$i-1] as $a
   | .[$i] as $b
   | {id:$b.id, dur:($b.dt - $a.dt)}]
| sort_by(-.dur)
| map([.dur, .id] | @tsv)
| .[]

#!/usr/bin/jq -rMf

# List modules in order of the gap between their timestamps.
# May show misleading outliers due to recently-frozen modules
# and in-progress bot passes.

# Usage: wget -q -O - http://status.ksp-ckan.space/status/netkan.json | slow-modules.jq

(now - 3*60*60) as $cutoff
| to_entries
| map(select(.value.frozen != true)
    | {id:.key, dt:(.value.last_checked[0:19]+"Z"|fromdate)}
    | select(.dt >= $cutoff))
| sort_by(.dt)
| [range(1;length) as $i
   | .[$i-1] as $a
   | .[$i] as $b
   | {id:$a.id, dur:($b.dt - $a.dt)}]
| sort_by(-.dur)


:pencil: Address review comment.

:pencil: Address review comment.

jsoniq version "1.0";
(:
 : Copyright 2014 28msec Inc.
 :)

(:~
 :
This module provides functionality for retrieving concepts
 : and labels.

 :
 : @author Federico Cavalieri
 :)
module namespace concepts = "http://28.io/modules/xbrl/concepts";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Name of the collection the concepts are stored in.
 :)
declare variable $concepts:col as xs:string := "concepts";

(:~
 : Name of the field pointing to the archive.
 :)
declare variable $concepts:ARCHIVE as xs:string := "Archive";

(:~
 : Name of the field pointing to the role.
 :)
declare variable $concepts:ROLE as xs:string := "Role";

(:~
 : Name of the field pointing to the name.
 :)
declare variable $concepts:NAME as xs:string := "Name";

(:~
 : Name of the field pointing to the labels.
 :)
declare variable $concepts:LABELS as xs:string := "Labels";

(:~
 : The default component link role.
 :)
declare variable $concepts:DEFAULT_COMPONENT_LINK_ROLE as xs:string :=
    "http://www.xbrl.org/2003/role/link";

(:~
 : Allow any component link role
 :)
declare variable $concepts:ANY_COMPONENT_LINK_ROLE as xs:string :=
    ":any component:";

(:~
 : A joker to ask for all concept names.
 :)
declare variable $concepts:ALL_CONCEPT_NAMES as xs:string := ":any concept:";

(:~
 : A joker to ask for all concept labels.
 :)
declare variable $concepts:ALL_CONCEPT_LABELS as xs:string := ":any concept label:";

(:~
 :
Retrieves all concepts.

 :
 : @return all concepts.
 :)
declare function concepts:concepts() as object*
{
  mw:find($concepts:col,{})
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives the archive numbers.
 : @param $component-roles the component roles.
 : @param $labels labels for exact matches.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archive-or-ids as item*,
    $component-roles as string*,
    $labels as string*
  ) as object*
{
  concepts:concepts($concept-names, $archive-or-ids, $component-roles, $labels, ())
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives the archive numbers.
 : @param $component-roles the component roles.
 : @param $labels labels for exact matches.
 : @param $options one option OnlyNames to only output concepts with the Name field.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archive-or-ids as item*,
    $component-roles as string*,
    $labels as string*,
    $options as object?
  ) as object*
{
  let $projection as object := if($options.OnlyNames eq true) then { Name: 1 } else {}
  let $archives := archives:aid($archive-or-ids)
  return
  if (exists($archives))
  then mw:find($concepts:col,
    {|
      {
        $concepts:ARCHIVE : { "$in" : [ $archives ] }
      },
      {
        $concepts:ROLE : { "$in" :
          [ $component-roles ]
        }
      }[not $component-roles = $concepts:ANY_COMPONENT_LINK_ROLE],
      {
        $concepts:NAME : { "$in" : [ $concept-names ] }
      }[not $concept-names = $concepts:ALL_CONCEPT_NAMES],
      {
        "Labels.Value" : { "$in" : [ $labels ] }
      }[not $labels = $concepts:ALL_CONCEPT_LABELS]
    |},
    $projection
  )
  else ()
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives the archive numbers.
 : @param $component-roles the component roles.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives as string*,
    $component-roles as string*
  ) as object*
{
  concepts:concepts(
    $concept-names,
    $archives,
    $component-roles,
    $concepts:ALL_CONCEPT_LABELS
  )
};

(:~
 :
Retrieves the concepts which:
 :  - concept name matches a given one;
 :  - archive number matches that of a given component;
 :  - component role matches that of a given component or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $component-or-ids the CIDs or the components themselves.
 : @param $labels labels for exact matches.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts-for-components(
    $concept-names as string*,
    $labels as string*,
    $component-or-ids as item*,
    $options as object?) as object*
{
  for $component in components:components($component-or-ids)
  return concepts:concepts($concept-names,
                           $component.Archive[not $$ instance of null],
                           $component.Role,
                           $labels,
                           $options)
};

jsoniq version "1.0";
(:
 : Copyright 2014 28msec Inc.
 :)

(:~
 :
This module provides functionality for retrieving concepts
 : and labels.

 :
 : @author Federico Cavalieri
 :)
module namespace concepts = "http://28.io/modules/xbrl/concepts";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Name of the collection the concepts are stored in.
 :)
declare variable $concepts:col as xs:string := "concepts";

(:~
 : Name of the field pointing to the archive.
 :)
declare variable $concepts:ARCHIVE as xs:string := "Archive";

(:~
 : Name of the field pointing to the role.
 :)
declare variable $concepts:ROLE as xs:string := "Role";

(:~
 : Name of the field pointing to the name.
 :)
declare variable $concepts:NAME as xs:string := "Name";

(:~
 : Name of the field pointing to the labels.
 :)
declare variable $concepts:LABELS as xs:string := "Labels";

(:~
 : The default component link role.
 :)
declare variable $concepts:DEFAULT_COMPONENT_LINK_ROLE as xs:string :=
    "http://www.xbrl.org/2003/role/link";

(:~
 : Allow any component link role
 :)
declare variable $concepts:ANY_COMPONENT_LINK_ROLE as xs:string :=
    "*";

(:~
 : A joker to ask for all concept names.
 :)
declare variable $concepts:ALL_CONCEPT_NAMES as xs:string := "";

(:~
 : A joker to ask for all concept labels.
 :)
declare variable $concepts:ALL_CONCEPT_LABELS as xs:string := "";

(:~
 :
Retrieves all concepts.

 :
 : @return all concepts.
 :)
declare function concepts:concepts() as object*
{
  mw:find($concepts:col,{})
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives the archive numbers.
 : @param $component-roles the component roles.
 : @param $labels labels for exact matches.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archive-or-ids as item*,
    $component-roles as string*,
    $labels as string*
  ) as object*
{
  concepts:concepts($concept-names, $archive-or-ids, $component-roles, $labels, ())
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives the archive numbers.
 : @param $component-roles the component roles.
 : @param $labels labels for exact matches.
 : @param $options one option OnlyNames to only output concepts with the Name field.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archive-or-ids as item*,
    $component-roles as string*,
    $labels as string*,
    $options as object?
  ) as object*
{
  let $projection as object := if($options.OnlyNames eq true) then { Name: 1 } else {}
  let $archives := archives:aid($archive-or-ids)
  return
  if (exists($archives))
  then mw:find($concepts:col,
    {|
      {
        $concepts:ARCHIVE : { "$in" : [ $archives ] }
      },
      {
        $concepts:ROLE : { "$in" :
          [ $component-roles ]
        }
      }[not $component-roles = $concepts:ANY_COMPONENT_LINK_ROLE],
      {
        $concepts:NAME : { "$in" : [ $concept-names ] }
      }[not $concept-names = $concepts:ALL_CONCEPT_NAMES],
      {
        "Labels.Value" : { "$in" : [ $labels ] }
      }[not $labels = $concepts:ALL_CONCEPT_LABELS]
    |},
    $projection
  )
  else ()
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives the archive numbers.
 : @param $component-roles the component roles.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives as string*,
    $component-roles as string*
  ) as object*
{
  concepts:concepts(
    $concept-names,
    $archives,
    $component-roles,
    $concepts:ALL_CONCEPT_LABELS
  )
};

(:~
 :
Retrieves the concepts which:
 :  - concept name matches a given one;
 :  - archive number matches that of a given component;
 :  - component role matches that of a given component or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $component-or-ids the CIDs or the components themselves.
 : @param $labels labels for exact matches.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts-for-components(
    $concept-names as string*,
    $labels as string*,
    $component-or-ids as item*,
    $options as object?) as object*
{
  for $component in components:components($component-or-ids)
  return concepts:concepts($concept-names,
                           $component.Archive[not $$ instance of null],
                           $component.Role,
                           $labels,
                           $options)
};


:green_heart: Fix syntax.

:green_heart: Fix syntax.

jsoniq version "1.0";

module namespace api = "http://apps.28.io/api";

import module namespace session    = "http://apps.28.io/session";
import module namespace resp       = "http://www.28msec.com/modules/http-response";
import module namespace sec-fiscal = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare function api:json-to-csv($objects as object*) as string
{
  let $flattened as object* := api:flatten-json-object($objects, ())
  return string-join(
    csv:serialize($flattened, { field-names: [ keys($flattened) ], serialize-null-as : "" }),
    "")
  };

declare function api:json-to-xml($objects as object*, $root-name as string) as element()*
{
  for $flattened-object as object in api:flatten-json-object($objects, { KeepArray: true })
  return element { $root-name } {
    for $key in keys($flattened-object)
    let $value := $flattened-object.$key
    return typeswitch($value)
           case array return
             let $key-with-s := if(substring($key, string-length($key), 1) eq "s")
                                then $key
                                else $key || "s"
             let $key-no-s := if(substring($key, string-length($key), 1) eq "s")
                                then substring($key, 1, string-length($key) - 1)
                                else $key
             return element {$key-with-s} { for $v in $value[] return element { $key-no-s } { $v } }
           default return element { $key } { $value }
  }
};

declare %private function api:flatten-json-object(
  $items as item*,
  $options as object?) as item*
{
  for $item in $items
  return typeswitch($item)
         case atomic return $item
         case array return let $atomic-items := flatten($item)[$$ instance of atomic]
                           return if ($options.KeepArray)
                                 then [ $atomic-items ]
                                  else string-join($atomic-items, ", ")
         case object return {|
             for $key in keys($item)
             return typeswitch($item.$key)
                    case object return api:flatten-json-object($item.$key, $options)
                    default return { $key: api:flatten-json-object($item.$key, $options) }
         |}
         default return ()
};

declare function api:validate-regexp($name as string, $value as string?, $regexp as string)
as ()
{
  if (exists($value))
  then
    if (matches($value, "^" || $regexp || "$"))
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " does not match reg. expression " || $regexp || ".");
  else ();
};

declare function api:validate-enum($name as string, $value as string?, $enum as string*)
as ()
{
  if (exists($value))
  then
    if ($value = $enum)
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " is not one of these: " || string-join($enum,", ") || ".");
  else ();
};

declare %private function api:sgpl($count as integer, $singular as string, $plural as string)
as string
{
    if ($count eq 1)
    then $count || " " || $singular
    else $count || " " || $plural
};

declare function api:passed($timestamp as xs:dateTime?)
as string
{
    if (empty($timestamp) or $timestamp eq null)
    then "Never"
    else
        let $now := current-dateTime()
        let $difference := $now - $timestamp
        let $years := years-from-duration($difference)
        let $month := months-from-duration($difference)
        let $days := days-from-duration($difference)
        let $hours := hours-from-duration($difference)
        let $minutes := minutes-from-duration($difference)
        return
            if ($years gt 0) then api:sgpl($years,"year ago","years ago")
            else if ($month gt 0) then api:sgpl($month,"month ago","month ago")
            else if ($days gt 0) then api:sgpl($days, "day ago", "days ago")
            else if ($hours gt 0) then api:sgpl($hours, "hour ago", "hours ago")
            else if ($minutes gt 0) then api:sgpl($minutes, "minute ago", "minutes ago")
            else if ($difference div xs:dayTimeDuration("PT1S") lt 0) then "future"
            else "Just Now"
};

declare function api:success() as object
{
  { "success" : true }
};

declare function api:success($data as object()) as object
{
  {|
     {"success" : true },
     $data
  |}
};

declare %an:sequential function api:check-and-return-results(
    $token as string?,
    $results as item*,
    $format as string?
) as item*
{
    switch(session:has-access($token, "data_sec"))
    case $session:ACCESS-ALLOWED return
        $results
    case $session:ACCESS-DENIED return {
          resp:status-code(403);
          session:error("access denied", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          resp:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()
};

declare function api:normalize-facts(
    $facts as object*) as object*
{
    for $fact in $facts
    return {|
        {
            "Aspects" : {|
                trim($fact.Aspects, ("xbrl:Unit")),
                { "sec:Archive" : $fact.Aspects."xbrl28:Archive" }[exists($fact.Aspects."xbrl28:Archive")]
            |}
        },
        trim($fact, ("Aspects", "_id")),
        { Unit: $fact.Aspects."xbrl:Unit" }[exists($fact.Aspects."xbrl:Unit") and not exists($fact.Unit)]
    |}
};

declare %an:sequential function api:serialize(
    $result as json-item,
    $comment as object,
    $serializers as object,
    $format as string?,
    $file-name as string) as item*
{
    switch ($format)
    case "xml" return {
        resp:serialization-parameters({"omit-xml-declaration" : false, indent : true });
        session:comment("xml", $comment),
        $serializers.to-xml($result)
    }
    case "text" case "csv" return {
        resp:content-type("text/csv");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    case "html" return {
        resp:content-type("text/html");
        let $csv as string := $serializers.to-csv($result)
        return api:csv-to-html($csv)
    }
    case "excel" return {
        resp:content-type("application/vnd.ms-excel");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    default return {
        resp:content-type("application/json");
        resp:serialization-parameters({"indent" : true});
        {|
            session:comment("json", $comment),
            $result
        |}
    }
};

declare %an:sequential function api:csv-to-html(
    $csv as string) as item*
{
    let $csv as string* := $csv
    let $rows := tokenize($csv, "\n")
    let $header := $rows[1]
    let $body := subsequence($rows, 2)
    let $display-cell := function($row as string, $cell-tag-name as string) {
      for tumbling window $cells as string* in tokenize($row, ",")
      start $start at $i when true
      only end $end at $j when not contains(replace($start, "\"\"", ""), "\"") or ($j gt $i and contains(replace($end, "\"\"", ""), "\""))
      let $cell := replace(string-join($cells, ","), "\"", "")
      return if(contains($cell, "http://") and (contains($cell, "28.io") or contains($cell, "rendering.secxbrl.info")))
      then element { $cell-tag-name } { Link }
      else element { $cell-tag-name } {$cell}
    }
    return

        <title>Cell Store REST API</title>

{$display-cell($header, "th")}

          {
            for $row as string in $body
            return
{$display-cell($row, "td")}

          }

};

declare function api:preprocess-fiscal-years($fiscal-years as string*) as integer*
{
  distinct-values(
    for $fy in $fiscal-years ! upper-case($$)
    return switch($fy)
           case "LATEST" return $sec-fiscal:LATEST_FISCAL_YEAR
           case "ALL" return $sec-fiscal:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
  )
};

declare function api:preprocess-fiscal-periods($fiscal-periods as string*) as string*
{
  distinct-values(
    for $fp in $fiscal-periods ! upper-case($$)
    return switch($fp)
           case "Q1"
           case "Q2"
           case "Q3"
           case "FY"
             return $fp
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIODS
           default
             return error(xs:QName("local:INVALID-PERIOD"),
               $fp || ": fiscalPeriod values must be one or more of Q1, Q2, Q3, FY, ALL")
  )
};

declare function api:preprocess-fiscal-period-types($fiscal-period-types as string*) as string*
{
  distinct-values(
    for $fpt in $fiscal-period-types
    return switch($fpt)
           case "instant"
           case "QTD"
           case "YTD"
             return $fpt
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIOD_TYPES
           default
             return error(xs:QName("local:INVALID-PERIOD-TYPE"),
               $fpt || ": fiscalPeriodType values must be one or more of instant, YTD, QTD")
  )
};

declare function api:preprocess-format($format as string?, $request-uri as string) as string?
{
  let $request-path := tokenize($request-uri, "\\?")[1]
  return lower-case(($format, substring-after($request-path, ".jq."))[1])
};

declare function api:preprocess-tags($tags as string*) as string*
{
  distinct-values($tags ! upper-case($$))
};

declare function api:preprocess-boolean($name as string, $value as string)
as boolean
{
  if ($value eq "")
  then true
  else
  {
      api:validate-enum($name, lower-case($value), ("true", "false"));
      boolean($value)
  }
};

jsoniq version "1.0";

module namespace api = "http://apps.28.io/api";

import module namespace session    = "http://apps.28.io/session";
import module namespace resp       = "http://www.28msec.com/modules/http-response";
import module namespace sec-fiscal = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare function api:json-to-csv($objects as object*) as string
{
  let $flattened as object* := api:flatten-json-object($objects, ())
  return string-join(
    csv:serialize($flattened, { field-names: [ keys($flattened) ], serialize-null-as : "" }),
    "")
  };

declare function api:json-to-xml($objects as object*, $root-name as string) as element()*
{
  for $flattened-object as object in api:flatten-json-object($objects, { KeepArray: true })
  return element { $root-name } {
    for $key in keys($flattened-object)
    let $value := $flattened-object.$key
    return typeswitch($value)
           case array return
             let $key-with-s := if(substring($key, string-length($key), 1) eq "s")
                                then $key
                                else $key || "s"
             let $key-no-s := if(substring($key, string-length($key), 1) eq "s")
                                then substring($key, 1, string-length($key) - 1)
                                else $key
             return element {$key-with-s} { for $v in $value[] return element { $key-no-s } { $v } }
           default return element { $key } { $value }
  }
};

declare %private function api:flatten-json-object(
  $items as item*,
  $options as object?) as item*
{
  for $item in $items
  return typeswitch($item)
         case atomic return $item
         case array return let $atomic-items := flatten($item)[$$ instance of atomic]
                           return if ($options.KeepArray)
                                 then [ $atomic-items ]
                                  else string-join($atomic-items, ", ")
         case object return {|
             for $key in keys($item)
             return typeswitch($item.$key)
                    case object return api:flatten-json-object($item.$key, $options)
                    default return { $key: api:flatten-json-object($item.$key, $options) }
         |}
         default return ()
};

declare function api:validate-regexp($name as string, $value as string?, $regexp as string)
as ()
{
  if (exists($value))
  then
    if (matches($value, "^" || $regexp || "$"))
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " does not match reg. expression " || $regexp || ".");
  else ();
};

declare function api:validate-enum($name as string, $value as string?, $enum as string*)
as ()
{
  if (exists($value))
  then
    if ($value = $enum)
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " is not one of these: " || string-join($enum,", ") || ".");
  else ();
};

declare %private function api:sgpl($count as integer, $singular as string, $plural as string)
as string
{
    if ($count eq 1)
    then $count || " " || $singular
    else $count || " " || $plural
};

declare function api:passed($timestamp as xs:dateTime?)
as string
{
    if (empty($timestamp) or $timestamp eq null)
    then "Never"
    else
        let $now := current-dateTime()
        let $difference := $now - $timestamp
        let $years := years-from-duration($difference)
        let $month := months-from-duration($difference)
        let $days := days-from-duration($difference)
        let $hours := hours-from-duration($difference)
        let $minutes := minutes-from-duration($difference)
        return
            if ($years gt 0) then api:sgpl($years,"year ago","years ago")
            else if ($month gt 0) then api:sgpl($month,"month ago","month ago")
            else if ($days gt 0) then api:sgpl($days, "day ago", "days ago")
            else if ($hours gt 0) then api:sgpl($hours, "hour ago", "hours ago")
            else if ($minutes gt 0) then api:sgpl($minutes, "minute ago", "minutes ago")
            else if ($difference div xs:dayTimeDuration("PT1S") lt 0) then "future"
            else "Just Now"
};

declare function api:success() as object
{
  { "success" : true }
};

declare function api:success($data as object()) as object
{
  {|
     {"success" : true },
     $data
  |}
};

declare %an:sequential function api:check-and-return-results(
    $token as string?,
    $results as item*,
    $format as string?
) as item*
{
    switch(session:has-access($token, "data_sec"))
    case $session:ACCESS-ALLOWED return
        $results
    case $session:ACCESS-DENIED return {
          resp:status-code(403);
          session:error("access denied", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          resp:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()
};

declare function api:normalize-facts(
    $facts as object*) as object*
{
    for $fact in $facts
    return {|
        {
            "Aspects" : {|
                trim($fact.Aspects, ("xbrl:Unit")),
                { "sec:Archive" : $fact.Aspects."xbrl28:Archive" }[exists($fact.Aspects."xbrl28:Archive")]
            |}
        },
        trim($fact, ("Aspects", "_id")),
        { Unit: $fact.Aspects."xbrl:Unit" }[exists($fact.Aspects."xbrl:Unit") and not exists($fact.Unit)]
    |}
};

declare %an:sequential function api:serialize(
    $result as json-item,
    $comment as object,
    $serializers as object,
    $format as string?,
    $file-name as string) as item*
{
    switch ($format)
    case "xml" return {
        resp:serialization-parameters({"omit-xml-declaration" : false, indent : true });
        session:comment("xml", $comment),
        $serializers.to-xml($result)
    }
    case "text" case "csv" return {
        resp:content-type("text/csv");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    case "html" return {
        resp:content-type("text/html");
        let $csv as string := $serializers.to-csv($result)
        return api:csv-to-html($csv)
    }
    case "excel" return {
        resp:content-type("application/vnd.ms-excel");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    default return {
        resp:content-type("application/json");
        resp:serialization-parameters({"indent" : true});
        {|
            session:comment("json", $comment),
            $result
        |}
    }
};

declare %an:sequential function api:csv-to-html(
    $csv as string) as item*
{
    let $csv as string* := $csv
    let $rows := tokenize($csv, "\n")
    let $header := $rows[1]
    let $body := subsequence($rows, 2)
    let $display-cell := function($row as string, $cell-tag-name as string) {
      for tumbling window $cells as string* in tokenize($row, ",")
      start $start at $i when true
      only end $end at $j when not contains(replace($start, "\"\"", ""), "\"") or ($j gt $i and contains(replace($end, "\"\"", ""), "\""))
      let $cell := replace(string-join($cells, ","), "\"", "")
      return if(contains($cell, "http://") and (contains($cell, "28.io") or contains($cell, "rendering.secxbrl.info")))
      then element $cell-tag-name { Link }
      else element $cell-tag-name {$cell}
    }
    return

        <title>Cell Store REST API</title>

{$display-cell($header, "th")}

          {
            for $row as string in $body
            return
{$display-cell($row, "td")}

          }

};

declare function api:preprocess-fiscal-years($fiscal-years as string*) as integer*
{
  distinct-values(
    for $fy in $fiscal-years ! upper-case($$)
    return switch($fy)
           case "LATEST" return $sec-fiscal:LATEST_FISCAL_YEAR
           case "ALL" return $sec-fiscal:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
  )
};

declare function api:preprocess-fiscal-periods($fiscal-periods as string*) as string*
{
  distinct-values(
    for $fp in $fiscal-periods ! upper-case($$)
    return switch($fp)
           case "Q1"
           case "Q2"
           case "Q3"
           case "FY"
             return $fp
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIODS
           default
             return error(xs:QName("local:INVALID-PERIOD"),
               $fp || ": fiscalPeriod values must be one or more of Q1, Q2, Q3, FY, ALL")
  )
};

declare function api:preprocess-fiscal-period-types($fiscal-period-types as string*) as string*
{
  distinct-values(
    for $fpt in $fiscal-period-types
    return switch($fpt)
           case "instant"
           case "QTD"
           case "YTD"
             return $fpt
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIOD_TYPES
           default
             return error(xs:QName("local:INVALID-PERIOD-TYPE"),
               $fpt || ": fiscalPeriodType values must be one or more of instant, YTD, QTD")
  )
};

declare function api:preprocess-format($format as string?, $request-uri as string) as string?
{
  let $request-path := tokenize($request-uri, "\\?")[1]
  return lower-case(($format, substring-after($request-path, ".jq."))[1])
};

declare function api:preprocess-tags($tags as string*) as string*
{
  distinct-values($tags ! upper-case($$))
};

declare function api:preprocess-boolean($name as string, $value as string)
as boolean
{
  if ($value eq "")
  then true
  else
  {
      api:validate-enum($name, lower-case($value), ("true", "false"));
      boolean($value)
  }
};


:green_heart: Fix test.

:green_heart: Fix test.


:bug: add support for fsa shortcuts submitted, fiscalYear, fiscalPeriod and fiscalPeriodType.

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace rules = "http://28.io/modules/xbrl/rules";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

import module namespace request = "http://www.28msec.com/modules/http-request";

declare function local:param-values($name as string) as string*
{
    switch(true)
     case $name eq "xbrl:Concept"
        return (request:param-values("concept"), request:param-values("xbrl:Concept"))

    (: sec profile :)
     case $name eq "sec:FiscalYear" and $profile-name eq "sec"
        return ($fiscalYear, request:param-values("sec:FiscalYear"))[$$ ne "LATEST"]
     case $name eq "sec:FiscalYear::type" and $profile-name eq "sec"
        return "integer"
     case $name eq "sec:FiscalPeriod" and $profile-name eq "sec"
        return ($fiscalPeriod, request:param-values("sec:FiscalPeriod"))
     case $name eq "sec:FiscalPeriodType" and $profile-name eq "sec"
        return ($fiscalPeriodType, request:param-values("sec:FiscalPeriodType"))
     case $name eq "dei:LegalEntityAxis" and $profile-name eq "sec"
        return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis")
     case $name eq "dei:LegalEntityAxis::default" return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis::default")

     case $name eq "xbrl:Entity" and $profile-name = ("sec", "japan") return (
         let $companies := multiplexer:entities(
            $profile-name,
            $eid,
            $cik,
            api:preprocess-tags($tag),
            $ticker,
            $sic)._id
        return if(empty(($cik,$tag,$ticker,$sic)) or exists($companies))
               then $companies
               else "dummy",
        request:param-values("xbrl:Entity"))
     case $name eq "xbrl28:Archive" and $profile-name = ("sec", "japan") return (
            let $prefix as string := ("sec"[$profile-name eq "sec"], "fsa"[$profile-name eq "japan"])
            let $fiscalYears := ($fiscalYear, request:param-values( $prefix || ":FiscalYear"))
            let $fiscalPeriods := local:param-values($prefix || ":FiscalPeriod")
            let $entities := entities:entities(local:param-values("xbrl:Entity"))
            return
                if($fiscalYears = "LATEST")
                then fiscal-core:latest-filings($entities, $fiscalPeriods)._id
                else (),
            $aid,
            request:param-values("xbrl28:Archive")
        )

     (: japan profile :)
     case $name eq "fsa:FiscalYear" and $profile-name eq "japan"
     return ($fiscalYear, request:param-values("fsa:FiscalYear"))[$$ ne "LATEST"]
     case $name eq "fsa:FiscalYear::type" and $profile-name eq "japan"
     return "integer"
     case $name eq "fsa:FiscalPeriod" and $profile-name eq "japan"
     return ($fiscalPeriod, request:param-values("fsa:FiscalPeriod"))
     case $name eq "fsa:FiscalPeriodType" and $profile-name eq "japan"
     return ($fiscalPeriodType, request:param-values("fsa:FiscalPeriodType"))

     default return request:param-values($name)
};

declare function local:param-names() as string*
{
    let $names := request:param-names()
    return distinct-values((
        (: generic xbrl :)
        $names[contains($$, ":")],
        "xbrl:Entity"[$names = ("eid")],
        "xbrl28:Archive"[$profile-name = ("sec", "japan")],
        "xbrl:Concept"[$names = "concept"],

        (: sec profile :)
        "sec:Accepted"[$profile-name eq "sec"],
        "sec:FiscalPeriod"[$profile-name eq "sec"],
        "sec:FiscalPeriodType"[$profile-name eq "sec"],
        "sec:FiscalYear"[$profile-name eq "sec"],
        "xbrl:Entity"[$profile-name eq "sec" and $names = ("cik", "tag", "ticker", "sic")],
        "dei:LegalEntityAxis"[$profile-name eq "sec"],
        "dei:LegalEntityAxis::default"[$profile-name eq "sec"],

        (: japan profile :)
        "fsa:Submitted"[$profile-name eq "japan"],
        "fsa:FiscalPeriod"[$profile-name eq "japan"],
        "fsa:FiscalPeriodType"[$profile-name eq "japan"],
        "fsa:FiscalYear"[$profile-name eq "japan"]
    ))
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube() as object
{
    let $hypercube-spec :=
    {|
        for $parameter in local:param-names()
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (local:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $type as string? :=
            (local:param-values($dimension-name || "::type"), local:param-values($dimension-name || ":type"))[1]

        let $values := local:param-values($dimension-name)
        let $typed-values := if (exists($type)) then local:cast-sequence($values[$$ ne "ALL"], $type) else $values

        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (local:param-values($dimension-name || "::default"), local:param-values($dimension-name || ":default"))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value

        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                if(not($all))
                then { "Domain" : [ $typed-values ] }[exists($typed-values)]
                else (),
                { "Default" : $typed-default-value }[$has-default]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($hypercube-spec)
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token             as string? external;
declare  %rest:case-insensitive                 variable $profile-name      as string  external := $config:profile-name;
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriodType  as string* external := ("instant", "YTD");
declare  %rest:case-insensitive %rest:distinct  variable $eid               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $validate          as boolean external := false;
declare  %rest:case-insensitive                 variable $labels            as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;
declare  %rest:case-insensitive                 variable $debug             as boolean external := false;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic,
        $eid,
        $aid)
let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $hypercube := local:hypercube()

let $facts :=
    let $options := {|
      {
        Hypercube : $hypercube,
        Validate: $validate
      },
      { "ConceptMaps" : $map }[exists($map)],
      { "Rules" : [ $rule ] }[exists($rule)],
      { "Concepts" : $report.Concepts }[exists($report) and exists($report.Concepts)]
    |}
    return if($profile-name eq "sec")
    then sec:facts-for($options)
    else facts:facts-for($options)

let $facts :=
  if(not $labels)
  then $facts
  else
    let $archives as string* := distinct-values($facts.Aspects."xbrl28:Archive")
    let $concept-names as string* := distinct-values($facts.Aspects."xbrl:Concept")
    let $concepts as object* :=
      (
          concepts:concepts($concept-names, $archives, $concepts:ANY_COMPONENT_LINK_ROLE),
          (reports:concepts(($report,$map)))[$$.Name = $concept-names]
      )
    let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $concepts:AMERICAN_ENGLISH )[1]
    let $roles as string* := ( $report.Role, $concepts:ANY_COMPONENT_LINK_ROLE )
    for $fact as object in $facts
    let $entityName as string := $entities[$$._id eq $fact.Aspects."xbrl:Entity"].Profiles.SEC.CompanyName
    return
    {|
      $fact,
      let $concept-labels as object? := facts:labels($fact, $roles, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
      let $standard-labels as object := conversion:get-standard-labels($fact, $entityName)
      return { Labels : {| $concept-labels, $standard-labels |} }
    |}

let $facts :=
  if($profile-name eq "sec")
  then api:normalize-facts($facts)
  else $facts

let $result := {
    NetworkIdentifier : "http://bizql.io/facts",
    TableName : "xbrl:Facts",
    FactTable : [ $facts ]
}
let $comment :=
{
    NumFacts: count($facts),
    TotalNumFacts: session:num-facts(),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $comment :=
    if($debug)
    then {|
        $comment,
        { DebugInfo: { Hypercube: $hypercube } }
    |}
    else $comment
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "facts")
return api:check-and-return-results($token, $results, $format)

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace rules = "http://28.io/modules/xbrl/rules";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

import module namespace request = "http://www.28msec.com/modules/http-request";

declare function local:param-values($name as string) as string*
{
    switch(true)
     case $name eq "xbrl:Concept"
        return (request:param-values("concept"), request:param-values("xbrl:Concept"))

     case $name eq "sec:FiscalYear" and $profile-name eq "sec"
        return ($fiscalYear, request:param-values("sec:FiscalYear"))[$$ ne "LATEST"]
     case $name eq "sec:FiscalYear::type" and $profile-name eq "sec"
        return "integer"
     case $name eq "sec:FiscalPeriod" and $profile-name eq "sec"
        return ($fiscalPeriod, request:param-values("sec:FiscalPeriod"))
     case $name eq "sec:FiscalPeriodType" and $profile-name eq "sec"
        return ($fiscalPeriodType, request:param-values("sec:FiscalPeriodType"))
     case $name eq "dei:LegalEntityAxis" and $profile-name eq "sec"
        return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis")
     case $name eq "dei:LegalEntityAxis::default" return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis::default")
     case $name eq "xbrl:Entity" and $profile-name eq "sec" return (
         let $companies := multiplexer:entities(
            $profile-name,
            $eid,
            $cik,
            api:preprocess-tags($tag),
            $ticker,
            $sic)._id
        return if(empty(($cik,$tag,$ticker,$sic)) or exists($companies))
               then $companies
               else "dummy",
        request:param-values("xbrl:Entity"))
     case $name eq "xbrl28:Archive" and $profile-name eq "sec" return (
            let $fiscalYears := ($fiscalYear, request:param-values("sec:FiscalYear"))
            let $fiscalPeriods := local:param-values("sec:FiscalPeriod")
            let $entities := entities:entities(local:param-values("xbrl:Entity"))
            return
                if($fiscalYears = "LATEST")
                then fiscal-core:latest-filings($entities, $fiscalPeriods)._id
                else (),
            $aid,
            request:param-values("xbrl28:Archive")
        )
     default return request:param-values($name)
};

declare function local:param-names() as string*
{
    let $names := request:param-names()
    return distinct-values((
        $names[contains($$, ":")],
        "xbrl:Concept"[$names = "concept"],

        "sec:Accepted"[$profile-name eq "sec"],
        "sec:FiscalPeriod"[$profile-name eq "sec"],
        "sec:FiscalPeriodType"[$profile-name eq "sec"],
        "sec:FiscalYear"[$profile-name eq "sec"],
        "xbrl:Entity"[$profile-name eq "sec" and $names = ("cik", "tag", "ticker", "sic")],
        "dei:LegalEntityAxis"[$profile-name eq "sec"],
        "dei:LegalEntityAxis::default"[$profile-name eq "sec"],
        "xbrl28:Archive"[$profile-name eq "sec"]))
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube() as object
{
    let $hypercube-spec :=
    {|
        for $parameter in local:param-names()
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (local:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $type as string? :=
            (local:param-values($dimension-name || "::type"), local:param-values($dimension-name || ":type"))[1]

        let $values := local:param-values($dimension-name)
        let $typed-values := if (exists($type)) then local:cast-sequence($values[$$ ne "ALL"], $type) else $values

        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (local:param-values($dimension-name || "::default"), local:param-values($dimension-name || ":default"))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value

        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                if(not($all))
                then { "Domain" : [ $typed-values ] }[exists($typed-values)]
                else (),
                { "Default" : $typed-default-value }[$has-default]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($hypercube-spec)
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token             as string? external;
declare  %rest:case-insensitive                 variable $profile-name      as string  external := $config:profile-name;
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriodType  as string* external := ("instant", "YTD");
declare  %rest:case-insensitive %rest:distinct  variable $eid               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $validate          as boolean external := false;
declare  %rest:case-insensitive                 variable $labels            as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;
declare  %rest:case-insensitive                 variable $debug             as boolean external := false;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic,
        $eid,
        $aid)
let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $hypercube := local:hypercube()

let $facts :=
    let $options := {|
      {
        Hypercube : $hypercube,
        Validate: $validate
      },
      { "ConceptMaps" : $map }[exists($map)],
      { "Rules" : [ $rule ] }[exists($rule)],
      { "Concepts" : $report.Concepts }[exists($report) and exists($report.Concepts)]
    |}
    return if($profile-name eq "sec")
    then sec:facts-for($options)
    else facts:facts-for($options)

let $facts :=
  if(not $labels)
  then $facts
  else
    let $archives as string* := distinct-values($facts.Aspects."xbrl28:Archive")
    let $concept-names as string* := distinct-values($facts.Aspects."xbrl:Concept")
    let $concepts as object* :=
      (
          concepts:concepts($concept-names, $archives, $concepts:ANY_COMPONENT_LINK_ROLE),
          (reports:concepts(($report,$map)))[$$.Name = $concept-names]
      )
    let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $concepts:AMERICAN_ENGLISH )[1]
    let $roles as string* := ( $report.Role, $concepts:ANY_COMPONENT_LINK_ROLE )
    for $fact as object in $facts
    let $entityName as string := $entities[$$._id eq $fact.Aspects."xbrl:Entity"].Profiles.SEC.CompanyName
    return
    {|
      $fact,
      let $concept-labels as object? := facts:labels($fact, $roles, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
      let $standard-labels as object := conversion:get-standard-labels($fact, $entityName)
      return { Labels : {| $concept-labels, $standard-labels |} }
    |}

let $facts :=
  if($profile-name eq "sec")
  then api:normalize-facts($facts)
  else $facts

let $result := {
    NetworkIdentifier : "http://bizql.io/facts",
    TableName : "xbrl:Facts",
    FactTable : [ $facts ]
}
let $comment :=
{
    NumFacts: count($facts),
    TotalNumFacts: session:num-facts(),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $comment :=
    if($debug)
    then {|
        $comment,
        { DebugInfo: { Hypercube: $hypercube } }
    |}
    else $comment
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "facts")
return api:check-and-return-results($token, $results, $format)


:lipstick: Refactor url functions.

:lipstick: Refactor url functions.

jsoniq version "1.0";
module namespace test = "http://apps.28.io/test";
import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace config = "http://apps.28.io/config";
import module namespace credentials = "http://www.28msec.com/modules/credentials";
import module namespace functx = "http://www.functx.com";
import module namespace backend = "http://apps.28.io/backend";

declare function test:is-dow30() as boolean{
  contains(credentials:credentials("MongoDB", "xbrl").db, "dow30")
};

declare function test:url($endpoint as string, $parameters as object) as string
{
    test:url($endpoint, $parameters, false)
};

declare function test:url($endpoint as string, $parameters as object, $includeToken as boolean) as string
{
  backend:url(
    $endpoint,
    {|
      { token: if($includeToken) then $config:test-token else "{{token}}" },
      $parameters
    |}
  )
};

declare %an:nondeterministic function test:invoke($endpoint as string, $parameters as object) as item*
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:get($url)
  return ($response.status, parse-json($response.body.content))
};

declare %an:nondeterministic function test:invoke-xml($endpoint as string, $parameters as object) as item*
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:get($url)
  return ($response.status, parse-xml($response.body.content))
};

declare %an:sequential function test:invoke-body($endpoint as string, $parameters as object, $body as string) as item*
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:post($url, $body, "application/x-www-form-urlencoded")
  return ($response.status, parse-json($response.body.content))
};

declare %an:nondeterministic function test:invoke-raw($endpoint as string, $parameters as object) as object
{
  let $url as string:= test:url($endpoint, $parameters, true)
  return http-client:get($url)
};

declare %an:nondeterministic function test:invoke-public($endpoint as string, $parameters as object) as item*
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:get($url)
  return ($response.status, parse-json($response.body.content))
};

declare %an:nondeterministic function test:get-expected-result(
  $expected-file as string
) as item*
{
  parse-json(
    http-client:get("http://" || request:server-name() || ":" || request:server-port() ||
                    "/test/" || $expected-file).body.content)
};

declare %an:nondeterministic function test:get-expected-result-xml(
  $expected-file as string
) as item*
{
  parse-xml(
    http-client:get("http://" || request:server-name() || ":" || request:server-port() ||
                    "/test/" || $expected-file).body.content)
};

declare %an:nondeterministic function test:invoke-and-assert-deep-equal(
  $endpoint as string,
  $parameters as object,
  $transform as function(object) as item*,
  $expected as item*
) as item
{
  test:invoke-and-assert-deep-equal($endpoint, $parameters, $transform, $expected, {})
};

(:
  Note on NoArrayOrder option: it allows for false positives in case arrays have duplicate elements. The number
  of occurrences of duplicate elements is not checked.
:)
declare %an:nondeterministic function test:invoke-and-assert-deep-equal(
  $endpoint as string,
  $parameters as object,
  $transform as function(object) as item*,
  $expected as item*,
  $options as object?
) as item
{
  let $request := switch($options.Format)
                  case "xml" return test:invoke-xml($endpoint, $parameters)
                  default return test:invoke($endpoint, $parameters)
  let $status as integer := $request[1]
  let $actual as item* := $transform($request[2])
  let $expected as item* := if($options.TrimIdField) then test:trim-ids($expected) else $expected
  let $actual as item* := if($options.TrimIdField) then test:trim-ids($actual) else $actual
  return if($options.NoArrayOrder)
          then test:assert-deep-equal-no-array-order($expected, $actual, $status, test:url($endpoint, $parameters))
          else test:assert-deep-equal($expected, $actual, $status, test:url($endpoint, $parameters))
};

declare %an:sequential function test:check-all-success($o as object) as object
{
  if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
  then {
    response:status-code(500);
    $o
  } else $o
};

declare %private function test:trim-ids($input as item*) as item*
{
  for $i in $input
  return typeswitch($i)
          case object return {|
                                 for $key in keys($i)
                                 where $key ne "_id"
                                 return {$key: test:trim-ids($i.$key) }
                               |}
          case array return [ test:trim-ids($i[]) ]
          default return $i
};

(:    return
        if ($actual eq $expected)
        then true
        else "false [Actual="||$actual||", Expected="||$expected ||"]":)

declare function test:assert-eq(
    $expected as atomic,
    $actual as atomic?,
    $status as integer,
    $url as string) as item
{
    switch(true)
    case $status ne 200 return { "url": $url, status: $status }
    case try {$expected eq $actual} catch * { false } return true
    default return
    {
        "url": $url,
        "expected": $expected,
        "actual": $actual
    }
};

declare function test:assert-deep-equal(
    $expected as item*,
    $actual as item*,
    $status as integer,
    $url as string) as item
{
    switch(true)
    case $status ne 200 return { "url": $url, status: $status }
    case functx:sequence-deep-equal($expected, $actual) return true
    default return
    {
        "url": $url,
        "expected": typeswitch($expected) case json-item return $expected default return serialize($expected),
        "actual": typeswitch($actual) case json-item return $actual default return serialize($actual)
    }
};

declare function test:assert-deep-equal-no-array-order(
    $expected as item*,
    $actual as item*,
    $status as integer,
    $url as string) as item
{
    switch(true)
    case $status ne 200 return { "url": $url, status: $status }
    case test:deep-equal-no-array-order($expected, $actual) return true
    default return
    {
        "url": $url,
        "expected": typeswitch($expected) case json-item return $expected default return serialize($expected),
        "actual": typeswitch($actual) case json-item return $actual default return serialize($actual)
    }
};

declare function test:deep-equal-no-array-order(
    $expected as item*,
    $actual as item*) as boolean
{
    if(count($expected) ne count($actual))
    then false
    else
      every $i in count($expected)
      satisfies
        let $expected := $expected[$i]
        let $actual := $actual[$i]
        return
        typeswitch(($expected, $actual))
        case object* return
          switch(true)
          case not (every $key in keys($expected) satisfies keys($actual) = $key)
            return false
          case not (every $key in keys($actual) satisfies keys($expected) = $key)
            return false
          default return
            let $recursive-calls :=
                for $key in keys($expected)
                return test:deep-equal-no-array-order($expected.$key, $actual.$key)
            return if (every $item in $recursive-calls satisfies $item)
                    then true
                    else false
        case array* return
          switch(true)
          case not size($expected) eq size($actual)
            return false
          case not (every $expected-item in $expected[] satisfies (
            some $actual-item in $actual[] satisfies test:deep-equal-no-array-order($expected-item, $actual-item)
          ))
            return false
          case not (every $actual-item in $actual[] satisfies (
            some $expected-item in $expected[] satisfies test:deep-equal-no-array-order($expected-item, $actual-item)
          ))
            return false
          default return true
        default return deep-equal($expected, $actual)
};

declare function test:assert-eq-array(
    $expected as array,
    $actual as array,
    $status as integer,
    $url as string) as item
{
    let $diff := try {
            {
                actual: [
                    for $a in flatten($actual)
                    where not($a = flatten($expected))
                    return $a ],
                actualDuplicates: [
                    for $a in flatten($actual)
                    where $a = flatten($expected) and count(flatten($actual)[$$ eq $a]) ne count(flatten($expected)[$$ eq $a])
                    return $a ],
                expected: [
                    for $e in flatten($expected)
                    where not($e = flatten($actual))
                    return $e ]
            }
        } catch * { "error: " || $err:description }

    return
        switch(true)
        case $status ne 200 return { "url": $url, status: $status }
        case empty(($diff.actual[],$diff.actualDuplicates[],$diff.expected[])) return true
        default return
        {
            "url": $url,
            "diff": $diff,
            "expected": $expected,
            "actual": $actual
        }
};

jsoniq version "1.0";
module namespace test = "http://apps.28.io/test";
import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace config = "http://apps.28.io/config";
import module namespace credentials = "http://www.28msec.com/modules/credentials";
import module namespace functx = "http://www.functx.com";

declare function test:is-dow30() as boolean{
  contains(credentials:credentials("MongoDB", "xbrl").db, "dow30")
};

declare function test:url($endpoint as string, $parameters as object) as string
{
    test:url($endpoint, $parameters, false)
};

declare function test:url($endpoint as string, $parameters as object, $includeToken as boolean) as string
{
    "http://" || request:server-name() || ":" || request:server-port() ||
    "/v1/_queries/public/api/"||$endpoint||".jq?_method=POST&token=" || (if($includeToken) then $config:test-token else "{{token}}") || "&"||
    string-join(
        for $key in keys($parameters)
        for $value as string in (flatten($parameters.$key) ! string($$))
        return ($key||"="||encode-for-uri($value)),
        "&")
};

declare %an:nondeterministic function test:invoke($endpoint as string, $parameters as object) as item*
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:get($url)
  return ($response.status, parse-json($response.body.content))
};

declare %an:nondeterministic function test:invoke-xml($endpoint as string, $parameters as object) as item*
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:get($url)
  return ($response.status, parse-xml($response.body.content))
};

declare %an:sequential function test:invoke-body($endpoint as string, $parameters as object, $body as string) as item*
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:post($url, $body, "application/x-www-form-urlencoded")
  return ($response.status, parse-json($response.body.content))
};

declare %an:nondeterministic function test:invoke-raw($endpoint as string, $parameters as object) as object
{
  let $url as string:= test:url($endpoint, $parameters, true)
  return http-client:get($url)
};

declare %an:nondeterministic function test:invoke-public($endpoint as string, $parameters as object) as item*
{
  let $url as string:= test:url($endpoint, $parameters, true)
  let $response as object := http-client:get($url)
  return ($response.status, parse-json($response.body.content))
};

declare %an:nondeterministic function test:get-expected-result(
  $expected-file as string
) as item*
{
  parse-json(
    http-client:get("http://" || request:server-name() || ":" || request:server-port() ||
                    "/test/" || $expected-file).body.content)
};

declare %an:nondeterministic function test:get-expected-result-xml(
  $expected-file as string
) as item*
{
  parse-xml(
    http-client:get("http://" || request:server-name() || ":" || request:server-port() ||
                    "/test/" || $expected-file).body.content)
};

declare %an:nondeterministic function test:invoke-and-assert-deep-equal(
  $endpoint as string,
  $parameters as object,
  $transform as function(object) as item*,
  $expected as item*
) as item
{
  test:invoke-and-assert-deep-equal($endpoint, $parameters, $transform, $expected, {})
};

(:
  Note on NoArrayOrder option: it allows for false positives in case arrays have duplicate elements. The number
  of occurrences of duplicate elements is not checked.
:)
declare %an:nondeterministic function test:invoke-and-assert-deep-equal(
  $endpoint as string,
  $parameters as object,
  $transform as function(object) as item*,
  $expected as item*,
  $options as object?
) as item
{
  let $request := switch($options.Format)
                  case "xml" return test:invoke-xml($endpoint, $parameters)
                  default return test:invoke($endpoint, $parameters)
  let $status as integer := $request[1]
  let $actual as item* := $transform($request[2])
  let $expected as item* := if($options.TrimIdField) then test:trim-ids($expected) else $expected
  let $actual as item* := if($options.TrimIdField) then test:trim-ids($actual) else $actual
  return if($options.NoArrayOrder)
          then test:assert-deep-equal-no-array-order($expected, $actual, $status, test:url($endpoint, $parameters))
          else test:assert-deep-equal($expected, $actual, $status, test:url($endpoint, $parameters))
};

declare %an:sequential function test:check-all-success($o as object) as object
{
  if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
  then {
    response:status-code(500);
    $o
  } else $o
};

declare %private function test:trim-ids($input as item*) as item*
{
  for $i in $input
  return typeswitch($i)
          case object return {|
                                 for $key in keys($i)
                                 where $key ne "_id"
                                 return {$key: test:trim-ids($i.$key) }
                               |}
          case array return [ test:trim-ids($i[]) ]
          default return $i
};

(:    return
        if ($actual eq $expected)
        then true
        else "false [Actual="||$actual||", Expected="||$expected ||"]":)

declare function test:assert-eq(
    $expected as atomic,
    $actual as atomic?,
    $status as integer,
    $url as string) as item
{
    switch(true)
    case $status ne 200 return { "url": $url, status: $status }
    case try {$expected eq $actual} catch * { false } return true
    default return
    {
        "url": $url,
        "expected": $expected,
        "actual": $actual
    }
};

declare function test:assert-deep-equal(
    $expected as item*,
    $actual as item*,
    $status as integer,
    $url as string) as item
{
    switch(true)
    case $status ne 200 return { "url": $url, status: $status }
    case functx:sequence-deep-equal($expected, $actual) return true
    default return
    {
        "url": $url,
        "expected": typeswitch($expected) case json-item return $expected default return serialize($expected),
        "actual": typeswitch($actual) case json-item return $actual default return serialize($actual)
    }
};

declare function test:assert-deep-equal-no-array-order(
    $expected as item*,
    $actual as item*,
    $status as integer,
    $url as string) as item
{
    switch(true)
    case $status ne 200 return { "url": $url, status: $status }
    case test:deep-equal-no-array-order($expected, $actual) return true
    default return
    {
        "url": $url,
        "expected": typeswitch($expected) case json-item return $expected default return serialize($expected),
        "actual": typeswitch($actual) case json-item return $actual default return serialize($actual)
    }
};

declare function test:deep-equal-no-array-order(
    $expected as item*,
    $actual as item*) as boolean
{
    if(count($expected) ne count($actual))
    then false
    else
      every $i in count($expected)
      satisfies
        let $expected := $expected[$i]
        let $actual := $actual[$i]
        return
        typeswitch(($expected, $actual))
        case object* return
          switch(true)
          case not (every $key in keys($expected) satisfies keys($actual) = $key)
            return false
          case not (every $key in keys($actual) satisfies keys($expected) = $key)
            return false
          default return
            let $recursive-calls :=
                for $key in keys($expected)
                return test:deep-equal-no-array-order($expected.$key, $actual.$key)
            return if (every $item in $recursive-calls satisfies $item)
                    then true
                    else false
        case array* return
          switch(true)
          case not size($expected) eq size($actual)
            return false
          case not (every $expected-item in $expected[] satisfies (
            some $actual-item in $actual[] satisfies test:deep-equal-no-array-order($expected-item, $actual-item)
          ))
            return false
          case not (every $actual-item in $actual[] satisfies (
            some $expected-item in $expected[] satisfies test:deep-equal-no-array-order($expected-item, $actual-item)
          ))
            return false
          default return true
        default return deep-equal($expected, $actual)
};

declare function test:assert-eq-array(
    $expected as array,
    $actual as array,
    $status as integer,
    $url as string) as item
{
    let $diff := try {
            {
                actual: [
                    for $a in flatten($actual)
                    where not($a = flatten($expected))
                    return $a ],
                actualDuplicates: [
                    for $a in flatten($actual)
                    where $a = flatten($expected) and count(flatten($actual)[$$ eq $a]) ne count(flatten($expected)[$$ eq $a])
                    return $a ],
                expected: [
                    for $e in flatten($expected)
                    where not($e = flatten($actual))
                    return $e ]
            }
        } catch * { "error: " || $err:description }

    return
        switch(true)
        case $status ne 200 return { "url": $url, status: $status }
        case empty(($diff.actual[],$diff.actualDuplicates[],$diff.expected[])) return true
        default return
        {
            "url": $url,
            "diff": $diff,
            "expected": $expected,
            "actual": $actual
        }
};


:pencil: Fix weight attribute.

:pencil: Fix weight attribute.

jsoniq version "1.0";
(:
 : Copyright 2014 28msec Inc.
 :)

(:~
 :
This module provides functionality for retrieving concepts
 : and labels.

 :
 : @author Federico Cavalieri
 :)
module namespace concepts = "http://28.io/modules/xbrl/concepts";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

declare namespace xbrli = "http://www.xbrl.org/2003/instance";
declare namespace xlink = "http://www.w3.org/1999/xlink";
declare namespace link = "http://www.xbrl.org/2003/linkbase";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Name of the collection the concepts are stored in.
 :)
declare variable $concepts:col as xs:string := "concepts";

(:~
 : Name of the field pointing to the archive.
 :)
declare variable $concepts:ARCHIVE as xs:string := "Archive";

(:~
 : Name of the field pointing to the role.
 :)
declare variable $concepts:ROLE as xs:string := "Role";

(:~
 : Name of the field pointing to the name.
 :)
declare variable $concepts:NAME as xs:string := "Name";

(:~
 : Name of the field pointing to the labels.
 :)
declare variable $concepts:LABELS as xs:string := "Labels";

(:~
 : The default component link role.
 :)
declare variable $concepts:DEFAULT_COMPONENT_LINK_ROLE as xs:string :=
    "http://www.xbrl.org/2003/role/link";

(:~
 : Allow any component link role
 :)
declare variable $concepts:ANY_COMPONENT_LINK_ROLE as xs:string :=
    ":any component:";

(:~
 : A joker to ask for all concept names.
 :)
declare variable $concepts:ALL_CONCEPT_NAMES as xs:string := ":any concept:";

(:~
 : A joker to ask for all concept labels.
 :)
declare variable $concepts:ALL_CONCEPT_LABELS as xs:string := ":any concept label:";

(:~
 : The concept report element
 :)
declare variable $concepts:CONCEPT_REPORT_ELEMENT as xs:string := "xbrl:Concept";

(:~
 : The period report element
 :)
declare variable $concepts:PERIOD_REPORT_ELEMENT as xs:string := "xbrl:Period";

(:~
 : The entity report element
 :)
declare variable $concepts:ENTITY_REPORT_ELEMENT as xs:string := "xbrl:Entity";

(:~
 : The unit report element
 :)
declare variable $concepts:UNIT_REPORT_ELEMENT as xs:string := "xbrl:Unit";

(:~
 : The archive report element
 :)
declare variable $concepts:ARCHIVE_REPORT_ELEMENT as xs:string := "xbrl28:Archive";

(:~
 : The implied table report element
 :)
declare variable $concepts:IMPLIED_TABLE_REPORT_ELEMENT as xs:string := "xbrl28:ImpliedTable";

(:~
 : The non-numeric table report element
 :)
declare variable $concepts:NON_NUMERIC_REPORT_ELEMENT as xs:string := "xbrl:NonNumeric";

(:~
 :
Retrieves all concepts.

 :
 : @return all concepts.
 :)
declare function concepts:concepts() as object*
{
  mw:find($concepts:col,{})
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives-or-ids the archives or their IDs.
 : @param $component-roles the component roles.
 : @param $labels labels for exact matches.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives-or-ids as item*,
    $component-roles as string*,
    $labels as string*
  ) as object*
{
  concepts:concepts($concept-names, $archives-or-ids, $component-roles, $labels, ())
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives-or-ids the archives or their IDs.
 : @param $component-roles the component roles.
 : @param $labels labels for exact matches.
 : @param $options one option OnlyNames to only output concepts with the Name field.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives-or-ids as item*,
    $component-roles as string*,
    $labels as string*,
    $options as object?
  ) as object*
{
  let $projection as object := if($options.OnlyNames eq true) then
  {
    Name: 1,
    IsAbstract: 1,
    IsTextBlock: 1
  } else {}
  let $archives := archives:aid($archives-or-ids)
  return
  if (exists($archives))
  then mw:find($concepts:col,
    {|
      {
        $concepts:ARCHIVE : { "$in" : [ $archives ] }
      },
      {
        $concepts:ROLE : { "$in" :
          [ $component-roles ]
        }
      }[not $component-roles = $concepts:ANY_COMPONENT_LINK_ROLE],
      {
        $concepts:NAME : { "$in" : [ $concept-names ] }
      }[not $concept-names = $concepts:ALL_CONCEPT_NAMES],
      {
        "Labels.Value" : { "$in" : [ $labels ] }
      }[not $labels = $concepts:ALL_CONCEPT_LABELS]
    |},
    $projection
  )
  else ()
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives-or-ids the archives or their IDs.
 : @param $component-roles the component roles.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives-or-ids as item*,
    $component-roles as string*
  ) as object*
{
  concepts:concepts(
    $concept-names,
    $archives-or-ids,
    $component-roles,
    $concepts:ALL_CONCEPT_LABELS
  )
};

(:~
 :
Retrieves the concepts which:
 :  - concept name matches a given one;
 :  - archive number matches that of a given component;
 :  - component role matches that of a given component or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $component-or-ids the CIDs or the components themselves.
 : @param $labels labels for exact matches.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts-for-components(
    $concept-names as string*,
    $labels as string*,
    $component-or-ids as item*,
    $options as object?) as object*
{
  for $component in components:components($component-or-ids)
  return concepts:concepts($concept-names,
                           $component.Archive[not $$ instance of null],
                           $component.Role,
                           $labels,
                           $options)
};

(:~
 :
Transforms a concept tree into XBRL arcs and locators.
 :    Those can then be added to an extended link.

 :
 : @param $trees the trees to transform.
 :
 : @return the locator and arcs elements
 :)
declare function concepts:tree-to-xml(
    $tree as object,
    $arcRole as string,
    $taxonomyName as string,
    $level as integer,
    $pos as integer
) as element()*
{
  let $children as object* := ($tree.To[], $tree.Members[])
  let $fromID as string := replace($tree.Name, ":", "_")
  let $fromLabel as string := $fromID || "_" || $level || "_" || $pos
  let $fromLocator as element() := <link:loc xlink:type="locator" xlink:href="{$taxonomyName}.xsd#{$fromID}" xlink:label="{$fromLabel}" xlink:title="{normalize-space($tree.Label)}" />

  return
  (
    $fromLocator,

    for $child at $pos in $children
    order by $child.Order ascending
    let $toID as string := replace($child.Name, ":", "_")
    let $toLabel as string := $toID || "_" || ($level + 1) || "_" || $pos
    let $order as decimal :=
      if(empty($child.Order) or $child.Order eq null)
      then 1
      else $child.Order
    let $arc as element() :=
      element {
        switch ($arcRole)
        case "http://www.xbrl.org/2003/arcrole/concept-label" return xs:QName("link:labelArc")
        case "http://www.xbrl.org/2003/arcrole/parent-child" return xs:QName("link:presentationArc")
        case "http://www.xbrl.org/2003/arcrole/summation-item" return xs:QName("link:calculationArc")
        case "http://xbrl.org/int/dim/arcrole/all"
        case "http://xbrl.org/int/dim/arcrole/hypercube-dimension"
        case "http://xbrl.org/int/dim/arcrole/dimension-domain"
        case "http://xbrl.org/int/dim/arcrole/dimension-default"
        case "http://xbrl.org/int/dim/arcrole/domain-member"
        return xs:QName("link:definitionArc")
        default return error(xs:QName("err:UnknownArcRole"), "arcrole " || $arcRole || " unknown or not supported.")
      }
      {
        attribute { "xlink:type" } { "arc" },
        attribute { "xlink:arcrole" } { $arcRole },
        attribute { "xlink:from" } { $fromLabel },
        attribute { "xlink:to" } { $toLabel },
        attribute { "use" } { "optional" },
        attribute { "order" } { $order },
        (attribute { "weight" } { $child.Weight } )[$arcRole eq "http://www.xbrl.org/2003/arcrole/summation-item"]
      }
    return
      (
        $arc,
        concepts:tree-to-xml($child, $arcRole, $taxonomyName, $level+1, $pos)
      )
  )
};

(:~
 :
Transforms concepts into XBRL definitions (XML Schema element definition).

 :
 : @param $concepts the concepts to transform.
 :
 : @return the XML Schema element definitions.
 :)
declare function concepts:to-xml(
    $concepts as object*) as element()*
{
  for $concept as object in $concepts
  where not $concept.Name = ($concepts:CONCEPT_REPORT_ELEMENT,
                             $concepts:ENTITY_REPORT_ELEMENT,
                             $concepts:PERIOD_REPORT_ELEMENT,
                             $concepts:UNIT_REPORT_ELEMENT,
                             $concepts:ARCHIVE_REPORT_ELEMENT,
                             $concepts:IMPLIED_TABLE_REPORT_ELEMENT,
                             $concepts:NON_NUMERIC_REPORT_ELEMENT)
  let $name as string := substring-after($concept.Name, ":")
  let $id as string := replace($concept.Name, ":", "_")
  let $substitutionGroup as string := $concept.SubstitutionGroup
  let $nillable as boolean := (boolean($concept.IsNillable), false)[1]
  let $abstract as boolean := (boolean($concept.IsAbstract), false)[1]
  let $periodType as attribute()? := if($concept.PeriodType) then attribute { "xbrli:periodType" } {$concept.PeriodType } else ()
  let $balance as attribute()? := if($concept.Balance) then attribute { "xbrli:balance" } {$concept.Balance } else ()
  let $type as string := $concept.DataType
  let $textBlock := $concept.IsTextBlock
  return
      <element xmlns="http://www.w3.org/2001/XMLSchema"
          id="{$id}"
          name="{$name}"
          nillable="{$nillable}"
          substitutionGroup="{$substitutionGroup}"
          abstract="{$abstract}"
          type="{$type}">{
              (
                  $periodType,
                  $balance
              )
      }</element>
};

jsoniq version "1.0";
(:
 : Copyright 2014 28msec Inc.
 :)

(:~
 :
This module provides functionality for retrieving concepts
 : and labels.

 :
 : @author Federico Cavalieri
 :)
module namespace concepts = "http://28.io/modules/xbrl/concepts";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

declare namespace xbrli = "http://www.xbrl.org/2003/instance";
declare namespace xlink = "http://www.w3.org/1999/xlink";
declare namespace link = "http://www.xbrl.org/2003/linkbase";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Name of the collection the concepts are stored in.
 :)
declare variable $concepts:col as xs:string := "concepts";

(:~
 : Name of the field pointing to the archive.
 :)
declare variable $concepts:ARCHIVE as xs:string := "Archive";

(:~
 : Name of the field pointing to the role.
 :)
declare variable $concepts:ROLE as xs:string := "Role";

(:~
 : Name of the field pointing to the name.
 :)
declare variable $concepts:NAME as xs:string := "Name";

(:~
 : Name of the field pointing to the labels.
 :)
declare variable $concepts:LABELS as xs:string := "Labels";

(:~
 : The default component link role.
 :)
declare variable $concepts:DEFAULT_COMPONENT_LINK_ROLE as xs:string :=
    "http://www.xbrl.org/2003/role/link";

(:~
 : Allow any component link role
 :)
declare variable $concepts:ANY_COMPONENT_LINK_ROLE as xs:string :=
    ":any component:";

(:~
 : A joker to ask for all concept names.
 :)
declare variable $concepts:ALL_CONCEPT_NAMES as xs:string := ":any concept:";

(:~
 : A joker to ask for all concept labels.
 :)
declare variable $concepts:ALL_CONCEPT_LABELS as xs:string := ":any concept label:";

(:~
 : The concept report element
 :)
declare variable $concepts:CONCEPT_REPORT_ELEMENT as xs:string := "xbrl:Concept";

(:~
 : The period report element
 :)
declare variable $concepts:PERIOD_REPORT_ELEMENT as xs:string := "xbrl:Period";

(:~
 : The entity report element
 :)
declare variable $concepts:ENTITY_REPORT_ELEMENT as xs:string := "xbrl:Entity";

(:~
 : The unit report element
 :)
declare variable $concepts:UNIT_REPORT_ELEMENT as xs:string := "xbrl:Unit";

(:~
 : The archive report element
 :)
declare variable $concepts:ARCHIVE_REPORT_ELEMENT as xs:string := "xbrl28:Archive";

(:~
 : The implied table report element
 :)
declare variable $concepts:IMPLIED_TABLE_REPORT_ELEMENT as xs:string := "xbrl28:ImpliedTable";

(:~
 : The non-numeric table report element
 :)
declare variable $concepts:NON_NUMERIC_REPORT_ELEMENT as xs:string := "xbrl:NonNumeric";

(:~
 :
Retrieves all concepts.

 :
 : @return all concepts.
 :)
declare function concepts:concepts() as object*
{
  mw:find($concepts:col,{})
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives-or-ids the archives or their IDs.
 : @param $component-roles the component roles.
 : @param $labels labels for exact matches.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives-or-ids as item*,
    $component-roles as string*,
    $labels as string*
  ) as object*
{
  concepts:concepts($concept-names, $archives-or-ids, $component-roles, $labels, ())
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives-or-ids the archives or their IDs.
 : @param $component-roles the component roles.
 : @param $labels labels for exact matches.
 : @param $options one option OnlyNames to only output concepts with the Name field.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives-or-ids as item*,
    $component-roles as string*,
    $labels as string*,
    $options as object?
  ) as object*
{
  let $projection as object := if($options.OnlyNames eq true) then
  {
    Name: 1,
    IsAbstract: 1,
    IsTextBlock: 1
  } else {}
  let $archives := archives:aid($archives-or-ids)
  return
  if (exists($archives))
  then mw:find($concepts:col,
    {|
      {
        $concepts:ARCHIVE : { "$in" : [ $archives ] }
      },
      {
        $concepts:ROLE : { "$in" :
          [ $component-roles ]
        }
      }[not $component-roles = $concepts:ANY_COMPONENT_LINK_ROLE],
      {
        $concepts:NAME : { "$in" : [ $concept-names ] }
      }[not $concept-names = $concepts:ALL_CONCEPT_NAMES],
      {
        "Labels.Value" : { "$in" : [ $labels ] }
      }[not $labels = $concepts:ALL_CONCEPT_LABELS]
    |},
    $projection
  )
  else ()
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives-or-ids the archives or their IDs.
 : @param $component-roles the component roles.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives-or-ids as item*,
    $component-roles as string*
  ) as object*
{
  concepts:concepts(
    $concept-names,
    $archives-or-ids,
    $component-roles,
    $concepts:ALL_CONCEPT_LABELS
  )
};

(:~
 :
Retrieves the concepts which:
 :  - concept name matches a given one;
 :  - archive number matches that of a given component;
 :  - component role matches that of a given component or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $component-or-ids the CIDs or the components themselves.
 : @param $labels labels for exact matches.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts-for-components(
    $concept-names as string*,
    $labels as string*,
    $component-or-ids as item*,
    $options as object?) as object*
{
  for $component in components:components($component-or-ids)
  return concepts:concepts($concept-names,
                           $component.Archive[not $$ instance of null],
                           $component.Role,
                           $labels,
                           $options)
};

(:~
 :
Transforms a concept tree into XBRL arcs and locators.
 :    Those can then be added to an extended link.

 :
 : @param $trees the trees to transform.
 :
 : @return the locator and arcs elements
 :)
declare function concepts:tree-to-xml(
    $tree as object,
    $arcRole as string,
    $taxonomyName as string,
    $level as integer,
    $pos as integer
) as element()*
{
  let $children as object* := ($tree.To[], $tree.Members[])
  let $fromID as string := replace($tree.Name, ":", "_")
  let $fromLabel as string := $fromID || "_" || $level || "_" || $pos
  let $fromLocator as element() := <link:loc xlink:type="locator" xlink:href="{$taxonomyName}.xsd#{$fromID}" xlink:label="{$fromLabel}" xlink:title="{normalize-space($tree.Label)}" />

  return
  (
    $fromLocator,

    for $child at $pos in $children
    order by $child.Order ascending
    let $toID as string := replace($child.Name, ":", "_")
    let $toLabel as string := $toID || "_" || ($level + 1) || "_" || $pos
    let $order as decimal :=
      if(empty($child.Order) or $child.Order eq null)
      then 1
      else $child.Order
    let $arc as element() :=
      element {
        switch ($arcRole)
        case "http://www.xbrl.org/2003/arcrole/concept-label" return xs:QName("link:labelArc")
        case "http://www.xbrl.org/2003/arcrole/parent-child" return xs:QName("link:presentationArc")
        case "http://www.xbrl.org/2003/arcrole/summation-item" return xs:QName("link:calculationArc")
        case "http://xbrl.org/int/dim/arcrole/all"
        case "http://xbrl.org/int/dim/arcrole/hypercube-dimension"
        case "http://xbrl.org/int/dim/arcrole/dimension-domain"
        case "http://xbrl.org/int/dim/arcrole/dimension-default"
        case "http://xbrl.org/int/dim/arcrole/domain-member"
        return xs:QName("link:definitionArc")
        default return error(xs:QName("err:UnknownArcRole"), "arcrole " || $arcRole || " unknown or not supported.")
      }
      {
        attribute { "xlink:type" } { "arc" },
        attribute { "xlink:arcrole" } { $arcRole },
        attribute { "xlink:from" } { $fromLabel },
        attribute { "xlink:to" } { $toLabel },
        attribute { "use" } { "optional" },
        attribute { "order" } { $order }
      }
    return
      (
        $arc,
        concepts:tree-to-xml($child, $arcRole, $taxonomyName, $level+1, $pos)
      )
  )
};

(:~
 :
Transforms concepts into XBRL definitions (XML Schema element definition).

 :
 : @param $concepts the concepts to transform.
 :
 : @return the XML Schema element definitions.
 :)
declare function concepts:to-xml(
    $concepts as object*) as element()*
{
  for $concept as object in $concepts
  where not $concept.Name = ($concepts:CONCEPT_REPORT_ELEMENT,
                             $concepts:ENTITY_REPORT_ELEMENT,
                             $concepts:PERIOD_REPORT_ELEMENT,
                             $concepts:UNIT_REPORT_ELEMENT,
                             $concepts:ARCHIVE_REPORT_ELEMENT,
                             $concepts:IMPLIED_TABLE_REPORT_ELEMENT,
                             $concepts:NON_NUMERIC_REPORT_ELEMENT)
  let $name as string := substring-after($concept.Name, ":")
  let $id as string := replace($concept.Name, ":", "_")
  let $substitutionGroup as string := $concept.SubstitutionGroup
  let $nillable as boolean := (boolean($concept.IsNillable), false)[1]
  let $abstract as boolean := (boolean($concept.IsAbstract), false)[1]
  let $periodType as attribute()? := if($concept.PeriodType) then attribute { "xbrli:periodType" } {$concept.PeriodType } else ()
  let $balance as attribute()? := if($concept.Balance) then attribute { "xbrli:balance" } {$concept.Balance } else ()
  let $type as string := $concept.DataType
  let $textBlock := $concept.IsTextBlock
  return
      <element xmlns="http://www.w3.org/2001/XMLSchema"
          id="{$id}"
          name="{$name}"
          nillable="{$nillable}"
          substitutionGroup="{$substitutionGroup}"
          abstract="{$abstract}"
          type="{$type}">{
              (
                  $periodType,
                  $balance
              )
      }</element>
};