:bug: fix missing most recent version api param.

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace session = "http://apps.28.io/session";
import module namespace request = "http://apps.28.io/request";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace rules = "http://28.io/modules/xbrl/rules";
import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace labels = "http://28.io/modules/xbrl/labels";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace conversion = "http://28.io/modules/xbrl/conversion";

declare option rest:response "first-item";

declare variable $local:additional-concepts as object* := (
  {
    Name: "sec:DefaultLegalEntity",
    Labels : [ {
        Role : "http://www.xbrl.org/2003/role/label",
        Language : "en-us",
        Value : "Default Legal Entity [Member]"
    } ]
  },
  {
    Name: "xbrl:NonNumeric",
    Labels : [ {
        Role : "http://www.xbrl.org/2003/role/label",
        Language : "en-us",
        Value : "Non Numeric"
    } ]
  }
);

declare function local:to-csv($o as object*) as string
{
    string-join(
        csv:serialize(
            for $o in $o
            let $a := $o.Aspects
            return {|
                for $k in keys($a)
                return { $k : $a.$k },
                { "Unit" :  $o."Unit" },
                { "Value" : $o.Value },
                { "Decimals" : "INF" },
                { "NumReports" : $o.NumReports }
            |},
            { serialize-null-as : "" }
        )
    )
};

declare function local:to-xml($o as object*) as node()*
{
    (
    session:comment("xml"),
    <FactTable NetworkIdentifier="http://xbrl.io/TotalForConceptGroupedByFiscalPeriodForAllEntities"
            TableName="xbrl:TotalForConceptGroupedByFiscalPeriodForAllEntities">{

        for $o in $o
        let $a := $o.Aspects
        return
            <Fact>{
                (<Aspects>{
                    for $k in keys($a)
                    return
                        <Aspect>
                            <Name>{$k}</Name>
                            <Value>{$a.$k}</Value>
                        </Aspect>
                }</Aspects>,
                <Unit>{$o.Unit}</Unit>,
                <Type>{$o.Value}</Type>,
                <Value>{$o.Value}</Value>,
                <Decimals>INF</Decimals>,
                <NumReports>{$o.NumReports}</NumReports>,
                if (exists($o.Debug))
                then
                    <Debug>{
                        for $a in $o.Debug[]
                        return <InstanceURL value="{$a.Value}">{$a.InstanceURL}</InstanceURL>
                    }</Debug>
                else ()
                )
            }</Fact>
    }</FactTable>)
};

(: Query parameters :)
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $profile-name      as string  external := $config:profile-name;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriodType  as string* external := ("instant", "YTD");
declare  %rest:case-insensitive %rest:distinct  variable $concept           as string  external := "us-gaap:Assets";
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $debug             as boolean external := false;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $eid               as string* external;
declare  %rest:case-insensitive                 variable $most-recent-version-aspect as string external :=
    switch($profile-name)
    case "sec" return "sec:Accepted"
    case "japan" return "fsa:Submitted"
    default return "NONE";

(: Post-processing :)
let $open as boolean := false

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  api:preprocess-tags($tag),
  $ticker,
  $sic,
  (),
  (),
  $aid,
  ())

let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $request as object := request:request(
  $profile-name,
  $entities,
  $request-uri,
  $format,
  $concept,
  $fiscalYear,
  $fiscalPeriod,
  $fiscalPeriodType,
  $cik,
  $tag,
  $ticker,
  $sic,
  $aid,
  $open)
let $hypercube := request:hypercube($request, $entities)

(: Object resolution :)
let $json-result :=
    let $facts :=
        let $options := {|
          {
            Hypercube : $hypercube
          },
          { "ConceptMaps" : $map }[exists($map)],
          { "Rules" : [ $rule ] }[exists($rule)],
          { "Concepts" : $report.Concepts }[exists($report) and exists($report.Concepts)],
          { MostRecentVersionAspect: $most-recent-version-aspect }[$most-recent-version-aspect ne "NONE"]
        |}
        return facts:facts-for($options)

    let $drilldown-axis as string* := keys($request.dimensions[])[starts-with($$, "us-gaap:")]

    let $archives as string* := distinct-values($facts.Aspects."xbrl28:Archive")
    let $facts :=
      for $fact in $facts
      let $drilldown-member as string? :=
        string-join(
          for $member in ( $drilldown-axis ! $fact.Aspects.$$ )
          order by $member
          return $member, "|")
      group by $fyf := ($fact.Aspects."sec:FiscalYear", $fact.Aspects."fsa:FiscalYear")[1],
               $fpf := ($fact.Aspects."sec:FiscalPeriod", $fact.Aspects."fsa:FiscalPeriod")[1],
               $unit := $fact.Aspects."xbrl:Unit",
               $drilldown-member
      order by $fyf
      return {|
          {
              Aspects : {|
                for $aspect in distinct-values(keys($fact.Aspects))
                where not($aspect = ("sec:Accepted", "xbrl28:Archive", "xbrl:Period"))
                return
                  if($aspect eq "xbrl:Entity")
                  then
                    { "xbrl:Entity": $tag || " Aggregation" }
                  else
                    { $aspect: distinct-values($fact.Aspects.$aspect) }
              |}
          },
          { "Value" : sum($fact.Value[$$ ne null]) },
          { "Type"  : "NumericValue" },
          { "Unit" : $unit },
          { "Decimals"  : "INF" },
          { "NumReports" : count(distinct-values($fact.$facts:ASPECTS."xbrl28:Archive")) },
          { "AuditTrails" : [
              {
                "Type" : "xbrl28:Aggregation",
                "Label" : "Aggregation",
                "Message" : "Aggregation for each year, period, unit and additional drilldown axis",
                "Data" :
                    {|
                      { DrilldownMembers: [ ( $drilldown-axis ! { Axis: $$, Member: distinct-values($fact.Aspects.$$) } ) ] },
                      { Members: $drilldown-member }[$debug]
                    |}
              },
              $fact.AuditTrails
            ]
          },

          if ($debug)
          then
              {
                  "Debug" : [ $fact !
                              {
                                  "InstanceURL" : archives:archives(distinct-values($$.$facts:ASPECTS."xbrl28:Archive")).InstanceURL,
                                  "Value" : $$.Value,
                                  "Decimals" : $$.Decimals
                              } ]
              }
          else ()
      |}

    let $facts :=
      let $concept-names as string* := (
        distinct-values((keys($facts.Aspects), values($facts.Aspects)[$$ instance of string]))
      )
      let $concepts as object* :=
        (
            concepts:concepts($concept-names, $archives, $concepts:ANY_COMPONENT_LINK_ROLE),
            $report.Concepts[][$$.Name = $concept-names],
            if($profile-name eq "sec") then $local:additional-concepts else ()
        )
      let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $labels:AMERICAN_ENGLISH )[1]
      let $nonFetchedEntities as string* := request:param-values("xbrl:Entity")[not $$ = entities:eid($entities)]
      let $entities as object* := ($entities, entities:entities($nonFetchedEntities))
      for $fact as object in $facts
      return
      {|
        $fact,
        let $concept-labels as object? := labels:labels-for-facts(
          $fact,
          $labels:STANDARD_LABEL_ROLE,
          $language,
          $concepts,
          $entities,
          ()
        )
        return { Labels : $concept-labels }
      |}

    let $facts := api:normalize-facts($facts)

    return $facts

let $result := {
                    NetworkIdentifier : "http://bizql.io/facts",
                    TableName : "xbrl:Facts",
                    FactTable : [ $json-result ]
               }
let $comment :=
  {|
      { NetworkIdentifier : "http://xbrl.io/TotalForConceptGroupedByFiscalPeriodForAllEntities" },
      { TableName : "xbrl:TotalForConceptGroupedByFiscalPeriodForAllEntities" },
      if($debug)
      then { DebugInfo: { Hypercube: $hypercube,
                          Request: $request } }
      else (),
      session:comment("json")
  |}
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://xbrl.io/TotalForConceptGroupedByFiscalPeriodForAllEntities"
            TableName="xbrl:TotalForConceptGroupedByFiscalPeriodForAllEntities">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}
let $results := api:serialize($result, $comment, $serializers, $request.format, "facts")
return $results

(:return
    switch ($format)
    case "xml" return {
        {
            serialization: { method: "xml", "omit-xml-declaration" : false, indent : true }
        },
        local:to-xml($json-result)
    }
    case "text" case "csv" return {
        {
            "content-type": "text/csv",
            headers: { "Content-Disposition": "attachment; filename=Total-" || $concept || ".csv" }
        },
        local:to-csv($json-result)
    }
    case "excel" return {
        {
            "content-type": "application/vnd.ms-excel",
            headers: { "Content-Disposition": "attachment; filename=Total-" || $concept || ".csv" }
        },
        local:to-csv($json-result)
    }
    default return {
        { serialization: { method: "json", indent : true } },
        {|
            { NetworkIdentifier : "http://xbrl.io/TotalForConceptGroupedByFiscalPeriodForAllEntities" },
            { TableName : "xbrl:TotalForConceptGroupedByFiscalPeriodForAllEntities" },
            { FactTable : [ $json-result ] },
            if($debug)
            then { DebugInfo: { Hypercube: $hypercube,
                                Request: $request } }
            else (),
            session:comment("json")
        |}
    }
:)

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace session = "http://apps.28.io/session";
import module namespace request = "http://apps.28.io/request";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace rules = "http://28.io/modules/xbrl/rules";
import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace labels = "http://28.io/modules/xbrl/labels";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace conversion = "http://28.io/modules/xbrl/conversion";

declare option rest:response "first-item";

declare variable $local:additional-concepts as object* := (
  {
    Name: "sec:DefaultLegalEntity",
    Labels : [ {
        Role : "http://www.xbrl.org/2003/role/label",
        Language : "en-us",
        Value : "Default Legal Entity [Member]"
    } ]
  },
  {
    Name: "xbrl:NonNumeric",
    Labels : [ {
        Role : "http://www.xbrl.org/2003/role/label",
        Language : "en-us",
        Value : "Non Numeric"
    } ]
  }
);

declare function local:to-csv($o as object*) as string
{
    string-join(
        csv:serialize(
            for $o in $o
            let $a := $o.Aspects
            return {|
                for $k in keys($a)
                return { $k : $a.$k },
                { "Unit" :  $o."Unit" },
                { "Value" : $o.Value },
                { "Decimals" : "INF" },
                { "NumReports" : $o.NumReports }
            |},
            { serialize-null-as : "" }
        )
    )
};

declare function local:to-xml($o as object*) as node()*
{
    (
    session:comment("xml"),
    <FactTable NetworkIdentifier="http://xbrl.io/TotalForConceptGroupedByFiscalPeriodForAllEntities"
            TableName="xbrl:TotalForConceptGroupedByFiscalPeriodForAllEntities">{

        for $o in $o
        let $a := $o.Aspects
        return
            <Fact>{
                (<Aspects>{
                    for $k in keys($a)
                    return
                        <Aspect>
                            <Name>{$k}</Name>
                            <Value>{$a.$k}</Value>
                        </Aspect>
                }</Aspects>,
                <Unit>{$o.Unit}</Unit>,
                <Type>{$o.Value}</Type>,
                <Value>{$o.Value}</Value>,
                <Decimals>INF</Decimals>,
                <NumReports>{$o.NumReports}</NumReports>,
                if (exists($o.Debug))
                then
                    <Debug>{
                        for $a in $o.Debug[]
                        return <InstanceURL value="{$a.Value}">{$a.InstanceURL}</InstanceURL>
                    }</Debug>
                else ()
                )
            }</Fact>
    }</FactTable>)
};

(: Query parameters :)
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $profile-name      as string  external := $config:profile-name;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriodType  as string* external := ("instant", "YTD");
declare  %rest:case-insensitive %rest:distinct  variable $concept           as string  external := "us-gaap:Assets";
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $debug             as boolean external := false;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $eid               as string* external;

(: Post-processing :)
let $open as boolean := false

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  api:preprocess-tags($tag),
  $ticker,
  $sic,
  (),
  (),
  $aid,
  ())

let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $request as object := request:request(
  $profile-name,
  $entities,
  $request-uri,
  $format,
  $concept,
  $fiscalYear,
  $fiscalPeriod,
  $fiscalPeriodType,
  $cik,
  $tag,
  $ticker,
  $sic,
  $aid,
  $open)
let $hypercube := request:hypercube($request, $entities)

(: Object resolution :)
let $json-result :=
    let $facts :=
        let $options := {|
          {
            Hypercube : $hypercube
          },
          { "ConceptMaps" : $map }[exists($map)],
          { "Rules" : [ $rule ] }[exists($rule)],
          { "Concepts" : $report.Concepts }[exists($report) and exists($report.Concepts)]
        |}
        return if($profile-name eq "sec")
        then sec:facts-for($options)
        else facts:facts-for($options)

    let $drilldown-axis as string* := keys($request.dimensions[])[starts-with($$, "us-gaap:")]

    let $archives as string* := distinct-values($facts.Aspects."xbrl28:Archive")
    let $facts :=
      for $fact in $facts
      let $drilldown-member as string? :=
        string-join(
          for $member in ( $drilldown-axis ! $fact.Aspects.$$ )
          order by $member
          return $member, "|")
      group by $fyf := ($fact.Aspects."sec:FiscalYear", $fact.Aspects."fsa:FiscalYear")[1],
               $fpf := ($fact.Aspects."sec:FiscalPeriod", $fact.Aspects."fsa:FiscalPeriod")[1],
               $unit := $fact.Aspects."xbrl:Unit",
               $drilldown-member
      order by $fyf
      return {|
          {
              Aspects : {|
                for $aspect in distinct-values(keys($fact.Aspects))
                where not($aspect = ("sec:Accepted", "xbrl28:Archive", "xbrl:Period"))
                return
                  if($aspect eq "xbrl:Entity")
                  then
                    { "xbrl:Entity": $tag || " Aggregation" }
                  else
                    { $aspect: distinct-values($fact.Aspects.$aspect) }
              |}
          },
          { "Value" : sum($fact.Value[$$ ne null]) },
          { "Type"  : "NumericValue" },
          { "Unit" : $unit },
          { "Decimals"  : "INF" },
          { "NumReports" : count(distinct-values($fact.$facts:ASPECTS."xbrl28:Archive")) },
          { "AuditTrails" : [
              {
                "Type" : "xbrl28:Aggregation",
                "Label" : "Aggregation",
                "Message" : "Aggregation for each year, period, unit and additional drilldown axis",
                "Data" :
                    {|
                      { DrilldownMembers: [ ( $drilldown-axis ! { Axis: $$, Member: distinct-values($fact.Aspects.$$) } ) ] },
                      { Members: $drilldown-member }[$debug]
                    |}
              },
              $fact.AuditTrails
            ]
          },

          if ($debug)
          then
              {
                  "Debug" : [ $fact !
                              {
                                  "InstanceURL" : archives:archives(distinct-values($$.$facts:ASPECTS."xbrl28:Archive")).InstanceURL,
                                  "Value" : $$.Value,
                                  "Decimals" : $$.Decimals
                              } ]
              }
          else ()
      |}

    let $facts :=
      let $concept-names as string* := (
        distinct-values((keys($facts.Aspects), values($facts.Aspects)[$$ instance of string]))
      )
      let $concepts as object* :=
        (
            concepts:concepts($concept-names, $archives, $concepts:ANY_COMPONENT_LINK_ROLE),
            $report.Concepts[][$$.Name = $concept-names],
            if($profile-name eq "sec") then $local:additional-concepts else ()
        )
      let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $labels:AMERICAN_ENGLISH )[1]
      let $nonFetchedEntities as string* := request:param-values("xbrl:Entity")[not $$ = entities:eid($entities)]
      let $entities as object* := ($entities, entities:entities($nonFetchedEntities))
      for $fact as object in $facts
      return
      {|
        $fact,
        let $concept-labels as object? := labels:labels-for-facts(
          $fact,
          $labels:STANDARD_LABEL_ROLE,
          $language,
          $concepts,
          $entities,
          ()
        )
        return { Labels : $concept-labels }
      |}

    let $facts := api:normalize-facts($facts)

    return $facts

let $result := {
                    NetworkIdentifier : "http://bizql.io/facts",
                    TableName : "xbrl:Facts",
                    FactTable : [ $json-result ]
               }
let $comment :=
  {|
      { NetworkIdentifier : "http://xbrl.io/TotalForConceptGroupedByFiscalPeriodForAllEntities" },
      { TableName : "xbrl:TotalForConceptGroupedByFiscalPeriodForAllEntities" },
      if($debug)
      then { DebugInfo: { Hypercube: $hypercube,
                          Request: $request } }
      else (),
      session:comment("json")
  |}
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://xbrl.io/TotalForConceptGroupedByFiscalPeriodForAllEntities"
            TableName="xbrl:TotalForConceptGroupedByFiscalPeriodForAllEntities">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}
let $results := api:serialize($result, $comment, $serializers, $request.format, "facts")
return $results

(:return
    switch ($format)
    case "xml" return {
        {
            serialization: { method: "xml", "omit-xml-declaration" : false, indent : true }
        },
        local:to-xml($json-result)
    }
    case "text" case "csv" return {
        {
            "content-type": "text/csv",
            headers: { "Content-Disposition": "attachment; filename=Total-" || $concept || ".csv" }
        },
        local:to-csv($json-result)
    }
    case "excel" return {
        {
            "content-type": "application/vnd.ms-excel",
            headers: { "Content-Disposition": "attachment; filename=Total-" || $concept || ".csv" }
        },
        local:to-csv($json-result)
    }
    default return {
        { serialization: { method: "json", indent : true } },
        {|
            { NetworkIdentifier : "http://xbrl.io/TotalForConceptGroupedByFiscalPeriodForAllEntities" },
            { TableName : "xbrl:TotalForConceptGroupedByFiscalPeriodForAllEntities" },
            { FactTable : [ $json-result ] },
            if($debug)
            then { DebugInfo: { Hypercube: $hypercube,
                                Request: $request } }
            else (),
            session:comment("json")
        |}
    }
:)


fixed removed archives variable.

fixed removed archives variable.

import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace rules = "http://28.io/modules/xbrl/rules";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace request = "http://www.28msec.com/modules/http-request";

declare function local:contains-aspect($name as string) as boolean
{
  exists(request:param-names()[starts-with($$, $name)])
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube(
    $entities as object*,
    $fiscalPeriods as string*,
    $fiscalYears as integer*,
    $aids as string*) as object
{
    let $shortcut-hypercube-spec :=
    {|
        let $concepts := request:param-values("concept")
        return { "xbrl:Concept" : { Domain : [ $concepts ] } }[exists($concepts)],
        fiscal-core:filter-override(
            $entities,
            $fiscalYears,
            $fiscalPeriods,
            $aids)
    |}
    let $main-hypercube-spec :=
    {|
        {
            "dei:LegalEntityAxis" : {
                "Domain" : [ "sec:DefaultLegalEntity" ],
                "Default" : "sec:DefaultLegalEntity"
            }
        }[not local:contains-aspect("dei:LegalEntityAxis")],

        for $parameter in (
            request:param-names(),
            keys($shortcut-hypercube-spec),
            "sec:Accepted",
            "sec:FiscalPeriod",
            "sec:FiscalYear")
        where contains($parameter, ":")
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (request:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $user-defined-type as string? :=
            (request:param-values($dimension-name || "::type"), request:param-values($dimension-name || ":type"))[1]
        let $type as string? :=
            switch(true)
            case exists($user-defined-type)
                return $user-defined-type
            case exists($shortcut-hypercube-spec.$dimension-name.Type)
                return $shortcut-hypercube-spec.$dimension-name.Type
            default
                return ()
        let $values := (request:param-values($dimension-name), $shortcut-hypercube-spec.$dimension-name.Domain[])
        let $typed-values := if (exists($type)) then local:cast-sequence($values, $type) else $values
        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (request:param-values($dimension-name || "::default"), request:param-values($dimension-name || ":default"))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value
        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                { "Domain" : [ $typed-values ] }[(exists($typed-values) and not($all))],
                { "Default" : $typed-default-value }[$has-default]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($main-hypercube-spec)
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token             as string? external;
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $validate          as boolean external := false;
declare  %rest:case-insensitive                 variable $labels            as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic)
let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $hypercube := local:hypercube($entities, $fiscalPeriod, $fiscalYear, $aid)

let $facts :=
  sec:facts-for(
    {|
      {
        Hypercube : $hypercube,
        Validate: $validate
      },
      { "ConceptMaps" : $map }[exists($map)],
      { "Rules" : [ $rule ] }[exists($rule)]
    |}
  )

let $facts :=
    if(empty(($cik,$tag,$ticker,$sic)) or exists($entities))
    then
        let $archives as string* := distinct-values($facts.Aspects."sec:Archive")
        let $concept-names := distinct-values($facts.Aspects."xbrl:Concept")
        let $concepts :=
            (
                concepts:concepts($concept-names, $archives, $concepts:ANY_COMPONENT_LINK_ROLE),
                (reports:concepts($report))[$$.Name = $concept-names]
            )
        for $fact in $facts
        let $entityName as string := $entities[$$._id eq $fact.Aspects."xbrl:Entity"].Profiles.SEC.CompanyName
        return {|
            $fact,
            { "EntityRegistrantName" : $entityName},
            if($labels)
            then
                let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $concepts:AMERICAN_ENGLISH )[1]
                let $roles as string* := ( $report.Role, $concepts:ANY_COMPONENT_LINK_ROLE )
                let $concept-labels as object? := facts:labels($fact, $roles, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
                let $standard-labels as object := conversion:get-standard-labels($fact, $entityName)
                return
                    { Labels : {| $concept-labels, $standard-labels |} }
            else ()
        |}
    else ()

let $facts := api:normalize-facts($facts)

let $result := {
    NetworkIdentifier : "http://bizql.io/facts",
    TableName : "xbrl:Facts",
    FactTable : [ $facts ]
}
let $comment :=
{
    NumFacts: count($facts),
    TotalNumFacts: session:num-facts(),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "facts")
return api:check-and-return-results($token, $results, $format)

import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace rules = "http://28.io/modules/xbrl/rules";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace request = "http://www.28msec.com/modules/http-request";

declare function local:contains-aspect($name as string) as boolean
{
  exists(request:param-names()[starts-with($$, $name)])
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube(
    $entities as object*,
    $fiscalPeriods as string*,
    $fiscalYears as integer*,
    $aids as string*) as object
{
    let $shortcut-hypercube-spec :=
    {|
        let $concepts := request:param-values("concept")
        return { "xbrl:Concept" : { Domain : [ $concepts ] } }[exists($concepts)],
        fiscal-core:filter-override(
            $entities,
            $fiscalYears,
            $fiscalPeriods,
            $aids)
    |}
    let $main-hypercube-spec :=
    {|
        {
            "dei:LegalEntityAxis" : {
                "Domain" : [ "sec:DefaultLegalEntity" ],
                "Default" : "sec:DefaultLegalEntity"
            }
        }[not local:contains-aspect("dei:LegalEntityAxis")],

        for $parameter in (
            request:param-names(),
            keys($shortcut-hypercube-spec),
            "sec:Accepted",
            "sec:FiscalPeriod",
            "sec:FiscalYear")
        where contains($parameter, ":")
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (request:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $user-defined-type as string? :=
            (request:param-values($dimension-name || "::type"), request:param-values($dimension-name || ":type"))[1]
        let $type as string? :=
            switch(true)
            case exists($user-defined-type)
                return $user-defined-type
            case exists($shortcut-hypercube-spec.$dimension-name.Type)
                return $shortcut-hypercube-spec.$dimension-name.Type
            default
                return ()
        let $values := (request:param-values($dimension-name), $shortcut-hypercube-spec.$dimension-name.Domain[])
        let $typed-values := if (exists($type)) then local:cast-sequence($values, $type) else $values
        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (request:param-values($dimension-name || "::default"), request:param-values($dimension-name || ":default"))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value
        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                { "Domain" : [ $typed-values ] }[(exists($typed-values) and not($all))],
                { "Default" : $typed-default-value }[$has-default]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($main-hypercube-spec)
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token             as string? external;
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $validate          as boolean external := false;
declare  %rest:case-insensitive                 variable $labels            as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic)
let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $hypercube := local:hypercube($entities, $fiscalPeriod, $fiscalYear, $aid)

let $facts :=
  sec:facts-for(
    {|
      {
        Hypercube : $hypercube,
        Validate: $validate
      },
      { "ConceptMaps" : $map }[exists($map)],
      { "Rules" : [ $rule ] }[exists($rule)]
    |}
  )

let $facts :=
    if(empty(($cik,$tag,$ticker,$sic)) or exists($entities))
    then
        let $concept-names := distinct-values($facts.Aspects."xbrl:Concept")
        let $concepts :=
            (
                concepts:concepts($concept-names, $archives."_id", $concepts:ANY_COMPONENT_LINK_ROLE),
                (reports:concepts($report))[$$.Name = $concept-names]
            )
        for $fact in $facts
        let $entityName as string := $entities[$$._id eq $fact.Aspects."xbrl:Entity"].Profiles.SEC.CompanyName
        return {|
            $fact,
            { "EntityRegistrantName" : $entityName},
            if($labels)
            then
                let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $concepts:AMERICAN_ENGLISH )[1]
                let $roles as string* := ( $report.Role, $concepts:ANY_COMPONENT_LINK_ROLE )
                let $concept-labels as object? := facts:labels($fact, $roles, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
                let $standard-labels as object := conversion:get-standard-labels($fact, $entityName)
                return
                    { Labels : {| $concept-labels, $standard-labels |} }
            else ()
        |}
    else ()

let $facts := api:normalize-facts($facts)

let $result := {
    NetworkIdentifier : "http://bizql.io/facts",
    TableName : "xbrl:Facts",
    FactTable : [ $facts ]
}
let $comment :=
{
    NumFacts: count($facts),
    TotalNumFacts: session:num-facts(),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "facts")
return api:check-and-return-results($token, $results, $format)


Update entities.jq

Update entities.jq


jq: Broaden components and error filters

  We do not filter out sub-components from components filters (e.g.
  `RemoteWatcher:Analysis` logs will be returned when filtering on
  `RemoteWatcher`) and logs containing an `err` attribute will be
  returned by the `error` filter.

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove notes content
def redactNote(obj):
  obj |=
    (.
    | if isempty(.metadata.content | objects) | not then .metadata.content |= "[redacted]" else . end
    | if isempty(.metadata.schema | objects) | not then .metadata.schema |= "[redacted]" else . end
    | if isempty(.remote | objects) | not then redactNote(.remote) else . end
    | if isempty(.moveFrom | objects) | not then redactNote(.moveFrom) else . end
    | if isempty(.overwrite | objects) | not then redactNote(.overwrite) else . end
    )
;

def cleanNote:
  .
  | if isempty(.doc | objects) | not then redactNote(.doc) else . end
  | if isempty(.was | objects) | not then redactNote(.was) else . end
  | if isempty(.remoteDoc | objects) | not then redactNote(.remoteDoc) else . end
  | if isempty(.note | objects) | not then redactNote(.note) else . end
  | if isempty(.file | objects) | not then redactNote(.file) else . end
  | if isempty(.err | objects) | not then .
    | .err |= (. | redactNote(.doc))
    else . end
  | if isempty(.change | objects) | not then .
    | if isempty(.change.doc | objects) | not then .change.doc |= (. | redactNote(.doc)) else . end
    | if isempty(.change.was | objects) | not then .change.was |= (. | redactNote(.was)) else . end
    else . end
;

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v) |
  del(.level) |
  del(.local) |
  del(.remote) |
  cleanNote |
  {time,component,msg,path}+(del(.time)|del(.component)|del(.msg)|del(.path));

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level or .err != null;
def error: select(is_error) | clean;
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: select(is_warn) | clean;
def warn_strict: select(is_warn_strict) | clean;
def info_level: 30;
def is_info: .level >= info_level;
def info: select(is_info) | clean;
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: select(is_debug) | clean;

# Components:
def component(pattern): select(.component | test(pattern));
def App: component("^App");
def Chokidar: component("^Chokidar|chokidar");
def Atom: component("^Atom|atom");
def Fs: component("^Fs|FS");
def LocalAnalysis: component("^LocalAnalysis|local/analysis");
def LocalChange: component("^LocalChange|local/change");
def LocalWatcher: component("^LocalWatcher");
def LocalWriter: component("^LocalWriter");
def Merge: component("^Merge");
def Metadata: component("^Metadata");
def Perfs: component("^Perfs|PerfReports");
def Pouch: component("^Pouch");
def Prep: component("^Prep");
def RemoteCozy: component("^RemoteCozy");
def RemoteUserActionRequired: component("^RemoteUserActionRequired|remote/user_action_required");
def RemoteWarningPoller: component("^RemoteWarningPoller");
def RemoteWatcher: component("^RemoteWatcher");
def RemoteWriter: component("^RemoteWriter");
def Sync: component("^Sync");

def Local: select(.component | test("^local|chokidar|atom"; "i"));
def Remote: select(.component | test("^remote"; "i"));

# Exclude "up-to-date" messages
#
#     yarn jq nouptodate path/to/logs*
#
def nouptodate: select(.msg | test("up[- ]to[- ]date") | not);

# Exclude initial scan messages
#
#     yarn jq noscan path/to/logs*
#
def noscan: select(.action != "scan");

# Exclude Proxy stuff:
#
#     yarn jq noproxy path/to/logs*
#
def noproxy: select(.component | test("GUI:proxy") | not);

# Find conflicts:
#
#     yarn jq -c conflict path/to/logs*
#
def is_conflict: .msg | test("resolveConflictAsync|Resolving conflict");
def conflict: clean | select(is_conflict);

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
# FIXME: Should not output the same line twice when both path & oldpath match.
def path(pattern):
  clean | select(
    (
      try (.path // (.doc | .path) // (.change | .doc | .path) // (.idConflict | .newDoc | .path) // (.event | .path)),
      try (.oldpath // (.was | .path) // (.idConflict | .existingDoc | .path) // (.event | .oldPath)),
      ""
    )
      | strings
      | test(pattern)
  );

def shortPath:
  (
    .
    | if isempty(.path | strings) | not then .path |= (. | split("\\") | last) else . end
    | if isempty(.path | strings) | not then .path |= (. | split("/") | last) else . end
    | if isempty(.oldpath | strings) | not then .oldpath |= (. | split("\\") | last) else . end
    | if isempty(.oldpath | strings) | not then .oldpath |= (. | split("/") | last) else . end
  );

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#    yarn jq 'before("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#    yarn jq 'after("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#
def time(pattern):
  select(.time | test(pattern));
def before(time): select(.time <= time);
def after(time): select(.time >= time);

# Drop time properties
def notime: del(.time);

# Filter inode number(s):
#
#    yarn jq 'ino(7852458)' path/to/logs*
#    yarn jq 'ino(7852458; 464672)' path/to/logs*
#
def ino(n): select((.ino // (.doc|.ino) // (.change|.doc|.ino)) as $ino | [n] | flatten | map(. == $ino) | any);

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path,oldpath,event}
    | if (.event | not) then del(.event) else . end
    | if (.event | type == "object") then del(.event) + (.event|{path, oldpath: .oldPath, action}) else . end
    | if .oldpath then . else del(.oldpath) end
    ;

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove notes content
def redactNote(obj):
  obj |=
    (.
    | if isempty(.metadata.content | objects) | not then .metadata.content |= "[redacted]" else . end
    | if isempty(.metadata.schema | objects) | not then .metadata.schema |= "[redacted]" else . end
    | if isempty(.remote | objects) | not then redactNote(.remote) else . end
    | if isempty(.moveFrom | objects) | not then redactNote(.moveFrom) else . end
    | if isempty(.overwrite | objects) | not then redactNote(.overwrite) else . end
    )
;

def cleanNote:
  .
  | if isempty(.doc | objects) | not then redactNote(.doc) else . end
  | if isempty(.was | objects) | not then redactNote(.was) else . end
  | if isempty(.remoteDoc | objects) | not then redactNote(.remoteDoc) else . end
  | if isempty(.note | objects) | not then redactNote(.note) else . end
  | if isempty(.file | objects) | not then redactNote(.file) else . end
  | if isempty(.err | objects) | not then .
    | .err |= (. | redactNote(.doc))
    else . end
  | if isempty(.change | objects) | not then .
    | if isempty(.change.doc | objects) | not then .change.doc |= (. | redactNote(.doc)) else . end
    | if isempty(.change.was | objects) | not then .change.was |= (. | redactNote(.was)) else . end
    else . end
;

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v) |
  del(.level) |
  del(.local) |
  del(.remote) |
  cleanNote |
  {time,component,msg,path}+(del(.time)|del(.component)|del(.msg)|del(.path));

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: select(is_error) | clean;
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: select(is_warn) | clean;
def warn_strict: select(is_warn_strict) | clean;
def info_level: 30;
def is_info: .level >= info_level;
def info: select(is_info) | clean;
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: select(is_debug) | clean;

# Components:
def component(pattern): select(.component | test(pattern));
def App: component("^App$");
def Chokidar: component("^Chokidar|chokidar$");
def Fs: component("^Fs|FS$");
def LocalAnalysis: component("^LocalAnalysis|local/analysis$");
def LocalChange: component("^LocalChange|local/change$");
def LocalWatcher: component("^LocalWatcher$");
def LocalWriter: component("^LocalWriter$");
def Merge: component("^Merge$");
def Metadata: component("^Metadata$");
def Perfs: component("^Perfs|PerfReports$");
def Pouch: component("^Pouch$");
def Prep: component("^Prep$");
def RemoteCozy: component("^RemoteCozy$");
def RemoteUserActionRequired: component("^RemoteUserActionRequired|remote/user_action_required$");
def RemoteWarningPoller: component("^RemoteWarningPoller$");
def RemoteWatcher: component("^RemoteWatcher$");
def RemoteWriter: component("^RemoteWriter$");
def Sync: component("^Sync$");

def Local: select(.component | test("^local|chokidar"; "i"));
def Remote: select(.component | test("^remote"; "i"));

# Exclude "up-to-date" messages
#
#     yarn jq nouptodate path/to/logs*
#
def nouptodate: select(.msg | test("up[- ]to[- ]date") | not);

# Exclude initial scan messages
#
#     yarn jq noscan path/to/logs*
#
def noscan: select(.action != "scan");

# Exclude Proxy stuff:
#
#     yarn jq noproxy path/to/logs*
#
def noproxy: select(.component | test("GUI:proxy") | not);

# Find conflicts:
#
#     yarn jq -c conflict path/to/logs*
#
def is_conflict: .msg | test("resolveConflictAsync|Resolving conflict");
def conflict: clean | select(is_conflict);

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
# FIXME: Should not output the same line twice when both path & oldpath match.
def path(pattern):
  clean | select(
    (
      try (.path // (.doc | .path) // (.change | .doc | .path) // (.idConflict | .newDoc | .path) // (.event | .path)),
      try (.oldpath // (.was | .path) // (.idConflict | .existingDoc | .path) // (.event | .oldPath)),
      ""
    )
      | strings
      | test(pattern)
  );

def shortPath:
  (
    .
    | if isempty(.path | strings) | not then .path |= (. | split("\\") | last) else . end
    | if isempty(.path | strings) | not then .path |= (. | split("/") | last) else . end
    | if isempty(.oldpath | strings) | not then .oldpath |= (. | split("\\") | last) else . end
    | if isempty(.oldpath | strings) | not then .oldpath |= (. | split("/") | last) else . end
  );

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#    yarn jq 'before("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#    yarn jq 'after("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#
def time(pattern):
  select(.time | test(pattern));
def before(time): select(.time <= time);
def after(time): select(.time >= time);

# Drop time properties
def notime: del(.time);

# Filter inode number(s):
#
#    yarn jq 'ino(7852458)' path/to/logs*
#    yarn jq 'ino(7852458; 464672)' path/to/logs*
#
def ino(n): select((.ino // (.doc|.ino) // (.change|.doc|.ino)) as $ino | [n] | flatten | map(. == $ino) | any);

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path,oldpath,event}
    | if (.event | not) then del(.event) else . end
    | if (.event | type == "object") then del(.event) + (.event|{path, oldpath: .oldPath, action}) else . end
    | if .oldpath then . else del(.oldpath) end
    ;

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );


Added comment

Added comment

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";
import module namespace mongo = "http://www.28msec.com/modules/mongodb";

(:
  Given an index name returns the name of the index first field
:)
declare function local:first-index-field($index as string) as string
{
  if (starts-with($index, "_"))
  then "_" || substring-before(substring($index, 2), "_")
  else substring-before($index, "_")
};

declare function local:check-index($conn as anyURI, $coll as string, $index as string) as object
{
  let $field := local:first-index-field($index)
  return subsequence(mongo:find($conn, $coll, {"$query": {$field: {"$exists": true}} , "$hint": $index}, {$field: 1}),1,1)
};

let $indexes:=
  let $conn := mw:connection()
  for $coll in keys($mw:ALL-INDEXES)
  for $index in members($mw:ALL-INDEXES($coll))
  return local:check-index($conn, $coll, $index)
return
  if (count($indexes) eq 21)
  then "OK"
  else error(QName("mw:TEST-FAIL"), "Expected 21 indexes, got " || count($indexes))

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";
import module namespace mongo = "http://www.28msec.com/modules/mongodb";

declare function local:first-index-field($index as string) as string
{
  if (starts-with($index, "_"))
  then "_" || substring-before(substring($index, 2), "_")
  else substring-before($index, "_")
};

declare function local:check-index($conn as anyURI, $coll as string, $index as string) as object
{
  let $field := local:first-index-field($index)
  return subsequence(mongo:find($conn, $coll, {"$query": {$field: {"$exists": true}} , "$hint": $index}, {$field: 1}),1,1)
};

let $indexes:=
  let $conn := mw:connection()
  for $coll in keys($mw:ALL-INDEXES)
  for $index in members($mw:ALL-INDEXES($coll))
  return local:check-index($conn, $coll, $index)
return
  if (count($indexes) eq 21)
  then "OK"
  else error(QName("mw:TEST-FAIL"), "Expected 21 indexes, got " || count($indexes))


Update facts.jq

Update facts.jq

import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace rules = "http://28.io/modules/xbrl/rules";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace request = "http://www.28msec.com/modules/http-request";

declare function local:param-values($name as string) as string*
{
    switch(true)
     case $name eq "xbrl:Concept"
        return (request:param-values("concept"), request:param-values("xbrl:Concept"))

     case $name eq "sec:FiscalYear" and $profile-name eq "sec"
        return ($fiscalYear, request:param-values("sec:FiscalYear"))[$$ ne "LATEST"]
     case $name eq "sec:FiscalYear::type" and $profile-name eq "sec"
        return "integer"
     case $name eq "sec:FiscalPeriod" and $profile-name eq "sec"
        return ($fiscalPeriod, request:param-values("sec:FiscalPeriod"))
     case $name eq "sec:FiscalPeriodType" and $profile-name eq "sec"
        return ($fiscalPeriodType, request:param-values("sec:FiscalPeriodType"))
     case $name eq "dei:LegalEntityAxis" and $profile-name eq "sec"
        return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis")
     case $name eq "dei:LegalEntityAxis::default" return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis::default")
     case $name eq "xbrl:Entity" and $profile-name eq "sec" return (
         let $companies := companies:companies(
            $cik,
            api:preprocess-tags($tag),
            $ticker,
            $sic)._id
        return if(empty(($cik,$tag,$ticker,$sic)) or exists($companies))
               then $companies
               else "dummy",
        request:param-values("xbrl:Entity"))
     case $name eq "xbrl28:Archive" and $profile-name eq "sec" return (
            let $fiscalYears := ($fiscalYear, request:param-values("sec:FiscalYear"))
            let $fiscalPeriods := local:param-values("sec:FiscalPeriod")
            let $entities := entities:entities(local:param-values("xbrl:Entity"))
            return
                if($fiscalYears = "LATEST")
                then fiscal-core:latest-filings($entities, $fiscalPeriods)._id
                else (),
            $aid,
            request:param-values("xbrl28:Archive")
        )
     default return request:param-values($name)
};

declare function local:param-names() as string*
{
    let $names := request:param-names()
    return distinct-values((
        $names[contains($$, ":")],
        "xbrl:Concept"[$names = "concept"],

        "sec:Accepted"[$profile-name eq "sec"],
        "sec:FiscalPeriod"[$profile-name eq "sec"],
        "sec:FiscalPeriodType"[$profile-name eq "sec"],
        "sec:FiscalYear"[$profile-name eq "sec"],
        "xbrl:Entity"[$profile-name eq "sec" and $names = ("cik", "tag", "ticker", "sic")],
        "dei:LegalEntityAxis"[$profile-name eq "sec"],
        "dei:LegalEntityAxis::default"[$profile-name eq "sec"],
        "xbrl28:Archive"[$profile-name eq "sec"]))
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube() as object
{
    let $hypercube-spec :=
    {|
        for $parameter in local:param-names()
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (local:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $type as string? :=
            (local:param-values($dimension-name || "::type"), local:param-values($dimension-name || ":type"))[1]

        let $values := local:param-values($dimension-name)
        let $typed-values := if (exists($type)) then local:cast-sequence($values, $type) else $values

        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (local:param-values($dimension-name || "::default"), local:param-values($dimension-name || ":default"))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value

        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                { "Domain" : [ $typed-values ] }[(exists($typed-values) and not($all))],
                { "Default" : $typed-default-value }[$has-default]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($hypercube-spec)
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token             as string? external;
declare  %rest:case-insensitive                 variable $profile-name      as string  external := "sec";
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriodTypes as string* external := ("instant", "YTD");
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $validate          as boolean external := false;
declare  %rest:case-insensitive                 variable $labels            as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic,
        $aid)
let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $hypercube := local:hypercube()

let $facts :=
    let $options := {|
      {
        Hypercube : $hypercube,
        Validate: $validate
      },
      { "ConceptMaps" : $map }[exists($map)],
      { "Rules" : [ $rule ] }[exists($rule)]
    |}
    return if($profile-name eq "sec")
    then sec:facts-for($options)
    else facts:facts-for($options)

let $facts :=
  if(not $labels)
  then $facts
  else
    let $archives as string* := distinct-values($facts.Aspects."xbrl28:Archive")
    let $concept-names as string* := distinct-values($facts.Aspects."xbrl:Concept")
    let $concepts as object* :=
      (
          concepts:concepts($concept-names, $archives, $concepts:ANY_COMPONENT_LINK_ROLE),
          (reports:concepts(($report,$map)))[$$.Name = $concept-names]
      )
    let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $concepts:AMERICAN_ENGLISH )[1]
    let $roles as string* := ( $report.Role, $concepts:ANY_COMPONENT_LINK_ROLE )
    for $fact as object in $facts
    let $entityName as string := $entities[$$._id eq $fact.Aspects."xbrl:Entity"].Profiles.SEC.CompanyName
    return
    {|
      $fact,
      let $concept-labels as object? := facts:labels($fact, $roles, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
      let $standard-labels as object := conversion:get-standard-labels($fact, $entityName)
      return { Labels : {| $concept-labels, $standard-labels |} }
    |}

let $facts :=
  if($profile-name eq "sec")
  then api:normalize-facts($facts)
  else $facts

let $result := {
    NetworkIdentifier : "http://bizql.io/facts",
    TableName : "xbrl:Facts",
    FactTable : [ $facts ]
}
let $comment :=
{
    NumFacts: count($facts),
    TotalNumFacts: session:num-facts(),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "facts")
return api:check-and-return-results($token, $results, $format)

import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace rules = "http://28.io/modules/xbrl/rules";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace request = "http://www.28msec.com/modules/http-request";

declare function local:param-values($name as string) as string*
{
    switch(true)
     case $name eq "xbrl:Concept"
        return (request:param-values("concept"), request:param-values("xbrl:Concept"))

     case $name eq "sec:FiscalYear" and $profile-name eq "sec"
        return ($fiscalYear, request:param-values("sec:FiscalYear"))[$$ ne "LATEST"]
     case $name eq "sec:FiscalYear::type" and $profile-name eq "sec"
        return "integer"
     case $name eq "sec:FiscalPeriod" and $profile-name eq "sec"
        return ($fiscalPeriod, request:param-values("sec:FiscalPeriod"))
     case $name eq "sec:FiscalPeriodType" and $profile-name eq "sec"
        return
         if(empty((request:param-values("sec:FiscalPeriodType"), request:param-values("sec:FiscalPeriodType::default"))))
         then ("instant", "YTD")
         else request:param-values("sec:FiscalPeriodType")
     case $name eq "dei:LegalEntityAxis" and $profile-name eq "sec"
        return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis")
     case $name eq "dei:LegalEntityAxis::default" return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis::default")
     case $name eq "xbrl:Entity" and $profile-name eq "sec" return (
         let $companies := companies:companies(
            $cik,
            api:preprocess-tags($tag),
            $ticker,
            $sic)._id
        return if(empty(($cik,$tag,$ticker,$sic)) or exists($companies))
               then $companies
               else "dummy",
        request:param-values("xbrl:Entity"))
     case $name eq "xbrl28:Archive" and $profile-name eq "sec" return (
            let $fiscalYears := ($fiscalYear, request:param-values("sec:FiscalYear"))
            let $fiscalPeriods := local:param-values("sec:FiscalPeriod")
            let $entities := entities:entities(local:param-values("xbrl:Entity"))
            return
                if($fiscalYears = "LATEST")
                then fiscal-core:latest-filings($entities, $fiscalPeriods)._id
                else (),
            $aid,
            request:param-values("xbrl28:Archive")
        )
     default return request:param-values($name)
};

declare function local:param-names() as string*
{
    let $names := request:param-names()
    return distinct-values((
        $names[contains($$, ":")],
        "xbrl:Concept"[$names = "concept"],

        "sec:Accepted"[$profile-name eq "sec"],
        "sec:FiscalPeriod"[$profile-name eq "sec"],
        "sec:FiscalPeriodType"[$profile-name eq "sec"],
        "sec:FiscalYear"[$profile-name eq "sec"],
        "xbrl:Entity"[$profile-name eq "sec" and $names = ("cik", "tag", "ticker", "sic")],
        "dei:LegalEntityAxis"[$profile-name eq "sec"],
        "dei:LegalEntityAxis::default"[$profile-name eq "sec"],
        "xbrl28:Archive"[$profile-name eq "sec"]))
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube() as object
{
    let $hypercube-spec :=
    {|
        for $parameter in local:param-names()
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (local:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $type as string? :=
            (local:param-values($dimension-name || "::type"), local:param-values($dimension-name || ":type"))[1]

        let $values := local:param-values($dimension-name)
        let $typed-values := if (exists($type)) then local:cast-sequence($values, $type) else $values

        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (local:param-values($dimension-name || "::default"), local:param-values($dimension-name || ":default"))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value

        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                { "Domain" : [ $typed-values ] }[(exists($typed-values) and not($all))],
                { "Default" : $typed-default-value }[$has-default]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($hypercube-spec)
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token             as string? external;
declare  %rest:case-insensitive                 variable $profile-name      as string  external := "sec";
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $validate          as boolean external := false;
declare  %rest:case-insensitive                 variable $labels            as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic,
        $aid)
let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $hypercube := local:hypercube()

let $facts :=
    let $options := {|
      {
        Hypercube : $hypercube,
        Validate: $validate
      },
      { "ConceptMaps" : $map }[exists($map)],
      { "Rules" : [ $rule ] }[exists($rule)]
    |}
    return if($profile-name eq "sec")
    then sec:facts-for($options)
    else facts:facts-for($options)

let $facts :=
  if(not $labels)
  then $facts
  else
    let $archives as string* := distinct-values($facts.Aspects."xbrl28:Archive")
    let $concept-names as string* := distinct-values($facts.Aspects."xbrl:Concept")
    let $concepts as object* :=
      (
          concepts:concepts($concept-names, $archives, $concepts:ANY_COMPONENT_LINK_ROLE),
          (reports:concepts(($report,$map)))[$$.Name = $concept-names]
      )
    let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $concepts:AMERICAN_ENGLISH )[1]
    let $roles as string* := ( $report.Role, $concepts:ANY_COMPONENT_LINK_ROLE )
    for $fact as object in $facts
    let $entityName as string := $entities[$$._id eq $fact.Aspects."xbrl:Entity"].Profiles.SEC.CompanyName
    return
    {|
      $fact,
      let $concept-labels as object? := facts:labels($fact, $roles, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
      let $standard-labels as object := conversion:get-standard-labels($fact, $entityName)
      return { Labels : {| $concept-labels, $standard-labels |} }
    |}

let $facts :=
  if($profile-name eq "sec")
  then api:normalize-facts($facts)
  else $facts

let $result := {
    NetworkIdentifier : "http://bizql.io/facts",
    TableName : "xbrl:Facts",
    FactTable : [ $facts ]
}
let $comment :=
{
    NumFacts: count($facts),
    TotalNumFacts: session:num-facts(),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "facts")
return api:check-and-return-results($token, $results, $format)