:bug: fix facts endpoint.

:bug: fix facts endpoint.

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace labels = "http://28.io/modules/xbrl/labels";
import module namespace rules = "http://28.io/modules/xbrl/rules";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

import module namespace request = "http://www.28msec.com/modules/http-request";

declare option rest:response "first-item";

declare variable $local:additional-concepts as object* := (
  {
    Name: "sec:DefaultLegalEntity",
    Labels : [ {
        Role : "http://www.xbrl.org/2003/role/label",
        Language : "en-us",
        Value : "Default Legal Entity [Member]"
    } ]
  },
  {
    Name: "xbrl:NonNumeric",
    Labels : [ {
        Role : "http://www.xbrl.org/2003/role/label",
        Language : "en-us",
        Value : "Non Numeric"
    } ]
  }
);

declare function local:param-values(
    $name as string,
    $entities as object*) as string*
{
    switch(true)
     case $name eq "xbrl:Concept"
        return (request:param-values("concept"), request:param-values("xbrl:Concept"))

    (: sec profile :)
     case $name eq "sec:FiscalYear" and $profile-name eq "sec"
        return ($fiscalYear, request:param-values("sec:FiscalYear"))[$$ ne "LATEST"]
     case $name eq "sec:FiscalYear::type" and $profile-name eq "sec"
        return "integer"
     case $name eq "sec:FiscalPeriod" and $profile-name eq "sec"
        return ($fiscalPeriod, request:param-values("sec:FiscalPeriod"))
     case $name eq "sec:FiscalPeriod::type" and $profile-name eq "sec"
        return "string"
     case $name eq "sec:FiscalPeriodType" and $profile-name eq "sec"
        return ($fiscalPeriodType, request:param-values("sec:FiscalPeriodType"))
     case $name eq "sec:FiscalPeriodType::type" and $profile-name eq "sec"
        return "string"
     case $name eq "dei:LegalEntityAxis" and $profile-name eq "sec"
        return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis")
     case $name eq "dei:LegalEntityAxis::default" return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis::default")

     case $name eq "xbrl:Entity" and $profile-name = ("sec", "japan")
         return (
                if(empty(($cik,$tag,$ticker,$sic)) or exists($entities))
                then entities:eid($entities)
                else "dummy",
                request:param-values("xbrl:Entity")
            )
     case $name eq "xbrl28:Archive" and $profile-name = ("sec", "japan") return (
            let $prefix as string :=
                switch($profile-name)
                case "sec" return "sec"
                case "japan" return "fsa"
                default return (: not reachable :) ()
            let $fiscalYears :=
                let $years := request:param-values( $prefix || ":FiscalYear")
                return if ($years) then $years else $fiscalYear
            let $fiscalPeriods := local:param-values($prefix || ":FiscalPeriod", $entities)
            return
                if($fiscalYears = "LATEST")
                then archives:aid(multiplexer:latest-filings($profile-name, $entities, $fiscalPeriods))
                else (),
            $aid,
            request:param-values("xbrl28:Archive")
        )

     (: japan profile :)
     case $name eq "fsa:FiscalYear" and $profile-name eq "japan"
     return ($fiscalYear, request:param-values("fsa:FiscalYear"))[$$ ne "LATEST"]
     case $name eq "fsa:FiscalYear::type" and $profile-name eq "japan"
     return "integer"
     case $name eq "fsa:FiscalPeriod" and $profile-name eq "japan"
     return ($fiscalPeriod, request:param-values("fsa:FiscalPeriod"))
     case $name eq "fsa:FiscalPeriod::type" and $profile-name eq "japan"
     return "string"
     case $name eq "fsa:FiscalPeriodType" and $profile-name eq "japan"
     return ($fiscalPeriodType, request:param-values("fsa:FiscalPeriodType"))
     case $name eq "fsa:FiscalPeriodType::type" and $profile-name eq "japan"
     return "string"
     case $name eq "fsa:ArchiveFiscalYear::type" and $profile-name eq "japan"
     return "integer"
     case $name eq "fsa:ArchiveFiscalPeriod::type" and $profile-name eq "japan"
     return "string"

     default return request:param-values($name)
};

declare function local:param-names() as string*
{
    let $names := request:param-names()
    return distinct-values((
        (: generic xbrl :)
        $names[contains($$, ":")],
        "xbrl:Entity"[$names = ("eid")],
        "xbrl28:Archive"[$profile-name = ("sec", "japan")],
        "xbrl:Concept"[$names = "concept"],

        (: sec profile :)
        "sec:Accepted"[$profile-name eq "sec"],
        "sec:FiscalPeriod"[$profile-name eq "sec"],
        "sec:FiscalPeriodType"[$profile-name eq "sec"],
        "sec:FiscalYear"[$profile-name eq "sec"],
        "xbrl:Entity"[$profile-name eq "sec" and
                      $names = ("cik", "tag", "ticker", "sic")],
        "dei:LegalEntityAxis"[$profile-name eq "sec"],
        "dei:LegalEntityAxis::default"[$profile-name eq "sec"],

        (: japan profile :)
        "fsa:Submitted"[$profile-name eq "japan"],
        "fsa:FiscalPeriod"[$profile-name eq "japan"],
        "fsa:FiscalPeriodType"[$profile-name eq "japan"],
        "fsa:FiscalYear"[$profile-name eq "japan"],
        "xbrl:Entity"[$profile-name eq "japan" and
                      $names = ("edinetcode", "tag", "ticker")]
    ))
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube($entities as object*) as object
{
    let $hypercube-spec :=
    {|
        for $parameter in local:param-names()
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (local:param-values($dimension-name, $entities) ! upper-case($$)) = "ALL"
        let $type as string? :=
            (local:param-values($dimension-name || "::type", $entities),
             local:param-values($dimension-name || ":type", $entities))[1]

        let $values := local:param-values($dimension-name, $entities)
        let $typed-values := if (exists($type)) then local:cast-sequence($values[$$ ne "ALL"], $type) else $values

        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (local:param-values($dimension-name || "::default", $entities),
                               local:param-values($dimension-name || ":default", $entities))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value

        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                { "Domain" : [ $typed-values ] }
                    [exists($typed-values) and not($all)],
                { "Default" : $typed-default-value }[$has-default],
                { "Default" : null }
                    [$all and (not $has-default) and exists($type)],
                { "Default" : "xbrl28:Domain" }
                    [$all and (not $has-default) and empty($type)]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($hypercube-spec, { IsOpen: $open })
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token             as string? external;
declare  %rest:case-insensitive                 variable $profile-name      as string  external := $config:profile-name;
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode        as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriodType  as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $eid               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $validate          as boolean external := false;
declare  %rest:case-insensitive                 variable $open              as boolean external := false;
declare  %rest:case-insensitive                 variable $labels            as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;
declare  %rest:case-insensitive                 variable $debug             as boolean external := false;
declare  %rest:case-insensitive                 variable $language          as string  external := "en-US";

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  api:preprocess-tags($tag),
  $ticker,
  $sic,
  (),
  (),
  $aid,
  ())

let $entities-not-found as boolean :=
  exists(($eid, $cik, $tag, $ticker, $sic)) and empty($entities)

let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $hypercube := local:hypercube($entities)

let $facts :=
    let $options := {|
      {
        Hypercube : $hypercube,
        Validate: $validate
      },
      { "ConceptMaps" : $map }[exists($map)],
      { "Rules" : [ $rule ] }[exists($rule)],
      { "Concepts" : $report.Concepts }[exists($report) and exists($report.Concepts)]
    |}
    return if($profile-name eq "sec")
    then sec:facts-for($options)
    else facts:facts-for($options)

let $facts :=
  if(not $labels)
  then $facts
  else
    let $archives as string* := distinct-values($facts.Aspects."xbrl28:Archive")
    let $concept-names as string* := (
      distinct-values((keys($facts.Aspects), values($facts.Aspects)[$$ instance of string]))
    )
    let $concepts as object* :=
      (
          concepts:concepts($concept-names, $archives, $concepts:ANY_COMPONENT_LINK_ROLE),
          $report.Concepts[][$$.Name = $concept-names],
          if($profile-name eq "sec") then $local:additional-concepts else ()
      )
    let $language as string := ( $language, $report.$components:DEFAULT-LANGUAGE , $labels:AMERICAN_ENGLISH )[1]
    let $nonFetchedEntities as string* := request:param-values("xbrl:Entity")[not $$ = entities:eid($entities)]
    let $entities as object* := ($entities, entities:entities($nonFetchedEntities))
    for $fact as object in $facts
    return
    {|
      $fact,
      let $concept-labels as object? := labels:labels-for-facts(
        $fact,
        $labels:STANDARD_LABEL_ROLE,
        $language,
        $concepts,
        $entities,
        ()
      )
      return { Labels : $concept-labels }
    |}

let $facts :=
  if($profile-name eq "sec")
  then api:normalize-facts($facts)
  else $facts

let $result := {
    NetworkIdentifier : "http://bizql.io/facts",
    TableName : "xbrl:Facts",
    FactTable : [ $facts ]
}
let $comment :=
{
    NumFacts: count($facts),
    TotalNumFacts: session:num-facts(),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $comment :=
    if($debug)
    then {|
        $comment,
        { DebugInfo: { Hypercube: $hypercube } }
    |}
    else $comment
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}
let $results := api:serialize($result, $comment, $serializers, $format, "facts")
return if($entities-not-found)
       then api:not-found("entity")
       else api:check-and-return-results($token, $results, $format)

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace labels = "http://28.io/modules/xbrl/labels";
import module namespace rules = "http://28.io/modules/xbrl/rules";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

import module namespace request = "http://www.28msec.com/modules/http-request";

declare option rest:response "first-item";

declare variable $local:additional-concepts as object* := (
  {
    Name: "sec:DefaultLegalEntity",
    Labels : [ {
        Role : "http://www.xbrl.org/2003/role/label",
        Language : "en-us",
        Value : "Default Legal Entity [Member]"
    } ]
  },
  {
    Name: "xbrl:NonNumeric",
    Labels : [ {
        Role : "http://www.xbrl.org/2003/role/label",
        Language : "en-us",
        Value : "Non Numeric"
    } ]
  }
);

declare function local:param-values(
    $name as string,
    $entities as object*) as string*
{
    switch(true)
     case $name eq "xbrl:Concept"
        return (request:param-values("concept"), request:param-values("xbrl:Concept"))

    (: sec profile :)
     case $name eq "sec:FiscalYear" and $profile-name eq "sec"
        return ($fiscalYear, request:param-values("sec:FiscalYear"))[$$ ne "LATEST"]
     case $name eq "sec:FiscalYear::type" and $profile-name eq "sec"
        return "integer"
     case $name eq "sec:FiscalPeriod" and $profile-name eq "sec"
        return ($fiscalPeriod, request:param-values("sec:FiscalPeriod"))
     case $name eq "sec:FiscalPeriod::type" and $profile-name eq "sec"
        return "string"
     case $name eq "sec:FiscalPeriodType" and $profile-name eq "sec"
        return ($fiscalPeriodType, request:param-values("sec:FiscalPeriodType"))
     case $name eq "sec:FiscalPeriodType::type" and $profile-name eq "sec"
        return "string"
     case $name eq "dei:LegalEntityAxis" and $profile-name eq "sec"
        return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis")
     case $name eq "dei:LegalEntityAxis::default" return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis::default")

     case $name eq "xbrl:Entity" and $profile-name = ("sec", "japan")
         return (
                if(empty(($cik,$tag,$ticker,$sic)) or exists($entities))
                then entities:eid($entities)
                else "dummy",
                request:param-values("xbrl:Entity")
            )
     case $name eq "xbrl28:Archive" and $profile-name = ("sec", "japan") return (
            let $prefix as string :=
                switch($profile-name)
                case "sec" return "sec"
                case "japan" return "fsa"
                default return (: not reachable :) ()
            let $fiscalYears :=
                let $years := request:param-values( $prefix || ":FiscalYear")
                return if ($years) then $years else $fiscalYear
            let $fiscalPeriods := local:param-values($prefix || ":FiscalPeriod", $entities)
            return
                if($fiscalYears = "LATEST")
                then archives:aid(multiplexer:latest-filings($profile-name, $entities, $fiscalPeriods))
                else (),
            $aid,
            request:param-values("xbrl28:Archive")
        )

     (: japan profile :)
     case $name eq "fsa:FiscalYear" and $profile-name eq "japan"
     return ($fiscalYear, request:param-values("fsa:FiscalYear"))[$$ ne "LATEST"]
     case $name eq "fsa:FiscalYear::type" and $profile-name eq "japan"
     return "integer"
     case $name eq "fsa:FiscalPeriod" and $profile-name eq "japan"
     return ($fiscalPeriod, request:param-values("fsa:FiscalPeriod"))
     case $name eq "fsa:FiscalPeriod::type" and $profile-name eq "japan"
     return "string"
     case $name eq "fsa:FiscalPeriodType" and $profile-name eq "japan"
     return ($fiscalPeriodType, request:param-values("fsa:FiscalPeriodType"))
     case $name eq "fsa:FiscalPeriodType::type" and $profile-name eq "japan"
     return "string"
     case $name eq "fsa:ArchiveFiscalYear::type" and $profile-name eq "japan"
     return "integer"
     case $name eq "fsa:ArchiveFiscalPeriod::type" and $profile-name eq "japan"
     return "string"

     default return request:param-values($name)
};

declare function local:param-names() as string*
{
    let $names := request:param-names()
    return distinct-values((
        (: generic xbrl :)
        $names[contains($$, ":")],
        "xbrl:Entity"[$names = ("eid")],
        "xbrl28:Archive"[$profile-name = ("sec", "japan")],
        "xbrl:Concept"[$names = "concept"],

        (: sec profile :)
        "sec:Accepted"[$profile-name eq "sec"],
        "sec:FiscalPeriod"[$profile-name eq "sec"],
        "sec:FiscalPeriodType"[$profile-name eq "sec"],
        "sec:FiscalYear"[$profile-name eq "sec"],
        "xbrl:Entity"[$profile-name eq "sec" and
                      $names = ("cik", "tag", "ticker", "sic")],
        "dei:LegalEntityAxis"[$profile-name eq "sec"],
        "dei:LegalEntityAxis::default"[$profile-name eq "sec"],

        (: japan profile :)
        "fsa:Submitted"[$profile-name eq "japan"],
        "fsa:FiscalPeriod"[$profile-name eq "japan"],
        "fsa:FiscalPeriodType"[$profile-name eq "japan"],
        "fsa:FiscalYear"[$profile-name eq "japan"],
        "xbrl:Entity"[$profile-name eq "japan" and
                      $names = ("edinetcode", "tag", "ticker")]
    ))
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube($entities as object*) as object
{
    let $hypercube-spec :=
    {|
        for $parameter in local:param-names()
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (local:param-values($dimension-name, $entities) ! upper-case($$)) = "ALL"
        let $type as string? :=
            (local:param-values($dimension-name || "::type", $entities),
             local:param-values($dimension-name || ":type", $entities))[1]

        let $values := local:param-values($dimension-name, $entities)
        let $typed-values := if (exists($type)) then local:cast-sequence($values[$$ ne "ALL"], $type) else $values

        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (local:param-values($dimension-name || "::default", $entities),
                               local:param-values($dimension-name || ":default", $entities))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value

        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                { "Domain" : [ $typed-values ] }
                    [exists($typed-values) and not($all)],
                { "Default" : $typed-default-value }[$has-default],
                { "Default" : null }
                    [$all and (not $has-default) and exists($type)],
                { "Default" : "xbrl28:Domain" }
                    [$all and (not $has-default) and empty($type)]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($hypercube-spec, { IsOpen: $open })
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token             as string? external;
declare  %rest:case-insensitive                 variable $profile-name      as string  external := $config:profile-name;
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode        as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriodType  as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $eid               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $validate          as boolean external := false;
declare  %rest:case-insensitive                 variable $open              as boolean external := false;
declare  %rest:case-insensitive                 variable $labels            as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;
declare  %rest:case-insensitive                 variable $debug             as boolean external := false;
declare  %rest:case-insensitive                 variable $language          as string  external := "en-US";

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  api:preprocess-tags($tag),
  $ticker,
  $sic,
  (),
  (),
  (),
  $aid)

let $entities-not-found as boolean :=
  exists(($eid, $cik, $tag, $ticker, $sic)) and empty($entities)

let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $hypercube := local:hypercube($entities)

let $facts :=
    let $options := {|
      {
        Hypercube : $hypercube,
        Validate: $validate
      },
      { "ConceptMaps" : $map }[exists($map)],
      { "Rules" : [ $rule ] }[exists($rule)],
      { "Concepts" : $report.Concepts }[exists($report) and exists($report.Concepts)]
    |}
    return if($profile-name eq "sec")
    then sec:facts-for($options)
    else facts:facts-for($options)

let $facts :=
  if(not $labels)
  then $facts
  else
    let $archives as string* := distinct-values($facts.Aspects."xbrl28:Archive")
    let $concept-names as string* := (
      distinct-values((keys($facts.Aspects), values($facts.Aspects)[$$ instance of string]))
    )
    let $concepts as object* :=
      (
          concepts:concepts($concept-names, $archives, $concepts:ANY_COMPONENT_LINK_ROLE),
          $report.Concepts[][$$.Name = $concept-names],
          if($profile-name eq "sec") then $local:additional-concepts else ()
      )
    let $language as string := ( $language, $report.$components:DEFAULT-LANGUAGE , $labels:AMERICAN_ENGLISH )[1]
    let $nonFetchedEntities as string* := request:param-values("xbrl:Entity")[not $$ = entities:eid($entities)]
    let $entities as object* := ($entities, entities:entities($nonFetchedEntities))
    for $fact as object in $facts
    return
    {|
      $fact,
      let $concept-labels as object? := labels:labels-for-facts(
        $fact,
        $labels:STANDARD_LABEL_ROLE,
        $language,
        $concepts,
        $entities,
        ()
      )
      return { Labels : $concept-labels }
    |}

let $facts :=
  if($profile-name eq "sec")
  then api:normalize-facts($facts)
  else $facts

let $result := {
    NetworkIdentifier : "http://bizql.io/facts",
    TableName : "xbrl:Facts",
    FactTable : [ $facts ]
}
let $comment :=
{
    NumFacts: count($facts),
    TotalNumFacts: session:num-facts(),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $comment :=
    if($debug)
    then {|
        $comment,
        { DebugInfo: { Hypercube: $hypercube } }
    |}
    else $comment
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}
let $results := api:serialize($result, $comment, $serializers, $format, "facts")
return if($entities-not-found)
       then api:not-found("entity")
       else api:check-and-return-results($token, $results, $format)


using companies:to-csv/xml functions added sic parameter (fix #168)

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace session = "http://apps.28.io/session";

let $format  := lower-case(request:param-values("format")[1])
let $ciks    := request:param-values("cik")
let $tags    := request:param-values("tag") ! upper-case($$) (: DOW30, SP500, FORTUNE100 :)
let $tickers := request:param-values("ticker")
let $sics    := request:param-values("sic")
let $entities :=
    for $entity in
        if (exists(($ciks, $tags, $tickers, $sics)))
        then
            for $entity in (
                companies:companies($ciks),
                companies:companies-for-tags($tags),
                companies:companies-for-tickers($tickers),
                companies:companies-for-SIC($sics))
            group by companies:eid($entity) (: duplicate elimination :)
            return $entity[1]
        else
            companies:companies()
    order by $entity.Profiles.SEC.CompanyName (: companies:name() ? :)
    return $entity
return
    switch ($format)
        case "xml"  return {
            response:content-type("application/xml");
            response:serialization-parameters({"omit-xml-declaration" : false});
            (session:comment("xml", {
                NumEntities: count($entities),
                TotalNumEntities: session:num-entities()
            }),
            <Entities>{
                companies:to-xml($entities)
            }</Entities>)
        }
        case "text" case "csv" return {
            response:content-type("application/csv");
            string-join(companies:to-csv($entities))
        }
        case "excel" return {
            response:content-type("application/vnd.ms-excel");
            response:header("Content-Disposition", "attachment; filename=entities.csv");
            string-join(companies:to-csv($entities))
        }
        default return {
            response:content-type("application/json");
            response:serialization-parameters({"indent" : false});
            {|
                { "Entities" : [ $entities ] },
                session:comment("json", {
                        NumEntities: count($entities),
                        TotalNumEntities: session:num-entities()
                    })
            |}
        }

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace session = "http://apps.28.io/session";

declare function local:to-xml($entities as object*) as node()*
{
  (session:comment("xml", {
        NumEntities: count($entities),
        TotalNumEntities: session:num-entities()
      }),
  <Entities>{
    for $e in $entities
    return
    <Entity>
        <ID>{$e._id}</ID>
        <Profile name="{$e.Profiles.SEC.Name}">
            <EntityRegistrantName>{$e.Profiles.SEC.CompanyName}</EntityRegistrantName>
            <CompanyType>{$e.Profiles.SEC.CompanyType}</CompanyType>
            <SIC>{$e.Profiles.SEC.SIC}</SIC>
            <SICDescription>{$e.Profiles.SEC.SICDescription}</SICDescription>
            <SICGroup>{$e.Profiles.SEC.SICGroup}</SICGroup>
            <Sector>{$e.Profiles.SEC.Sector}</Sector>
            <IsTrust>{$e.Profiles.SEC.IsTrust}</IsTrust>
            <Tickers>{
                for $t in $e.Profiles.SEC.Tickers[]
                return <Ticker>{$t}</Ticker>
            }</Tickers>
            <Tags>
            {
                for $t in $e.Profiles.SEC.Tags[]
                return <Tag>{$t}</Tag>
            }
            </Tags>
        </Profile>
    </Entity>
  }</Entities>)
};

declare function local:to-csv($entities as object*) as string*
{
    csv:serialize(
        for $e in $entities
        return  {
            ID : $e._id,
            Profile : "SEC",
            EntityName : $e.Profiles.SEC.CompanyName,
            CompanyType : $e.Profiles.SEC.CompanyType,
            SIC : $e.Profiles.SEC.SIC,
            SICDescription : $e.Profiles.SEC.SICDescription,
            SICGroup :$e.Profiles.SEC.SICGroup,
            Sector : $e.Profiles.SEC.Sector,
            IsTrust : $e.Profiles.SEC.IsTrust,
            Tickers : string-join($e.Profiles.SEC.Tickers[], " "),
            Tags : string-join($e.Profiles.SEC.Tags[], " ")
        },
        { serialize-null-as : "" }
    )
};

let $format  := lower-case(request:param-values("format")[1])
let $ciks    := request:param-values("cik")
let $tags    := request:param-values("tag") ! upper-case($$) (: DOW30, SP500, FORTUNE100 :)
let $tickers := request:param-values("ticker")
let $entities :=
    for $entity in
        if (exists(($ciks, $tags, $tickers)))
        then
            for $entity in (companies:companies($ciks),
                       companies:companies-for-tags($tags),
                       companies:companies-for-tickers($tickers))
            group by companies:eid($entity) (: duplicate elimination :)
            return $entity[1]
        else
            companies:companies()
    order by $entity.Profiles.SEC.CompanyName (: companies:name() ? :)
    return $entity
return
    switch ($format)
        case "xml"  return {
            response:content-type("application/xml");
            response:serialization-parameters({"omit-xml-declaration" : false});
            local:to-xml($entities)
        }
        case "text" case "csv" return {
            response:content-type("application/csv");
            string-join(local:to-csv($entities))
        }
        case "excel" return {
            response:content-type("application/vnd.ms-excel");
            response:header("Content-Disposition", "attachment; filename=entities.csv");
            string-join(local:to-csv($entities))
        }
        default return {
            response:content-type("application/json");
            response:serialization-parameters({"indent" : false});
            {|
                { "Entities" : [ $entities ] },
                session:comment("json", {
                        NumEntities: count($entities),
                        TotalNumEntities: session:num-entities()
                    })
            |}
        }


:fire: Remove unused module.

:fire: Remove unused module.

jsoniq version "1.0";
(:
 : Copyright 2012-2013 28msec Inc.
 :)

(:~
 :
This module provides functionality for creating and manipulating fact tables.
 :

 :
 : @author Charles Hoffman
 : @author Matthias Brantner
 : @author Dennis Knochenwefel
 : @author Ghislain Fourny
 :)
module namespace fact-tables = "http://28.io/modules/xbrl/fact-tables";

import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace components = "http://28.io/modules/xbrl/components";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 :
 :
Generates a fact table containing facts associated with
 : all concepts in the template, and with all associated dimensions
 : as specified in the template.

 :
 : @param $template-or-model-or-model-id a template (can be a model) or an MID.
 :
 : @return a sequence of facts with populated default dimension values.
 :)
declare function fact-tables:fact-table-as-objects(
  $template-or-model-or-model-id as item) as object*
{
  fact-tables:fact-table-as-objects($template-or-model-or-model-id, ())
};

(:~
 :
 :
Generates a fact table containing facts associated with
 : all concepts in the template, and with all associated dimensions
 : as specified in the template.

 :
 : @param $template-or-model-or-model-id a template (can be a model) or an MID.
 : @param $options further optional parameters:
 : - concept-map a concept map or its name (no concept map used if absent).
 : - archives an array of AIDs to use instead of the ones specified in the template.
 : @return a sequence of facts with populated default dimension values.
 :)
declare function fact-tables:fact-table-as-objects(
  $template-or-model-or-model-id as item,
  $options as object?) as object*
{
  (: @TODO: why is ModelStructure used here? does this still exist? :)
  let $model := fact-tables:resolve-component-or-template($template-or-model-or-model-id)
  let $table := $model.ModelStructure.Table
  let $default-dimension-value-map := {|
    {
      "xbrl:Concept" : null,
      "xbrl:Entity" : null,
      "xbrl:Period" : null,
      "xbrl:Unit" : null
    },
    $table.Aspects[][not $$.Name = ("xbrl:Concept", "xbrl:Entity", "xbrl:Period", "xbrl:Unit")] ! { $$.Name : $$.Members[[1]][ends-with($$.Label, "[Domain]")].Name }
  |}
  let $non-optional-dimensions := keys($default-dimension-value-map)[$default-dimension-value-map.($$) eq null]
  let $line-items := descendant-objects($table.LineItems).Concepts[].Name
  let $concept-map := $options.concept-map
  let $archives := (($options.archives, [ $model.AccessionNumber ])[1])[]
  return if (exists($concept-map))
  then
    let $all-mapped-concepts := $line-items ! $concept-map.mappings.($$)[]
    let $all-facts-by-archive-and-mapped-concept := {|
        for $facts in facts:facts-for-archives-and-concepts($archives,
                                                            $all-mapped-concepts,
                                                            $options)
        group by $archive := $facts.$facts:ASPECTS."xbrl28:Archive"
        return { $archive : {|
            for $facts-for-archive in $facts
            group by $concept := $facts-for-archive.$facts:ASPECTS."xbrl:Concept"
            return { $concept : [ $facts-for-archive ] }
        |} }
    |}
    for $archive in $archives
    for $line-item in $line-items
    let $mapped-concepts := $concept-map.mappings.($line-item)
    let $mapped-facts := {
      variable $nb := size($mapped-concepts);
      variable $i := 1;
      variable $found := false;
      variable $result := ();
      while (not $found and $i le $nb)
      {
        $result := $all-facts-by-archive-and-mapped-concept.$archive.($mapped-concepts[[$i]])[];
        if (empty($result))
        then
          $i := $i + 1;
        else
          $found := true;
      }
      $result
    }
    for $fact in $mapped-facts
    where (
      every $key in keys($fact.Aspects)
      satisfies exists($default-dimension-value-map.$key)
    ) and (
      every $key in $non-optional-dimensions
      satisfies exists($fact.Aspects.$key)
    )
    return
      copy $populated := $fact
      modify (
        replace value of json $populated.Aspects."xbrl:Concept" with $line-item,
        for $dimension in $table.Aspects[].Name
        let $value := $populated.Aspects.($dimension)
        return if (empty($value))
               then insert json { $dimension: $default-dimension-value-map.$dimension }
                    into $populated.Aspects
               else ()
      )
      return $populated
  else
    for $fact in facts:facts-for-archives-and-concepts($archives, $line-items, $options)
    where (
      every $key in keys($fact.Aspects)
      satisfies exists($default-dimension-value-map.$key)
    ) and (
      every $key in $non-optional-dimensions
      satisfies exists($fact.Aspects.$key)
    )
    return
      copy $populated := $fact
      modify
        for $dimension in $table.Aspects[].Name
        let $value := $populated.Aspects.($dimension)
        return if (empty($value))
               then insert json { $dimension: $default-dimension-value-map.$dimension }
                    into $populated.Aspects
               else ()
      return $populated
};

(:~
 :
Generates a fact table by populating the supplied template.

 :
 : @param $template-or-model-or-model-id a template (can be a model) or an MID.
 :
 : @return the populated template.
 :)
declare function fact-tables:fact-table-populated(
  $template-or-model-or-model-id as item) as object
{
  fact-tables:fact-table-populated($template-or-model-or-model-id, ())
};

(:~
 :
Generates a fact table by populating the supplied template.

 :
 : @param $template-or-model-or-model-id a template (can be a model) or an MID.
 : @param $options further optional parameters:
 : - concept-map a concept map or its name (no concept map used if absent).
 : - archives an array of AIDs to use instead of the ones specified in the template.
 :
 : @return the populated template.
 :)
declare function fact-tables:fact-table-populated(
  $template-or-model-or-model-id as item,
  $options as object?) as object
{
  let $model := fact-tables:resolve-component-or-template($template-or-model-or-model-id)
  let $fact-table-by-concepts := {|
    for $f in fact-tables:fact-table-as-objects($model, $options)
    group by $concept := $f.Aspects."xbrl:Concept"
    return { $concept : [ $f ] }
  |}
  return
    copy $result := $model
    modify
      let $table := $result.ModelStructure.Table
      for $line-item in descendant-objects($table.LineItems).Concepts[]
      let $name := $line-item.Name
      return insert json { "Facts" : $fact-table-by-concepts.$name } into $line-item
    return $result
};

(:~
 :
 :
Generates a fact table containing facts associated with
 : all concepts in the template, and with all associated dimensions
 : as specified in the template.

 :
 : @param $template-or-model-or-model-id a template or a model object or an MID.
 :
 : @return an array of arrays containing flat facts.
 :)
declare function fact-tables:fact-table(
  $template-or-model-or-model-id as item) as array
{
  fact-tables:fact-table($template-or-model-or-model-id, ())
};

(:~
 :
 :
Generates a fact table containing facts associated with
 : all concepts in the template, and with all associated dimensions
 : as specified in the template.

 :
 : @param $template-or-model-or-model-id a template or a model object or an MID.
 : @param $options further optional parameters:
 : - concept-map a concept map or its name (no concept map used if absent).
 : - archives an array of AIDs to use instead of the ones specified in the template.
 :
 : @return an array of arrays containing flat facts.
 :)
declare function fact-tables:fact-table(
  $template-or-model-or-model-id as item,
  $options as object?) as array
{
  let $model := fact-tables:resolve-component-or-template($template-or-model-or-model-id)
  let $table := $model.ModelStructure.Table
  let $dimensions := distinct-values(("xbrl:Concept", "xbrl:Entity", "xbrl:Period", "xbrl:Unit", $table.Aspects[].Name))
  return  [
    [ $dimensions, "Value", "Unit" ],
    for $fact in fact-tables:fact-table-as-objects($template-or-model-or-model-id, $options)
    return [
      $dimensions ! $fact.Aspects.($$),
      $fact.Value
    ]
  ]
};

(:~
 :
 :)
declare %private function fact-tables:resolve-component-or-template($template-or-model-or-model-id as item)
as object
{
  (components:components($template-or-model-or-model-id),
   if($template-or-model-or-model-id instance of object) then $template-or-model-or-model-id else ())[1]
};

(:~
 :
 :
Filters the fact table by removing dimensions and removing facts associating
 : these dimensions with a non-default value.

 :
 : @param $fact-table a fact table.
 : @param $dimensions the dimensions to restrict.
 : @param $model-or-id the model corresponding to the fact table.
 :
 : @return the filtered fact table.
 :)
declare function fact-tables:collapse($fact-table as array, $model-or-id as item, $dimensions as string*)
as array
{
  [
    let $model := components:components($model-or-id)
    let $table := $model.ModelStructure.Table
    let $all-dimensions := $table.Aspects[]
    let $positions-to-filter := for $i in 1 to count($all-dimensions)
                                where $all-dimensions[$i].Name = $dimensions or empty($dimensions)
                                return $i
    let $default-values := for $i in $positions-to-filter
                           return $all-dimensions[$i].Members[[1]].Name
    return
    for $fact at $position in $fact-table[]
    where $position eq 1 or (every $p in 1 to count($positions-to-filter)
                             satisfies $fact[[3 + $positions-to-filter[$p]]] eq $default-values[$p])
    return copy $copy := $fact
           modify for $i in $positions-to-filter return delete json $copy[[3 + $i]]
           return $copy
  ]
};

(:~
 :
 :
Filters the fact table by removing all dimensions and removing facts associating
 : these dimensions with a non-default value.

 :
 : @param $fact-table a fact table.
 : @param $model-or-id the model corresponding to the fact table.
 :
 : @return the filtered fact table.
 :)
declare function fact-tables:collapse($fact-table as array, $model-or-id as item)
as array
{
  fact-tables:collapse($fact-table, $model-or-id, ())
};


jq: Introduce ino filter [ci skip]

jq: Introduce ino filter [ci skip]

Works with nested fields like path filter does.

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v);

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: clean | select(is_error);
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: clean | select(is_warn);
def warn_strict: clean | select(is_warn_strict);
def info_level: 30;
def is_info: .level >= info_level;
def info: clean | select(is_info);
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: clean | select(is_debug);

# Components:
def chokidar: select(.component == "Chokidar");
def LocalChange: select(.component == "local/change");
def LocalWatcher: select(.component == "LocalWatcher");
def Merge: select(.component == "Merge");
def Metadata: select(.component == "Metadata");
def Pouch: select(.component == "Pouch");
def Sync: select(.component == "Sync");

# Find conflicts:
#
#     yarn jq -c conflicts path/to/logs*
#
def is_conflict: .msg == "resolveConflictAsync";
def conflicts: clean | select(is_conflict) | {time,path};

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
def path(pattern):
  clean | select(
    (
      (.path // (.doc | .path) // (.change | .doc | .path)),
      (.oldpath // (.was | .path)),
      ""
    )
      | strings
      | test(pattern)
  );

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#
def time(pattern):
  select(.time | test(pattern));

# Filter inode number(s):
#
#    yarn jq 'ino(7852458)' path/to/logs*
#    yarn jq 'ino(7852458; 464672)' path/to/logs*
#
def ino(n): select((.ino // (.doc|.ino) // (.change|.doc|.ino)) as $ino | [n] | flatten | map(. == $ino) | any);

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path};

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v);

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: clean | select(is_error);
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: clean | select(is_warn);
def warn_strict: clean | select(is_warn_strict);
def info_level: 30;
def is_info: .level >= info_level;
def info: clean | select(is_info);
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: clean | select(is_debug);

# Components:
def chokidar: select(.component == "Chokidar");
def LocalChange: select(.component == "local/change");
def LocalWatcher: select(.component == "LocalWatcher");
def Merge: select(.component == "Merge");
def Metadata: select(.component == "Metadata");
def Pouch: select(.component == "Pouch");
def Sync: select(.component == "Sync");

# Find conflicts:
#
#     yarn jq -c conflicts path/to/logs*
#
def is_conflict: .msg == "resolveConflictAsync";
def conflicts: clean | select(is_conflict) | {time,path};

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
def path(pattern):
  clean | select(
    (
      (.path // (.doc | .path) // (.change | .doc | .path)),
      (.oldpath // (.was | .path)),
      ""
    )
      | strings
      | test(pattern)
  );

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#
def time(pattern):
  select(.time | test(pattern));

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path};

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );


Extract variables for each property of the claim

# Parse JSON data returned by Special:Entity API of Wikidata
# and convert it to CSV format

# headers
[
  "Entity",
  "Group",
  "Value Name",
  "Value Type",
  "Value",
  "Rank",
  "Start Time",
  "End Time"
]
,
(
  # records
  .entities
  | .[]
  | [
      "",
      "",
      "",
      "",
      "",
      "",
      "",
      ""
    ] as $empty_record
  | .id as $entity
  | (
      .aliases
      | .[]
      | .[]
      | [
          $entity,
          "alias",
          "language",
          .language,
          .value,
          "",
          "",
          ""
        ]
    ),
    $empty_record,
    (
      .descriptions
      | .[]
      | [
          $entity,
          "description",
          "language",
          .language,
          .value,
          "",
          "",
          ""
        ]
    ),
    $empty_record,
    (
      .labels
      | .[]
      | [
          $entity,
          "label",
          "language",
          .language,
          .value,
          "",
          "",
          ""
        ]
    ),
    $empty_record,
    (
      .claims
      | .[]
      | .[]
      | .rank as $rank
      | .qualifiers.P580[0].datavalue.value.time as $start_time
      | .qualifiers.P582[0].datavalue.value.time as $end_time
      | .mainsnak
      | .property as $value_name
      | .datatype as $value_type
      | (
          .datavalue.value
          | (
                 scalars
              // ."numeric-id"
              // .amount
              // .latitude
              // .longitude
              // .altitude
              // .text
              // .time
              // ""
            )
        ) as $value
      | [
          $entity,
          "claim",
          $value_name,
          $value_type,
          $value,
          $rank,
          $start_time,
          $end_time
        ]
    ),
    $empty_record,
    (
      .sitelinks
      | .[]
      | [
          $entity,
          "sitelink",
          "site",
          .site,
          .title,
          "",
          "",
          ""
        ]
    )
)
# Note: @csv requires --raw-output parameter on jq command line
| @csv

# Parse JSON data returned by Special:Entity API of Wikidata
# and convert it to CSV format

# headers
[
  "Entity",
  "Group",
  "Value Name",
  "Value Type",
  "Value",
  "Rank",
  "Start Time",
  "End Time"
]
,
(
  # records
  .entities
  | .[]
  | [
      "",
      "",
      "",
      "",
      "",
      "",
      "",
      ""
    ] as $empty_record
  | .id as $entity
  | (
      .aliases
      | .[]
      | .[]
      | [
          $entity,
          "alias",
          "language",
          .language,
          .value,
          "",
          "",
          ""
        ]
    ),
    $empty_record,
    (
      .descriptions
      | .[]
      | [
          $entity,
          "description",
          "language",
          .language,
          .value,
          "",
          "",
          ""
        ]
    ),
    $empty_record,
    (
      .labels
      | .[]
      | [
          $entity,
          "label",
          "language",
          .language,
          .value,
          "",
          "",
          ""
        ]
    ),
    $empty_record,
    (
      .claims
      | .[]
      | .[]
      | .rank as $rank
      | .qualifiers.P580[0].datavalue.value.time as $start_time
      | .qualifiers.P582[0].datavalue.value.time as $end_time
      | .mainsnak
      | [
          $entity,
          "claim",
          .property,
          .datatype,
          (
            .datavalue.value
            | (
                   scalars
                // ."numeric-id"
                // .amount
                // .latitude
                // .longitude
                // .altitude
                // .text
                // .time
                // ""
              )
          ),
          $rank,
          $start_time,
          $end_time
        ]
    ),
    $empty_record,
    (
      .sitelinks
      | .[]
      | [
          $entity,
          "sitelink",
          "site",
          .site,
          .title,
          "",
          "",
          ""
        ]
    )
)
# Note: @csv requires --raw-output parameter on jq command line
| @csv