Handle BolusWizard512 records

Handle BolusWizard512 records

# This is a jq filter that converts the output of
# https://github.com/ecc1/medtronic/cmd/pumphistory
# to the schema that openaps expects.
# Usage:
#   pumphistory | jq -f openaps.jq > history.json

# Convert a Go duration to minutes.
def duration_to_minutes:
  if test("^(\\d+h)?(\\d+m)?0s$") then
    capture("^((?<h>\\d+)h)?((?<m>\\d+)m)?0s$") |
    60*(.h // 0 | tonumber) + (.m // 0 | tonumber)
  else
    ("unexpected duration: " + .) | error
  end;

# Output a single JSON array.
[
  # Treat null as an empty array.
  (. // []) |
  # Perform the following on each element of the input array.
  .[] |
  # Filter out fake meter entries.
  select((.Type == "BGReceived" and .Info.MeterID == "000000") | not) |
  # Start with the timestamp field, common to all record types,
  # and the type, which is the same as the decocare type in many cases.
  {
    timestamp: .Time,
    _type: .Type
  } +
  # Add type-specific fields.
  if .Type == "TempBasalDuration" then
    {
      "duration (min)": .Info | duration_to_minutes
    }
  elif .Type == "TempBasalRate" then
    {
      _type: "TempBasal",
      temp: .Info.Type | ascii_downcase,
      rate: .Info.Value
    }
  elif .Type == "Bolus" then
    {
      amount: .Info.Amount,
      programmed: .Info.Programmed,
      unabsorbed: .Info.Unabsorbed,
      duration: .Info.Duration | duration_to_minutes
    }
  elif .Type == "BolusWizard" or .Type == "BolusWizard512" then
    {
      _type: "BolusWizard",
      bg: (.Info.Glucose // 0),
      carb_input: (.Info.CarbInput // 0),
      bg_target_low: .Info.TargetLow,
      bg_target_high: .Info.TargetHigh,
      sensitivity: .Info.Sensitivity,
      carb_ratio: .Info.CarbRatio,
      correction_estimate: .Info.Correction,
      food_estimate: .Info.Food,
      unabsorbed_insulin_total: .Info.Unabsorbed,
      bolus_estimate: .Info.Bolus
    }
  elif .Type == "Prime" then
    {
      amount: .Info.Manual,
      fixed: .Info.Fixed,
      type: (if .Info.Fixed == 0 then "manual" else "fixed" end),
    }
  elif .Type == "BGReceived" then
    {
      glucose: .Info.Glucose,
      meterID: .Info.MeterID
    }
  elif .Type == "SuspendPump" then
    {
      _type: "PumpSuspend"
    }
  elif .Type == "ResumePump" then
    {
      _type: "PumpResume"
    }
  elif .Type == "BatteryChange" then
    {
      _type: "Battery"
    }
  elif .Type == "Rewind" then
    {
    }
  # Add additional cases here as needed.
  # WARNING: if DailyTotal records are added,
  # they will not appear in chronological order.
  else
    # Warn about record types being skipped.
    ("skipping " + .Type + " record") | debug | empty
  end
]

# This is a jq filter that converts the output of
# https://github.com/ecc1/medtronic/cmd/pumphistory
# to the schema that openaps expects.
# Usage:
#   pumphistory | jq -f openaps.jq > history.json

# Convert a Go duration to minutes.
def duration_to_minutes:
  if test("^(\\d+h)?(\\d+m)?0s$") then
    capture("^((?<h>\\d+)h)?((?<m>\\d+)m)?0s$") |
    60*(.h // 0 | tonumber) + (.m // 0 | tonumber)
  else
    ("unexpected duration: " + .) | error
  end;

# Output a single JSON array.
[
  # Treat null as an empty array.
  (. // []) |
  # Perform the following on each element of the input array.
  .[] |
  # Filter out fake meter entries.
  select((.Type == "BGReceived" and .Info.MeterID == "000000") | not) |
  # Start with the timestamp field, common to all record types,
  # and the type, which is the same as the decocare type in many cases.
  {
    timestamp: .Time,
    _type: .Type
  } +
  # Add type-specific fields.
  if .Type == "TempBasalDuration" then
    {
      "duration (min)": .Info | duration_to_minutes
    }
  elif .Type == "TempBasalRate" then
    {
      _type: "TempBasal",
      temp: .Info.Type | ascii_downcase,
      rate: .Info.Value
    }
  elif .Type == "Bolus" then
    {
      amount: .Info.Amount,
      programmed: .Info.Programmed,
      unabsorbed: .Info.Unabsorbed,
      duration: .Info.Duration | duration_to_minutes
    }
  elif .Type == "BolusWizard" then
    {
      bg: (.Info.Glucose // 0),
      carb_input: (.Info.CarbInput // 0),
      bg_target_low: .Info.TargetLow,
      bg_target_high: .Info.TargetHigh,
      sensitivity: .Info.Sensitivity,
      carb_ratio: .Info.CarbRatio,
      correction_estimate: .Info.Correction,
      food_estimate: .Info.Food,
      unabsorbed_insulin_total: .Info.Unabsorbed,
      bolus_estimate: .Info.Bolus
    }
  elif .Type == "Prime" then
    {
      amount: .Info.Manual,
      fixed: .Info.Fixed,
      type: (if .Info.Fixed == 0 then "manual" else "fixed" end),
    }
  elif .Type == "BGReceived" then
    {
      glucose: .Info.Glucose,
      meterID: .Info.MeterID
    }
  elif .Type == "SuspendPump" then
    {
      _type: "PumpSuspend"
    }
  elif .Type == "ResumePump" then
    {
      _type: "PumpResume"
    }
  elif .Type == "BatteryChange" then
    {
      _type: "Battery"
    }
  elif .Type == "Rewind" then
    {
    }
  # Add additional cases here as needed.
  # WARNING: if DailyTotal records are added,
  # they will not appear in chronological order.
  else
    # Warn about record types being skipped.
    ("skipping " + .Type + " record") | debug | empty
  end
]


fixed Bug: Returning BOTH Q4 AND FY

fixed Bug: Returning BOTH Q4 AND FY

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace sec-fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace core = "http://xbrl.io/modules/bizql/profiles/sec/core";

import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:modify-hypercube(
    $hypercube as object,
    $options as object?) as object
{
    let $options :=
    {|
        for $dimension in keys(($hypercube.Aspects, $options))
        let $hypercube-metadata := $hypercube.Aspects.$dimension
        let $new-metadata := $options.$dimension
        return {
            $dimension : if(exists($new-metadata))
            then
                $new-metadata
            else {|
                { Type : $hypercube.Aspects.$dimension.Type }[$hypercube-metadata.Kind eq "TypedDimension"],
                let $hypercube-domain := descendant-objects(values($hypercube-metadata.Domains)).Name
                where exists($hypercube-domain)
                return { Domain : [ $hypercube-domain ] },
                let $hypercube-default := $hypercube-metadata.Default
                where exists($hypercube-default)
                return { Default : $hypercube-default }
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($options)
};

declare function local:to-csv($o as object*) as string?
{
    if (exists($o)) (: bug in csv:serialize :)
    then
    string-join(
        csv:serialize(
            for $o in $o
            let $a := $o.Aspects
            return {|
                for $k in keys($a)
                return { $k : $a.$k },
                { "Value" : $o.Value },
                if (exists($o."Unit"))
                then { "Unit" :  $o."Unit" }
                else (),
                if (exists($o.Decimals))
                then { "Decimals" : $o.Decimals }
                else(),
                { "EntityRegistrantName" : $o.EntityRegistrantName },
                if (exists($o.ReportedConcept))
                then { "ReportedConcept" : $o.ReportedConcept }
                else ()
            |},
            { serialize-null-as : "" }
        )
    )
    else ()
};

declare function local:audittrail-to-xml($audit as item) as element()
{
    <AuditTrails>{
        for $a in flatten($audit)
        return (
            <Type>{$a.Type}</Type>,
            <Label>{$a.Label}</Label>,
            <Message>{$a.Message}</Message>,
            <Data>{
                if (exists($a.Data.OriginalConcept))
                then <OriginalConcept>{$a.Data.OriginalConcept}</OriginalConcept>
                else (),
                if (exists($a.Data.OutputConcept))
                then <OutputConcept>{$a.Data.OutputConcept}</OutputConcept>
                else (),
                if (exists($a.Data.AuditTrails))
                then local:audittrail-to-xml($a.Data.AuditTrails)
                else ()
            }</Data>
        )
    }</AuditTrails>
};

declare function local:to-xml($o as object*) as node()*
{
    (session:comment("xml", {
        NumFacts: count($o),
        TotalNumFacts: session:num-facts(),
        TotalNumArchives: session:num-archives(),
        TotalNumEntities: session:num-entities()
    }),
    <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
        for $o in $o
        let $a := $o.Aspects
        return
            <Fact>{
                <Aspects>{
                    for $k in keys($a)
                    return
                        <Aspect>
                            <Name>{$k}</Name>
                            <Value>{$a.$k}</Value>
                        </Aspect>
                }</Aspects>,
                <Value>{$o.Value}</Value>,
                <Type>{$o.Type}</Type>,
                if (exists($o.Unit))
                then <Unit>{$o.Unit}</Unit>
                else(),
                if (exists($o.Decimals))
                then <Decimals>{$o.Decimals}</Decimals>
                else (),
                <EntityRegistrantName>{$o.EntityRegistrantName}</EntityRegistrantName>,
                if (exists($o.AuditTrails))
                then local:audittrail-to-xml($o."AuditTrails")
                else ()
            }</Fact>
    }</FactTable>)
};

declare function local:contains-aspect($name as string) as boolean
{
  some $x in (request:param-names() ! starts-with($$, $name))
  satisfies $x
};

declare function local:cast($value as string, $type as string) as atomic
{
  switch ($type)
  case "integer" return $value cast as integer
  default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube($fiscalYears as string*, $fiscalPeriods as string*) as object
{
    hypercubes:user-defined-hypercube({|
        let $concepts := (request:param-values("concept"), request:param-values("xbrl:Concept"))
        return
            if (exists($concepts))
            then { "xbrl:Concept" : { Domain : [ $concepts ] } }
            else (),

        if (not(request:param-names() = "dei:LegalEntityAxis"))
        then {
                "dei:LegalEntityAxis" : {
                    "Domain" : [ "sec:DefaultLegalEntity" ],
                    "Default" : "sec:DefaultLegalEntity"
                }
        } else (),

        if (not(local:contains-aspect("sec:Accepted")))
        then { "sec:Accepted" : {  } } else (),

        if (not(local:contains-aspect("sec:FiscalYear")) and not(local:contains-aspect("fiscalYear")))
        then { "sec:FiscalYear" : {  } } else { "sec:FiscalYear" : { Type: "xs:integer", Domain : [ $fiscalYears ! local:cast($$, "integer") ] } },

        if (not(local:contains-aspect("sec:FiscalPeriod")) and not(local:contains-aspect("fiscalPeriod")))
        then { "sec:FiscalPeriod" : {  } } else { "sec:FiscalPeriod" : { Domain : [ $fiscalPeriods ] } },

        for $p in request:param-names()
        where contains($p, ":") and not(starts-with($p, "xbrl:Concept"))
        group by $dimension-name := if (ends-with(lower-case($p), "::default"))
                                    then substring-before($p, "::default")
                                    else if (ends-with(lower-case($p), ":default"))
                                    then substring-before($p, ":default")
                                    else if (ends-with(lower-case($p), "::type"))
                                    then substring-before($p, "::type")
                                    else $p
        let $all := (request:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $type := (request:param-values($dimension-name || "::type"), request:param-values($dimension-name || ":type"))[1]
        return
        {
            $dimension-name : {|
                let $v := request:param-values($dimension-name)
                return
                    if (exists($v) and not($all))
                    then { "Domain" : [ if (exists($type)) then local:cast($v, $type) else $v ] }
                    else (),

            let $is-default := ($p = $dimension-name || "::default") or ($p = $dimension-name || ":default")
            let $default := (request:param-values($dimension-name || "::default"), request:param-values($dimension-name || ":default"))[1]
            return
                (if ($is-default)
                then {
                    "Default" : if (exists($type)) then local:cast($default, $type) else $default
                  } else (),
                if (exists($type))
                then { "Type" : $type }
                else ())
            |}
        }
    |})
};

declare function local:facts(
    $archives as object*,
    $map as string?,
    $rules as string?,
    $fiscalYears as string*,
    $fiscalPeriods as string*) as object*
{
    let $hypercube := local:modify-hypercube(
        local:hypercube($fiscalYears, $fiscalPeriods),
        {
            "sec:Archive" : {
                Type: "string",
                Domain : [archives:aid($archives)]
            }
        }
    )
    for $fact in core:facts-for(
        {|
            { Hypercube : $hypercube },
            if (exists($map)) then { "ConceptMaps" : $map } else (),
            if (exists($rules)) then { "Rules" : $rules } else ()
        |}
    )
    return {|
        { Aspects : {|
            for $a in keys($fact.Aspects)
            where $a ne "xbrl:Unit"
            return { $a : $fact.Aspects.$a }
        |} },
        { Type: $fact.Type },
        if (exists($fact.Aspects."xbrl:Unit"))
        then { Unit: $fact.Aspects."xbrl:Unit" }
        else  (),
        if (exists($fact.Decimals))
        then { Decimals: $fact.Decimals }
        else (),
        { Value: $fact.Value },
        { "EntityRegistrantName" : companies:companies($fact.Aspects."xbrl:Entity").Profiles.SEC.CompanyName},
        if (exists(flatten($fact.AuditTrails[])))
        then { "AuditTrails" : $fact.AuditTrails }
        else ()
    |}
};

declare function local:filings(
    $ciks as string*,
    $tags as string*,
    $tickers as string*,
    $sics as string*,
    $fp as string*,
    $fy as string*) as object*
{
    let $entities := if ($tags = "ALL") then companies:companies()
                                        else (
                                            companies:companies($ciks),
                                            companies:companies-for-tags($tags),
                                            companies:companies-for-tickers($tickers),
                                            companies:companies-for-SIC($sics)
                                        )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then sec-fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else sec-fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                        case "ALL" return  $sec-fiscal:ALL_FISCAL_YEARS
                    default return $fy
                )
    for $fp in $fp
    return sec-fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy cast as integer)
};

session:audit-call();

let $format      := lower-case(request:param-values("format")[1])

(: choose
    1. entities (using ciks, tags, tickers, sics), FY, and FP ) or
    2. accession numbers
:)
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $sec-fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids     := request:param-values("aid")
let $map      := request:param-values("map")[1]
let $rules    := request:param-values("rules")[1]
let $archives := (
                    local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                    archives:archives($aids)
                 )
let $entities := companies:companies($archives.Entity)
return
    switch(true)
    case empty($archives) return {
        response:status-code(400);
        session:error("entities or archives not found (valid parameters: cik, ticker, tag, sic, aid)", $format)
    }
    default return
      switch(session:check-access($entities, "data_sec"))
      case $session:ACCESS-ALLOWED return {
        let $facts := local:facts($archives, $map, $rules, $fiscalYears, $fiscalPeriods)
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($facts)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=facts.csv");
                local:to-csv($facts)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=fact.csv");
                local:to-csv($facts)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { NetworkIdentifier : "http://bizql.io/facts" },
                    { TableName : "xbrl:Facts" },
                    { FactTable : [ $facts ] },
                    session:comment("json", {
                            NumFacts: count($facts),
                            TotalNumFacts: session:num-facts(),
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })
                |}
            }
       }
    case $session:ACCESS-DENIED return {
          response:status-code(403);
          session:error("accessing filings of an entity that is not in the DOW30", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          response:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace sec-fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace core = "http://xbrl.io/modules/bizql/profiles/sec/core";

import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:modify-hypercube(
    $hypercube as object,
    $options as object?) as object
{
    let $options :=
    {|
        for $dimension in keys(($hypercube.Aspects, $options))
        let $hypercube-metadata := $hypercube.Aspects.$dimension
        let $new-metadata := $options.$dimension
        return {
            $dimension : if(exists($new-metadata))
            then
                $new-metadata
            else {|
                { Type : $hypercube.Aspects.$dimension.Type }[$hypercube-metadata.Kind eq "TypedDimension"],
                let $hypercube-domain := descendant-objects(values($hypercube-metadata.Domains)).Name
                where exists($hypercube-domain)
                return { Domain : [ $hypercube-domain ] },
                let $hypercube-default := $hypercube-metadata.Default
                where exists($hypercube-default)
                return { Default : $hypercube-default }
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($options)
};

declare function local:to-csv($o as object*) as string?
{
    if (exists($o)) (: bug in csv:serialize :)
    then
    string-join(
        csv:serialize(
            for $o in $o
            let $a := $o.Aspects
            return {|
                for $k in keys($a)
                return { $k : $a.$k },
                { "Value" : $o.Value },
                if (exists($o."Unit"))
                then { "Unit" :  $o."Unit" }
                else (),
                if (exists($o.Decimals))
                then { "Decimals" : $o.Decimals }
                else(),
                { "EntityRegistrantName" : $o.EntityRegistrantName },
                if (exists($o.ReportedConcept))
                then { "ReportedConcept" : $o.ReportedConcept }
                else ()
            |},
            { serialize-null-as : "" }
        )
    )
    else ()
};

declare function local:audittrail-to-xml($audit as item) as element()
{
    <AuditTrails>{
        for $a in flatten($audit)
        return (
            <Type>{$a.Type}</Type>,
            <Label>{$a.Label}</Label>,
            <Message>{$a.Message}</Message>,
            <Data>{
                if (exists($a.Data.OriginalConcept))
                then <OriginalConcept>{$a.Data.OriginalConcept}</OriginalConcept>
                else (),
                if (exists($a.Data.OutputConcept))
                then <OutputConcept>{$a.Data.OutputConcept}</OutputConcept>
                else (),
                if (exists($a.Data.AuditTrails))
                then local:audittrail-to-xml($a.Data.AuditTrails)
                else ()
            }</Data>
        )
    }</AuditTrails>
};

declare function local:to-xml($o as object*) as node()*
{
    (session:comment("xml", {
        NumFacts: count($o),
        TotalNumFacts: session:num-facts(),
        TotalNumArchives: session:num-archives(),
        TotalNumEntities: session:num-entities()
    }),
    <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
        for $o in $o
        let $a := $o.Aspects
        return
            <Fact>{
                <Aspects>{
                    for $k in keys($a)
                    return
                        <Aspect>
                            <Name>{$k}</Name>
                            <Value>{$a.$k}</Value>
                        </Aspect>
                }</Aspects>,
                <Value>{$o.Value}</Value>,
                <Type>{$o.Type}</Type>,
                if (exists($o.Unit))
                then <Unit>{$o.Unit}</Unit>
                else(),
                if (exists($o.Decimals))
                then <Decimals>{$o.Decimals}</Decimals>
                else (),
                <EntityRegistrantName>{$o.EntityRegistrantName}</EntityRegistrantName>,
                if (exists($o.AuditTrails))
                then local:audittrail-to-xml($o."AuditTrails")
                else ()
            }</Fact>
    }</FactTable>)
};

declare function local:contains-aspect($name as string) as boolean
{
  some $x in (request:param-names() ! starts-with($$, $name))
  satisfies $x
};

declare function local:cast($value as string, $type as string) as atomic
{
  switch ($type)
  case "integer" return $value cast as integer
  default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube() as object
{
    hypercubes:user-defined-hypercube({|
        let $concepts := (request:param-values("concept"), request:param-values("xbrl:Concept"))
        return
            if (exists($concepts))
            then { "xbrl:Concept" : { Domain : [ $concepts ] } }
            else (),

        if (not(request:param-names() = "dei:LegalEntityAxis"))
        then {
                "dei:LegalEntityAxis" : {
                    "Domain" : [ "sec:DefaultLegalEntity" ],
                    "Default" : "sec:DefaultLegalEntity"
                }
        } else (),

        if (not(local:contains-aspect("sec:Accepted")))
        then { "sec:Accepted" : {  } } else (),

        if (not(local:contains-aspect("sec:FiscalYear")))
        then { "sec:FiscalYear" : {  } } else (),

        if (not(local:contains-aspect("sec:FiscalPeriod")))
        then { "sec:FiscalPeriod" : {  } } else (),

        for $p in request:param-names()
        where contains($p, ":") and not(starts-with($p, "xbrl:Concept"))
        group by $dimension-name := if (ends-with(lower-case($p), "::default"))
                                    then substring-before($p, "::default")
                                    else if (ends-with(lower-case($p), ":default"))
                                    then substring-before($p, ":default")
                                    else if (ends-with(lower-case($p), "::type"))
                                    then substring-before($p, "::type")
                                    else $p
        let $all := (request:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $type := (request:param-values($dimension-name || "::type"), request:param-values($dimension-name || ":type"))[1]
        return
        {
            $dimension-name : {|
                let $v := request:param-values($dimension-name)
                return
                    if (exists($v) and not($all))
                    then { "Domain" : [ if (exists($type)) then local:cast($v, $type) else $v ] }
                    else (),

            let $is-default := ($p = $dimension-name || "::default") or ($p = $dimension-name || ":default")
            let $default := (request:param-values($dimension-name || "::default"), request:param-values($dimension-name || ":default"))[1]
            return
                (if ($is-default)
                then {
                    "Default" : if (exists($type)) then local:cast($default, $type) else $default
                  } else (),
                if (exists($type))
                then { "Type" : $type }
                else ())
            |}
        }
    |})
};

declare function local:facts(
    $archives as object*,
    $map as string?,
    $rules as string?) as object*
{
    let $hypercube := local:modify-hypercube(
        local:hypercube(),
        {
            "sec:Archive" : {
                Type: "string",
                Domain : [archives:aid($archives)]
            }
        }
    )
    for $fact in core:facts-for(
        {|
            { Hypercube : $hypercube },
            if (exists($map)) then { "ConceptMaps" : $map } else (),
            if (exists($rules)) then { "Rules" : $rules } else ()
        |}
    )
    return {|
        { Aspects : {|
            for $a in keys($fact.Aspects)
            where $a ne "xbrl:Unit"
            return { $a : $fact.Aspects.$a }
        |} },
        { Type: $fact.Type },
        if (exists($fact.Aspects."xbrl:Unit"))
        then { Unit: $fact.Aspects."xbrl:Unit" }
        else  (),
        if (exists($fact.Decimals))
        then { Decimals: $fact.Decimals }
        else (),
        { Value: $fact.Value },
        { "EntityRegistrantName" : companies:companies($fact.Aspects."xbrl:Entity").Profiles.SEC.CompanyName},
        if (exists(flatten($fact.AuditTrails[])))
        then { "AuditTrails" : $fact.AuditTrails }
        else ()
    |}
};

declare function local:filings(
    $ciks as string*,
    $tags as string*,
    $tickers as string*,
    $sics as string*,
    $fp as string*,
    $fy as string*) as object*
{
    let $entities := if ($tags = "ALL") then companies:companies()
                                        else (
                                            companies:companies($ciks),
                                            companies:companies-for-tags($tags),
                                            companies:companies-for-tickers($tickers),
                                            companies:companies-for-SIC($sics)
                                        )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then sec-fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else sec-fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                        case "ALL" return  $sec-fiscal:ALL_FISCAL_YEARS
                    default return $fy
                )
    for $fp in $fp
    return sec-fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy cast as integer)
};

session:audit-call();

let $format      := lower-case(request:param-values("format")[1])

(: choose
    1. entities (using ciks, tags, tickers, sics), FY, and FP ) or
    2. accession numbers
:)
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $sec-fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids     := request:param-values("aid")
let $map      := request:param-values("map")[1]
let $rules    := request:param-values("rules")[1]
let $archives := (
                    local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                    archives:archives($aids)
                 )
let $entities := companies:companies($archives.Entity)
return
    switch(true)
    case empty($archives) return {
        response:status-code(400);
        session:error("entities or archives not found (valid parameters: cik, ticker, tag, sic, aid)", $format)
    }
    default return
      switch(session:check-access($entities, "data_sec"))
      case $session:ACCESS-ALLOWED return {
        let $facts := local:facts($archives, $map, $rules)
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($facts)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=facts.csv");
                local:to-csv($facts)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=fact.csv");
                local:to-csv($facts)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { NetworkIdentifier : "http://bizql.io/facts" },
                    { TableName : "xbrl:Facts" },
                    { FactTable : [ $facts ] },
                    session:comment("json", {
                            NumFacts: count($facts),
                            TotalNumFacts: session:num-facts(),
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })
                |}
            }
       }
    case $session:ACCESS-DENIED return {
          response:status-code(403);
          session:error("accessing filings of an entity that is not in the DOW30", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          response:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()


Refactored calls on filings.jq.

Refactored calls on filings.jq.

import module namespace http-request = "http://www.28msec.com/modules/http/request";

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/test";

import module namespace csv = "http://zorba.io/modules/json-csv";

import module namespace archives = "http://28.io/modules/xbrl/archives";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace filings = "http://28.io/modules/xbrl/profiles/sec/filings";

import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token         as string? external;
declare  %rest:env                              variable $request-uri   as string  external;
declare  %rest:case-insensitive                 variable $format        as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $eid           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cik           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker        as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear    as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod  as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid           as string* external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := $config:profile-name;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities :=
    switch($profile-name)
    case "sec" return companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic)
    default return ()
let $archives as object* :=
    switch($profile-name)
    case "sec" return fiscal-core:filings(
        $entities,
        $fiscalPeriod,
        $fiscalYear,
        $aid)
    default return
        if(exists($eid)) then archives:archives-for-entities($eid)
                              else archives:archives()
let $summaries :=
    switch($profile-name)
    case "sec" return
        for $f in filings:summaries($archives)
        order by $f.Accepted descending
        return $f
    default return
        for $a in $archives
        return {
            AID: $a._id,
            Entity: $a.Entity
        }
let $summaries :=
  switch($profile-name)
  case "sec" return
    for $archive in $summaries
    return {|
      project($archive, "AccessionNumber"),
      {
        Components: backend:url("components",
          {
              aid: encode-for-uri($archive.AccessionNumber),
              format: $format,
              profile-name: $profile-name
          })
      },
      trim($archive, "AccessionNumber")
    |}
  default return
    for $archive in $summaries
    return {|
      $archive,
      {
        Components: backend:url("components",
          {
              aid: encode-for-uri($archive.AID),
              format: $format,
              profile-name: $profile-name
          })
      }
    |}

let $result := { "Archives" : [ $summaries ] }
let $comment :=
{
    NumArchives: count($summaries),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        switch($profile-name)
        case "sec" return
            <Filings>{
                filings:summaries-to-xml($res.Archives[])
            }</Filings>
        default return
            <Archives>{
                for $a in $res.Archives[]
                return <Archive>
                    <AID>{$a.AID}</AID>
                    <Entity>{$a.Entity}</Entity>
                </Archive>
            }
            </Archives>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec" return
            string-join(filings:summaries-to-csv($res.Archives[]))
        default return
            string-join(
                csv:serialize($res.Archives[], { serialize-null-as : "" }
            ),
        "")
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "filings")
return api:check-and-return-results($token, $results, $format)

import module namespace http-request = "http://www.28msec.com/modules/http/request";

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace csv = "http://zorba.io/modules/json-csv";

import module namespace archives = "http://28.io/modules/xbrl/archives";

import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace filings = "http://28.io/modules/xbrl/profiles/sec/filings";

import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token         as string? external;
declare  %rest:env                              variable $request-uri   as string  external;
declare  %rest:case-insensitive                 variable $format        as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $eid           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cik           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker        as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear    as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod  as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid           as string* external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := $config:profile-name;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities :=
    switch($profile-name)
    case "sec" return companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic)
    default return ()
let $archives as object* :=
    switch($profile-name)
    case "sec" return fiscal-core:filings(
        $entities,
        $fiscalPeriod,
        $fiscalYear,
        $aid)
    default return
        if(exists($eid)) then archives:archives-for-entities($eid)
                              else archives:archives()
let $summaries :=
    switch($profile-name)
    case "sec" return
        for $f in filings:summaries($archives)
        order by $f.Accepted descending
        return $f
    default return
        for $a in $archives
        return {
            AID: $a._id,
            Entity: $a.Entity
        }
let $summaries :=
  switch($profile-name)
  case "sec" return
    for $archive in $summaries
    return {|
      project($archive, "AccessionNumber"),
      {
        Components: "http://" || http-request:server-name() || ":" || http-request:server-port() ||
          "/v1/_queries/public/api/components.jq?_method=POST&aid="||encode-for-uri($archive.AccessionNumber) ||
          "&format=" || $format || "&profile-name=" || $profile-name ||
          "&token=" || http-request:parameter-values("token")
      },
      trim($archive, "AccessionNumber")
    |}
  default return
    for $archive in $summaries
    return {|
      $archive,
      {
        Components: "http://" || http-request:server-name() || ":" || http-request:server-port() ||
          "/v1/_queries/public/api/components.jq?_method=POST&aid="||encode-for-uri($archive.AID) ||
          "&format=" || $format || "&profile-name=" || $profile-name ||
          "&token=" || http-request:parameter-values("token")
      }
    |}

let $result := { "Archives" : [ $summaries ] }
let $comment :=
{
    NumArchives: count($summaries),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        switch($profile-name)
        case "sec" return
            <Filings>{
                filings:summaries-to-xml($res.Archives[])
            }</Filings>
        default return
            <Archives>{
                for $a in $res.Archives[]
                return <Archive>
                    <AID>{$a.AID}</AID>
                    <Entity>{$a.Entity}</Entity>
                </Archive>
            }
            </Archives>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec" return
            string-join(filings:summaries-to-csv($res.Archives[]))
        default return
            string-join(
                csv:serialize($res.Archives[], { serialize-null-as : "" }
            ),
        "")
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "filings")
return api:check-and-return-results($token, $results, $format)


Fixing component-by-concept lookup.

Fixing component-by-concept lookup.

jsoniq version "1.0";
(:
 : Copyright 2012-2014 28msec Inc.
 :)

(:~
 :
This module provides functionality for retrieving components.

 :
 :
Components help organizing the facts contained in archives
 : in smaller parts that "make sense" together. A component is identified
 : with a CID (component ID).

 :
 :
A component is made of networks (see the networks module) and hypercubes
 : (see the hypercubes module).

 :
 :
With this module, you can retrieve all components, all components belonging
 : to one or several archives. You can retrieve the CID of a component or retrieve
 : the components associated with a couple of CIDs.

 :
 :)
module namespace components = "http://28.io/modules/xbrl/components";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace networks = "http://28.io/modules/xbrl/networks";
import module namespace resolution = "http://28.io/modules/xbrl/resolution";
import module namespace layout = "http://28.io/modules/xbrl/layout";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";

import module namespace accountant = "http://28.io/modules/xbrl/profiles/accountant/converter";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Name of the collection the components are stored in.
 :)
declare variable $components:col as xs:string := "components";

(:~
 : Name of the field pointing to the archive.
 :)
declare variable $components:ARCHIVE as xs:string := "Archive";

(:~
 : Name of the field pointing to the role.
 :)
declare variable $components:ROLE as xs:string := "Role";

(:~
 : Name of the field pointing to the default language.
 :)
declare variable $components:DEFAULT-LANGUAGE as xs:string := "DefaultLanguage";

(:~
 :
Retrieves all components.

 :
 : @return all components.
 :)
declare function components:components() as object*
{
  let $conn := components:connection()
  return mongo:find($conn, $components:col, {})
};

(:~
 :
Retrieves the components with the given CIDs.

 :
 : @param $component-or-ids the CIDs or the components themselves.
 :
 : @return the components whose _id field matches one of these CIDs.
 :)
declare function components:components($component-or-ids as item*) as object*
{
  let $ids as string* :=
    for $s in $component-or-ids
    where $s instance of string
    return $s
  let $components as object* :=
    for $s in $component-or-ids
    where $s instance of object
    return $s
  return
    (
      $components,
      if (exists($ids))
      then
        let $conn := components:connection()
        return mongo:find($conn, $components:col, { "_id" : { "$in" : [ $ids ! components:cid($$) ] } })
      else ()
    )
};

(:~
 :
Retrieves all components that belong to the supplied archives.

 :
 : @param $archive-or-id an archive or its AID.
 :
 : @return all components in the archive with this AID.
 :)
declare function components:components-for-archives($archive-or-ids as item*) as object*
{
  let $conn := components:connection()
  for $archive-or-id in $archive-or-ids
  return mongo:find($conn, $components:col, { $components:ARCHIVE: archives:aid($archive-or-id) })
};

(:~
 : Retrieves all components for the supplied filings and roles.
 :
 : @param $archives-or-ids a sequence of archives or AIDs.
 : @param $roles a sequence of role URIs.
 :
 : @return the components.
 :)
declare function components:components-for-archives-and-roles(
    $archives-or-ids as item*,
    $roles as string*) as object*
{
    let $aids := archives:aid($archives-or-ids)
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: { "$in" : [ $aids ] },
            "Role": { "$in" : [ $roles ] }
        })
};

(:~
 : Retrieves all components for the supplied filings and that use the supplied concepts.
 :
 : @param $archives-or-ids a sequence of archives or AIDs.
 : @param $concepts a sequence of concept names.
 :
 : @return the components.
 :)
declare function components:components-for-archives-and-concepts(
    $archives-or-ids as item*,
    $concepts as string*) as object*
{
    let $aids as string* := archives:aid($archives-or-ids)
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    let $concepts := mongo:find($conn, "concepts",
        {|
            (
                { "Name" : { "$in" : [ $concepts ] } },
                { "Archive" : { "$in" : [ $aids ] } }
            )
        |})
    let $roles := $concepts.Role
    for $component in components:components-for-archives-and-roles($aids, $roles)
    where (some $concept in $concepts satisfies $concept.Archive eq $component.Archive and $concept.Role eq $component.Role)
    return $component
};

(:~
 :
Retrieves all definition models in a component.

 :
 : @param $component a component object.
 :
 : @return all definition models
 :)
declare function components:definition-models-for-components(
    $components as object*) as object*
{
    $components.DefinitionModels[]
};

(:~
 :
Retrieves definition models in a component
 : that have the supplied names.

 :
 : @param $component a component object.
 : @param $names a sequence of names.
 :
 : @return the definition models.
 :)
declare function components:definition-models-for-components(
    $components as object*,
    $names as string*) as object
{
    $components.DefinitionModels[][$$.Name = $names]
};

(:~
 :
Selects a table from the component.

 :
 : @param $component a component object.
 : @param $options
some optional parameters, including:

 :

 :
HypercubeName: a string specifying which hypercube to use. By default, one of the non-default hypercubes will be arbitrarily chosen. If none
 :  is available, the default hypercube will be picked.

 :

 :
 : @error components:HYPERCUBE-DOES-NOT-EXIST if the specified hypercube is not found.
 : @return a definition model
 :)
declare function components:select-table(
    $component as object,
    $options as object?) as object
{
    let $user-chosen-table as object? := hypercubes:hypercubes-for-components($component, $options.HypercubeName)
    let $non-implicit-table as object? := hypercubes:hypercubes-for-components($component)[$$.Name ne "xbrl:DefaultHypercube"][1]
    let $implicit-table as object := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube")
    return
        switch(true)
        case exists($options.HypercubeName) and empty($user-chosen-table)
            return error(QName("components:HYPERCUBE-DOES-NOT-EXIST"), $options.HypercubeName || " : Hypercube does not exist.")
        case exists($user-chosen-table)
            return $user-chosen-table
        case exists($non-implicit-table)
            return $non-implicit-table
        default return $implicit-table
};

(:~
 :
 :
Retrieves all facts that are relevant to the
 : supplied component, and populates them with the default dimension values
 : when missing.

 :
The default hypercube will be taken, and the concept maps and rules will
 : be used if any.

 :
 : @param $component a component object.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:facts(
    $component as object
)
as object*
{
  components:facts($component, {})
};

(:~
 :
 :
Retrieves all facts that are relevant to the
 : supplied component, and populates them with the default dimension values
 : when missing.

 :
 :
 : @param $report-or-id a report or its RID,
 : @param $options standard fact retrieving options as well as
 :

 :
FilterOverride: hypercube dimension specs (as defined as hypercubes:user-defined-hypercube)
 :   to override filters in the report's hypercube.

 :
HypercubeName: picks a different hypercube than the default one (xbrl:DefaultHypercube).

 :

 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:facts(
    $component as object,
    $options as object?
)
as object*
{
  let $concept-map := networks:networks-for-components-and-short-names(
      $component,
      "ConceptMap")
  let $rules := $component.Rules
  let $overriden-hypercube as object? :=
        if(exists($options.Hypercube))
        then $options.Hypercube
        else components:select-table($component, $options)
    let $hypercube :=
    if(exists($options.FilterOverride))
    then hypercubes:modify-hypercube(
      $overriden-hypercube,
      $options.FilterOverride
    )
    else $overriden-hypercube
  return hypercubes:facts(
          $hypercube,
          {|
            $options,
            { "ConceptMaps": $concept-map }[exists($concept-map)],
            { "Rules": $rules }[exists($rules)]
          |}
         )
};

(:~
 :
 :
Retrieves the spreadsheet-like rendering for the component.

 :
The default definition model will be taken.

 :
The default hypercube will be taken, and the concept maps and rules will
 : be used if any.

 :
 : @param $component a component object.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:spreadsheet(
    $component as object
)
as object*
{
  components:spreadsheet($component, {})
};

(:~
 :
 :
Retrieves the spreadsheet-like rendering for the component.

 :
 : @param $report-or-id a report or its RID,
 : @param $options standard fact retrieving options as well as
 :

 :
FilterOverride: hypercube dimension specs (as defined as hypercubes:user-defined-hypercube)
 :   to override filters in the report's hypercube.

 :
HypercubeName: picks a different hypercube than the default one (xbrl:DefaultHypercube).

 :

 :
 :
 : @error err:XPTY0004 if zero, or more than one definition models are found.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:spreadsheet(
    $component as object,
    $options as object?
)
as object*
{
    let $definition-model as object :=
        switch(true)
        case exists($options.DefinitionModel)
            return $options.DefinitionModel
        case (exists($options.DefinitionModelName))
            return components:definition-models-for-components($component, $options.DefinitionModelName)
        default
            return components:definition-models-for-components($component)
    let $overriden-hypercube as object? :=
        if(exists($options.Hypercube))
        then $options.Hypercube
        else components:select-table($component, $options)
    let $concept-map as object?:= networks:networks-for-components-and-short-names(
      $component,
      "ConceptMap")
    let $rules as array? := $component.Rules
    let $new-options as object :=
        {|
            $options,
            {"ConceptMaps": $concept-map }[exists($concept-map)],
            { "Rules": $rules }[exists($rules)],
            {
                Hypercube:
                    if(exists($options.FilterOverride))
                    then hypercubes:modify-hypercube(
                        $overriden-hypercube,
                        $options.FilterOverride
                    )
                    else $overriden-hypercube
            }[exists($overriden-hypercube)]
        |}
    let $structural-model as object := resolution:resolve(
        $definition-model,
        $component,
        $new-options)
    let $layout-model as object := layout:layout($structural-model, $new-options)
    let $accountant-model as object := accountant:flatten-row-headers($layout-model)
    return
        if($options.FlattenRows)
        then $accountant-model
        else $layout-model
};

(:~
 : Return the number of hypercubes in each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of hypercubes
 :)
declare function components:num-hypercubes($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumHypercubes
};

(:~
 : Return the number of networks in each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of networks
 :)
declare function components:num-networks($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumNetworks
};

(:~
 : Return the number of (distinct) explicit dimensions in each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of dimensions
 :)
declare function components:num-explicit-dimensions($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumDistinctExplicitDimensions
};

(:~
 : Return the number of (distinct) domains in each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of domains
 :)
declare function components:num-domains($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumDistinctDomains
};

(:~
 : Return the number of (distinct) members in each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of members
 :)
declare function components:num-members($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumDistinctMembers
};

(:~
 : Return the number of (distinct) concrete primary items being
 : in a hypercube for each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of primary items
 :)
declare function components:num-concrete-primary-items-in-hypercubes($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumDistinctConcretePrimaryItemsInHypercubes
};

(:~
 : Return the number of (distinct) abstract primary items being
 : in a hypercube for each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of primary items
 :)
declare function components:num-abstract-primary-items-in-hypercubes($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumDistinctAbstractPrimaryItemsInHypercubes
};

(:~
 : Return the number of (distinct) concrete primary items not being
 : in a hypercube for each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of primary items
 :)
declare function components:num-concrete-primary-items-not-in-hypercubes($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumDistinctConcretePrimaryItemsNotInHypercubes
};

(:~
 : Return the number of (distinct) abstract primary items not being
 : in a hypercube for each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of primary items
 :)
declare function components:num-distinct-abstract-primary-items-not-in-hypercubes($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumDistinctAbstractPrimaryItemsNotInHypercubes
};

(:~
 :
 :
Converts the input to a normalized component id (CID). The input
 : can be either an CID, or an component object which contains an _id.

 :
 : @param $component-or-id an component object or an CID.
 :
 : @error components:INVALID_PARAMETER if the CID or component is not valid.
 :
 : @return the normalized CID.
 :)
declare function components:cid($component-or-id as item) as atomic
{
  typeswitch ($component-or-id)
  case object return
    let $id := $component-or-id._id
    return if(exists($id))
           then $id
           else error(QName("components:INVALID_PARAMETER"),
                      "Invalid component provided (no _id field)")
  case $id as atomic return $id
  default return error(
      QName("components:INVALID_PARAMETER"),
      "Invalid component or id (must be an object or an atomic): "
      || serialize($component-or-id))
};

(:~
 :)
declare %private %an:strictlydeterministic function components:connection() as anyURI
{
  let $credentials :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return if (empty($credentials))
             then error(QName("components:CONNECTION-FAILED"), "no xbrl MongoDB configured")
             else $credentials
  return
    try {
      mongo:connect($credentials)
    } catch mongo:* {
      error(QName("components:CONNECTION-FAILED"), $err:description)
    }
};

jsoniq version "1.0";
(:
 : Copyright 2012-2014 28msec Inc.
 :)

(:~
 :
This module provides functionality for retrieving components.

 :
 :
Components help organizing the facts contained in archives
 : in smaller parts that "make sense" together. A component is identified
 : with a CID (component ID).

 :
 :
A component is made of networks (see the networks module) and hypercubes
 : (see the hypercubes module).

 :
 :
With this module, you can retrieve all components, all components belonging
 : to one or several archives. You can retrieve the CID of a component or retrieve
 : the components associated with a couple of CIDs.

 :
 :)
module namespace components = "http://28.io/modules/xbrl/components";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace networks = "http://28.io/modules/xbrl/networks";
import module namespace resolution = "http://28.io/modules/xbrl/resolution";
import module namespace layout = "http://28.io/modules/xbrl/layout";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";

import module namespace accountant = "http://28.io/modules/xbrl/profiles/accountant/converter";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Name of the collection the components are stored in.
 :)
declare variable $components:col as xs:string := "components";

(:~
 : Name of the field pointing to the archive.
 :)
declare variable $components:ARCHIVE as xs:string := "Archive";

(:~
 : Name of the field pointing to the role.
 :)
declare variable $components:ROLE as xs:string := "Role";

(:~
 : Name of the field pointing to the default language.
 :)
declare variable $components:DEFAULT-LANGUAGE as xs:string := "DefaultLanguage";

(:~
 :
Retrieves all components.

 :
 : @return all components.
 :)
declare function components:components() as object*
{
  let $conn := components:connection()
  return mongo:find($conn, $components:col, {})
};

(:~
 :
Retrieves the components with the given CIDs.

 :
 : @param $component-or-ids the CIDs or the components themselves.
 :
 : @return the components whose _id field matches one of these CIDs.
 :)
declare function components:components($component-or-ids as item*) as object*
{
  let $ids as string* :=
    for $s in $component-or-ids
    where $s instance of string
    return $s
  let $components as object* :=
    for $s in $component-or-ids
    where $s instance of object
    return $s
  return
    (
      $components,
      if (exists($ids))
      then
        let $conn := components:connection()
        return mongo:find($conn, $components:col, { "_id" : { "$in" : [ $ids ! components:cid($$) ] } })
      else ()
    )
};

(:~
 :
Retrieves all components that belong to the supplied archives.

 :
 : @param $archive-or-id an archive or its AID.
 :
 : @return all components in the archive with this AID.
 :)
declare function components:components-for-archives($archive-or-ids as item*) as object*
{
  let $conn := components:connection()
  for $archive-or-id in $archive-or-ids
  return mongo:find($conn, $components:col, { $components:ARCHIVE: archives:aid($archive-or-id) })
};

(:~
 : Retrieves all components for the supplied filings and roles.
 :
 : @param $archives-or-ids a sequence of archives or AIDs.
 : @param $roles a sequence of role URIs.
 :
 : @return the components.
 :)
declare function components:components-for-archives-and-roles(
    $archives-or-ids as item*,
    $roles as string*) as object*
{
    let $aids := archives:aid($archives-or-ids)
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: { "$in" : [ $aids ] },
            "Role": { "$in" : [ $roles ] }
        })
};

(:~
 : Retrieves all components for the supplied filings and that use the supplied concepts.
 :
 : @param $archives-or-ids a sequence of archives or AIDs.
 : @param $concepts a sequence of concept names.
 :
 : @return the components.
 :)
declare function components:components-for-archives-and-concepts(
    $archives-or-ids as item*,
    $concepts as string*) as object*
{
    let $aids as string* := archives:aid($archives-or-ids)
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    let $ids  := mongo:find($conn, "concepts",
        {|
            (
                { "Name" : { "$in" : [ $concepts ] } },
                { "Archive" : { "$in" : [ $aids ] } }
            )
        |}).Component
    return components:components($ids)
};

(:~
 :
Retrieves all definition models in a component.

 :
 : @param $component a component object.
 :
 : @return all definition models
 :)
declare function components:definition-models-for-components(
    $components as object*) as object*
{
    $components.DefinitionModels[]
};

(:~
 :
Retrieves definition models in a component
 : that have the supplied names.

 :
 : @param $component a component object.
 : @param $names a sequence of names.
 :
 : @return the definition models.
 :)
declare function components:definition-models-for-components(
    $components as object*,
    $names as string*) as object
{
    $components.DefinitionModels[][$$.Name = $names]
};

(:~
 :
Selects a table from the component.

 :
 : @param $component a component object.
 : @param $options
some optional parameters, including:

 :

 :
HypercubeName: a string specifying which hypercube to use. By default, one of the non-default hypercubes will be arbitrarily chosen. If none
 :  is available, the default hypercube will be picked.

 :

 :
 : @error components:HYPERCUBE-DOES-NOT-EXIST if the specified hypercube is not found.
 : @return a definition model
 :)
declare function components:select-table(
    $component as object,
    $options as object?) as object
{
    let $user-chosen-table as object? := hypercubes:hypercubes-for-components($component, $options.HypercubeName)
    let $non-implicit-table as object? := hypercubes:hypercubes-for-components($component)[$$.Name ne "xbrl:DefaultHypercube"][1]
    let $implicit-table as object := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube")
    return
        switch(true)
        case exists($options.HypercubeName) and empty($user-chosen-table)
            return error(QName("components:HYPERCUBE-DOES-NOT-EXIST"), $options.HypercubeName || " : Hypercube does not exist.")
        case exists($user-chosen-table)
            return $user-chosen-table
        case exists($non-implicit-table)
            return $non-implicit-table
        default return $implicit-table
};

(:~
 :
 :
Retrieves all facts that are relevant to the
 : supplied component, and populates them with the default dimension values
 : when missing.

 :
The default hypercube will be taken, and the concept maps and rules will
 : be used if any.

 :
 : @param $component a component object.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:facts(
    $component as object
)
as object*
{
  components:facts($component, {})
};

(:~
 :
 :
Retrieves all facts that are relevant to the
 : supplied component, and populates them with the default dimension values
 : when missing.

 :
 :
 : @param $report-or-id a report or its RID,
 : @param $options standard fact retrieving options as well as
 :

 :
FilterOverride: hypercube dimension specs (as defined as hypercubes:user-defined-hypercube)
 :   to override filters in the report's hypercube.

 :
HypercubeName: picks a different hypercube than the default one (xbrl:DefaultHypercube).

 :

 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:facts(
    $component as object,
    $options as object?
)
as object*
{
  let $concept-map := networks:networks-for-components-and-short-names(
      $component,
      "ConceptMap")
  let $rules := $component.Rules
  let $overriden-hypercube as object? :=
        if(exists($options.Hypercube))
        then $options.Hypercube
        else components:select-table($component, $options)
    let $hypercube :=
    if(exists($options.FilterOverride))
    then hypercubes:modify-hypercube(
      $overriden-hypercube,
      $options.FilterOverride
    )
    else $overriden-hypercube
  return hypercubes:facts(
          $hypercube,
          {|
            $options,
            { "ConceptMaps": $concept-map }[exists($concept-map)],
            { "Rules": $rules }[exists($rules)]
          |}
         )
};

(:~
 :
 :
Retrieves the spreadsheet-like rendering for the component.

 :
The default definition model will be taken.

 :
The default hypercube will be taken, and the concept maps and rules will
 : be used if any.

 :
 : @param $component a component object.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:spreadsheet(
    $component as object
)
as object*
{
  components:spreadsheet($component, {})
};

(:~
 :
 :
Retrieves the spreadsheet-like rendering for the component.

 :
 : @param $report-or-id a report or its RID,
 : @param $options standard fact retrieving options as well as
 :

 :
FilterOverride: hypercube dimension specs (as defined as hypercubes:user-defined-hypercube)
 :   to override filters in the report's hypercube.

 :
HypercubeName: picks a different hypercube than the default one (xbrl:DefaultHypercube).

 :

 :
 :
 : @error err:XPTY0004 if zero, or more than one definition models are found.
 :
 : @return a sequence of facts with populated dimension values.
 :)
declare function components:spreadsheet(
    $component as object,
    $options as object?
)
as object*
{
    let $definition-model as object :=
        switch(true)
        case exists($options.DefinitionModel)
            return $options.DefinitionModel
        case (exists($options.DefinitionModelName))
            return components:definition-models-for-components($component, $options.DefinitionModelName)
        default
            return components:definition-models-for-components($component)
    let $overriden-hypercube as object? :=
        if(exists($options.Hypercube))
        then $options.Hypercube
        else components:select-table($component, $options)
    let $concept-map as object?:= networks:networks-for-components-and-short-names(
      $component,
      "ConceptMap")
    let $rules as array? := $component.Rules
    let $new-options as object :=
        {|
            $options,
            {"ConceptMaps": $concept-map }[exists($concept-map)],
            { "Rules": $rules }[exists($rules)],
            {
                Hypercube:
                    if(exists($options.FilterOverride))
                    then hypercubes:modify-hypercube(
                        $overriden-hypercube,
                        $options.FilterOverride
                    )
                    else $overriden-hypercube
            }[exists($overriden-hypercube)]
        |}
    let $structural-model as object := resolution:resolve(
        $definition-model,
        $component,
        $new-options)
    let $layout-model as object := layout:layout($structural-model, $new-options)
    let $accountant-model as object := accountant:flatten-row-headers($layout-model)
    return
        if($options.FlattenRows)
        then $accountant-model
        else $layout-model
};

(:~
 : Return the number of hypercubes in each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of hypercubes
 :)
declare function components:num-hypercubes($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumHypercubes
};

(:~
 : Return the number of networks in each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of networks
 :)
declare function components:num-networks($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumNetworks
};

(:~
 : Return the number of (distinct) explicit dimensions in each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of dimensions
 :)
declare function components:num-explicit-dimensions($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumDistinctExplicitDimensions
};

(:~
 : Return the number of (distinct) domains in each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of domains
 :)
declare function components:num-domains($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumDistinctDomains
};

(:~
 : Return the number of (distinct) members in each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of members
 :)
declare function components:num-members($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumDistinctMembers
};

(:~
 : Return the number of (distinct) concrete primary items being
 : in a hypercube for each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of primary items
 :)
declare function components:num-concrete-primary-items-in-hypercubes($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumDistinctConcretePrimaryItemsInHypercubes
};

(:~
 : Return the number of (distinct) abstract primary items being
 : in a hypercube for each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of primary items
 :)
declare function components:num-abstract-primary-items-in-hypercubes($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumDistinctAbstractPrimaryItemsInHypercubes
};

(:~
 : Return the number of (distinct) concrete primary items not being
 : in a hypercube for each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of primary items
 :)
declare function components:num-concrete-primary-items-not-in-hypercubes($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumDistinctConcretePrimaryItemsNotInHypercubes
};

(:~
 : Return the number of (distinct) abstract primary items not being
 : in a hypercube for each of the given components.
 :
 : @param $components-or-ids list of archives or IDs
 :
 : @return the said number of primary items
 :)
declare function components:num-distinct-abstract-primary-items-not-in-hypercubes($components-or-ids as item*) as integer*
{
  components:components($components-or-ids) ! $$.Statistics.NumDistinctAbstractPrimaryItemsNotInHypercubes
};

(:~
 :
 :
Converts the input to a normalized component id (CID). The input
 : can be either an CID, or an component object which contains an _id.

 :
 : @param $component-or-id an component object or an CID.
 :
 : @error components:INVALID_PARAMETER if the CID or component is not valid.
 :
 : @return the normalized CID.
 :)
declare function components:cid($component-or-id as item) as atomic
{
  typeswitch ($component-or-id)
  case object return
    let $id := $component-or-id._id
    return if(exists($id))
           then $id
           else error(QName("components:INVALID_PARAMETER"),
                      "Invalid component provided (no _id field)")
  case $id as atomic return $id
  default return error(
      QName("components:INVALID_PARAMETER"),
      "Invalid component or id (must be an object or an atomic): "
      || serialize($component-or-id))
};

(:~
 :)
declare %private %an:strictlydeterministic function components:connection() as anyURI
{
  let $credentials :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return if (empty($credentials))
             then error(QName("components:CONNECTION-FAILED"), "no xbrl MongoDB configured")
             else $credentials
  return
    try {
      mongo:connect($credentials)
    } catch mongo:* {
      error(QName("components:CONNECTION-FAILED"), $err:description)
    }
};