fixed common stock shares outstanding imputation. removed default zero rule from bfi.

jsoniq version "1.0";
import module namespace http = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";

declare variable $id := "BasicFinancialInformation";
declare variable $report := parse-json(http:get("http://" || request:server-name() || ":" || request:server-port() || "/v1/_queries/public/export/fac.jq?_method=POST").body.content);
declare variable $desc :=
  "This report extends the fundamental accounting concepts report. Documentation about the fundamental accounting concepts defined in this report can be found at http://fundamentalaccountingconcepts.wikispaces.com/home";
declare variable $role := "http://xbrl.io/basic-financial-information";
declare variable $additionalRules :=
  (
    {
        "Id" : "gi_CommonStockSharesOutstanding",
        "Type" : "xbrl28:formula",
        "Label" : "CommonStockSharesOutstanding imputation",
        "Formula" : "(: workaround for bug: https://github.com/28msec/xbrl-infosetgenerator/issues/7 :)
                     let $aligned-filter-no-fiscal as object :=
    copy $new := $aligned-filter
    modify (
	  if($new.Aspects.\"sec:FiscalPeriod\") then delete json $new.Aspects.\"sec:FiscalPeriod\" else (),
	  if($new.Aspects.\"sec:FiscalYear\") then delete json $new.Aspects.\"sec:FiscalYear\" else ()
	)
	return $new
for $facts in (facts:facts-for-internal((
      \"fac:CommonStockSharesOutstanding\"
    ), $hypercube, $aligned-filter, $concept-maps, $rules, $cache, $options),
	facts:facts-for-internal((
      \"dei:EntityCommonStockSharesOutstanding\"
    ), $hypercube, $aligned-filter-no-fiscal, $concept-maps, $rules, $cache, $options)
	)
let $aligned-period := year-from-date(xs:date(( facts:duration-for-fact($facts).End, facts:instant-for-fact($facts))[1]))
group by $canonical-filter-string :=
  facts:canonically-serialize-object($facts, ($facts:CONCEPT, \"_id\", \"IsInDefaultHypercube\", \"Type\", \"Value\", \"Decimals\", \"AuditTrails\", \"xbrl28:Type\", \"Balance\", $facts:PERIOD))
  , $aligned-period
let $warnings as string* := ()
let $CommonStockSharesOutstanding as object* := $facts[$$.$facts:ASPECTS.$facts:CONCEPT eq \"fac:CommonStockSharesOutstanding\"]
let $EntityCommonStockSharesOutstanding as object* := $facts[$$.$facts:ASPECTS.$facts:CONCEPT eq \"dei:EntityCommonStockSharesOutstanding\"]
let $warnings := ($warnings, if(count($CommonStockSharesOutstanding) gt 1)
                             then if(count(distinct-values($CommonStockSharesOutstanding.Value)) gt 1)
                                  then \"Cell collision with conflicting values for concept CommonStockSharesOutstanding.\"
                                  else \"Cell collision with consistent values for concept CommonStockSharesOutstanding.\"
                             else (),
							 if(count($EntityCommonStockSharesOutstanding) gt 1)
                             then if(count(distinct-values($EntityCommonStockSharesOutstanding.Value)) gt 1)
                                  then \"Cell collision with conflicting values for concept EntityCommonStockSharesOutstanding.\"
                                  else \"Cell collision with consistent values for concept EntityCommonStockSharesOutstanding.\"
                             else ())
let $CommonStockSharesOutstanding as object? := $CommonStockSharesOutstanding[1]
let $EntityCommonStockSharesOutstanding as object? := $EntityCommonStockSharesOutstanding[1]
let $_unit := ($facts.$facts:ASPECTS.$facts:UNIT)[1]
return
  switch (true)
  case exists($CommonStockSharesOutstanding) return $CommonStockSharesOutstanding
  case (exists($EntityCommonStockSharesOutstanding) and true)
  return
    let $computed-value := rules:decimal-value($EntityCommonStockSharesOutstanding)
    let $audit-trail-message as string* :=
      rules:fact-trail({\"Aspects\": { \"xbrl:Unit\" : $_unit, \"xbrl:Concept\" : \"fac:CommonStockSharesOutstanding\" }, Value: $computed-value }) || \" = \" ||
         rules:fact-trail($EntityCommonStockSharesOutstanding, \"EntityCommonStockSharesOutstanding\") || \" [\" || (facts:duration-for-fact($facts).End, facts:instant-for-fact($facts)) || \"]\"
    let $audit-trail-message as string* := ($audit-trail-message, $warnings)
    let $source-facts as object* := ($EntityCommonStockSharesOutstanding)
    return
      if(string(number($computed-value)) != \"NaN\" and not($computed-value instance of xs:boolean) and $computed-value ne xs:integer($computed-value))
      then
        copy $newfact :=
          rules:create-computed-fact(
            $EntityCommonStockSharesOutstanding,
            \"fac:CommonStockSharesOutstanding\",
            $computed-value,
            $rule,
            $audit-trail-message,
            $source-facts,
            $options)
        modify (
            replace value of json $newfact(\"Decimals\") with 2
          )
        return $newfact
      else
        rules:create-computed-fact(
          $EntityCommonStockSharesOutstanding,
          \"fac:CommonStockSharesOutstanding\",
          $computed-value,
          $rule,
          $audit-trail-message,
          $source-facts,
            $options)
  default return ()",
        "ComputableConcepts" : [ "fac:CommonStockSharesOutstanding" ]
      },
      {
          "Id" : "decimals_inf",
          "Type" : "xbrl28:formula",
          "Label" : "Align INF decimals",
          "Formula" : "for $fact in facts:facts-for-internal($concepts, $hypercube, $aligned-filter, $concept-maps, $rules, $cache, $options)
                             return
                               if(string($fact.Decimals) eq \"INF\")
                               then
                                 let $concept := $fact.Aspects.\"xbrl:Concept\"
                                 let $audit-trail :=
                                   {
                                     \"Type\" : \"xbrl28:decimals-inf-alignment\",
                                     \"Label\" : \"Aligning Decimals to 3 instead of INF\",
                                     \"Message\" : $concept || \" = 0\",
                                     \"Data\" : {
                                       \"OutputConcept\" : $concept
                                     }
                                   }
                                 return
                                 copy $f := $fact
                                 modify (
                                     replace value of json $f.Decimals with 3,
                                     if(exists($f.AuditTrails))
                                     then append json $audit-trail into $f.AuditTrails
                                     else insert json { AuditTrails: [ $audit-trail ] } into $f
                                 )
                                 return $f
                               else $fact"
        }
  );

(: Presentation :)
let $presentation := $report.Networks[][$$.ShortName eq "Presentation"]
let $concepts :=
    {
      "fac:CommonStockSharesAuthorized": {
        "Name" : "fac:CommonStockSharesAuthorized",
        "Label" : "Common Stock Shares (authorized)",
        "Id" : "e7c449eb-4e1b-4d20-af35-b783cfdab03e"
      },
      "fac:CommonStockSharesIssued": {
        "Name" : "fac:CommonStockSharesIssued",
        "Label" : "Common Stock Shares (issued)",
        "Id" : "a8a4ae32-0424-437e-a645-428398f95f9c"
      },
      "fac:CommonStockSharesOutstanding": {
        "Name" : "fac:CommonStockSharesOutstanding",
        "Label" : "Common Stock Shares (outstanding)",
        "Id" : "f51e72c7-a3ce-4639-99b0-47ca36d54a7b"
      }
    }
let $genInfo := $presentation.Trees."fac:FundamentalAccountingConceptsLineItems".To."fac:FundamentalAccountingConceptsHierarchy".To."fac:GeneralInformationHierarchy"
return
  {
    replace value of json $presentation.LinkRole with $role;
    replace value of json $genInfo.To with {| ( $genInfo.To, $concepts ) |};
  }

(: Hypercube :)
let $conceptsDomain := $report.Hypercubes."xbrl:DefaultHypercube".Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain"
let $concepts :=
    {
      "fac:CommonStockSharesAuthorized": {
        "Name" : "fac:CommonStockSharesAuthorized",
        "Label" : "Common Stock Shares (authorized)",
        "Id" : "e7c449eb-4e1b-4d20-af35-b783cfdab03e"
      },
      "fac:CommonStockSharesIssued": {
        "Name" : "fac:CommonStockSharesIssued",
        "Label" : "Common Stock Shares (issued)",
        "Id" : "a8a4ae32-0424-437e-a645-428398f95f9c"
      },
      "fac:CommonStockSharesOutstanding": {
        "Name" : "fac:CommonStockSharesOutstanding",
        "Label" : "Common Stock Shares (outstanding)",
        "Id" : "f51e72c7-a3ce-4639-99b0-47ca36d54a7b"
      }
    }
return
  replace value of json $conceptsDomain.Members with {| $conceptsDomain.Members, $concepts |};

(: ConceptMap :)
let $conceptMap := $report.Networks[][$$.ShortName eq "ConceptMap"]
let $mappings :=
    {
      "fac:CommonStockSharesAuthorized" : {
        "Name" : "fac:CommonStockSharesAuthorized",
        "Label" : "Common Stock Shares (authorized)",
        "To" : {
          "us-gaap:CommonStockSharesAuthorized": {
            "Name" : "us-gaap:CommonStockSharesAuthorized",
            "Id" : "8ed0a312-9891-4d3d-b459-67604dce2124"
          }
        },
        "Id" : "372f39c9-c8ed-4654-a424-6f033bb0ddd2"

      },
      "fac:CommonStockSharesIssued" : {
        "Name" : "fac:CommonStockSharesIssued",
        "Label" : "Common Stock Shares (issued)",
        "To" : {
          "us-gaap:CommonStockSharesIssued": {
            "Name" : "us-gaap:CommonStockSharesIssued",
            "Id" : "1359b87e-5f75-49f3-849b-d410cc399b50"
          }
        },
        "Id" : "66226139-512f-4edd-8c18-6d7e307a1f66"
      },
      "fac:CommonStockSharesOutstanding" : {
        "Name" : "fac:CommonStockSharesOutstanding",
        "Label" : "Common Stock Shares (outstanding)",
        "To" : {
          "dei:EntityCommonStockSharesOutstanding": {
            "Name" : "dei:EntityCommonStockSharesOutstanding",
            "Id" : "b8d27670-5caa-411f-8305-860c0613c31f",
            "Order" : 1
          },
          "us-gaap:CommonStockSharesOutstanding" : {
            "Id" : "a91849bd-0aa2-4f1c-8dcb-c85537bf36a8",
            "Name" : "us-gaap:CommonStockSharesOutstanding",
            "Order" : 2
          }
        },
        "Id" : "91b87448-a8dd-4739-9cdf-2cdaef6a4ff6"
      }
    }
return
  {
    replace value of json $conceptMap.LinkRole with $role;
    replace value of json $conceptMap.Trees with {| $conceptMap.Trees, $mappings |};
  }

(: DefinitionModel :)
replace value of json $report.DefinitionModels[1].Breakdowns.y[1].BreakdownTrees[1].LinkRole with $role;

(: remove default_zero rule :)
replace value of json $report.Rules with [ $report.Rules[][$$.Id ne "default_zero"] ];

{ "_id": $id,
  Archive: null,
  Label: "Basic Financial Information",
  Description: $desc,
  Prefix: "fac",
  Role: $role,
  Owner: "charlie@prudena.com",
  LastModified: string(current-dateTime()),
  ACL: [],
  Networks: $report.Networks,
  Hypercubes: $report.Hypercubes,
  DefinitionModels: $report.DefinitionModels,
  Rules: [ for $rule in ($report.Rules[], $additionalRules)
           order by $rule.ComputableConcepts[][1], $rule.Id descending empty least
           return $rule ],
  Filters: {
    cik: [  ],
    tag: [ "DOW30" ],
    fiscalYear : [ 2013 ],
    fiscalPeriod : [ "FY" ],
    sic : [  ]
  }
}

jsoniq version "1.0";
import module namespace http = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";

declare variable $id := "BasicFinancialInformation";
declare variable $report := parse-json(http:get("http://" || request:server-name() || ":" || request:server-port() || "/v1/_queries/public/export/fac.jq?_method=POST").body.content);
declare variable $desc :=
  "This report extends the fundamental accounting concepts report. Documentation about the fundamental accounting concepts defined in this report can be found at http://fundamentalaccountingconcepts.wikispaces.com/home";
declare variable $role := "http://xbrl.io/basic-financial-information";
declare variable $additionalRules :=
  (
    {
        "Id" : "gi_CommonStockSharesOutstanding",
        "Type" : "xbrl28:formula",
        "Label" : "CommonStockSharesOutstanding imputation",
        "Formula" : "(: workaround for bug: https://github.com/28msec/xbrl-infosetgenerator/issues/7 :)
                     let $aligned-filter := trim($aligned-filter, (\"Profiles.SEC.Fiscal\"))
                     for $facts in  facts:facts-for-internal((
                             \"fac:CommonStockSharesOutstanding\"), $hypercube, $aligned-filter, $concept-maps, $rules, $cache, $options)
                     group by $canonical-filter-string :=
                                 facts:canonically-serialize-object($facts, ($facts:CONCEPT, \"_id\", \"IsInDefaultHypercube\", \"Type\", \"Value\", \"Decimals\", \"AuditTrails\", \"xbrl28:Type\", \"Balance\"))
                     for $CommonStockSharesOutstandingByArchive in $facts[$$.$facts:ASPECTS.$facts:CONCEPT eq \"fac:CommonStockSharesOutstanding\"]
                     group by $CommonStockSharesOutstandingByArchive.Archive
                     return
                         let $source-fact :=
                             (
                                 for $CommonStockSharesOutstanding in $CommonStockSharesOutstandingByArchive
                                 order by $CommonStockSharesOutstanding.$facts:ASPECTS.$facts:PERIOD descending
                                 return $CommonStockSharesOutstanding
                             )[1]
                         let $value := rules:decimal-value($source-fact)
                         let $original-concept := $source-fact.AuditTrails[][$$.Type eq \"xbrl28:concept-maps\"].Data.OriginalConcept
                         let $audit-trail-message :=
                            rules:fact-trail({ \"Aspects\" : { \"xbrl:Unit\" : \"shares\", \"xbrl:Concept\" : $original-concept }, Value: $value }) || \" (as of latest practicable date: \" || $source-fact.$facts:ASPECTS.$facts:PERIOD
                            || \")\"
                         return
                             rules:create-computed-fact(
                                 $source-fact,
                                 \"fac:CommonStockSharesOutstanding\",
                                 $value,
                                 $rule,
                                 $audit-trail-message,
                                 $source-fact,
                                 $options)",
        "ComputableConcepts" : [ "fac:CommonStockSharesOutstanding" ]
      },
      {
          "Id" : "decimals_inf",
          "Type" : "xbrl28:formula",
          "Label" : "Align INF decimals",
          "Formula" : "for $fact in facts:facts-for-internal($concepts, $hypercube, $aligned-filter, $concept-maps, $rules, $cache, $options)
                             return
                               if(string($fact.Decimals) eq \"INF\")
                               then
                                 let $concept := $fact.Aspects.\"xbrl:Concept\"
                                 let $audit-trail :=
                                   {
                                     \"Type\" : \"xbrl28:decimals-inf-alignment\",
                                     \"Label\" : \"Aligning Decimals to 3 instead of INF\",
                                     \"Message\" : $concept || \" = 0\",
                                     \"Data\" : {
                                       \"OutputConcept\" : $concept
                                     }
                                   }
                                 return
                                 copy $f := $fact
                                 modify (
                                     replace value of json $f.Decimals with 3,
                                     if(exists($f.AuditTrails))
                                     then append json $audit-trail into $f.AuditTrails
                                     else insert json { AuditTrails: [ $audit-trail ] } into $f
                                 )
                                 return $f
                               else $fact"
        }
  );

(: Presentation :)
let $presentation := $report.Networks[][$$.ShortName eq "Presentation"]
let $concepts :=
    {
      "fac:CommonStockSharesAuthorized": {
        "Name" : "fac:CommonStockSharesAuthorized",
        "Label" : "Common Stock Shares (authorized)",
        "Id" : "e7c449eb-4e1b-4d20-af35-b783cfdab03e"
      },
      "fac:CommonStockSharesIssued": {
        "Name" : "fac:CommonStockSharesIssued",
        "Label" : "Common Stock Shares (issued)",
        "Id" : "a8a4ae32-0424-437e-a645-428398f95f9c"
      },
      "fac:CommonStockSharesOutstanding": {
        "Name" : "fac:CommonStockSharesOutstanding",
        "Label" : "Common Stock Shares (outstanding)",
        "Id" : "f51e72c7-a3ce-4639-99b0-47ca36d54a7b"
      }
    }
let $genInfo := $presentation.Trees."fac:FundamentalAccountingConceptsLineItems".To."fac:FundamentalAccountingConceptsHierarchy".To."fac:GeneralInformationHierarchy"
return
  {
    replace value of json $presentation.LinkRole with $role;
    replace value of json $genInfo.To with {| ( $genInfo.To, $concepts ) |};
  }

(: Hypercube :)
let $conceptsDomain := $report.Hypercubes."xbrl:DefaultHypercube".Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain"
let $concepts :=
    {
      "fac:CommonStockSharesAuthorized": {
        "Name" : "fac:CommonStockSharesAuthorized",
        "Label" : "Common Stock Shares (authorized)",
        "Id" : "e7c449eb-4e1b-4d20-af35-b783cfdab03e"
      },
      "fac:CommonStockSharesIssued": {
        "Name" : "fac:CommonStockSharesIssued",
        "Label" : "Common Stock Shares (issued)",
        "Id" : "a8a4ae32-0424-437e-a645-428398f95f9c"
      },
      "fac:CommonStockSharesOutstanding": {
        "Name" : "fac:CommonStockSharesOutstanding",
        "Label" : "Common Stock Shares (outstanding)",
        "Id" : "f51e72c7-a3ce-4639-99b0-47ca36d54a7b"
      }
    }
return
  replace value of json $conceptsDomain.Members with {| $conceptsDomain.Members, $concepts |};

(: ConceptMap :)
let $conceptMap := $report.Networks[][$$.ShortName eq "ConceptMap"]
let $mappings :=
    {
      "fac:CommonStockSharesAuthorized" : {
        "Name" : "fac:CommonStockSharesAuthorized",
        "Label" : "Common Stock Shares (authorized)",
        "To" : {
          "us-gaap:CommonStockSharesAuthorized": {
            "Name" : "us-gaap:CommonStockSharesAuthorized",
            "Id" : "8ed0a312-9891-4d3d-b459-67604dce2124"
          }
        },
        "Id" : "372f39c9-c8ed-4654-a424-6f033bb0ddd2"

      },
      "fac:CommonStockSharesIssued" : {
        "Name" : "fac:CommonStockSharesIssued",
        "Label" : "Common Stock Shares (issued)",
        "To" : {
          "us-gaap:CommonStockSharesIssued": {
            "Name" : "us-gaap:CommonStockSharesIssued",
            "Id" : "1359b87e-5f75-49f3-849b-d410cc399b50"
          }
        },
        "Id" : "66226139-512f-4edd-8c18-6d7e307a1f66"
      },
      "fac:CommonStockSharesOutstanding" : {
        "Name" : "fac:CommonStockSharesOutstanding",
        "Label" : "Common Stock Shares (outstanding)",
        "To" : {
          "dei:EntityCommonStockSharesOutstanding": {
            "Name" : "dei:EntityCommonStockSharesOutstanding",
            "Id" : "b8d27670-5caa-411f-8305-860c0613c31f"
          }
        },
        "Id" : "91b87448-a8dd-4739-9cdf-2cdaef6a4ff6"
      }
    }
return
  {
    replace value of json $conceptMap.LinkRole with $role;
    replace value of json $conceptMap.Trees with {| $conceptMap.Trees, $mappings |};
  }

(: DefinitionModel :)
replace value of json $report.DefinitionModels[1].Breakdowns.y[1].BreakdownTrees[1].LinkRole with $role;

(: adapt default_zero rule :)
let $defaultZeroRule := $report.Rules[][$$.Id eq "default_zero"]
(: add concepts to the list of $numeric-instant-concepts :)
let $formula := replace($defaultZeroRule.Formula,
                        "\"fac:LiabilitiesAndEquity\"\\)",
                        "\"fac:LiabilitiesAndEquity\", \"fac:CommonStockSharesAuthorized\", \"fac:CommonStockSharesIssued\", \"fac:CommonStockSharesOutstanding\")"
                       )
(: add concepts to the list of $concepts-in-report :)
let $formula := replace($formula,
                        "\"fac:Validations\"\\)",
                        "\"fac:Validations\", \"fac:CommonStockSharesAuthorized\", \"fac:CommonStockSharesIssued\", \"fac:CommonStockSharesOutstanding\")"
                       )
return
  {
    replace value of json $defaultZeroRule.Formula with $formula;
  }

{ "_id": $id,
  Archive: null,
  Label: "Basic Financial Information",
  Description: $desc,
  Prefix: "fac",
  Role: $role,
  Owner: "charlie@prudena.com",
  LastModified: string(current-dateTime()),
  ACL: [],
  Networks: $report.Networks,
  Hypercubes: $report.Hypercubes,
  DefinitionModels: $report.DefinitionModels,
  Rules: [ for $rule in ($report.Rules[], $additionalRules)
           order by $rule.ComputableConcepts[][1], $rule.Id descending empty least
           return $rule ],
  Filters: {
    cik: [  ],
    tag: [ "DOW30" ],
    fiscalYear : [ 2013 ],
    fiscalPeriod : [ "FY" ],
    sic : [  ]
  }
}


Update drugfilter.jq

Update drugfilter.jq

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv

#start from the .results array
.results

#Creates new object with .safetyreportid string and .patient.drug array
|map(with_entries(select(.key == "safetyreportid")) + (.patient.drug[]))

# Flattens .openfda and turns openfda arrays into strings delimited with ;
|map(with_entries(select(.key != "openfda")) + (.openfda//{openfda:{"NA":"NA"}}|with_entries(.value = ([.value[]|tostring]|sort|join(";")) |.key |= "openfda_"  + .)))

#Flattens .activesubstance array
|map(with_entries(select(.key != "activesubstance")) + (.activesubstance))

#Grabs all keys, and builds CSV, to fix jagged array
|(map(keys)|add|unique) as $keys| $keys, (.[]|[.[$keys[]]])|@csv

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv

#start from the .results array
.results

#Creates new object with .safetyreportid string and .patient.drug array
|map(with_entries(select(.key == "safetyreportid")) + (.patient.drug[]))

# Flattens .openfda and turns openfda arrays into strings delimited with ;
|map(with_entries(select(.key != "openfda")) + (.openfda//{openfda:{"NA":"NA"}}|with_entries(.value = ([.value[]|tostring]|sort|join(";")) |.key |= "openfda_"  + .)))

#Flattens .activesubstance array
|map(with_entries(select(.key != "activesubstance")) + (.activesubstance))

#Iterates through array of objects to return a stream of objects instead of a single array
|.[]


Update api.jq

Update api.jq

jsoniq version "1.0";

module namespace api = "http://apps.28.io/api";

import module namespace session    = "http://apps.28.io/session";
import module namespace resp       = "http://www.28msec.com/modules/http-response";
import module namespace sec-fiscal = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

declare function api:validate-regexp($name as string, $value as string?, $regexp as string)
as ()
{
  if (exists($value))
  then
    if (matches($value, "^" || $regexp || "$"))
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " does not match reg. expression " || $regexp || ".");
  else ();
};

declare function api:validate-enum($name as string, $value as string?, $enum as string*)
as ()
{
  if (exists($value))
  then
    if ($value = $enum)
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " is not one of these: " || string-join($enum,", ") || ".");
  else ();
};

declare %private function api:sgpl($count as integer, $singular as string, $plural as string)
as string
{
    if ($count eq 1)
    then $count || " " || $singular
    else $count || " " || $plural
};

declare function api:passed($timestamp as xs:dateTime?)
as string
{
    if (empty($timestamp) or $timestamp eq null)
    then "Never"
    else
        let $now := current-dateTime()
        let $difference := $now - $timestamp
        let $years := years-from-duration($difference)
        let $month := months-from-duration($difference)
        let $days := days-from-duration($difference)
        let $hours := hours-from-duration($difference)
        let $minutes := minutes-from-duration($difference)
        return
            if ($years gt 0) then api:sgpl($years,"year ago","years ago")
            else if ($month gt 0) then api:sgpl($month,"month ago","month ago")
            else if ($days gt 0) then api:sgpl($days, "day ago", "days ago")
            else if ($hours gt 0) then api:sgpl($hours, "hour ago", "hours ago")
            else if ($minutes gt 0) then api:sgpl($minutes, "minute ago", "minutes ago")
            else if ($difference div xs:dayTimeDuration("PT1S") lt 0) then "future"
            else "Just Now"
};

declare function api:success() as object
{
  { "success" : true }
};

declare function api:success($data as object()) as object
{
  {|
     {"success" : true },
     $data
  |}
};

declare %an:sequential function api:check-and-return-results(
    $token as string?,
    $results as item*,
    $format as string?
) as item*
{
    switch(session:has-access($token, "data_sec"))
    case $session:ACCESS-ALLOWED return
        $results
    case $session:ACCESS-DENIED return {
          resp:status-code(403);
          session:error("access denied", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          resp:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()
};

declare function api:normalize-facts(
    $facts as object*) as object*
{
    for $fact in $facts
    return {|
        {
            "Aspects" : {|
                trim($fact.Aspects, ("xbrl:Unit"))
            |}
        },
        trim($fact, ("Aspects", "_id")),
        { Unit: $fact.Aspects."xbrl:Unit" }[exists($fact.Aspects."xbrl:Unit")]
    |}
};

declare %an:sequential function api:serialize(
    $result as json-item,
    $comment as object,
    $serializers as object,
    $format as string?,
    $file-name as string) as item*
{
    switch ($format)
    case "xml" return {
        resp:serialization-parameters({"omit-xml-declaration" : false, indent : true });
        session:comment("xml", $comment),
        $serializers.to-xml($result)
    }
    case "text" case "csv" return {
        resp:content-type("text/csv");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    case "excel" return {
        resp:content-type("application/vnd.ms-excel");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    default return {
        resp:content-type("application/json");
        resp:serialization-parameters({"indent" : true});
        {|
            session:comment("json", $comment),
            $result
        |}
    }
};

declare function api:preprocess-fiscal-years($fiscal-years as string*) as integer*
{
  distinct-values(
    for $fy in $fiscal-years ! upper-case($$)
    return switch($fy)
           case "LATEST" return $sec-fiscal:LATEST_FISCAL_YEAR
           case "ALL" return $sec-fiscal:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
  )
};

declare function api:preprocess-fiscal-periods($fiscal-periods as string*) as string*
{
  distinct-values(
    for $fp in $fiscal-periods ! upper-case($$)
    return switch($fp)
           case "Q1"
           case "Q2"
           case "Q3"
           case "FY"
             return $fp
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIODS
           default
             return error(xs:QName("local:INVALID-PERIOD"),
               $fp || ": fiscalPeriod values must be one or more of Q1, Q2, Q3, FY, ALL")
  )
};

declare function api:preprocess-fiscal-period-types($fiscal-period-types as string*) as string*
{
  distinct-values(
    for $fpt in $fiscal-period-types ! upper-case($$)
    return switch($fpt)
           case "INSTANT"
           case "QTD"
           case "YTD"
             return $fpt
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIOD_TYPES
           default
             return error(xs:QName("local:INVALID-PERIOD-TYPE"),
               $fpt || ": fiscalPeriodType values must be one or more of instant, YTD, QTD")
  )
};

declare function api:preprocess-format($format as string?, $request-uri as string) as string?
{
  let $request-path := tokenize($request-uri, "\\?")[1]
  return lower-case(($format, substring-after($request-path, ".jq."))[1])
};

declare function api:preprocess-tags($tags as string*) as string*
{
  distinct-values($tags ! upper-case($$))
};

declare function api:preprocess-boolean($name as string, $value as string)
as boolean
{
  if ($value eq "")
  then true
  else
  {
      api:validate-enum($name, lower-case($value), ("true", "false"));
      boolean($value)
  }
};

jsoniq version "1.0";

module namespace api = "http://apps.28.io/api";

import module namespace session    = "http://apps.28.io/session";
import module namespace resp       = "http://www.28msec.com/modules/http-response";
import module namespace sec-fiscal = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

declare function api:validate-regexp($name as string, $value as string?, $regexp as string)
as ()
{
  if (exists($value))
  then
    if (matches($value, "^" || $regexp || "$"))
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " does not match reg. expression " || $regexp || ".");
  else ();
};

declare function api:validate-enum($name as string, $value as string?, $enum as string*)
as ()
{
  if (exists($value))
  then
    if ($value = $enum)
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " is not one of these: " || string-join($enum,", ") || ".");
  else ();
};

declare %private function api:sgpl($count as integer, $singular as string, $plural as string)
as string
{
    if ($count eq 1)
    then $count || " " || $singular
    else $count || " " || $plural
};

declare function api:passed($timestamp as xs:dateTime?)
as string
{
    if (empty($timestamp) or $timestamp eq null)
    then "Never"
    else
        let $now := current-dateTime()
        let $difference := $now - $timestamp
        let $years := years-from-duration($difference)
        let $month := months-from-duration($difference)
        let $days := days-from-duration($difference)
        let $hours := hours-from-duration($difference)
        let $minutes := minutes-from-duration($difference)
        return
            if ($years gt 0) then api:sgpl($years,"year ago","years ago")
            else if ($month gt 0) then api:sgpl($month,"month ago","month ago")
            else if ($days gt 0) then api:sgpl($days, "day ago", "days ago")
            else if ($hours gt 0) then api:sgpl($hours, "hour ago", "hours ago")
            else if ($minutes gt 0) then api:sgpl($minutes, "minute ago", "minutes ago")
            else if ($difference div xs:dayTimeDuration("PT1S") lt 0) then "future"
            else "Just Now"
};

declare function api:success() as object
{
  { "success" : true }
};

declare function api:success($data as object()) as object
{
  {|
     {"success" : true },
     $data
  |}
};

declare %an:sequential function api:check-and-return-results(
    $token as string?,
    $results as item*,
    $format as string?
) as item*
{
    switch(session:has-access($token, "data_sec"))
    case $session:ACCESS-ALLOWED return
        $results
    case $session:ACCESS-DENIED return {
          resp:status-code(403);
          session:error("access denied", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          resp:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()
};

declare function api:normalize-facts(
    $facts as object*) as object*
{
    for $fact in $facts
    return {|
        {
            "Aspects" : {|
                trim($fact.Aspects, ("xbrl:Unit"))
            |}
        },
        trim($fact, ("Aspects", "_id")),
        { Unit: $fact.Aspects."xbrl:Unit" }[exists($fact.Aspects."xbrl:Unit")]
    |}
};

declare %an:sequential function api:serialize(
    $result as json-item,
    $comment as object,
    $serializers as object,
    $format as string?,
    $file-name as string) as item*
{
    switch ($format)
    case "xml" return {
        resp:serialization-parameters({"omit-xml-declaration" : false, indent : true });
        session:comment("xml", $comment),
        $serializers.to-xml($result)
    }
    case "text" case "csv" return {
        resp:content-type("text/csv");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    case "excel" return {
        resp:content-type("application/vnd.ms-excel");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    default return {
        resp:content-type("application/json");
        resp:serialization-parameters({"indent" : true});
        {|
            session:comment("json", $comment),
            $result
        |}
    }
};

declare function api:preprocess-fiscal-years($fiscal-years as string*) as integer*
{
  distinct-values(
    for $fy in $fiscal-years ! upper-case($$)
    return switch($fy)
           case "LATEST" return $sec-fiscal:LATEST_FISCAL_YEAR
           case "ALL" return $sec-fiscal:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
  )
};

declare function api:preprocess-fiscal-periods($fiscal-periods as string*) as string*
{
  distinct-values(
    for $fp in $fiscal-periods ! upper-case($$)
    return switch($fp)
           case "Q1"
           case "Q2"
           case "Q3"
           case "FY"
             return $fp
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIODS
           default
             return error(xs:QName("local:INVALID-PERIOD"),
               $fp || ": fiscalPeriod values must be one or more of Q1, Q2, Q3, FY, ALL")
  )
};

declare function api:preprocess-fiscal-period-types($fiscal-period-types as string*) as string*
{
  distinct-values(
    for $fpt in $fiscal-period-types ! upper-case($$)
    return switch($fpt)
           case "instant"
           case "QTD"
           case "YTD"
             return $fpt
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIOD_TYPES
           default
             return error(xs:QName("local:INVALID-PERIOD-TYPE"),
               $fpt || ": fiscalPeriodType values must be one or more of instant, YTD, QTD")
  )
};

declare function api:preprocess-format($format as string?, $request-uri as string) as string?
{
  let $request-path := tokenize($request-uri, "\\?")[1]
  return lower-case(($format, substring-after($request-path, ".jq."))[1])
};

declare function api:preprocess-tags($tags as string*) as string*
{
  distinct-values($tags ! upper-case($$))
};

declare function api:preprocess-boolean($name as string, $value as string)
as boolean
{
  if ($value eq "")
  then true
  else
  {
      api:validate-enum($name, lower-case($value), ("true", "false"));
      boolean($value)
  }
};


Further POST removals

Further POST removals

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace config = "http://apps.28.io/config";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace random = "http://zorba.io/modules/random";

declare variable $newId as string := random:uuid();
declare variable $newReport as object :=
    {
        "_id" : $newId,
        "Archive" : null,
        "Label" : "Add test",
        "Description" : "",
        "Owner" : "support@28.io",
        "Role" : "http://reports.28.io",
        "Prefix" : "t",
        "Networks" : [ {
            "LinkName" : "link:presentationLink",
            "LinkRole" : "http://reports.28.io",
            "ArcName" : "link:presentationArc",
            "ArcRole" : "http://www.xbrl.org/2003/arcrole/parent-child",
            "Kind" : "InterConceptTreeNetwork",
            "ShortName" : "Presentation",
            "CyclesAllowed" : "undirected",
            "Trees" : [
                {
                    "Id" : "3ef962e3-a913-464e-94be-f78a689d15ae",
                    "Name" : "t:ReportLineItems",
                    "Label" : "test",
                    "Order" : 1
                }
            ]
        } ],
        "Hypercubes" : {
            "xbrl:DefaultHypercube" : {
                "Name" : "xbrl:DefaultHypercube",
                "Label" : "test [Table]",
                "Aspects" : {
                    "xbrl:Concept" : {
                        "Name" : "xbrl:Concept",
                        "Label" : "Concept",
                        "Members" : [
                            {
                                "Name" : "t:ReportLineItems",
                                "Label" : "test",
                                "IsAbstract" : true
                            }
                        ]
                    }
                }
            }
        },
        "Rules" : [  ],
        "Filters" : {
            "cik" : [  ],
            "tag" : [ "DOW30" ],
            "fiscalYear" : [ 2014 ],
            "fiscalPeriod" : [ "FY" ],
            "fiscalPeriodType" : [ "instant", "YTD" ],
            "sic" : [  ]
        },
        "LastModified" : "2014-11-20T10:41:17.75774Z"
    };

declare %private function local:url($parameters as object?, $endpoint as string, $includeToken as boolean) as string
{
    "http://" || request:server-name() || ":" || request:server-port() ||
    "/v1/_queries/public/reports/" || $endpoint || ".jq?token=" || (if($includeToken) then $config:test-token else "{{token}}") || "&"||
    string-join(
        for $key in keys($parameters)
        for $value as string in (flatten($parameters.$key) ! string($$))
        return ($key||"="||$value),
        "&")
};

declare %private function local:check-response($url as string, $response as object, $body-checks as object?) as item
{
    let $status as integer := $response.status
    let $body as object := parse-json($response.body.content)
    return
        if($status eq 200 and (every $key in keys($body-checks) satisfies $body.$key eq $body-checks.$key))
        then true()
        else
            {
                "url": $url,
                "status": $status,
                "reponse": $body
            }
};

declare %an:sequential function local:add-report() as item*
{
    let $parameters := {
    }
    let $endpoint as string := "add-report"
    let $url := local:url($parameters, $endpoint, true())
    let $request := http-client:post($url, serialize($newReport))
    return local:check-response(
                local:url($parameters, $endpoint, false()),
                $request,
                {
                    "_id": $newId,
                    "Label": $newReport.Label
                })
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    "add-report": local:add-report()
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace config = "http://apps.28.io/config";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace random = "http://zorba.io/modules/random";

declare variable $newId as string := random:uuid();
declare variable $newReport as object :=
    {
        "_id" : $newId,
        "Archive" : null,
        "Label" : "Add test",
        "Description" : "",
        "Owner" : "support@28.io",
        "Role" : "http://reports.28.io",
        "Prefix" : "t",
        "Networks" : [ {
            "LinkName" : "link:presentationLink",
            "LinkRole" : "http://reports.28.io",
            "ArcName" : "link:presentationArc",
            "ArcRole" : "http://www.xbrl.org/2003/arcrole/parent-child",
            "Kind" : "InterConceptTreeNetwork",
            "ShortName" : "Presentation",
            "CyclesAllowed" : "undirected",
            "Trees" : [
                {
                    "Id" : "3ef962e3-a913-464e-94be-f78a689d15ae",
                    "Name" : "t:ReportLineItems",
                    "Label" : "test",
                    "Order" : 1
                }
            ]
        } ],
        "Hypercubes" : {
            "xbrl:DefaultHypercube" : {
                "Name" : "xbrl:DefaultHypercube",
                "Label" : "test [Table]",
                "Aspects" : {
                    "xbrl:Concept" : {
                        "Name" : "xbrl:Concept",
                        "Label" : "Concept",
                        "Members" : [
                            {
                                "Name" : "t:ReportLineItems",
                                "Label" : "test",
                                "IsAbstract" : true
                            }
                        ]
                    }
                }
            }
        },
        "Rules" : [  ],
        "Filters" : {
            "cik" : [  ],
            "tag" : [ "DOW30" ],
            "fiscalYear" : [ 2014 ],
            "fiscalPeriod" : [ "FY" ],
            "fiscalPeriodType" : [ "instant", "YTD" ],
            "sic" : [  ]
        },
        "LastModified" : "2014-11-20T10:41:17.75774Z"
    };

declare %private function local:url($parameters as object?, $endpoint as string, $includeToken as boolean) as string
{
    "http://" || request:server-name() || ":" || request:server-port() ||
    "/v1/_queries/public/reports/" || $endpoint || ".jq?_method=POST&token=" || (if($includeToken) then $config:test-token else "{{token}}") || "&"||
    string-join(
        for $key in keys($parameters)
        for $value as string in (flatten($parameters.$key) ! string($$))
        return ($key||"="||$value),
        "&")
};

declare %private function local:check-response($url as string, $response as object, $body-checks as object?) as item
{
    let $status as integer := $response.status
    let $body as object := parse-json($response.body.content)
    return
        if($status eq 200 and (every $key in keys($body-checks) satisfies $body.$key eq $body-checks.$key))
        then true()
        else
            {
                "url": $url,
                "status": $status,
                "reponse": $body
            }
};

declare %an:sequential function local:add-report() as item*
{
    let $parameters := {
    }
    let $endpoint as string := "add-report"
    let $url := local:url($parameters, $endpoint, true())
    let $request := http-client:post($url, serialize($newReport))
    return local:check-response(
                local:url($parameters, $endpoint, false()),
                $request,
                {
                    "_id": $newId,
                    "Label": $newReport.Label
                })
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    "add-report": local:add-report()
})


Fixed facst.jq to work with 3.2

Fixed facst.jq to work with 3.2

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace sec-fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";

import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-csv($o as object*) as string?
{
    if (exists($o)) (: bug in csv:serialize :)
    then
    string-join(
        csv:serialize(
            for $o in $o
            let $a := $o.Aspects
            return {|
                for $k in keys($a)
                return { $k : $a.$k },
                { "Value" : $o.Value },
                if (exists($o."Unit"))
                then { "Unit" :  $o."Unit" }
                else (),
                if (exists($o.Decimals))
                then { "Decimals" : $o.Decimals }
                else(),
                { "EntityRegistrantName" : $o.EntityRegistrantName },
                if (exists($o.ReportedConcept))
                then { "ReportedConcept" : $o.ReportedConcept }
                else ()
            |},
            { serialize-null-as : "" }
        )
    )
    else ()
};

declare function local:to-xml($o as object*)
{
    (session:comment("xml"),
    <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
        for $o in $o
        let $a := $o.Aspects
        return
            <Fact>{
                <Aspects>{
                    for $k in keys($a)
                    return
                        <Aspect>
                            <Name>{$k}</Name>
                            <Value>{$a.$k}</Value>
                        </Aspect>
                }</Aspects>,
                <Value>{$o.Value}</Value>,
                <Type>{$o.Type}</Type>,
                if (exists($o.Unit))
                then <Unit>{$o.Unit}</Unit>
                else(),
                if (exists($o.Decimals))
                then <Decimals>{$o.Decimals}</Decimals>
                else (),
                <EntityRegistrantName>{$o.EntityRegistrantName}</EntityRegistrantName>,
                if (exists($o.ReportedConcept))
                then <ReportedConcept>{$o."ReportedConcept"}</ReportedConcept>
                else ()
            }</Fact>
    }</FactTable>)
};

declare function local:facts(
    $entities,
    $archives,
    $fiscalPeriods,
    $fiscalYears,
    $concepts,
    $dimensions,
    $map)
{
    for $entity in $entities
    (: compute latest reported fiscalYear if none is specified :)
    let $years  :=  distinct-values(
                        for $f in $fiscalYears
                        return
                            switch ($f)
                            case "LATEST" return
                                for $p in $fiscalPeriods
                                return
                                    if ($p eq "FY")
                                    then sec-fiscal:latest-reported-fiscal-period($entity, "10-K").year
                                    else sec-fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                            case "ALL" return  $sec-fiscal:ALL_FISCAL_YEARS
                            default return $f
                    )
    let $aspects :=
        {|
            { "xbrl:Concept" : $concepts },
            { "xbrl:Entity" : $entity._id },
            (: This is because of a bug that will be fixed later (hypercube members do not get considered) :)
            for $d in values($dimensions)
            let $members := descendant-objects(values($d)).Name
            where exists($members)
            return { $d.Name: [ $members ] }
        |}
    let $hypercube := copy $h := hypercubes:dimensionless-hypercube()
                      modify (
                          insert json (
                            if (values($dimensions).Name = "dei:LegalEntityAxis")
                            then { "dei:LegalEntityAxis" : values($dimensions)[$$.Name eq "dei:LegalEntityAxis"] }
                            else
                            {|
                                "dei:LegalEntityAxis" ! {
                                $$ : {
                                        Name : $$,
                                        Default : "sec:DefaultLegalEntity",
                                        Domains: {
                                            "sec:DefaultLegalEntity" : {
                                                Name: "sec:DefaultLegalEntity"
                                            }
                                        }
                                    }
                                }
                            |})
                          into $h.Aspects,
                          for $d in $dimensions[not values($$).Name = "dei:LegalEntityAxis"]
                          return insert json $d into $h.Aspects
                      )
                      return $h
    let $options :=
            {|
                if (exists($map))
                then { "concept-maps" : $map }
                else (),
                { Hypercube : $hypercube }
            |}
    return
    for $fact in
        for $f in
            (
                sec-fiscal:facts-for-aspects-and-fiscal-periods-and-years(
                    $aspects, $fiscalPeriods, $years, $options)
            )
        group by $f.Profiles.SEC.Fiscal.Year,
                 $f.Profiles.SEC.Fiscal.Period,
                 $f.Aspects."xbrl:Concept"
        let $latest-accepted := max(distinct-values($f.Profiles.SEC.Fiscal.Acceptance))
        return if (empty($latest-accepted))
               then $f
               else $f[$$.Profiles.SEC.Fiscal.Acceptance eq $latest-accepted]
    return {|
        { Aspects : {|
            for $a in keys($fact.Aspects)
            where $a ne "xbrl:Unit"
            return { $a : $fact.Aspects.$a },
            { "bizql:FiscalPeriod" : $fact.Profiles.SEC.Fiscal.Period },
            { "bizql:FiscalYear" : $fact.Profiles.SEC.Fiscal.Year }
        |} },
        { Type: $fact.Type },
        if (exists($fact.Aspects."xbrl:Unit"))
        then { Unit: $fact.Aspects."xbrl:Unit" }
        else  (),
        if (exists($fact.Decimals))
        then { Decimals: $fact.Decimals }
        else (),
        { Value: $fact.Value },
        { "EntityRegistrantName" : $entity.Profiles.SEC.CompanyName },
        if (exists($map))
        then { "ReportedConcept" : $fact.AuditTrails[].Data.OriginalConcept[1] }
        else ()
    |}
};

let $format      := lower-case(request:param-values("format")[1])

(: choose
    1. entities (using ciks, tags, tickers, sics), FY, and FP ) or
    2. accession numbers
:)
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y cast as integer
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $sec-fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids := request:param-values("aid")
let $dimensions :=  for $p in request:param-names()
                    where contains($p, ":")
                    group by $dimension-name := if (ends-with(lower-case($p), ":default"))
                                                then
                                                    if (ends-with(lower-case($p), "::default"))
                                                    then substring-before($p, "::default")
                                                    else substring-before($p, ":default")
                                                else $p
                    let $default := ($p = $dimension-name || "::default") or ($p = $dimension-name || ":default")
                    let $all := (request:param-values($dimension-name) ! upper-case($$)) = "ALL"
                    return
                    {
                       $dimension-name : {|
                            { Name : $dimension-name },
                            if ($default)
                            then { Default :
                                (request:param-values($dimension-name || "::default"),
                                 request:param-values($dimension-name || ":default"))[1] }
                            else (),
                            if ($all)
                            then ()
                            else {
                                Domains : {
                                    "sec:ImplicitDomain" : {
                                        Name: "sec:ImplicitDomain",
                                        Members: {|
                                            (request:param-values($dimension-name)) ! { $$ : { Name: $$ } }
                                        |}
                                    }
                                }
                            }
                        |}
                    }
let $concepts := request:param-values("concept")
let $map      := request:param-values("map")[1]
return
  if (empty($concepts))
  then {
    response:status-code(400);
    session:error("concept: missing parameter", $format)
  } else
    let $archives := (
        archives:archives($aids)
    )
    let $entities := (
        companies:companies($ciks),
        companies:companies-for-tags($tags),
        companies:companies-for-tickers($tickers),
        companies:companies-for-SIC($sics),
        if (exists($archives)) then companies:companies($archives.Entity) else ()
    )
    return switch(true)
      case empty($entities) and empty($archives) return {
        response:status-code(404);
        session:error("entities or archives not found (valid parameters: cik, ticker, tag, sic, aid)", $format)
      }
      case not (session:only-dow30($entities) or session:valid()) return {
        response:status-code(401);
        session:error("accessing facts of an entity that is not in the DOW30", $format)
      }
      default return
        let $facts := local:facts(
            $entities, $archives,
            $fiscalPeriods, $fiscalYears,
            $concepts, $dimensions, $map)
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($facts)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=facts.csv");
                local:to-csv($facts)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=fact.csv");
                local:to-csv($facts)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { NetworkIdentifier : "http://bizql.io/facts" },
                    { TableName : "xbrl:Facts" },
                    { FactTable : [ $facts ] },
                    session:comment("json")
                |}
            }

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace sec-fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";

import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-csv($o as object*) as string?
{
    if (exists($o)) (: bug in csv:serialize :)
    then
    string-join(
        csv:serialize(
            for $o in $o
            let $a := $o.Aspects
            return {|
                for $k in keys($a)
                return { $k : $a.$k },
                { "Value" : $o.Value },
                if (exists($o."Unit"))
                then { "Unit" :  $o."Unit" }
                else (),
                if (exists($o.Decimals))
                then { "Decimals" : $o.Decimals }
                else(),
                { "EntityRegistrantName" : $o.EntityRegistrantName },
                if (exists($o.ReportedConcept))
                then { "ReportedConcept" : $o.ReportedConcept }
                else ()
            |},
            { serialize-null-as : "" }
        )
    )
    else ()
};

declare function local:to-xml($o as object*)
{
    (session:comment("xml"),
    <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
        for $o in $o
        let $a := $o.Aspects
        return
            <Fact>{
                <Aspects>{
                    for $k in keys($a)
                    return
                        <Aspect>
                            <Name>{$k}</Name>
                            <Value>{$a.$k}</Value>
                        </Aspect>
                }</Aspects>,
                <Value>{$o.Value}</Value>,
                <Type>{$o.Type}</Type>,
                if (exists($o.Unit))
                then <Unit>{$o.Unit}</Unit>
                else(),
                if (exists($o.Decimals))
                then <Decimals>{$o.Decimals}</Decimals>
                else (),
                <EntityRegistrantName>{$o.EntityRegistrantName}</EntityRegistrantName>,
                if (exists($o.ReportedConcept))
                then <ReportedConcept>{$o."ReportedConcept"}</ReportedConcept>
                else ()
            }</Fact>
    }</FactTable>)
};

declare function local:facts(
    $entities,
    $archives,
    $fiscalPeriods,
    $fiscalYears,
    $concepts,
    $dimensions,
    $map)
{
    for $entity in $entities
    (: compute latest reported fiscalYear if none is specified :)
    let $years  :=  distinct-values(
                        for $f in $fiscalYears
                        return
                            switch ($f)
                            case "LATEST" return
                                for $p in $fiscalPeriods
                                return
                                    if ($p eq "FY")
                                    then sec-fiscal:latest-reported-fiscal-period($entity, "10-K").year
                                    else sec-fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                            case "ALL" return  $sec-fiscal:ALL_FISCAL_YEARS
                            default return $f
                    )
    let $fiscalPeriods := distinct-values(
                            $fiscalPeriods !
                                (switch($$)
                                case "Q1" return ("Q1","YTD1")
                                case "Q2" return ("Q2","YTD2")
                                case "Q3" return ("Q3","YTD3")
                                default return ("Q4","FY")))
    let $aspects :=
        {|
            { "xbrl:Concept" : $concepts },
            { "xbrl:Entity" : $entity._id },
            (: This is because of a bug that will be fixed later (hypercube members do not get considered) :)
            for $d in values($dimensions)
            let $members := descendant-objects(values($d)).Name
            where exists($members)
            return { $d.Name: [ $members ] }
        |}
    let $hypercube := copy $h := hypercubes:dimensionless-hypercube()
                      modify (
                          insert json (
                            if (values($dimensions).Name = "dei:LegalEntityAxis")
                            then { "dei:LegalEntityAxis" : values($dimensions)[$$.Name eq "dei:LegalEntityAxis"] }
                            else
                            {|
                                "dei:LegalEntityAxis" ! {
                                $$ : {
                                        Name : $$,
                                        Default : "sec:DefaultLegalEntity",
                                        Domains: {
                                            "sec:DefaultLegalEntity" : {
                                                Name: "sec:DefaultLegalEntity"
                                            }
                                        }
                                    }
                                }
                            |})
                          into $h.Aspects,
                          for $d in $dimensions[not values($$).Name = "dei:LegalEntityAxis"]
                          return insert json $d into $h.Aspects
                      )
                      return $h
    let $options :=
            {|
                if (exists($map))
                then { "concept-maps" : $map }
                else (),
                { Hypercube : $hypercube }
            |}
    return
    for $fact in
        for $f in
            (
                sec-fiscal:facts-for-aspects-and-fiscal-periods-and-years(
                    $aspects, $fiscalPeriods, $years, $options)
            )
        group by $f.Profiles.SEC.Fiscal.Year,
                 $f.Profiles.SEC.Fiscal.Period,
                 $f.Aspects."xbrl:Concept"
        let $latest-accepted := max(distinct-values($f.Profiles.SEC.Fiscal.Acceptance))
        return if (empty($latest-accepted))
               then $f
               else $f[$$.Profiles.SEC.Fiscal.Acceptance eq $latest-accepted]
    return {|
        { Aspects : {|
            for $a in keys($fact.Aspects)
            where $a ne "xbrl:Unit"
            return { $a : $fact.Aspects.$a },
            { "bizql:FiscalPeriod" : $fact.Profiles.SEC.Fiscal.Period },
            { "bizql:FiscalYear" : $fact.Profiles.SEC.Fiscal.Year }
        |} },
        { Type: $fact.Type },
        if (exists($fact.Aspects."xbrl:Unit"))
        then { Unit: $fact.Aspects."xbrl:Unit" }
        else  (),
        if (exists($fact.Decimals))
        then { Decimals: $fact.Decimals }
        else (),
        { Value: $fact.Value },
        { "EntityRegistrantName" : $entity.Profiles.SEC.CompanyName },
        if (exists($map))
        then { "ReportedConcept" : $fact.AuditTrails[].Data.OriginalConcept[1] }
        else ()
    |}
};

let $format      := lower-case(request:param-values("format")[1])

(: choose
    1. entities (using ciks, tags, tickers, sics), FY, and FP ) or
    2. accession numbers
:)
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y cast as integer
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $sec-fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids := request:param-values("aid")
let $dimensions :=  for $p in request:param-names()
                    where contains($p, ":")
                    group by $dimension-name := if (ends-with(lower-case($p), ":default"))
                                                then
                                                    if (ends-with(lower-case($p), "::default"))
                                                    then substring-before($p, "::default")
                                                    else substring-before($p, ":default")
                                                else $p
                    let $default := ($p = $dimension-name || "::default") or ($p = $dimension-name || ":default")
                    let $all := (request:param-values($dimension-name) ! upper-case($$)) = "ALL"
                    return
                    {
                       $dimension-name : {|
                            { Name : $dimension-name },
                            if ($default)
                            then { Default :
                                (request:param-values($dimension-name || "::default"),
                                 request:param-values($dimension-name || ":default"))[1] }
                            else (),
                            if ($all)
                            then ()
                            else {
                                Domains : {
                                    "sec:ImplicitDomain" : {
                                        Name: "sec:ImplicitDomain",
                                        Members: {|
                                            (request:param-values($dimension-name)) ! { $$ : { Name: $$ } }
                                        |}
                                    }
                                }
                            }
                        |}
                    }
let $concepts := request:param-values("concept")
let $map      := request:param-values("map")[1]
return
  if (empty($concepts))
  then {
    response:status-code(400);
    session:error("concept: missing parameter", $format)
  } else
    let $archives := (
        archives:archives($aids)
    )
    let $entities := (
        companies:companies($ciks),
        companies:companies-for-tags($tags),
        companies:companies-for-tickers($tickers),
        companies:companies-for-SIC($sics),
        if (exists($archives)) then companies:companies($archives.Entity) else ()
    )
    return switch(true)
      case empty($entities) and empty($archives) return {
        response:status-code(404);
        session:error("entities or archives not found (valid parameters: cik, ticker, tag, sic, aid)", $format)
      }
      case not (session:only-dow30($entities) or session:valid()) return {
        response:status-code(401);
        session:error("accessing facts of an entity that is not in the DOW30", $format)
      }
      default return
        let $facts := local:facts(
            $entities, $archives,
            $fiscalPeriods, $fiscalYears,
            $concepts, $dimensions, $map)
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($facts)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=facts.csv");
                local:to-csv($facts)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=fact.csv");
                local:to-csv($facts)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { NetworkIdentifier : "http://bizql.io/facts" },
                    { TableName : "xbrl:Facts" },
                    { FactTable : [ $facts ] },
                    session:comment("json")
                |}
            }


:pencil: Add tests.

:pencil: Add tests.

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-spreadsheet($expected as integer, $params as object) as item
{
    let $endpoint := "spreadsheet-for-report"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].TableSet[].TableCells.Facts[][].Value)
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-report-does-not-exist($params as object) as item
{
    let $endpoint := "spreadsheet-for-report"
    let $request := test:invoke($endpoint, $params)
    let $status as integer := $request[1]
    return if ($status eq 404) then true else { url: test:url($endpoint, $params), unexpectedResponse: $request[2] }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    cocacola: local:test-spreadsheet(93, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2013",
        fiscalPeriod:"Q1"}),
    tickerrole: local:test-spreadsheet(93, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1"}),
    tickerconcept: local:test-spreadsheet(372, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"ALL",
        fiscalPeriodType:["instant", "YTD"]
    }),
    tickerconcept-qtd: local:test-spreadsheet(192, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"ALL",
        fiscalPeriodType:["instant", "QTD"]
    }),
    tickerfyfprole: local:test-spreadsheet(187, {
        report:"FundamentalAccountingConcepts",
        ticker:["ko","wmt"],
        fiscalYear:"2013",
        fiscalPeriod:"FY",
        fiscalPeriodType:["instant", "YTD"],
        eliminate:"true"}),
    reportDoesntExist: local:test-report-does-not-exist({
        report:"report-not-found",
        ticker:"MSFT",
        fiscalYear:"ALL",
        fiscalPeriod:"FY"}),
    GeneralInformation: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-GeneralInformation",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          360
        ),
    GeneralInformation-periods: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-GeneralInformation",
              ticker: "ko",
              fiscalYear:"ALL",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          60
        ),
    CashFlowStatement: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-CashFlowStatement",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          330
        ),
    NetCashFlowBreakdown: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-NetCashFlowBreakdown",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          120
        ),
    BalanceSheetClassified: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-BalanceSheetClassified",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          377
        ),
    IncomeStatementMultiStep: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-IncomeStatementMultiStep",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          13
        ),
    StatementOfComprehensiveIncome: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-StatementOfComprehensiveIncome",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          90
        ),
    ComprehensiveIncomeBreakdown: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-ComprehensiveIncomeBreakdown",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          90
        ),
    NetIncomeLossBreakdown: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-NetIncomeLossBreakdown",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          90
        ),
    NetIncomeLossAvailableToCommonBreakdown: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-NetIncomeLossAvailableToCommonBreakdown",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          450
        ),
    KeyRatios: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-KeyRatios",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          120
        )
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-spreadsheet($expected as integer, $params as object) as item
{
    let $endpoint := "spreadsheet-for-report"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].TableSet[].TableCells.Facts[][].Value)
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-report-does-not-exist($params as object) as item
{
    let $endpoint := "spreadsheet-for-report"
    let $request := test:invoke($endpoint, $params)
    let $status as integer := $request[1]
    return if ($status eq 404) then true else { url: test:url($endpoint, $params), unexpectedResponse: $request[2] }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    cocacola: local:test-spreadsheet(93, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2013",
        fiscalPeriod:"Q1"}),
    tickerrole: local:test-spreadsheet(93, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1"}),
    tickerconcept: local:test-spreadsheet(372, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"ALL",
        fiscalPeriodType:["instant", "YTD"]
    }),
    tickerconcept-qtd: local:test-spreadsheet(192, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"ALL",
        fiscalPeriodType:["instant", "QTD"]
    }),
    tickerfyfprole: local:test-spreadsheet(187, {
        report:"FundamentalAccountingConcepts",
        ticker:["ko","wmt"],
        fiscalYear:"2013",
        fiscalPeriod:"FY",
        fiscalPeriodType:["instant", "YTD"],
        eliminate:"true"}),
    reportDoesntExist: local:test-report-does-not-exist({
        report:"report-not-found",
        ticker:"MSFT",
        fiscalYear:"ALL",
        fiscalPeriod:"FY"})
})