merging claudio's original pr

merging claudio's original pr

def _semver_obj2obj($req):
  if   . == $req  then true
  elif .major != $req.major    then false
  elif .minor != $req.minor and .minor != "x" and .minor != "*" then false
  elif .patch != $req.patch and .patch != "x" and .patch != "*"   then false
  elif $req.minor == null and ( .minor == "x" or .minor == "*" ) then false
  elif $req.patch == null and ( .patch == "x" or .patch == "*" ) then false
  elif $req.major == null and $req.minor == null and $req.patch == null  then false
  else true end;

def _ver2obj:
  if   type == "object" then .
  elif type == "string" and test("(?<major>[0-9]+)(\\.(?<minor>[0-9x*]+))?(\\.?(?<patch>[0-9x*]+))?") then capture("(?<major>[0-9]+)(\\.(?<minor>[0-9x*]+))?(\\.?(?<patch>[0-9x*]+))?")
  elif type == "string" and . == "" then {major: null, minor:null, patch:null}
  elif type == "number" then {minor:floor,patch:(.-floor)}
  else {major: .} end;

# Returns true if input version spec semantically matches the requested version
def semver($req):
  if $req == null or $req == "" then false
  elif . == $req then true
  else _ver2obj|_semver_obj2obj($req|_ver2obj) end;

error: pathspec 'docker-image-src/4.4/semver.jq' did not match any file(s) known to git


Create drugfilter.jq

Create drugfilter.jq

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv

#start from the .results array
.results

#Creates new object with .safetyreportid string and .patient.drug array
|map(with_entries(select(.key == "safetyreportid")) + (.patient.drug[]))

# Flattens .openfda and turns openfda arrays into strings delimited with ;
|map(with_entries(select(.key != "openfda")) + (.openfda//{openfda:{"NA":"NA"}}|with_entries(.value = ([.value[]|tostring]|join(";")) |.key |= "openfda_"  + .)))

#Flattens .activesubstance array
|map(with_entries(select(.key != "activesubstance")) + (.activesubstance))

#Iterates through array of objects to return a stream of objects instead of a single array
|.[]

error: pathspec 'drugfilter.jq' did not match any file(s) known to git


:white_check_mark: add test for fixed ambiguous concepts bug.

import module namespace test = "http://apps.28.io/test";

let $is-all := test:is-all()
return
test:check-all-success({|
    {
        ambiguous-concepts-fix: test:invoke-and-assert-deep-equal(
          "facttable-for-component",
          {
            aid: "TD2014110700368",
            role: "http://disclosure.edinet-fsa.go.jp/role/jppfs/rol_QuarterlyConsolidatedStatementOfCashFlows-indirect"
          },
          function($res as item*) as item* { count($res.FactTable[]) },
          78
        )
    }[$is-all]
|})

error: pathspec 'queries/public/test/edinet/facttable-for-component.jq' did not match any file(s) known to git


:pencil: Add migration script.

:pencil: Add migration script.

import module namespace export = "http://apps.28.io/reports-export";

for $report in collection("reports")
where empty($report.Concepts)
return {
  if($report.Hypercubes instance of object)
  then
    replace value of json $report.Hypercubes
    with [ values($report.Hypercubes) ];
  else {}

  let $rep := export:cleanup($report)
  return db:edit($report, $rep);
}

error: pathspec 'queries/private/migration/db10.jq' did not match any file(s) known to git


added missing module

added missing module

jsoniq version "1.0";

module namespace accountant-converter = "http://28.io/modules/xbrl/profiles/accountant/converter";

(:~
 :
Flattens the row headers of the layout, leading to a rendering more familiar to
 : accountants.

 :
 : @param the original layout model.
 : @return the layout model with row headers flattened.
 :)
declare function accountant-converter:flatten-row-headers($layout-model as object) as object
{
  copy $l := $layout-model
  modify
    for $table in $l.TableSet[]
    let $breakdown := $table.TableHeaders.y[]
    let $headers as array := $breakdown.GroupCells
    let $cells as array := $table.TableCells.Facts
    let $result := accountant-converter:flatten-headers({ Headers: $headers, Cells: $cells}, 1)
(:    return insert json { Debug: $result } into $breakdown:)
    return (replace value of json $breakdown.GroupCells with $result.Headers,
            replace value of json $table.TableCells.Facts with $result.Cells)
  return $l
};

declare %private function accountant-converter:flatten-headers(
    $headers-and-cells as object,
    $depth as integer) as object?
{
    switch(true)

    case size($headers-and-cells.Headers) le 1
    return copy $result := $headers-and-cells
           modify for $header in $result.Headers[][]
                  return (insert json { Depth: $depth } into $header,
                          delete json $header.CellSpan,
                          delete json $header.RollUp)
           return $result

    case size($headers-and-cells.Headers[[1]]) le 0
    return {
        Headers: null,
        Cells: $headers-and-cells.Cells
    }

    default return
        let $headers as array := $headers-and-cells.Headers
        let $cells as array := $headers-and-cells.Cells
        let $first-header as object := $headers[[1]][[1]]
        let $span as integer* := $first-header.CellSpan
        let $is-rightmost as boolean := size($headers) eq 1

        let $cells-to-the-right := accountant-converter:take-span([$headers[][position() gt 1]], $span)
        let $roll-up-position := (for $cell at $i in ($cells-to-the-right[[1]])[]
                                  where $cell.RollUp
                                  return $i, 0)[1]
        let $roll-up-span := for $cell in ($cells-to-the-right[[1]])[]
                             where $cell.RollUp
                             return $cell.CellSpan
        let $cells-to-the-right-no-rollup:= switch($roll-up-position)
                                            case 1 return accountant-converter:remove-span($cells-to-the-right, $roll-up-span)
                                            case $span return accountant-converter:take-span($cells-to-the-right, $span - $roll-up-span)
                                            case 0 return $cells-to-the-right
                                            default return []

        let $children-cells := [$cells[][position() le $span]]
        let $converted-children := accountant-converter:flatten-headers(
            {
                Headers: $cells-to-the-right-no-rollup,
                Cells: $children-cells
            },
            $depth + 1
        )

        let $bottom := accountant-converter:remove-span($headers, $span)
        let $bottom-cells := [$cells[][position() gt $span]]
        let $converted-bottom := accountant-converter:flatten-headers(
            {
                Headers: $bottom,
                Cells: $bottom-cells
            },
            $depth
        )

        let $new-first-header as object := copy $n := $first-header
                                           modify (insert json {
                                                        Depth: $depth,
                                                        IsAbstract: size($cells-to-the-right[[1]]) gt 1 or not $cells-to-the-right[[1]][[1]].RollUp
                                                   } into $n,
                                                   delete json $n.CellSpan,
                                                   delete json $n.RollUp)
                                           return $n

        return {
            (:OriginalCells: $cells,
            CellsToTheRight: $cells-to-the-right,
            RollUpPosition: $roll-up-position,
            RollUpSpan: $roll-up-span,
            CellsToTheRightNoRollUp: $cells-to-the-right-no-rollup,
            ChildrenCells: $children-cells,:)
            Headers: [
                [
                    if($roll-up-position = (0, 1))
                    then $new-first-header
                    else (),
                    $converted-children.Headers[][],
                    if($roll-up-position eq $span and $roll-up-position ne 1)
                    then $new-first-header
                    else (),
                    $converted-bottom.Headers[][]
                ]
            ],
            Cells: [
                if(not $is-rightmost and $roll-up-position eq 0)
                then [ (1 to size($cells[[1]])) ! null ]
                else (),
                $converted-children.Cells[],
                $converted-bottom.Cells[]
            ]
        }
};

declare %private function accountant-converter:take-span($headers as array, $span as integer) as array
{
    [
        for $column as array in $headers[]
        return [ for $n in 1 to size($column)
                 let $first-n-headers := $column[][position() le $n]
                 let $sum := sum($first-n-headers.CellSpan)
                 where $sum eq $span
                 return $first-n-headers
        ]
    ]
};

declare %private function accountant-converter:remove-span($headers as array, $span as integer) as array
{
    [
        for $column as array in $headers[]
        return [ for $n in 1 to size($column)
                 let $first-n-headers := $column[][position() le $n]
                 let $sum := sum($first-n-headers ! ($$.CellSpan, 1)[1])
                 where $sum eq $span
                 return $column[][position() gt $n]
        ]
    ]
};

error: pathspec 'profiles/accountant/converter.jq' did not match any file(s) known to git


Object sorting functionality

Object sorting functionality

# Sort entries in objects by either key or value.
def byKeyAsc:
	to_entries
	| sort_by(.key)
	| from_entries;

def byKeyDesc:
	to_entries
	| sort_by(.key)
	| reverse
	| from_entries;

def byValueAsc:
	to_entries
	| sort_by(.value)
	| from_entries;

def byValueDesc:
	to_entries
	| sort_by(.value)
	| reverse
	| from_entries;

error: pathspec 'jq/main.jq' did not match any file(s) known to git


Consistency

Consistency

. | to_entries | sort_by(.key) | map(
  "  - `" + .key + "`  \n\n"
  + (.value.description | split("\n") | map("    " + . + "\n") | join("")) + "\n\n"
  + "    **Default**: " + (
      if .value.value == "" or .value.value == []
      then "*empty*"
      elif (.value.value | type) == "array"
      then "`" + (.value.value | join(" ")) + "`"
      else "`" + (.value.value | tostring) + "`" end)
  + "\n\n"
) | join("")

. | to_entries | sort_by(.key) | map(
  "  - `" + .key + "`  \n"
  + (.value.description | split("\n") | map("    " + . + "\n") | join("")) + "\n\n"
  + "    **Default**: " + (
      if .value.value == "" or .value.value == []
      then "*empty*"
      elif (.value.value | type) == "array"
      then "`" + (.value.value | join(" ")) + "`"
      else "`" + (.value.value | tostring) + "`" end)
  + "\n\n"
) | join("")


added report parameters endpoint

added report parameters endpoint

jsoniq version "1.0";

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace entities ="http://xbrl.io/modules/bizql/entities";
import module namespace mongo = "http://www.28msec.com/modules/mongodb";

let $years :=
    for $year in 2009 to fn:year-from-date(current-date())
    order by $year descending
    return $year

let $periods := ("FY", "Q3", "Q2", "Q1")

let $sics :=
    let $c := mongo:connect("xbrl", {})
    return
        for $s in mongo:find($c, "sics")
        return {
            ID: $s.ID,
            Description: $s.Description,
            Sector: $s.Sector
        }

let $tags := ("DOW30", "SP500", "FORTUNE100", "PJI")

let $entities :=
    for $entity in entities:entities()
    return {
        name: $entity.Profiles.SEC.CompanyName,
        tickers: $entity.Profiles.SEC.Tickers,
        cik: $entity.Profiles.SEC.CIK,
        tags: $entity.Profiles.SEC.Tags,
        sic: $entity.Profiles.SEC.SIC
    }

return
    {
        response:content-type("application/json");
        response:serialization-parameters({"indent" : true});

        {
            years: [ $years ],
            periods: [ $periods ],
            sics: [ $sics ],
            tags: [ $tags ],
            entities: [ $entities ]
        }
    }

error: pathspec 'queries/public/reports/parameters.jq' did not match any file(s) known to git


Add jq filter to convert pump history to openaps-compatible form

# This is a jq filter that converts the output of
# https://github.com/ecc1/medtronic/cmd/pumphistory
# to the schema that openaps expects.
# Usage:
#   pumphistory | jq -f openaps.jq > history.json

# Convert a Go duration of the form Nm0s to minutes.
# This will fail (in the tonumber conversion)
# if the input has an hours or nonzero seconds component.
def duration_to_minutes:
  . | rtrimstr("m0s") | tonumber;

# Output a single JSON array.
[
  # Perform the following on each element of the input array.
  .[] |
  # Start with the timestamp field, common to all record types.
  {
    timestamp: .Time,
  } +
  # Add type-specific fields.
  if .Type == "TempBasalDuration" then
    {
      _type: "TempBasalDuration",
      "duration (min)": .Info | duration_to_minutes
    }
  elif .Type == "TempBasalRate" then
    {
      _type: "TempBasal",
      temp: .Info.Type | ascii_downcase,
      rate: .Info.Value
    }
  elif .Type == "Bolus" then
    {
      _type: "Bolus",
      amount: .Info.Amount
    }
  # Add additional cases here as needed.
  else
    # Warn about record types being skipped.
    ("skipping " + .Type + " record") | debug | empty
  end
]

error: pathspec 'cmd/pumphistory/openaps.jq' did not match any file(s) known to git


Adding test for entities.

Adding test for entities.

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace test = "http://apps.28.io/test";
import module namespace config = "http://apps.28.io/config";

declare variable $local:expected as object :=
    {
      "all" : [ "http://info.edinet-fsa.go.jp E01225-000", "http://info.edinet-fsa.go.jp E01264-000", "http://info.edinet-fsa.go.jp E02166-000", "http://info.edinet-fsa.go.jp E02274-000", "http://info.edinet-fsa.go.jp E02513-000", "http://info.edinet-fsa.go.jp E02529-000", "http://info.edinet-fsa.go.jp E04147-000", "http://info.edinet-fsa.go.jp E04425-000", "http://info.edinet-fsa.go.jp E04430-000", "http://info.edinet-fsa.go.jp EDINET-000" ]
    };

declare %an:nondeterministic function local:test-example1() as item
{
  let $expected :=
  [ {
    "_id" : "http://info.edinet-fsa.go.jp E01225-000",
    "Archives" : "http://" || request:server-name() || ":" || request:server-port() ||
      "/v1/_queries/public/api/filings.jq?_method=POST&token=" || $config:test-token || "&eid=http%3A%2F%2Finfo.edinet-fsa.go.jp%20E01225-000&format=&profile-name=japan&fiscalYear=ALL&fiscalPeriod=ALL",
    "Profiles" : {
      "JAPAN" : {
        "Name" : "JAPAN",
        "EDINETCode" : "E01225-000",
        "SubmitterName" : "新日鐵住金株式会社",
        "SubmitterNameAlphabetic" : "NIPPON STEEL & SUMITOMO METAL CORPORATION",
        "SubmitterNamePhonetic" : "シンニッテツスミキンカブシキガイシャ",
        "SubmitterType" : "内国法人・組合",
        "SIC" : "5401",
        "Province" : "千代田区丸の内二丁目６番１号",
        "Industry" : "Iron & Steel",
        "CapitalStock" : 419524,
        "AccountClosingDate" : " 3.31",
        "IsListed" : true,
        "IsConsolidated" : true,
        "Tags" : [ "NIKKEI" ]
      }
    }
  } ]
  let $endpoint := "entities"
  let $params := {cik:"E01225"}
  let $request := test:invoke($endpoint, $params)
  let $status as integer := $request[1]
  let $actual as array? := $request[2].Entities
  return test:assert-deep-equal($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-entities(
    $expected as array,
    $params as object) as item
{
    let $endpoint := "entities"
    let $request := test:invoke($endpoint, $params)
    let $actual as array := [ $request[2].Entities[]._id ]
    let $status as integer := $request[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

let $dow30 := test:is-dow30()
return local:check({
    all: local:test-entities($local:expected.all, {}),
    example1: local:test-example1()
})

error: pathspec 'queries/public/test/edinet/entities.jq' did not match any file(s) known to git


:pencil: Add new file.

:pencil: Add new file.

jsoniq version "1.0";
module namespace marklogic-wrapper = "http://28.io/modules/xbrl/marklogic-wrapper";

import module namespace marklogic = "http://28.io/modules/marklogic";

declare function marklogic-wrapper:find($collection as string, $query as object) as object()*
{
  let $query := marklogic-wrapper:to-marklogic-query($query)
  return marklogic:qbe("xbrl", $collection, $query)
};

declare function marklogic-wrapper:find($collection as string, $query as object, $offset as integer?, $limit as integer?) as object()*
{
  let $query := marklogic-wrapper:to-marklogic-query($query)
  return
    if(empty($offset) or empty($limit))
    then marklogic:qbe("xbrl", $collection, $query)
    else marklogic:qbe("xbrl", $collection, $query, $offset, $limit)
};

declare function marklogic-wrapper:count($collection as string) as integer
{
    if($collection eq "") then -1 else -1
};

(:~
 : Converts a data-store-neutral 28 query to a mongo query:
 : 1. Arrays get converted to $or querys.
 : 2. $ operators are left untouched (e.g., $or, $exists).
 : 3. Conversion is applied recursively in objects and arrays.
 :)
declare %private function marklogic-wrapper:to-marklogic-query($query as object) as item?
{
  if(keys($query) = "$or")
  then {
    "$or" : [
      for $q as object in $query."$or"[]
      return marklogic-wrapper:to-marklogic-query($q)
    ]
  }
  else
  let $flat as object := marklogic-wrapper:flatten-query($query)
  let $unboxed-arrays as object := marklogic-wrapper:unbox-size-one-arrays($flat)
  let $converted-arrays as object := marklogic-wrapper:convert-arrays($unboxed-arrays)
  return $converted-arrays
};

(:~
 :)
declare %private function marklogic-wrapper:flatten-query($query as object) as object
{
  {|
    for $key in keys($query)
    let $value := $query.$key
    return typeswitch($value)
           case object return let $flattened-value := marklogic-wrapper:flatten-query($value)
                              return {|
                                for $key2 in keys($flattened-value)
                                return { $key || "." || $key2 : $flattened-value.$key2 }
                              |}
           default return { $key : $value }
  |}
};

(:~
 :)
declare %private function marklogic-wrapper:unbox-size-one-arrays($query as object) as object
{
  {|
    for $key in keys($query)
    let $value := $query.$key
    return typeswitch($value)
           case array return if(size($value) eq 1)
                             then { $key : $value[] }
                             else { $key : $value }
           default return { $key : $value }
  |}
};

(:~
 :)
declare %private function marklogic-wrapper:convert-arrays($query as object) as object
{
  if(some $value in values($query) satisfies $value instance of array)
  then {
      "$and" : [
        for $key in keys($query)
        let $value := $query.$key
        return typeswitch($value)
               case array return
                 { "$or" : [ for $v in $value[]
                             return marklogic-wrapper:expand-query({ $key : $v })
                           ]
                 }
               default return marklogic-wrapper:expand-query({ $key : $value })
       ]
    }
  else marklogic-wrapper:expand-query($query)
};

declare %private function marklogic-wrapper:expand-query($query as object) as object
{
  {|
    for $key as string in keys($query)
    let $tokens as string+ := tokenize($key, "\\.")
    let $prefix as string := head($tokens)
    let $suffix as string := string-join(tail($tokens), ".")
    group by $prefix
    return if($suffix = "")
           then project($query, $prefix)
           else {
              $prefix : {| marklogic-wrapper:expand-query({|
                for $s in $suffix[$$ ne ""]
                return { $s: $query.($prefix || "." || $s) }|})
              |}
            }
  |}
};

declare function marklogic-wrapper:search(
    $collection as string,
    $full-text as string,
    $full-text-fields as string*,
    $full-text-offset as integer?,
    $full-text-limit as integer?
) as object*
{
    marklogic-wrapper:find(
        $collection,
        {
          "$or" : [
            for $f in $full-text-fields
            return marklogic-wrapper:expand-query({ $f : { "$case-sensitive": false, "$word" : $full-text } })
          ]
        },
        $full-text-offset,
        $full-text-limit
    )
};

error: pathspec 'queries/modules/io/28/modules/xbrl/marklogic-wrapper.jq' did not match any file(s) known to git


Fallback functionality

Fallback functionality

# If a condition is met, apply and return the fallback filter.
def unlessNullFallback(f; fallback):
	if type != "null" then
		f
	else
		fallback
	end;

def ifNull(fallback):
	unlessNullFallback(.; fallback);

def unlessNull(f):
	unlessNullFallback(f; .);

def nullAsObject:
	ifNull({});


:arrow_up: Add new module.

:arrow_up: Add new module.

jsoniq version "1.0";
module namespace data-source-wrapper = "http://28.io/modules/xbrl/data-source-wrapper";

import module namespace config = "http://apps.28.io/config";
import module namespace mongo-wrapper = "http://28.io/modules/xbrl/mongo-wrapper";

declare function data-source-wrapper:find(
    $collection as string,
    $query as object
) as object()*
{
  switch($config:query-db.category)
  case "MongoDB" return mongo-wrapper:find($collection, $query)
  default return ()
};

declare function data-source-wrapper:find(
    $collection as string,
    $query as object,
    $projection as object
) as object()*
{
  switch($config:query-db.category)
  case "MongoDB" return mongo-wrapper:find($collection, $query, $projection)
  default return ()
};

declare function data-source-wrapper:text-search(
    $collection as string,
    $query as object) as object*
{
  switch($config:query-db.category)
  case "MongoDB" return mongo-wrapper:run-cmd-deterministic(
    {|
      { "text" : $collection },
      $query
    |}).results[].obj
  default return ()
};

error: pathspec 'queries/modules/io/28/modules/xbrl/data-source-wrapper.jq' did not match any file(s) known to git


Update static/vendors/ace-builds/demo/kitchen-sink/docs/jsoniq.jq

Signed-off-by: Bernard Ojengwa <fccb4cc00c1ba7e51bf5c2b5f77130b9fe8d917e@gmail.com>

error: pathspec 'static/vendors/ace-builds/demo/kitchen-sink/docs/jsoniq.jq' did not match any file(s) known to git


added query to export basic financial information report (prudena)

jsoniq version "1.0";
import module namespace http = "http://zorba.io/modules/http-client";
import module namespace request = "http://www.28msec.com/modules/http-request";

declare variable $id := "BasicFinancialInformation";
declare variable $report := parse-json(http:get("http://" || request:server-name() || ":" || request:server-port() || "/v1/_queries/public/export/fac.jq?_method=POST").body.content);
declare variable $desc :=
  "Documentation about the fundamental accounting concepts defined in this report can be found at http://fundamentalaccountingconcepts.wikispaces.com/home";
declare variable $role := "http://xbrl.io/basic-financial-information";
declare variable $additionalRules :=
  (
    {
        "Id" : "gi_CommonStockSharesOutstanding",
        "Type" : "xbrl28:formula",
        "Label" : "CommonStockSharesOutstanding imputation",
        "Formula" : "(: workaround for bug: https://github.com/28msec/xbrl-infosetgenerator/issues/7 :)
                     let $aligned-filter := trim($aligned-filter, (\"Profiles.SEC.Fiscal\"))
                     for $facts in  facts:facts-for-internal((
                             \"fac:CommonStockSharesOutstanding\"), $hypercube, $aligned-filter, $concept-maps, $rules, $cache, $options)
                     group by $canonical-filter-string :=
                                 facts:canonically-serialize-object($facts, ($facts:CONCEPT, \"_id\", \"IsInDefaultHypercube\", \"Type\", \"Value\", \"Decimals\", \"AuditTrails\", \"xbrl28:Type\", \"Balance\"))
                     for $CommonStockSharesOutstandingByArchive in $facts[$$.$facts:ASPECTS.$facts:CONCEPT eq \"fac:CommonStockSharesOutstanding\"]
                     group by $CommonStockSharesOutstandingByArchive.Archive
                     return
                         let $source-fact :=
                             (
                                 for $CommonStockSharesOutstanding in $CommonStockSharesOutstandingByArchive
                                 order by $CommonStockSharesOutstanding.$facts:ASPECTS.$facts:PERIOD descending
                                 return $CommonStockSharesOutstanding
                             )[1]
                         let $value := rules:decimal-value($source-fact)
                         let $original-concept := $source-fact.AuditTrails[][$$.Type eq \"xbrl28:concept-maps\"].Data.OriginalConcept
                         let $audit-trail-message :=
                            rules:fact-trail({ \"Aspects\" : { \"xbrl:Unit\" : \"shares\", \"xbrl:Concept\" : $original-concept }, Value: $value }) || \" (as of latest practicable date: \" || $source-fact.$facts:ASPECTS.$facts:PERIOD
                            || \")\"
                         return
                             rules:create-computed-fact(
                                 $source-fact,
                                 \"fac:CommonStockSharesOutstanding\",
                                 $value,
                                 $rule,
                                 $audit-trail-message,
                                 $source-fact,
                                 $options)",
        "ComputableConcepts" : [ "fac:CommonStockSharesOutstanding" ]
      },
      {
          "Id" : "decimals_inf",
          "Type" : "xbrl28:formula",
          "Label" : "Align INF decimals",
          "Formula" : "for $fact in facts:facts-for-internal($concepts, $hypercube, $aligned-filter, $concept-maps, $rules, $cache, $options)
                             return
                               if(string($fact.Decimals) eq \"INF\")
                               then
                                 let $concept := $fact.Aspects.\"xbrl:Concept\"
                                 let $audit-trail :=
                                   {
                                     \"Type\" : \"xbrl28:decimals-inf-alignment\",
                                     \"Label\" : \"Aligning Decimals to 3 instead of INF\",
                                     \"Message\" : $concept || \" = 0\",
                                     \"Data\" : {
                                       \"OutputConcept\" : $concept
                                     }
                                   }
                                 return
                                 copy $f := $fact
                                 modify (
                                     replace value of json $f.Decimals with 3,
                                     if(exists($f.AuditTrails))
                                     then replace value of json $f.AuditTrails with [ $audit-trail ]
                                     else insert json { AuditTrails: [ $audit-trail ] } into $f
                                 )
                                 return $f
                               else $fact"
        }
  );

(: Presentation :)
let $presentation := $report.Networks[][$$.ShortName eq "Presentation"]
let $concepts :=
    {
      "fac:CommonStockSharesAuthorized": {
        "Name" : "fac:CommonStockSharesAuthorized",
        "Label" : "Common Stock Shares (authorized)",
        "Id" : "e7c449eb-4e1b-4d20-af35-b783cfdab03e"
      },
      "fac:CommonStockSharesIssued": {
        "Name" : "fac:CommonStockSharesIssued",
        "Label" : "Common Stock Shares (issued)",
        "Id" : "a8a4ae32-0424-437e-a645-428398f95f9c"
      },
      "fac:CommonStockSharesOutstanding": {
        "Name" : "fac:CommonStockSharesOutstanding",
        "Label" : "Common Stock Shares (outstanding)",
        "Id" : "f51e72c7-a3ce-4639-99b0-47ca36d54a7b"
      }
    }
let $genInfo := $presentation.Trees."fac:FundamentalAccountingConceptsLineItems".To."fac:FundamentalAccountingConceptsHierarchy".To."fac:GeneralInformationHierarchy"
return
  {
    replace value of json $presentation.LinkRole with $role;
    replace value of json $genInfo.To with {| ( $genInfo.To, $concepts ) |};
  }

(: Hypercube :)
let $conceptsDomain := $report.Hypercubes."xbrl:DefaultHypercube".Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain"
let $concepts :=
    {
      "fac:CommonStockSharesAuthorized": {
        "Name" : "fac:CommonStockSharesAuthorized",
        "Label" : "Common Stock Shares (authorized)",
        "Id" : "e7c449eb-4e1b-4d20-af35-b783cfdab03e"
      },
      "fac:CommonStockSharesIssued": {
        "Name" : "fac:CommonStockSharesIssued",
        "Label" : "Common Stock Shares (issued)",
        "Id" : "a8a4ae32-0424-437e-a645-428398f95f9c"
      },
      "fac:CommonStockSharesOutstanding": {
        "Name" : "fac:CommonStockSharesOutstanding",
        "Label" : "Common Stock Shares (outstanding)",
        "Id" : "f51e72c7-a3ce-4639-99b0-47ca36d54a7b"
      }
    }
return
  replace value of json $conceptsDomain.Members with {| $conceptsDomain.Members, $concepts |};

(: ConceptMap :)
let $conceptMap := $report.Networks[][$$.ShortName eq "ConceptMap"]
let $mappings :=
    {
      "fac:CommonStockSharesAuthorized" : {
        "Name" : "fac:CommonStockSharesAuthorized",
        "Label" : "Common Stock Shares (authorized)",
        "To" : {
          "us-gaap:CommonStockSharesAuthorized": {
            "Name" : "us-gaap:CommonStockSharesAuthorized",
            "Id" : "8ed0a312-9891-4d3d-b459-67604dce2124"
          }
        },
        "Id" : "372f39c9-c8ed-4654-a424-6f033bb0ddd2"

      },
      "fac:CommonStockSharesIssued" : {
        "Name" : "fac:CommonStockSharesIssued",
        "Label" : "Common Stock Shares (issued)",
        "To" : {
          "us-gaap:CommonStockSharesIssued": {
            "Name" : "us-gaap:CommonStockSharesIssued",
            "Id" : "1359b87e-5f75-49f3-849b-d410cc399b50"
          }
        },
        "Id" : "66226139-512f-4edd-8c18-6d7e307a1f66"
      },
      "fac:CommonStockSharesOutstanding" : {
        "Name" : "fac:CommonStockSharesOutstanding",
        "Label" : "Common Stock Shares (outstanding)",
        "To" : {
          "dei:EntityCommonStockSharesOutstanding": {
            "Name" : "dei:EntityCommonStockSharesOutstanding",
            "Id" : "b8d27670-5caa-411f-8305-860c0613c31f"
          }
        },
        "Id" : "91b87448-a8dd-4739-9cdf-2cdaef6a4ff6"
      }
    }
return
  {
    replace value of json $conceptMap.LinkRole with $role;
    replace value of json $conceptMap.Trees with {| $conceptMap.Trees, $mappings |};
  }

(: DefinitionModel :)
replace value of json $report.DefinitionModels[1].Breakdowns.y[1].BreakdownTrees[1].LinkRole with $role;

(: adapt default_zero rule :)
let $defaultZeroRule := $report.Rules[][$$.Id eq "default_zero"]
let $formula := replace($defaultZeroRule.Formula,
                        "\"fac:LiabilitiesAndEquity\"\\)",
                        "\"fac:LiabilitiesAndEquity\", \"us-gaap:CommonStockSharesAuthorized\", \"us-gaap:CommonStockSharesIssued\", \"dei:EntityCommonStockSharesOutstanding\")"
                       )
return
  {
    replace value of json $defaultZeroRule.Formula with $formula;
  }

{ "_id": $id,
  Archive: null,
  Label: "Basic Financial Information",
  Description: $desc,
  Prefix: "fac",
  Role: $role,
  Owner: "charlie@prudena.com",
  LastModified: string(current-dateTime()),
  ACL: [],
  Networks: $report.Networks,
  Hypercubes: $report.Hypercubes,
  DefinitionModels: $report.DefinitionModels,
  Rules: [ for $rule in ($report.Rules[], $additionalRules)
           order by $rule.ComputableConcepts[][1], $rule.Id descending empty least
           return $rule ],
  Filters: {
    cik: [  ],
    tag: [ "DOW30" ],
    fiscalYear : [ 2013 ],
    fiscalPeriod : [ "FY" ],
    sic : [  ]
  }
}

error: pathspec 'queries/public/export/bfi.jq' did not match any file(s) known to git


Update spreadsheet-for-component.jq

Update spreadsheet-for-component.jq

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-spreadsheet($expected as integer, $params as object) as item
{
    let $endpoint := "spreadsheet-for-component"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].TableSet[].TableCells.Facts[][].Value)
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    cocacola: local:test-spreadsheet(1, {
        ticker:"ko"
    }),
    tickerrole: local:test-spreadsheet(60, {
        ticker:"ko",
        networkIdentifier:"http://www.thecocacolacompany.com/role/ConsolidatedStatementsOfIncome"
    }),
    tickerconcept: local:test-spreadsheet(76, {
        ticker:"ko",
        fiscalYear:"2013",
        concept:"us-gaap:Assets"
    }),
    tickerfyfprole: local:test-spreadsheet(70, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        networkIdentifier:"http://www.thecoca-colacompany.com/role/CondensedConsolidatedBalanceSheets"
    }),
    generic: local:test-spreadsheet(76, {
        profile-name: "generic",
        aid: "0000021344-14-000008",
        role:"http://www.thecocacolacompany.com/role/ConsolidatedBalanceSheets"
    })
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-spreadsheet($expected as integer, $params as object) as item
{
    let $endpoint := "spreadsheet-for-component"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].TableSet[].TableCells.Facts[][].Value)
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    cocacola: local:test-spreadsheet(1, {
        ticker:"ko"
    }),
    tickerrole: local:test-spreadsheet(60, {
        ticker:"ko",
        networkIdentifier:"http://www.thecocacolacompany.com/role/ConsolidatedStatementsOfIncome"
    }),
    tickerconcept: local:test-spreadsheet(76, {
        ticker:"ko",
        fiscalYear:"2013",
        concept:"us-gaap:Assets"
    }),
    tickerfyfprole: local:test-spreadsheet(70, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        networkIdentifier:"http://www.thecoca-colacompany.com/role/CondensedConsolidatedBalanceSheets"
    }),
    generic: local:test-spreadsheet(36, {
        profile-name: "generic",
        aid: "0000021344-14-000008",
        role:"http://www.thecocacolacompany.com/role/ConsolidatedBalanceSheets"
    })
})


Add jq helper functions to remember how to handle mixed logs

function jqdbg {
  jq -R '. as $raw | try fromjson catch $raw'
}

function strip_log_with_errors {
  awk -F'{' '{ st = index($0,"{"); print "{"substr($0,st+1)}' $1 | jq -R '. as $raw | try fromjson catch $raw'
}
function strip_log {
  awk -F'{' '{ st = index($0,"{"); print "{"substr($0,st+1)}' $1 | jq -R 'fromjson?'
}

error: pathspec '.bashrc.jq' did not match any file(s) known to git


Create mongodb-aggregation.jq

Create mongodb-aggregation.jq

(:
 : JSONiq Query
 :)
for $answers in collection("answers")
group by $user-id := $answers.owner.user_id
let $count := count($answers)
let $avg-score := avg($answers.score)
order by $avg-score, $count descending
let $titles := for $answer in $answers
               for $question in collection("faq")
               where $answer.question_id eq $question.question_id
               return $question.title
return {
  name: $answers[1].owner.display_name,
  reputation: $avg-score,
  count: $count,
  titles: [distinct-values($titles)]
}

error: pathspec 'mongodb-aggregation.jq' did not match any file(s) known to git


import module namespace facts = "http://xbrl.io/modules/bizql/facts";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace sec-fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-xml($o as object*)
{
    <Results>{
        for $o in $o
        return
            <Result>
                <Concept>{$o.Concept}</Concept>
                <FiscalYearFocus>{$o.FiscalYearFocus}</FiscalYearFocus>
                <FiscalPeriodFocus>{$o.FiscalPeriodFocus}</FiscalPeriodFocus>
                <Unit>{$o.Unit}</Unit>
                <Value>{$o.Value}</Value>
                <Reports>{$o.Report}</Reports>
            </Result>
    }</Results>
};

let $format  := lower-case(substring-after(request:path(), ".jq.")) (: text, xml, or json (default) :)
let $period  := let $period := upper-case(request:param-values("period")[1])
                return  if ($period = ("Q1", "Q2", "Q3", "FY"))
                        then $period
                        else error(xs:QName("local:INVALID-PERIOD"), $period || ": period value must be one of Q1, Q2, Q3, FY")
let $concept := let $concept := request:param-values("concept")[1]
                return  if (empty($concept))
                        then error(xs:QName("local:INVALID-CONCEPT"), "concept parameter missing")
                        else $concept
let $map     := let $map := lower-case(request:param-values("map")[1])
                return if ($map eq "fundamentalaccountingconcepts")
                       then "FundamentalAccountingConcepts"
                       else error(xs:QName("local:INVALID-MAP"), $map || ": map parameter invalid")

let $json-result :=
    (: Computes the total assets of all filing companies, grouped by fiscal year and currency :)
    let $facts :=
        (: Getting all facts, but restricting to just one per entity and per fiscal year :)
        for $fact in sec-fiscal:facts-for-entities-and-concepts-and-fiscal-periods-and-years(
            entities:entities(),
            $concept,
            $period,
            (),
            {|
                if (exists($map))
                then { "concept-maps" : $map }
                else ()
            |}
        )
        group by $fyf := sec-fiscal:fiscal-year($fact),
                 $fpf := sec-fiscal:fiscal-period($fact),
                 $entity:= facts:entity-for-fact($fact)
        return ($fact[$$.Aspects."xbrl:Unit" eq "iso4217:USD"], $fact)[1]
    for $fact in $facts
    group by $fyf := sec-fiscal:fiscal-year($fact),
             $fpf := sec-fiscal:fiscal-period($fact),
             $unit := $fact.Aspects."xbrl:Unit"
    order by $fyf
    return {
        Concept : $concept,
        FiscalYearFocus : $fyf,
        FiscalPeriodFocus : $fpf,
        Unit : $unit,
        Value : format-number(sum($fact.Value[$$ ne null]), "###,000"),
        Reports : count(distinct-values($fact.Archive))
    }
return
    switch ($format)
    case "xml" return {
        response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
        local:to-xml($json-result)
    }
    case "text" return {
        response:content-type("text/csv");
        string-join(csv:serialize($json-result))
    }
    default return {
        response:content-type("application/json");
        response:serialization-parameters({"indent" : true});
        [ $json-result ]
    }

error: pathspec 'queries/public/api/TotalForConceptGroupedByFiscalPeriodForAllEntities.jq' did not match any file(s) known to git


Migration query for rights.

Migration query for rights.

append json "anyone" into collection("Rights")[$$._id eq "data_sec"].roles;

error: pathspec 'queries/private/FixRights.jq' did not match any file(s) known to git


merging claudio's original pr

merging claudio's original pr

def _semver_obj2obj($req):
  if   . == $req  then true
  elif .major != $req.major    then false
  elif .minor != $req.minor and .minor != "x" and .minor != "*" then false
  elif .patch != $req.patch and .patch != "x" and .patch != "*"   then false
  elif $req.minor == null and ( .minor == "x" or .minor == "*" ) then false
  elif $req.patch == null and ( .patch == "x" or .patch == "*" ) then false
  elif $req.major == null and $req.minor == null and $req.patch == null  then false
  else true end;

def _ver2obj:
  if   type == "object" then .
  elif type == "string" and test("(?<major>[0-9]+)(\\.(?<minor>[0-9x*]+))?(\\.?(?<patch>[0-9x*]+))?") then capture("(?<major>[0-9]+)(\\.(?<minor>[0-9x*]+))?(\\.?(?<patch>[0-9x*]+))?")
  elif type == "string" and . == "" then {major: null, minor:null, patch:null}
  elif type == "number" then {minor:floor,patch:(.-floor)}
  else {major: .} end;

# Returns true if input version spec semantically matches the requested version
def semver($req):
  if $req == null or $req == "" then false
  elif . == $req then true
  else _ver2obj|_semver_obj2obj($req|_ver2obj) end;

error: pathspec 'docker-image-src/4.4/semver.jq' did not match any file(s) known to git


Add jq filter to convert autotune.json files into args for setbasals

# This is a filter that converts the output of autotune
# to an argument list for the setbasals command.
# Usage:
#   setbasals $(jq -r -f autotune.jq < autotune.json)
# Don't forget the -r (--raw-output) flag!

.[] | (.start | rtrimstr(":00")), .rate

error: pathspec 'cmd/setbasals/autotune.jq' did not match any file(s) known to git


jq: Filter logs by path/oldpath [ci skip]

# You can use jq to analyse multiple huge log files at once:
#
#   `yarn jq find_issues path/to/logs.txt*`
#
# The `yarn jq` script doesn't use `--slurp` so JSON lines are filtered one by
# one and memory usage is kept below 100MB.
#
# But we should be able to use it in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
#
# TODO: Find a way to make aggregation work, e.g.:
#   `yarn jq 'find_issues|.msg' | yarn jq:frequencies`

# Errors
def error_level: 50;
def is_error: .level >= error_level;
def select_error: select(is_error);
def find_errors: select_error|{component,path,msg,time};

# Warnings
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def select_warn: select(is_warn);
def select_warn_strict: select(is_warn_strict);
def find_warns: select_warn|{component,path,msg,time};
def find_warns_strict: select_warn_strict|{component,path,msg,time};

# Conflicts
def is_conflict: .msg == "resolveConflictAsync";
def select_conflict: select(is_conflict);
def find_conflicts: select_conflict|{path,time};

# Non-issues
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Issues
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def select_issue: select(is_issue);
def find_issues: select_issue|{component,path,msg,time,level};

# Path filtering
def filter_path(pattern): select((.path,.oldpath,"")|strings|test(pattern));

# Config info
def find_client_info:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Utils
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );

# You can use jq to analyse multiple huge log files at once:
#
#   `yarn jq find_issues path/to/logs.txt*`
#
# The `yarn jq` script doesn't use `--slurp` so JSON lines are filtered one by
# one and memory usage is kept below 100MB.
#
# But we should be able to use it in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
#
# TODO: Find a way to make aggregation work, e.g.:
#   `yarn jq 'find_issues|.msg' | yarn jq:frequencies`

# Errors
def error_level: 50;
def is_error: .level >= error_level;
def select_error: select(is_error);
def find_errors: select_error|{component,path,msg,time};

# Warnings
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def select_warn: select(is_warn);
def select_warn_strict: select(is_warn_strict);
def find_warns: select_warn|{component,path,msg,time};
def find_warns_strict: select_warn_strict|{component,path,msg,time};

# Conflicts
def is_conflict: .msg == "resolveConflictAsync";
def select_conflict: select(is_conflict);
def find_conflicts: select_conflict|{path,time};

# Non-issues
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Issues
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def select_issue: select(is_issue);
def find_issues: select_issue|{component,path,msg,time};

# Config info
def find_client_info:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Utils
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );


Add a jq script to analyse ping results.

Contributed by: stephane@bortzmeyer.org

map(.result) | flatten(1) | map(.rtt) | length as $total |
 "Median: " + (sort |
      if length % 2 == 0 then .[length/2] else .[(length-1)/2] end | tostring),
 "Average: " + (map(select(. != null)) | add/length | tostring) + " ms",
 "Min: " + (map(select(. != null)) | min | tostring) + " ms",
 "Max: " + (max | tostring) + " ms",
 "Failures: " + (map(select(. == null)) | (length*100/$total) | tostring) + " %"

error: pathspec 'contrib/ping-report.jq' did not match any file(s) known to git


Add Github release jq template.

Add Github release jq template.

Create actually JSON file with:
jq -n --arg changelog "$(cat ch1_5_7.json)" -f gh-1_5_7.jq > gh-1_5_7.json

[Documentation]

{
  "tag_name": "v1.5.6",
  "name": "The Xenophon release",
  "draft": true,
  "prerelease": false,
  "body": $changelog
}

error: pathspec 'general/docs/changelogs/gh-1_5_7.jq' did not match any file(s) known to git