Remove unused query

Remove unused query

import module namespace req = "http://www.28msec.com/modules/http/request";
import module namespace res = "http://www.28msec.com/modules/http/response";

declare variable $keyword-map := {
(:    default: "difault" :)
};

declare function local:camel-case($name as string) as string {
    let $name := if(exists($keyword-map($name))) then
$keyword-map($name) else $name
    let $tokens :=
        for $token at $i in tokenize($name, "-")
        return (
            if($i eq 1) then
                lower-case(substring($token, 1, 1))
            else
                upper-case(substring($token, 1, 1))
        ) || substring($token, 2)
    return string-join($tokens, "")
};

declare function local:signature($parameters as object*) as string {
    string-join(
        for $parameter in $parameters
        where not($parameter.paramType eq "header" and
jn:size($parameter.enum) eq 1)
        return local:camel-case($parameter.name)
    , ", ")
};

declare function local:queries($parameters as object*) as object* {
    for $param in $parameters
    where $param.paramType eq "query"
    return $param
};

declare function local:singletons($parameters as object*) as object*{
    for $parameter in $parameters
    where jn:size($parameter.enum) eq 1
    return {
        "key": local:camel-case($parameter.name),
        "value": $parameter.enum()[1]
    }
};

declare function local:path($path as string, $parameters as object*) as string {
    let $segments :=
        for $segment in $parameters
        where $segment.paramType eq "path"
        return $segment.name
    return
        local:compute-path($path, $segments) };

declare function local:compute-path($path as string, $segments as string*) as string {
    if(empty($segments)) then
        "'" || $path || "'"
    else
        let $segment := $segments[1]
        let $new-path := replace(
            $path, "\\{" || $segment || "\\}", "' + parameters['" || local:camel-case($segment) || "'] + '"
        )
        return local:compute-path(
            $new-path,
            subsequence($segments, 2)
        )
};

declare function local:has-body($parameters as object*) as boolean {
    exists(
        for $param in $parameters
        where $param.paramType eq "body"
        return $param
    )
};

declare function local:has-headers($parameters as object*) as boolean {
    exists(
        for $param in $parameters
        where $param.paramType eq "header"
        return $param
    )
};

declare function local:javascript($d as object, $new-module as boolean) as string {
"angular.module('" || $d.module || "'" || (if($new-module) then ", []" else "") || ")
/**
 * " || (if($d.doc.description) then $d.doc.description else "Description Missing") || "
 */
.factory('" || $d.service || "', function($q, $http, $rootScope){
    /**
     * @class " || $d.service || "
     * @param {string} domain - The project domain
     */
    return function(domain) {
        if(typeof(domain) !== 'string') {
            throw new Error('Domain parameter must be specified as a string.');
        }

        var root = '" || $d.doc.resourcePath || "';

        this.$on = function($scope, path, handler) {
            var url = domain + path;
            $scope.$on(url, function(event, data){
                handler(data);
            });
            return this;
        };

        this.$broadcast = function(path, data){
            var url = domain + path;
            $rootScope.$broadcast(url, data);
            return this;
        };

" || string-join(
        for $operation in $d.operations()

        return "        /**
         * " || $operation.doc.notes || "
         * @method
         * @name " || $d.service || "#" || $operation.name || "
         * " || string-join(for $param in $operation.doc.parameters() return "@param {" || $param.type || "} " || $param.name || " - " || $param.description || ",
         * ")|| "
         */
        this." || $operation.name || " = function(parameters){
            var deferred = $q.defer();" ||
        string-join(for $singleton in $operation.singletons()
        return "
            var " || $singleton.key || " = '" || $singleton.value || "';", "
        ")
        || "
            var that = this;
            var path = " || $operation.path || "
            var url = domain + path;
            var params = {};
            " ||
        string-join(for $query in $operation.queries()
        return if($query.required eq false) then
        if(exists($query.pattern)) then
            "for(var prop in parameters)
                if (parameters.hasOwnProperty(prop))
                    if(/^" || $query.pattern || "$/ig.test(prop))
                        params[prop]=parameters[prop];"
        else "params['" || $query.name || "'] = parameters['" || local:camel-case($query.name) || "'];"
        else "if(parameters['" || local:camel-case($query.name) || "'] === undefined) {
                deferred.reject(new Error('The " || local:camel-case($query.name) || " parameter is required'));
                return deferred.promise;
            } else {
                params['" || $query.name || "'] = parameters['" || local:camel-case($query.name) || "'];
            }", "
            ")
            ||
        (if($operation.hasBody)
        then "
            var body = parameters['" || $operation.body || "'];"
        else "
            var body = null;")
            || "
            var method = '" || $operation.method || "'.toUpperCase();
            if (parameters.$method)
            {
                params['_method'] = parameters.$method;
                method = '" || (if($operation.hasBody) then "POST" else "GET") || "';
            }
            var cached = parameters.$cache && parameters.$cache.get(url);
            if(method === 'GET' && cached !== undefined && parameters.$refresh !== true) {
                deferred.resolve(cached);
            } else {
            $http({
                method: method,
                url: url,
                params: params" || (if($operation.hasBody) then ",
data: body" else "")  || (if($operation.hasHeaders) then ", headers: "
|| $operation.headers else "") || ",
                cache: (parameters.$refresh !== true)
            })
            .success(function(data, status, headers, config){
                deferred.resolve(data);
                " || (
                        if($operation.method eq "POST" or $operation.method eq "PUT" or $operation.method eq "DELETE" or $operation.method eq "PATCH") then
                             "//cache.removeAll();"
                        else if($operation.method eq "GET") then
                            "//that.$broadcast(url);
                if(parameters.$cache !== undefined) parameters.$cache.put(url, data, parameters.$cacheItemOpts ?
parameters.$cacheItemOpts : {});"
                        else ""
                    ) || "
            })
            .error(function(data, status, headers, config){
                deferred.reject({data: data, status: status, headers: headers, config: config});" || "
                //cache.removeAll();
            })
            ;
            }
            return deferred.promise;
        };"
        , "

") || "
    };
});"
};

res:set-content-type("text/plain");

let $input :=
let $doc     := jn:parse-json(req:text-content())
let $module  := req:parameter-values("module") let $service := req:parameter-values("service") return {
    doc: $doc,
    module: $module,
    service: $service,
    operations: [
        for $api in $doc.apis()
        for $operation in $api.operations()
        return  {
            doc: $operation,
            name: $operation.nickname,
            method: $operation.method,
            signature: local:signature($operation.parameters()),
            queries: [local:queries($operation.parameters())],
            singletons: [local:singletons($operation.parameters())],
            path: local:path($api.path, $operation.parameters()),
            hasBody: local:has-body($operation.parameters()),
            hasHeaders: local:has-headers($operation.parameters()),
            headers: "{" ||
                (
                    for $param in $operation.parameters()
                    where $param.paramType eq "header"
                    return "'" || $param.name || "': parameters['" || local:camel-case($param.name) || "']"
                )
            || "}",
            body:
                for $param in $operation.parameters()
                where $param.paramType eq "body"
                return local:camel-case($param.name)
        }
    ]
}
return
    local:javascript($input,(string-join(req:parameter-values("new-module", "true"), "") cast as boolean))


Fixed xbrl entity.

Fixed xbrl entity.

import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace rules = "http://28.io/modules/xbrl/rules";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace request = "http://www.28msec.com/modules/http-request";

declare function local:param-values($name as string) as string*
{
    switch(true)
     case $name eq "xbrl:Concept"
        return (request:param-values("concept"), request:param-values("xbrl:Concept"))
     case $name eq "sec:FiscalYear" and $profile-name eq "sec"
        return string(api:preprocess-fiscal-years(($fiscalYear, request:param-values("sec:FiscalYear"))))
     case $name eq "sec:FiscalPeriod" and $profile-name eq "sec"
        return api:preprocess-fiscal-periods(($fiscalPeriod, request:param-values("sec:FiscalPeriod")))
     case $name eq "dei:LegalEntityAxis" and $profile-name eq "sec"
        return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis")
     case $name eq "dei:LegalEntityAxis::default" return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis::default")
     case $name eq "xbrl:Entity" and $profile-name eq "sec" return (
         let $companies := companies:companies(
            $cik,
            api:preprocess-tags($tag),
            $ticker,
            $sic)._id
        return if(empty(($cik,$tag,$ticker,$sic)) or exists($companies))
               then $companies
               else "dummy",
        request:param-values("xbrl:Entity"))
     case $name eq "sec:Archive" and $profile-name eq "sec" return (
            let $fiscalYears := local:param-values("sec:FiscalYear")
            let $fiscalPeriods := local:param-values("sec:FiscalPeriod")
            let $entities := local:param-values("xbrl:Entity")
            return
                if($fiscalYears = "LATEST")
                then fiscal-core:latest-filings($entities, $fiscalPeriods)
                else (),
            $aid,
            request:param-values("sec:Archive")
        )
     default return request:param-values($name)
};

declare function local:param-names() as string*
{
    let $names := request:param-names()
    return distinct-values((
        $names[contains($$, ":")],
        "sec:Accepted"[$profile-name eq "sec"],
        "sec:FiscalPeriod"[$profile-name eq "sec" and $names = "fiscalPeriod"],
        "sec:FiscalYear"[$profile-name eq "sec" and $names = "fiscalYear"],
        "xbrl:Concept"[$names = "concept"],
        "xbrl:Entity"[$names = ("cik", "tag", "ticker", "sic")],
        "sec:DefaultLegalEntity"[$profile-name eq "sec"]))
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube() as object
{
    let $hypercube-spec :=
    {|
        for $parameter in local:param-names()
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (local:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $type as string? :=
            (local:param-values($dimension-name || "::type"), local:param-values($dimension-name || ":type"))[1]

        let $values := local:param-values($dimension-name)
        let $typed-values := if (exists($type)) then local:cast-sequence($values, $type) else $values

        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (local:param-values($dimension-name || "::default"), local:param-values($dimension-name || ":default"))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value

        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                { "Domain" : [ $typed-values ] }[(exists($typed-values) and not($all))],
                { "Default" : $typed-default-value }[$has-default]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($hypercube-spec)
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token             as string? external;
declare  %rest:case-insensitive                 variable $profile-name      as string  external := "sec";
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $validate          as boolean external := false;
declare  %rest:case-insensitive                 variable $labels            as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic)
let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $hypercube := local:hypercube()

let $facts :=
    let $options := {|
      {
        Hypercube : $hypercube,
        Validate: $validate
      },
      { "ConceptMaps" : $map }[exists($map)],
      { "Rules" : [ $rule ] }[exists($rule)]
    |}
    return if($profile-name eq "sec")
    then sec:facts-for($options)
    else facts:facts-for($options)

let $facts :=
  let $archives as string* := distinct-values($facts.Aspects."sec:Archive")
  let $concept-names as string* := distinct-values($facts.Aspects."xbrl:Concept")
  let $concepts as object* :=
      (
          concepts:concepts($concept-names, $archives, $concepts:ANY_COMPONENT_LINK_ROLE),
          (reports:concepts($report))[$$.Name = $concept-names]
      )
  let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $concepts:AMERICAN_ENGLISH )[1]
  let $roles as string* := ( $report.Role, $concepts:ANY_COMPONENT_LINK_ROLE )
  for $fact as object in $facts
  let $entityName as string := $entities[$$._id eq $fact.Aspects."xbrl:Entity"].Profiles.SEC.CompanyName
  return
    if(not $labels)
    then $fact
    else {|
      $fact,
      let $concept-labels as object? := facts:labels($fact, $roles, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
      let $standard-labels as object := conversion:get-standard-labels($fact, $entityName)
      return { Labels : {| $concept-labels, $standard-labels |} }
    |}

let $facts := api:normalize-facts($facts)

let $result := {
    NetworkIdentifier : "http://bizql.io/facts",
    TableName : "xbrl:Facts",
    Hypercube: $hypercube,
    FactTable : [ $facts ]
}
let $comment :=
{
    NumFacts: count($facts),
    TotalNumFacts: session:num-facts(),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "facts")
return api:check-and-return-results($token, $results, $format)

import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace rules = "http://28.io/modules/xbrl/rules";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace request = "http://www.28msec.com/modules/http-request";

declare function local:param-values($name as string) as string*
{
    switch(true)
     case $name eq "xbrl:Concept"
        return (request:param-values("concept"), request:param-values("xbrl:Concept"))
     case $name eq "sec:FiscalYear" and $profile-name eq "sec"
        return string(api:preprocess-fiscal-years(($fiscalYear, request:param-values("sec:FiscalYear"))))
     case $name eq "sec:FiscalPeriod" and $profile-name eq "sec"
        return api:preprocess-fiscal-periods(($fiscalPeriod, request:param-values("sec:FiscalPeriod")))
     case $name eq "dei:LegalEntityAxis" and $profile-name eq "sec"
        return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis")
     case $name eq "dei:LegalEntityAxis::default" return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis::default")
     case $name eq "xbrl:Entity" and $profile-name eq "sec" return (
         let $companies := companies:companies(
            $cik,
            api:preprocess-tags($tag),
            $ticker,
            $sic)._id
        return if(empty(($cik,$tag,$ticker,$sic)) or exists($companies))
               then $companies
               else "dummy",
        request:param-values("xbrl:Entity"))
     case $name eq "sec:Archive" and $profile-name eq "sec" return (
            let $fiscalYears := local:param-values("sec:FiscalYear")
            let $fiscalPeriods := local:param-values("sec:FiscalPeriod")
            let $entities := local:param-values("xbrl:Entity")
            return
                if($fiscalYears = "LATEST")
                then fiscal-core:latest-filings($entities, $fiscalPeriods)
                else (),
            $aid,
            request:param-values("sec:Archive")
        )
     default return request:param-values($name)
};

declare function local:param-names() as string*
{
    let $names := request:param-names()
    return distinct-values((
        $names[contains($$, ":")],
        "sec:Accepted"[$profile-name eq "sec"],
        "sec:FiscalPeriod"[$profile-name eq "sec" and $names = "fiscalPeriod"],
        "sec:FiscalYear"[$profile-name eq "sec" and $names = "fiscalYear"],
        "xbrl:Concept"[$names = "concept"],
        "sec:DefaultLegalEntity"[$profile-name eq "sec"]))
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube() as object
{
    let $hypercube-spec :=
    {|
        for $parameter in local:param-names()
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (local:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $type as string? :=
            (local:param-values($dimension-name || "::type"), local:param-values($dimension-name || ":type"))[1]

        let $values := local:param-values($dimension-name)
        let $typed-values := if (exists($type)) then local:cast-sequence($values, $type) else $values

        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (local:param-values($dimension-name || "::default"), local:param-values($dimension-name || ":default"))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value

        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                { "Domain" : [ $typed-values ] }[(exists($typed-values) and not($all))],
                { "Default" : $typed-default-value }[$has-default]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($hypercube-spec)
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token             as string? external;
declare  %rest:case-insensitive                 variable $profile-name      as string  external := "generic";
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $validate          as boolean external := false;
declare  %rest:case-insensitive                 variable $labels            as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic)
let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $hypercube := local:hypercube()

let $facts :=
    let $options := {|
      {
        Hypercube : $hypercube,
        Validate: $validate
      },
      { "ConceptMaps" : $map }[exists($map)],
      { "Rules" : [ $rule ] }[exists($rule)]
    |}
    return if($profile-name eq "sec")
    then sec:facts-for($options)
    else facts:facts-for($options)

let $facts :=
  let $archives as string* := distinct-values($facts.Aspects."sec:Archive")
  let $concept-names as string* := distinct-values($facts.Aspects."xbrl:Concept")
  let $concepts as object* :=
      (
          concepts:concepts($concept-names, $archives, $concepts:ANY_COMPONENT_LINK_ROLE),
          (reports:concepts($report))[$$.Name = $concept-names]
      )
  let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $concepts:AMERICAN_ENGLISH )[1]
  let $roles as string* := ( $report.Role, $concepts:ANY_COMPONENT_LINK_ROLE )
  for $fact as object in $facts
  let $entityName as string := $entities[$$._id eq $fact.Aspects."xbrl:Entity"].Profiles.SEC.CompanyName
  return
    if(not $labels)
    then $fact
    else {|
      $fact,
      let $concept-labels as object? := facts:labels($fact, $roles, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
      let $standard-labels as object := conversion:get-standard-labels($fact, $entityName)
      return { Labels : {| $concept-labels, $standard-labels |} }
    |}

let $facts := api:normalize-facts($facts)

let $result := {
    NetworkIdentifier : "http://bizql.io/facts",
    TableName : "xbrl:Facts",
    FactTable : [ $facts ]
}
let $comment :=
{
    NumFacts: count($facts),
    TotalNumFacts: session:num-facts(),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "facts")
return api:check-and-return-results($token, $results, $format)


update jq

update jq

# from this directory, run ./run-ffprobe-jq-song-length.sh

recurse(.[] | flatten)

# from this directory, run ./run-ffprobe-jq-song-length.sh

reduce to_entries[] as $pair ([]; . + [$pair]) | .[]


Added <p/>.

Added <p/>.

jsoniq version "1.0";

(:
 : Copyright 2006-2011 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

(:===========================================================================:)

(:~
 : This module provides an API for parsing and serializing CSV (comma-separated
 : values) files.
 : See RFC 4180,
 : "Common Format and MIME Type for Comma-Separated Values (CSV) Files."
 : @project Zorba/CSV
 :)

module namespace csv = "http://zorba.io/modules/json-csv";

declare namespace err = "http://www.w3.org/2005/xqt-errors";
declare namespace zerr = "http://zorba.io/errors";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:===========================================================================:)

(:~
 : Parses a CSV (comma-separated values) string using the given options.
 : A newline (U+000A), optionally preceeded by a carriage-return (U+000D),
 : terminates lines, aka, "records."
 :

 : Quoted values are always considered strings;
 : unquoted values are attempted to be cast to other types, e.g., integer
 : (unless the cast-unquoted-values option is false).
 : Casting is attempted in the following order:
 : integer, decimal, double, and boolean.
 : If casting fails, the value is considered a string.
 : Header field names are always considered strings even if unquoted.
 :

 : In addition to the "normal" values of
 : true and false for boolean,
 : T and Y are also considered "true"
 : and F and N are also considered "false."
 :
 : @param $csv The CSV string to parse.
 : @param $options The options to use:
 :

 :
cast-unquoted-values

 :

 :        Whether to attempt to cast unquoted values to
 :        integer, decimal, double, or boolean;
 :        default: true.
 :

 :
extra-name

 :

 :        The field name for extra values, if any;
 :        default: none (error zerr:ZCSV0003 is raised).
 :        If this option is given and a line contains one or more extra values
 :        (that is, values that have no corresponding field names),
 :        then the extra values are assigned as the values
 :        for fields having extra-name as their names.
 :

 :        If extra-name contains a # (U+0023),
 :        then the # is substituted with the field number
 :        (where field numbers start at 1).
 :        If extra-name does not contains a #,
 :        then the field number is appended.
 :

 :
field-names

 :

 :        A JSON array of strings denoting field names;
 :        default: none.
 :        If this option is given, then the first CSV line is assumed not to be
 :        a header line; if omitted, then the first CSV line is assumed to be a
 :        header line and the field names are taken from this line.
 :

 :
missing-value

 :

 :        What should happen when a missing value is detected;
 :        default: "null".
 :        A "missing" value is one of:
 :

 :
Two consecutive separator characters.

 :
A separator character as either the first
 :              or last character on a line.

 :
Fewer values than the number of field names.

 :

 :        When a missing value is detected,
 :        the value of this option determines what happens:
 :

 :
"error"

 :
Error zerr:ZCSV0002 is raised.

 :
"omit"

 :
Both the value and its key are omitted from the result
 :                object.

 :
"null"

 :
The value is set to null.

 :

 :

 :
quote-char

 :

 :        The single ASCII character that may be used to quote values;
 :        default: " (U+0022).
 :

 :
quote-escape

 :

 :        The single ASCII character used to escape quote-char;
 :        default: same as quote-char.
 :        If quote-escape equals quote-char,
 :        it means that quote-char must be doubled to escape it.
 :        If quote-escape does not equal quote-char,
 :        it means that quote-escape is used to escape
 :        quote-char.
 :        For example, a quote-char of " (U+0022)
 :        and a quote-escape of \ (U+005C) means that
 :        quotes will be escaped by \".
 :

 :
separator

 :

 :        The single ASCII character used to separate values;
 :        default: , (U+002C).
 :

 :

 : @return a sequence of zero or more JSON objects where each key is a field
 : name and each value is a parsed value.
 : @error zerr:ZCSV0001 if the quote-char,
 : quote-escape, or separator option is given
 : and it's not a single ASCII character.
 : @error zerr:ZCSV0002 if a missing value is detected and the
 : missing-value option is "error".
 : @error zerr:ZCSV0003 if an extra value is detected and the
 : extra-name option is not set.
 :)
declare function csv:parse( $csv as string, $options as object() )
  as object()* external;

(:~
 : Parses a CSV (comma-separated values) string using the default options.
 : A newline (U+000A), optionally preceeded by a carriage-return (U+000D),
 : terminates lines, aka, "records."
 :

 : Quoted values are always considered strings;
 : unquoted values are attempted to be cast to other types, e.g., integer
 : (unless the cast-unquoted-values option is false).
 : Casting is attempted in the following order:
 : integer, decimal, double, and boolean.
 : If casting fails, the value is considered a string.
 : Header field names are always considered strings even if unquoted.
 :

 : In addition to the "normal" values of
 : true and false for boolean,
 : T and Y are also considered "true"
 : and F and N are also considered "false."
 :

 : The default options are:
 :

 :
cast-unquoted-values

 :

 :        Whether to attempt to cast unquoted values to
 :        integer, decimal, double, or boolean;
 :        default: true.
 :

 :
extra-name

 :

 :        The field name for extra values, if any;
 :        default: none (error zerr:ZCSV0003 is raised).
 :

 :
field-names

 :

 :        A JSON array of strings denoting field names;
 :        default: none.
 :        The first CSV line is assumed to be a header line
 :        and the field names are taken from this line.
 :

 :
missing-value

 :

 :        What should happen when a missing value is detected;
 :        default: "null".
 :        A "missing" value is one of:
 :

 :
Two consecutive quote-char characters.

 :
A quote-char character as either the first
 :              or last character on a line.

 :
Fewer values than the number of field names.

 :

 :        When a missing value is detected,
 :        the value is set to null.
 :

 :
quote-char

 :

 :        The single ASCII character that may be used to quote values;
 :        default: " (U+0022).
 :

 :
quote-escape

 :

 :        The single ASCII character used to escape quote-char;
 :        default: same as quote-char.
 :        This means that an escaped quote is doubled as "".
 :

 :
separator

 :

 :        The single ASCII character used to separate values;
 :        default: , (U+002C).
 :

 :

 :
 : @param $csv The CSV string to parse.
 : @return a sequence of zero or more JSON objects where each key is a field
 : name and each value is a parsed value.
 : @error zerr:ZCSV0003 if an extra value is detected.
 :)
declare function csv:parse( $csv as string )
  as object()*
{
  csv:parse( $csv, {} )
};

(:~
 : Serializes a sequence of JSON objects as CSV (comma-separated values) using
 : the given options.
 :
 : @param $obj The sequence of JSON objects to serialize.
 : @param $options The options to use:
 :

 :
field-names

 :

 :        A JSON array of strings denoting field names;
 :        default: none.
 :        If this option is not set,
 :        the field names are taken from the first JSON object
 :        and the order of the fields is implementation dependent.
 :        If this option is set,
 :        the fields are serielized in the order they are in the array.
 :        In either case,
 :        every JSON object must have the same keys as the first object.
 :

 :
serialize-boolean-as

 :

 :        What strings to serialize true and false as;
 :        default: true and false.
 :        This must be a sub-object with the two keys
 :        "true" and "false", e.g.:
 :        { "true" : "Y", "false" : "N" }.
 :

 :
serialize-header

 :

 :        Whether a header line is included;
 :        default: true.
 :        If true,
 :        the first string result is the header line
 :        comprised of all the objects' keys' names;
 :        if false,
 :        the heder line is not returned.
 :

 :
serialize-null-as

 :

 :        What string to serialize JSON null values as;
 :        default: null.
 :

 :
quote-char

 :

 :        The single ASCII character that may be used to quote values;
 :        default: " (U+0022).
 :

 :
quote-escape

 :

 :        The single ASCII character used to escape quote-char;
 :        default: same as quote-char.
 :        If quote-escape equals quote-char,
 :        it means that quote-char must be doubled to escape it.
 :        If quote-escape does not equal quote-char,
 :        it means that quote-escape is used to escape
 :        quote-char.
 :        For example, a quote-char of " (U+0022)
 :        and a quote-escape of \ (U+005C) means that
 :        quotes will be escaped by \".
 :

 :
separator

 :

 :        The single ASCII character used to separate values;
 :        default: , (U+002C).
 :

 :

 : @return a sequence of strings where each string corresponds to a JSON
 : object, aka, "record."
 :)
declare function csv:serialize( $obj as object()*, $options as object() )
  as string* external;

(:~
 : Serializes a sequence of JSON objects as CSV (comma-separated values) using
 : the default options.
 : The default options are:
 :

 :
field-names

 :

 :        A JSON array of strings denoting field names;
 :        default: none.
 :        The field names are taken from the first JSON object
 :        and the order of the fields is implementation dependent.
 :

 :
serialize-boolean-as

 :

 :        What strings to serialize true and false as;
 :        default: true and false.
 :

 :
serialize-header

 :

 :        Whether a header line is included;
 :        default: true.
 :        The first string result is the header line
 :        comprised of all the objects' keys' names.
 :

 :
serialize-null-as

 :

 :        What string to serialize JSON null values as;
 :        default: null.
 :

 :
quote-char

 :

 :        The single ASCII character that may be used to quote values;
 :        default: " (U+0022).
 :

 :
quote-escape

 :

 :        The single ASCII character used to escape quote-char;
 :        default: same as quote-char.
 :        This means that quote-char is doubled to escape it.
 :

 :
separator

 :

 :        The single ASCII character used to separate values;
 :        default: , (U+002C).
 :

 :

 :
 : @param $obj The sequence of JSON objects to serialize.
 : @return a sequence of strings where each string corresponds to a JSON
 : object, aka, "record."
 :)
declare function csv:serialize( $obj as object()* )
  as string*
{
  csv:serialize( $obj, {} )
};

(:===========================================================================:)

(: vim:set syntax=xquery et sw=2 ts=2: :)


improved entities query according to new functionality

import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-xml($entities as object*) as element()*
{
  <Entities>{
    for $e in $entities
    return
    <Entity>
        <ID>{$e._id}</ID>
        <Profile name="{$e.Profiles.SEC.Name}">
            <CompanyName>{$e.Profiles.SEC.CompanyName}</CompanyName>
            <CompanyType>{$e.Profiles.SEC.CompanyType}</CompanyType>
            <SIC>{$e.Profiles.SEC.SIC}</SIC>
            <SICDescription>{$e.Profiles.SEC.SICDescription}</SICDescription>
            <SICGroup>{$e.Profiles.SEC.SICGroup}</SICGroup>
            <Sector>{$e.Profiles.SEC.Sector}</Sector>
            <IsTrust>{$e.Profiles.SEC.IsTrust}</IsTrust>
            <Tickers>{
                for $t in $e.Profiles.SEC.Tickers[]
                return <Ticker>{$t}</Ticker>
            }</Tickers>
            <Tags>
            {
                for $t in $e.Profiles.SEC.Tags[]
                return <Tag>{$t}</Tag>
            }
            </Tags>
        </Profile>
    </Entity>
  }</Entities>
};

declare function local:to-csv($entities as object*) as string*
{
    csv:serialize(
        for $e in $entities
        return  {
            ID : $e._id,
            Profile : "SEC",
            CompanyName : $e.Profiles.SEC.CompanyName,
            CompanyType : $e.Profiles.SEC.CompanyType,
            SIC : $e.Profiles.SEC.SIC,
            SICDescription : $e.Profiles.SEC.SICDescription,
            SICGroup :$e.Profiles.SEC.SICGroup,
            Sector : $e.Profiles.SEC.Sector,
            IsTrust : $e.Profiles.SEC.IsTrust,
            Tickers : string-join($e.Profiles.SEC.Tickers[], " "),
            Tags : string-join($e.Profiles.SEC.Tags[], " ")
        }
    )
};

let $format  := lower-case(substring-after(request:path(), ".jq.")) (: text, xml, or json (default) :)
let $ciks    := request:param-values("cik")
let $indexes := request:param-values("index") ! upper-case($$) (: DOW30, SP500, FORTUNE100 :)
let $tickers := request:param-values("ticker")
let $entities :=
    for $entity in
        if (exists(($ciks, $indexes, $tickers)))
        then
            for $entity in (companies:companies($ciks),
                       companies:companies-for-tags($indexes),
                       companies:companies-for-tickers($tickers))
            group by companies:eid($entity) (: duplicate elimination :)
            return $entity[1]
        else
            entities:entities() (: companies:companies() ? :)
    order by $entity.Profiles.SEC.CompanyName (: companies:name() ? :)
    return $entity
return
    switch ($format)
        case "xml"  return local:to-xml($entities)
        case "text"  return string-join(local:to-csv($entities))
        default return [ $entities ]

import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace sec = "http://xbrl.io/modules/bizql/profiles/sec/core";

declare function local:to-xml($entities as object*) as element()*
{
  <Entities>{
    for $e in $entities
    return
    <Entity>
        <ID>{$e._id}</ID>
        <Profile name="{$e.Profiles.SEC.Name}">
            <CompanyName>{$e.Profiles.SEC.CompanyName}</CompanyName>
            <CompanyType>{$e.Profiles.SEC.CompanyType}</CompanyType>
            <SIC>{$e.Profiles.SEC.SIC}</SIC>
            <SICDescription>{$e.Profiles.SEC.SICDescription}</SICDescription>
            <SICGroup>{$e.Profiles.SEC.SICGroup}</SICGroup>
            <Sector>{$e.Profiles.SEC.Sector}</Sector>
            <IsTrust>{$e.Profiles.SEC.IsTrust}</IsTrust>
            <Tickers>{
                for $t in $e.Profiles.SEC.Tickers[]
                return <Ticker>{$t}</Ticker>
            }</Tickers>
            <Tags>
            {
                for $t in $e.Profiles.SEC.Tags[]
                return <Tag>{$t}</Tag>
            }
            </Tags>
        </Profile>
    </Entity>
  }</Entities>
};

declare function local:to-csv($entities as object*) as string*
{
    csv:serialize(
        for $e in $entities
        return  {
            ID : $e._id,
            Profile : "SEC",
            CompanyName : $e.Profiles.SEC.CompanyName,
            CompanyType : $e.Profiles.SEC.CompanyType,
            SIC : $e.Profiles.SEC.SIC,
            SICDescription : $e.Profiles.SEC.SICDescription,
            SICGroup :$e.Profiles.SEC.SICGroup,
            Sector : $e.Profiles.SEC.Sector,
            IsTrust : $e.Profiles.SEC.IsTrust,
            Tickers : string-join($e.Profiles.SEC.Tickers[], " "),
            Tags : string-join($e.Profiles.SEC.Tags[], " ")
        }
    )
};

declare function local:only-dow30($entities as object*) as boolean
{
    count(
        for $e in $entities
        where count($e.Profiles.SEC.Tags[]) gt 0 and $e.Profiles.SEC.Tags[] = "DOW30"
        return $e) eq count($entities)
};

declare function local:pad-ciks($ciks as string*) as string*
{
    for $cik in $ciks
    let $cik := normalize-space($cik)
    where $cik castable as integer
    return
        format-integer($cik cast as integer, "0000000000") cast as string
};

let $format  := lower-case(substring-after(request:path(), ".jq.")) (: text, xml, or json (default) :)
let $ciks    := local:pad-ciks(request:param-values("cik")) ! sec:normalize-cik($$)
let $indexes := request:param-values("index") ! upper-case($$) (: DOW30, SP500, FORTUNE100 :)
let $tickers := request:param-values("ticker")
let $entities :=
    for $e in
        if (exists($ciks) or exists($indexes) or exists($tickers))
        then
            for $e in (entities:entities($ciks),
                       companies:companies-for-tags($indexes),
                       companies:companies-for-tickers($tickers))
            group by $e._id (: duplicate elimination :)
            return $e[1]
        else
            entities:entities()
    order by $e.Profiles.SEC.CompanyName
    return  $e
return
    switch ($format)
        case "xml"  return local:to-xml($entities)
        case "text"  return string-join(local:to-csv($entities), "")
        default return [ $entities ]


Update rxnfilter.jq

Update rxnfilter.jq

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv
.[]
#start from the .results array
|.results

#Creates new object with .safetyreportid string and .patient.reaction array
|map(with_entries(select(.key == "safetyreportid", .key == "receiptdate")) + (.patient.reaction[]))

# Grabs all unique keys and spits out CSV
|["receiptdate","safetyreportid","reactionmeddrapt","reactionmeddraversionpt","reactionoutcome"] as $keys
|$keys, (.[]|[.[$keys[]]])|@csv

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv
.[]
#start from the .results array
|.results

#Creates new object with .safetyreportid string and .patient.reaction array
|map(with_entries(select(.key == "safetyreportid")) + (.patient.reaction[]))

# Grabs all unique keys and spits out CSV
|["safetyreportid","reactionmeddrapt","reactionmeddraversionpt","reactionoutcome"] as $keys
|$keys, (.[]|[.[$keys[]]])|@csv


:green_heart: Fix test.

:green_heart: Fix test.

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facts"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-404($params as object) as item
{
    let $endpoint := "facts"
    let $res as object := test:invoke-raw($endpoint, $params)
    return if($res.status eq 404)
           then true
           else {
              url: test:url($endpoint, $params),
              unexpectedResponse: $res
           }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw($endpoint, $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "SEC Acceptance Date [Axis],Archive [Axis],Fiscal Period [Axis],Fiscal Year [Axis],Fiscal Period Type [Axis],Period [Axis],Reporting Entity [Axis],Concept [Axis],Unit [Axis],Legal Entity [Axis],Value,Decimals\r\n20131024121047,0000021344-13-000050,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO,\"Cash and Cash Equivalents, at Carrying Value\",USD,Default Legal Entity [Member],11118000000,-6\r\n20131024121047,0000021344-13-000050,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO,Assets,USD,Default Legal Entity [Member],89432000000,-6\r\n"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res,
        expected: $expectedLines
    }
};

declare %an:nondeterministic function local:test-labels-aids() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: "disc:AdvertisingCostsPolicyTextBlock",
        report: "Disclosures",
        format: "json",
        fiscalYear: 2014,
        fiscalPeriod: "FY",
        labels: true,
        aid: "0000858877-14-000029"
    }
    let $res := test:invoke($endpoint, $params)
    let $actual := [
            for $labels in $res[2].FactTable[].Labels
            return (keys($labels) ! $labels.$$)
        ]
    let $expected := [ "Archive [Axis]", "SEC Acceptance Date [Axis]", "Fiscal Year [Axis]", "Non-Numeric [Member]", "Default Legal Entity [Member]", "Unit [Axis]", "Advertising Costs, Policy", "Fiscal Period [Axis]", "Concept [Axis]", "Reporting Entity [Axis]", "Legal Entity [Axis]", "Fiscal Period Type [Axis]", "Period [Axis]", "From July 28, 2013 to July 26, 2014", "CISCO SYSTEMS, INC." ]
    let $status as integer := $res[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

local:check({
    cocacola-latest:
    (: this test will fail and needs to be updated if a newer report has been filed.
       current latest filing: 2014 :)
    local:test-facttable(462, {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    cocacola-all: local:test-facttable(468, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-all-q3: local:test-facttable(341, {
        ticker:"ko",
        fiscalYear: 2014,
        fiscalPeriod: [ "Q3" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-instant: local:test-facttable(163, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant" ]
    }),
    cocacola-ytd: local:test-facttable(305, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-qtd: local:test-facttable(90, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "Q2" ],
        fiscalPeriodType: [ "QTD" ]
    }),
    cocacolaCSVLabels: local:test-labels(),
    ciscoLabelsByAid: local:test-labels-aids(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(30, {
        tag:"DOW30",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-404({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    }),
    quartersfact: local:test-facttable(0, { (: this fact is not reported in the raw XML filing :)
        aid: "0000051143-13-000007",
        concept: "disc:AssetRetirementObligationsPolicy",
        fiscalPeriod: "Q3",
        fiscalYear: "2013",
        map: "Disclosures"
    }),
    quartersfact2: local:test-facttable(1, {
        aid: "0001193125-12-207154",
        concept: "disc:CashFlowOperatingCapitalTableTextBlock",
        fiscalPeriod: "Q1",
        fiscalYear: "2012",
        map: "Disclosures"
    }),
    generic: local:test-facttable(2, {
        "xbrl:Entity":"http://www.sec.gov/CIK 0000021344",
        "sec:Accepted":"20140227132423",
        concept:"us-gaap:Assets",
        profile-name: "generic"
    }),
    count: test:invoke-and-assert-deep-equal(
          "facts",
          {
              "ticker" : "ko",
              "fiscalYear" : ["2012", "2013", "2014"],
              "fiscalPeriod" : "FY",
              "count" : "true"
          },
          function($res as item*) as item* { $res.Statistics.NumFacts },
          1405
        )
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facts"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-404($params as object) as item
{
    let $endpoint := "facts"
    let $res as object := test:invoke-raw($endpoint, $params)
    return if($res.status eq 404)
           then true
           else {
              url: test:url($endpoint, $params),
              unexpectedResponse: $res
           }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw($endpoint, $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "SEC Acceptance Date [Axis],Archive [Axis],Fiscal Period [Axis],Fiscal Year [Axis],Fiscal Period Type [Axis],Period [Axis],Reporting Entity [Axis],Concept [Axis],Legal Entity [Axis],Unit,Value,Decimals\r\n20131024121047,0000021344-13-000050,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO,\"Cash and Cash Equivalents, at Carrying Value\",Default Legal Entity [Member],USD,11118000000,-6\r\n20131024121047,0000021344-13-000050,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO,Assets,Default Legal Entity [Member],USD,89432000000,-6\r\n"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res,
        expected: $expectedLines
    }
};

declare %an:nondeterministic function local:test-labels-aids() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: "disc:AdvertisingCostsPolicyTextBlock",
        report: "Disclosures",
        format: "json",
        fiscalYear: 2014,
        fiscalPeriod: "FY",
        labels: true,
        aid: "0000858877-14-000029"
    }
    let $res := test:invoke($endpoint, $params)
    let $actual := [
            for $labels in $res[2].FactTable[].Labels
            return (keys($labels) ! $labels.$$)
        ]
    let $expected := [ "Archive [Axis]", "SEC Acceptance Date [Axis]", "Fiscal Year [Axis]", "Non-Numeric [Member]", "Default Legal Entity [Member]", "Unit [Axis]", "Advertising Costs, Policy", "Fiscal Period [Axis]", "Concept [Axis]", "Reporting Entity [Axis]", "Legal Entity [Axis]", "Fiscal Period Type [Axis]", "Period [Axis]", "From July 28, 2013 to July 26, 2014", "CISCO SYSTEMS, INC." ]
    let $status as integer := $res[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

local:check({
    cocacola-latest:
    (: this test will fail and needs to be updated if a newer report has been filed.
       current latest filing: 2014 :)
    local:test-facttable(462, {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    cocacola-all: local:test-facttable(468, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-all-q3: local:test-facttable(341, {
        ticker:"ko",
        fiscalYear: 2014,
        fiscalPeriod: [ "Q3" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-instant: local:test-facttable(163, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant" ]
    }),
    cocacola-ytd: local:test-facttable(305, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-qtd: local:test-facttable(90, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "Q2" ],
        fiscalPeriodType: [ "QTD" ]
    }),
    cocacolaCSVLabels: local:test-labels(),
    ciscoLabelsByAid: local:test-labels-aids(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(30, {
        tag:"DOW30",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-404({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    }),
    quartersfact: local:test-facttable(0, { (: this fact is not reported in the raw XML filing :)
        aid: "0000051143-13-000007",
        concept: "disc:AssetRetirementObligationsPolicy",
        fiscalPeriod: "Q3",
        fiscalYear: "2013",
        map: "Disclosures"
    }),
    quartersfact2: local:test-facttable(1, {
        aid: "0001193125-12-207154",
        concept: "disc:CashFlowOperatingCapitalTableTextBlock",
        fiscalPeriod: "Q1",
        fiscalYear: "2012",
        map: "Disclosures"
    }),
    generic: local:test-facttable(2, {
        "xbrl:Entity":"http://www.sec.gov/CIK 0000021344",
        "sec:Accepted":"20140227132423",
        concept:"us-gaap:Assets",
        profile-name: "generic"
    }),
    count: test:invoke-and-assert-deep-equal(
          "facts",
          {
              "ticker" : "ko",
              "fiscalYear" : ["2012", "2013", "2014"],
              "fiscalPeriod" : "FY",
              "count" : "true"
          },
          function($res as item*) as item* { $res.Statistics.NumFacts },
          1405
        )
})


Update rxnfilter.jq

Update rxnfilter.jq

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv
.[]
#start from the .results array
|.results

#Creates new object with .safetyreportid string and .patient.reaction array
|map(with_entries(select(.key == "safetyreportid")) + (.patient.reaction[]))

# Grabs all unique keys and spits out CSV
|["safetyreportid","reactionmeddrapt","reactionmeddraversionpt","reactionoutcome"] as $keys
|$keys, (.[]|[.[$keys[]]])|@csv

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv
.[]
#start from the .results array
|.results

#Creates new object with .safetyreportid string and .patient.reaction array
|map(with_entries(select(.key == "safetyreportid")) + (.patient.reaction[]))

# Grabs all unique keys and spits out CSV
|(map(keys)|add|unique) as $keys| $keys, (.[]|[.[$keys[]]])|@csv


Update patientfilter.jq

Update patientfilter.jq

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv
.[]
# start from the .results array
|.results

# remove .patient.drug array
|map(del(.patient.drug))

# remove .patient.reaction array
|map(del(.patient.reaction))

# flattens .patient object and renames child keys with patient_ prefix
|map(with_entries(select(.key != "patient")) +
       (.patient|with_entries(.key = "patient_" + .key)))

# flattens all the objects and renames child keys
|map(with_entries(select(.key != "primarysource" and
                         .key != "sender" and
                         .key != "receiver" and
                         .key != "patient_summary" and
                         .key != "reportduplicate" and
                         .key != "patient_patientdeath")) +
       (.primarysource//{primarysource:{"NA":"NA"}}|with_entries(.key = "primarysource_" + .key)) +
       (.sender//{sender:{"NA":"NA"}}|with_entries(.key = "sender_" + .key)) +
       (.receiver//{receiver:{"NA":"NA"}}|with_entries(.key = "receiver_" + .key)) +
       (.patient_summary//{patient_summary:{"NA":"NA"}}|with_entries(.key = "patient_summary_" + .key)) +
       (.report_duplicate//{report_duplicate:{"NA":"NA"}}|with_entries(.key = "report_duplicate_" + .key))+
       (.patient_patientdeath//{patient_patientdeath:{"NA":"NA"}}|with_entries(.key = "patient_patientdeath_" + .key)))

|map(del(.primarysource_primarysource))
|map(del(.sender_sender))
|map(del(.receiver_receiver))
|map(del(.report_duplicate_report_duplicate))
|map(del(.patient_summary_patient_summary))
|map(del(.patient_patientdeath_patient_patientdeath))

# grabs all unique keys and spits out CSV
|(map(keys)|add|unique) as $keys| $keys, (.[]|[.[$keys[]]])|@csv

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv
.[]
# start from the .results array
|.results

# remove .patient.drug array
|map(del(.patient.drug))

# remove .patient.reaction array
|map(del(.patient.reaction))

# flattens .patient object and renames child keys with patient_ prefix
|map(with_entries(select(.key != "patient")) +
       (.patient|with_entries(.key = "patient_" + .key)))

# flattens all the objects and renames child keys
|map(with_entries(select(.key != "primarysource" and
                         .key != "sender" and
                         .key != "receiver" and
                         .key != "patient_summary" and
                         .key != "reportduplicate" and
                         .key != "patient_patientdeath")) +
       (.primarysource//{primarysource:{"NA":"NA"}}|with_entries(.key = "primarysource_" + .key)) +
       (.sender//{sender:{"NA":"NA"}}|with_entries(.key = "sender_" + .key)) +
       (.receiver//{receiver:{"NA":"NA"}}|with_entries(.key = "receiver_" + .key)) +
       (.patient_summary//{patient_summary:{"NA":"NA"}}|with_entries(.key = "patient_summary_" + .key)) +
       (.report_duplicate//{report_duplicate:{"NA":"NA"}}|with_entries(.key = "report_duplicate_" + .key))+
       (.patient_patientdeath//{patient_patientdeath:{"NA":"NA"}}|with_entries(.key = "patient_patientdeath_" + .key)))

|map(del(.primarysource_primary_source))
|map(del(.sender_sender))
|map(del(.receiver_receiver))
|map(del(.report_duplicate_report_duplicate))
|map(del(.patient_summary_patient_summary))
|map(del(.patient_patientdeath_patient_patientdeath))

# grabs all unique keys and spits out CSV
|(map(keys)|add|unique) as $keys| $keys, (.[]|[.[$keys[]]])|@csv


Made audit-call non-sequential (DEBUG ONLY)

jsoniq version "1.0";

module namespace session = "http://apps.28.io/session";

import module namespace random = "http://zorba.io/modules/random";
import module namespace user = "http://apps.28.io/user";
import module namespace api = "http://apps.28.io/api";
import module namespace req = "http://www.28msec.com/modules/http-request";
import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace cred = "http://www.28msec.com/modules/credentials";

declare variable $session:tokens := "Tokens";
declare variable $session:VALID-TOKEN := "[a-z0-9\\-]+";
declare variable $session:ACCESS-ALLOWED := 0;
declare variable $session:ACCESS-DENIED := 1;
declare variable $session:ACCESS-AUTH-REQUIRED := 2;

(: a short lived token used in a UI session - e.g. the accounts login for app.secxbrl.info :)
declare variable $session:TOKEN-TYPE-LOGIN := "ui-login";
(: a short or long lived token used in an application :)
declare variable $session:TOKEN-TYPE-APP := "app";

declare variable $session:comment := "Generated by 28.io, for more information see http://www.28.io/xbrl (" || fn:current-dateTime() || ")";

declare function session:comment($format as string) as item*
{
    switch ($format)
    case "json" return { "Comment" : $session:comment }
    case "xml" return
        (
            comment { $session:comment },
            comment { req:server-name() || req:uri() }
        )
    default return error()
};

declare function session:comment($format as string, $stats as object)
  as item*
{
    switch ($format)
    case "json" return
        {|
            session:comment($format),
            { "Statistics" : $stats }
        |}
    case "xml" return
        (
            session:comment($format),
            comment { "Statistics: " || string-join(for $k in keys($stats) return concat ($k, ": ", $stats($k)), ", ") }
        )
    default return error()
};

declare function session:connect() as xs:anyURI
{
    let $cred := cred:credentials("MongoDB", "xbrl")
    return mongo:connect($cred)
};

declare function session:num-facts() as xs:integer
{
    let $conn := session:connect()
    return mongo:count($conn, "facts")
};

declare function session:num-archives() as xs:integer
{
    let $conn := session:connect()
    return mongo:count($conn, "archives")
};

declare function session:num-components() as xs:integer
{
    let $conn := session:connect()
    return mongo:count($conn, "components")
};

declare function session:num-concepts() as xs:integer
{
    let $conn := session:connect()
    return mongo:count($conn, "concepts")
};

declare function session:num-entities() as xs:integer
{
    let $conn := session:connect()
    return mongo:count($conn, "entities")
};

declare %an:sequential function session:start($user-id as xs:string, $expiration-date as xs:dateTime, $token-type as  xs:string)
as string
{
    session:cleanup();

    variable $token := random:uuid();
    variable $session :=
      {
        "_id"           : $token,
        user-id         : $user-id,
        expiration-date : $expiration-date,
        token-type      : $token-type
      };
    db:insert($session:tokens, $session);

    $token
};

declare function session:get($token as string)
as string
{
    api:validate-regexp("token", $token, $session:VALID-TOKEN);

    variable $session := find($session:tokens, { "_id" : $token });

    if (exists($session))
    then
        if ($session.expiration-date gt current-dateTime())
        then $session.user-id
        else fn:error(xs:QName("session:expired"), "Your session has expired at " || $session.expiration-date )
    else fn:error(xs:QName("session:expired"), "Your session is not valid")
};

declare %an:sequential function session:terminate($token as string)
as empty-sequence()
{
    api:validate-regexp("token", $token, $session:VALID-TOKEN);

    variable $session := find($session:tokens, { "_id" : $token });

    if (exists($session))
    then db:delete($session);
    else ();
};

declare %an:sequential function session:cleanup()
as empty-sequence()
{
    for $session in collection($session:tokens)
    where $session.expiration-date le current-dateTime()
    return db:delete($session);
};

declare function session:ensure-valid($token as string)
as string
{
    session:get($token) (: validates token :)
};

declare function session:is-valid($token as string?)
as boolean
{
    if (exists($token))
    then try {{ session:get($token); true }} catch * {{ false }} (: validates token :)
    else false
};

declare function session:ensure-right($token as string, $right-id as string)
as string
{
    variable $user-id := session:get($token); (: validates token :)
    if (user:is-authorized($user-id, $right-id))
    then $user-id
    else fn:error(xs:QName("session:missing-authorization"), "User does not have required right " || $right-id)
};

declare function session:has-right($token as string?, $right-id as string)
as boolean
{
    variable $user-id := session:get($token); (: validates token :)
    if (user:is-authorized($user-id, $right-id))
    then true
    else false
};

declare function session:has-access($token as string?, $right-id as string)
as integer
{
	try {{
	    if (exists($token))
	    then
	    {
	      if (session:has-right($token, $right-id))
	      then $session:ACCESS-ALLOWED
	      else $session:ACCESS-DENIED
	    }
	    else $session:ACCESS-AUTH-REQUIRED
	}}
	catch * {{
	    $session:ACCESS-AUTH-REQUIRED
	}}
};

declare function session:error($msg as string, $format as string?) as item
{
	switch ($format)
    case "xml" return <error><message>{$msg}</message></error>
	case "text" or "csv" or "excel" return $msg
    default return
	{
		"success" : false,
		"description" : $msg
	}
};

declare function session:audit-call($token as string?) as empty-sequence()
{
    let $dist-aspects := [ "xbrl:Concept", "xbrl:Entity", "xbrl:Period" ]
    let $facts := {
        KeyAspects : $dist-aspects,
        Aspects : {
            "xbrl:Concept" : "secxbrl:ClientIP",
            "xbrl:Entity" : if (session:is-valid($token)) then user:get-by-id(session:get($token)).email else "Anonymous",
            "xbrl:Period" : string(fn:current-dateTime()),
            "xbrl:Unit" : "xbrl:NonNumeric",
            "secxbrl:Query" : req:path()
        },
        Type : "NonNumericValue",
        Value : (req:header-value("X-FORWARDED-FOR"), req:remote-addr())[1]
    }
    return
        ()
        (:db:insert("audit", $facts);:)
};

jsoniq version "1.0";

module namespace session = "http://apps.28.io/session";

import module namespace random = "http://zorba.io/modules/random";
import module namespace user = "http://apps.28.io/user";
import module namespace api = "http://apps.28.io/api";
import module namespace req = "http://www.28msec.com/modules/http-request";
import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace cred = "http://www.28msec.com/modules/credentials";

declare variable $session:tokens := "Tokens";
declare variable $session:VALID-TOKEN := "[a-z0-9\\-]+";
declare variable $session:ACCESS-ALLOWED := 0;
declare variable $session:ACCESS-DENIED := 1;
declare variable $session:ACCESS-AUTH-REQUIRED := 2;

(: a short lived token used in a UI session - e.g. the accounts login for app.secxbrl.info :)
declare variable $session:TOKEN-TYPE-LOGIN := "ui-login";
(: a short or long lived token used in an application :)
declare variable $session:TOKEN-TYPE-APP := "app";

declare variable $session:comment := "Generated by 28.io, for more information see http://www.28.io/xbrl (" || fn:current-dateTime() || ")";

declare function session:comment($format as string) as item*
{
    switch ($format)
    case "json" return { "Comment" : $session:comment }
    case "xml" return
        (
            comment { $session:comment },
            comment { req:server-name() || req:uri() }
        )
    default return error()
};

declare function session:comment($format as string, $stats as object)
  as item*
{
    switch ($format)
    case "json" return
        {|
            session:comment($format),
            { "Statistics" : $stats }
        |}
    case "xml" return
        (
            session:comment($format),
            comment { "Statistics: " || string-join(for $k in keys($stats) return concat ($k, ": ", $stats($k)), ", ") }
        )
    default return error()
};

declare function session:connect() as xs:anyURI
{
    let $cred := cred:credentials("MongoDB", "xbrl")
    return mongo:connect($cred)
};

declare function session:num-facts() as xs:integer
{
    let $conn := session:connect()
    return mongo:count($conn, "facts")
};

declare function session:num-archives() as xs:integer
{
    let $conn := session:connect()
    return mongo:count($conn, "archives")
};

declare function session:num-components() as xs:integer
{
    let $conn := session:connect()
    return mongo:count($conn, "components")
};

declare function session:num-concepts() as xs:integer
{
    let $conn := session:connect()
    return mongo:count($conn, "concepts")
};

declare function session:num-entities() as xs:integer
{
    let $conn := session:connect()
    return mongo:count($conn, "entities")
};

declare %an:sequential function session:start($user-id as xs:string, $expiration-date as xs:dateTime, $token-type as  xs:string)
as string
{
    session:cleanup();

    variable $token := random:uuid();
    variable $session :=
      {
        "_id"           : $token,
        user-id         : $user-id,
        expiration-date : $expiration-date,
        token-type      : $token-type
      };
    db:insert($session:tokens, $session);

    $token
};

declare function session:get($token as string)
as string
{
    api:validate-regexp("token", $token, $session:VALID-TOKEN);

    variable $session := find($session:tokens, { "_id" : $token });

    if (exists($session))
    then
        if ($session.expiration-date gt current-dateTime())
        then $session.user-id
        else fn:error(xs:QName("session:expired"), "Your session has expired at " || $session.expiration-date )
    else fn:error(xs:QName("session:expired"), "Your session is not valid")
};

declare %an:sequential function session:terminate($token as string)
as empty-sequence()
{
    api:validate-regexp("token", $token, $session:VALID-TOKEN);

    variable $session := find($session:tokens, { "_id" : $token });

    if (exists($session))
    then db:delete($session);
    else ();
};

declare %an:sequential function session:cleanup()
as empty-sequence()
{
    for $session in collection($session:tokens)
    where $session.expiration-date le current-dateTime()
    return db:delete($session);
};

declare function session:ensure-valid($token as string)
as string
{
    session:get($token) (: validates token :)
};

declare function session:is-valid($token as string?)
as boolean
{
    if (exists($token))
    then try {{ session:get($token); true }} catch * {{ false }} (: validates token :)
    else false
};

declare function session:ensure-right($token as string, $right-id as string)
as string
{
    variable $user-id := session:get($token); (: validates token :)
    if (user:is-authorized($user-id, $right-id))
    then $user-id
    else fn:error(xs:QName("session:missing-authorization"), "User does not have required right " || $right-id)
};

declare function session:has-right($token as string?, $right-id as string)
as boolean
{
    variable $user-id := session:get($token); (: validates token :)
    if (user:is-authorized($user-id, $right-id))
    then true
    else false
};

declare function session:has-access($token as string?, $right-id as string)
as integer
{
	try {{
	    if (exists($token))
	    then
	    {
	      if (session:has-right($token, $right-id))
	      then $session:ACCESS-ALLOWED
	      else $session:ACCESS-DENIED
	    }
	    else $session:ACCESS-AUTH-REQUIRED
	}}
	catch * {{
	    $session:ACCESS-AUTH-REQUIRED
	}}
};

declare function session:error($msg as string, $format as string?) as item
{
	switch ($format)
    case "xml" return <error><message>{$msg}</message></error>
	case "text" or "csv" or "excel" return $msg
    default return
	{
		"success" : false,
		"description" : $msg
	}
};

declare %an:sequential function session:audit-call($token as string?) as empty-sequence()
{
    let $dist-aspects := [ "xbrl:Concept", "xbrl:Entity", "xbrl:Period" ]
    let $facts := {
        KeyAspects : $dist-aspects,
        Aspects : {
            "xbrl:Concept" : "secxbrl:ClientIP",
            "xbrl:Entity" : if (session:is-valid($token)) then user:get-by-id(session:get($token)).email else "Anonymous",
            "xbrl:Period" : string(fn:current-dateTime()),
            "xbrl:Unit" : "xbrl:NonNumeric",
            "secxbrl:Query" : req:path()
        },
        Type : "NonNumericValue",
        Value : (req:header-value("X-FORWARDED-FOR"), req:remote-addr())[1]
    }
    return db:insert("audit", $facts);
};


jq: Find by path even with nested fields [ci skip]

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v);

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: clean | select(is_error);
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: clean | select(is_warn);
def warn_strict: clean | select(is_warn_strict);
def info_level: 30;
def is_info: .level >= info_level;
def info: clean | select(is_info);
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: clean | select(is_debug);

# Components:
def chokidar: select(.component == "Chokidar");
def LocalChange: select(.component == "local/change");
def LocalWatcher: select(.component == "LocalWatcher");
def Merge: select(.component == "Merge");
def Metadata: select(.component == "Metadata");
def Pouch: select(.component == "Pouch");
def Sync: select(.component == "Sync");

# Find conflicts:
#
#     yarn jq -c conflicts path/to/logs*
#
def is_conflict: .msg == "resolveConflictAsync";
def conflicts: clean | select(is_conflict) | {time,path};

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
def path(pattern):
  clean | select(
    (
      (.path // (.doc | .path) // (.change | .doc | .path)),
      (.oldpath // (.was | .path)),
      ""
    )
      | strings
      | test(pattern)
  );

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#
def time(pattern):
    select(.time | test(pattern));

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path};

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v);

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: clean | select(is_error);
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: clean | select(is_warn);
def warn_strict: clean | select(is_warn_strict);
def info_level: 30;
def is_info: .level >= info_level;
def info: clean | select(is_info);
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: clean | select(is_debug);

# Components:
def chokidar: select(.component == "Chokidar");
def LocalChange: select(.component == "local/change");
def LocalWatcher: select(.component == "LocalWatcher");
def Merge: select(.component == "Merge");
def Metadata: select(.component == "Metadata");
def Pouch: select(.component == "Pouch");
def Sync: select(.component == "Sync");

# Find conflicts:
#
#     yarn jq -c conflicts path/to/logs*
#
def is_conflict: .msg == "resolveConflictAsync";
def conflicts: clean | select(is_conflict) | {time,path};

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
def path(pattern): clean | select((.path,.oldpath,"") | strings | test(pattern));

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#
def time(pattern):
    select(.time | test(pattern));

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path};

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );