Update api.jq

Update api.jq

jsoniq version "1.0";

module namespace api = "http://apps.28.io/api";

import module namespace session    = "http://apps.28.io/session";
import module namespace resp       = "http://www.28msec.com/modules/http-response";
import module namespace sec-fiscal = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

declare function api:validate-regexp($name as string, $value as string?, $regexp as string)
as ()
{
  if (exists($value))
  then
    if (matches($value, "^" || $regexp || "$"))
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " does not match reg. expression " || $regexp || ".");
  else ();
};

declare function api:validate-enum($name as string, $value as string?, $enum as string*)
as ()
{
  if (exists($value))
  then
    if ($value = $enum)
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " is not one of these: " || string-join($enum,", ") || ".");
  else ();
};

declare %private function api:sgpl($count as integer, $singular as string, $plural as string)
as string
{
    if ($count eq 1)
    then $count || " " || $singular
    else $count || " " || $plural
};

declare function api:passed($timestamp as xs:dateTime?)
as string
{
    if (empty($timestamp) or $timestamp eq null)
    then "Never"
    else
        let $now := current-dateTime()
        let $difference := $now - $timestamp
        let $years := years-from-duration($difference)
        let $month := months-from-duration($difference)
        let $days := days-from-duration($difference)
        let $hours := hours-from-duration($difference)
        let $minutes := minutes-from-duration($difference)
        return
            if ($years gt 0) then api:sgpl($years,"year ago","years ago")
            else if ($month gt 0) then api:sgpl($month,"month ago","month ago")
            else if ($days gt 0) then api:sgpl($days, "day ago", "days ago")
            else if ($hours gt 0) then api:sgpl($hours, "hour ago", "hours ago")
            else if ($minutes gt 0) then api:sgpl($minutes, "minute ago", "minutes ago")
            else if ($difference div xs:dayTimeDuration("PT1S") lt 0) then "future"
            else "Just Now"
};

declare function api:success() as object
{
  { "success" : true }
};

declare function api:success($data as object()) as object
{
  {|
     {"success" : true },
     $data
  |}
};

declare %an:sequential function api:check-and-return-results(
    $token as string?,
    $results as item*,
    $format as string?
) as item*
{
    switch(session:has-access($token, "data_sec"))
    case $session:ACCESS-ALLOWED return
        $results
    case $session:ACCESS-DENIED return {
          resp:status-code(403);
          session:error("access denied", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          resp:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()
};

declare function api:normalize-facts(
    $facts as object*) as object*
{
    for $fact in $facts
    return {|
        {
            "Aspects" : {|
                trim($fact.Aspects, ("xbrl:Unit"))
            |}
        },
        trim($fact, ("Aspects", "_id")),
        { Unit: $fact.Aspects."xbrl:Unit" }[exists($fact.Aspects."xbrl:Unit")]
    |}
};

declare %an:sequential function api:serialize(
    $result as json-item,
    $comment as object,
    $serializers as object,
    $format as string?,
    $file-name as string) as item*
{
    switch ($format)
    case "xml" return {
        resp:serialization-parameters({"omit-xml-declaration" : false, indent : true });
        session:comment("xml", $comment),
        $serializers.to-xml($result)
    }
    case "text" case "csv" return {
        resp:content-type("text/csv");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    case "excel" return {
        resp:content-type("application/vnd.ms-excel");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    default return {
        resp:content-type("application/json");
        resp:serialization-parameters({"indent" : true});
        {|
            session:comment("json", $comment),
            $result
        |}
    }
};

declare function api:preprocess-fiscal-years($fiscal-years as string*) as integer*
{
  distinct-values(
    for $fy in $fiscal-years ! upper-case($$)
    return switch($fy)
           case "LATEST" return $sec-fiscal:LATEST_FISCAL_YEAR
           case "ALL" return $sec-fiscal:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
  )
};

declare function api:preprocess-fiscal-periods($fiscal-periods as string*) as string*
{
  distinct-values(
    for $fp in $fiscal-periods ! upper-case($$)
    return switch($fp)
           case "Q1"
           case "Q2"
           case "Q3"
           case "FY"
             return $fp
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIODS
           default
             return error(xs:QName("local:INVALID-PERIOD"),
               $fp || ": fiscalPeriod values must be one or more of Q1, Q2, Q3, FY, ALL")
  )
};

declare function api:preprocess-format($format as string?, $request-uri as string) as string?
{
  let $request-path := tokenize($request-uri, "\\?")[1]
  return lower-case(($format, substring-after($request-path, ".jq."))[1])
};

declare function api:preprocess-tags($tags as string*) as string*
{
  distinct-values($tags ! upper-case($$))
};

declare function api:preprocess-boolean($name as string, $value as string)
as boolean
{
  if ($value eq "")
  then true
  else
  {
      api:validate-enum($name, lower-case($value), ("true", "false"));
      boolean($value)
  }
};

jsoniq version "1.0";

module namespace api = "http://apps.28.io/api";

import module namespace session    = "http://apps.28.io/session";
import module namespace resp       = "http://www.28msec.com/modules/http-response";
import module namespace sec-fiscal = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

declare function api:validate-regexp($name as string, $value as string?, $regexp as string)
as ()
{
  if (exists($value))
  then
    if (matches($value, "^" || $regexp || "$"))
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " does not match reg. expression " || $regexp || ".");
  else ();
};

declare function api:validate-enum($name as string, $value as string?, $enum as string*)
as ()
{
  if (exists($value))
  then
    if ($value = $enum)
    then ();
    else fn:error(xs:QName("api:bad-parameter"), "Provided parameter " || $name || " with value " || $value || " is not one of these: " || string-join($enum,", ") || ".");
  else ();
};

declare %private function api:sgpl($count as integer, $singular as string, $plural as string)
as string
{
    if ($count eq 1)
    then $count || " " || $singular
    else $count || " " || $plural
};

declare function api:passed($timestamp as xs:dateTime?)
as string
{
    if (empty($timestamp) or $timestamp eq null)
    then "Never"
    else
        let $now := current-dateTime()
        let $difference := $now - $timestamp
        let $years := years-from-duration($difference)
        let $month := months-from-duration($difference)
        let $days := days-from-duration($difference)
        let $hours := hours-from-duration($difference)
        let $minutes := minutes-from-duration($difference)
        return
            if ($years gt 0) then api:sgpl($years,"year ago","years ago")
            else if ($month gt 0) then api:sgpl($month,"month ago","month ago")
            else if ($days gt 0) then api:sgpl($days, "day ago", "days ago")
            else if ($hours gt 0) then api:sgpl($hours, "hour ago", "hours ago")
            else if ($minutes gt 0) then api:sgpl($minutes, "minute ago", "minutes ago")
            else if ($difference div xs:dayTimeDuration("PT1S") lt 0) then "future"
            else "Just Now"
};

declare function api:success() as object
{
  { "success" : true }
};

declare function api:success($data as object()) as object
{
  {|
     {"success" : true },
     $data
  |}
};

declare %an:sequential function api:check-and-return-results(
    $token as string?,
    $results as item*,
    $format as string?
) as item*
{
    switch(session:has-access($token, "data_sec"))
    case $session:ACCESS-ALLOWED return
        $results
    case $session:ACCESS-DENIED return {
          resp:status-code(403);
          session:error("access denied", $format)
       }
    case $session:ACCESS-AUTH-REQUIRED return {
          resp:status-code(401);
          session:error("authentication required or session expired", $format)
       }
    default return error()
};

declare function api:normalize-facts(
    $facts as object*) as object*
{
    for $fact in $facts
    return {|
        {
            "Aspects" : {|
                trim($fact.Aspects, ("xbrl:Unit"))
            |}
        },
        trim($fact, ("Aspects", "_id")),
        { Unit: $fact.Aspects."xbrl:Unit" }[exists($fact.Aspects."xbrl:Unit")]
    |}
};

declare %an:sequential function api:serialize(
    $result as json-item,
    $comment as object,
    $serializers as object,
    $format as string?,
    $file-name as string) as item*
{
    switch ($format)
    case "xml" return {
        resp:serialization-parameters({"omit-xml-declaration" : false, indent : true });
        session:comment("xml", $comment),
        $serializers.to-xml($result)
    }
    case "text" case "csv" return {
        resp:content-type("text/csv");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    case "excel" return {
        resp:content-type("application/vnd.ms-excel");
        resp:header("Content-Disposition", "attachment; filename=" || $file-name || ".csv");
        $serializers.to-csv($result)
    }
    default return {
        resp:content-type("application/json");
        resp:serialization-parameters({"indent" : true});
        {|
            session:comment("json", $comment),
            $result
        |}
    }
};

declare function api:preprocess-fiscal-years($fiscal-years as string*) as integer*
{
  distinct-values(
    for $fy in $fiscal-years ! upper-case($$)
    return switch($fy)
           case "LATEST" return $sec-fiscal:LATEST_FISCAL_YEAR
           case "ALL" return $sec-fiscal:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
  )
};

declare function api:preprocess-fiscal-periods($fiscal-periods as string*) as string*
{
  distinct-values(
    for $fp in $fiscal-periods ! upper-case($$)
    return switch($fp)
           case "Q1"
           case "YTD1"
           case "Q2"
           case "QTD2"
           case "YTD2"
           case "Q3"
           case "QTD3"
           case "YTD3"
           case "FY"
           case "QTD4"
           case "YTD4"
             return $fp
           case "ALL"
             return $sec-fiscal:ALL_FISCAL_PERIODS
           default
             return error(xs:QName("local:INVALID-PERIOD"),
               $fp || ": fiscalPeriod values must be one or more of Q1, YTD1, Q2, QTD2, YTD2, Q3, QTD3, YTD3, FY, QTD4, YTD4, ALL")
  )
};

declare function api:preprocess-format($format as string?, $request-uri as string) as string?
{
  let $request-path := tokenize($request-uri, "\\?")[1]
  return lower-case(($format, substring-after($request-path, ".jq."))[1])
};

declare function api:preprocess-tags($tags as string*) as string*
{
  distinct-values($tags ! upper-case($$))
};

declare function api:preprocess-boolean($name as string, $value as string)
as boolean
{
  if ($value eq "")
  then true
  else
  {
      api:validate-enum($name, lower-case($value), ("true", "false"));
      boolean($value)
  }
};


Limit frontmatter to only one page.

Limit frontmatter to only one page.

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    (. == {} or . == [] or . == false or . == null or . == 0) | not
;

# Group the values of properties of an array of objects. For example, [{"a":1,
# "b":2}, {"a":3}] turns into {"a":[1, 3], "b":[3]}. This can be used to
# partition an array; try, for example:
# [1,2,3,4] | map({(if .%2==0 then "even" else "odd" end):.}) | group
def group:
    reduce ([.[] | to_entries[]] | group_by(.key))[] as $group
    ( {}
    ; .[ $group[0].key ] |= (. // []) + [ $group[].value ]
    )
;

# Insert a child element at a particular path.
# Like `setpath/2`, but instead of making objects like `{"a":{"b":{…}}}`, this
# makes objects like `{"contents":[{"name":"a", "contents":[{"name":"b",…}]}]}`
def insert($path; $child):
    if ($path | length) > 0 then
        if ([.contents[]? | select(.name == $path[0])] | length) > 0 then
            (.contents[] | select(.name == $path[0])) |= (. | insert($path[1:]; $child))
        else
            (.contents |= (. // []) + [ {} | .name = $path[0] | insert($path[1:]; $child)])
        end
    else
        . * $child
    end
;

# Merge together `filetree.json` and `*.meta.json` files, with behaviour
# depending on the filename of the object. Use with the `--null-input` switch.
def process_files:
    reduce inputs as $input
        (   {}
        ;   ($input | input_filename) as $f |
            if ($f | endswith(".meta.json")) then
                insert
                ( $f | ltrimstr($prefix) | rtrimstr(".meta.json") | split("/")
                ; {"meta": $input} )
            else
                reduce $input[] as $entry
                ( .
                ;   insert
                    ( $entry.path | split("/")
                    ; $entry )
                )
            end
        )
;

def classify_frontmatter:
    [ .[] | { ( if .name == "index.md" or .meta.frontmatter then "frontmatter" else "contents" end ) : . }]
    | group
    | .contents |= (. // [])
    | .frontmatter |= (. // [])
;

def classify_draft:
    [.[] | { (if .meta.draft then "drafts" else "contents" end ) : . }]
    | group
    | .contents |= (. // [])
    | .drafts |= (. // [])
;

def classify_resource:
    [ .[] | { ( if (.meta | bool) or (.contents | bool) then "contents" else "resources" end ) : . }]
    | group
    | .contents |= (. // [])
    | .resources |= (. // [])
;

def add_links:
    if (.path and (.path | endswith(".md")))
    then
        .link = (.path | rtrimstr(".md") | . + ".html")
    else
        .contents[]? |= add_links
    end
;

def add_frontmatter:
    if .contents | bool then
        (.contents | classify_frontmatter | (.contents[]? |= add_frontmatter) ) as $p
        | .frontmatter = ($p["frontmatter"][0] // {})
        | .contents = ($p["frontmatter"][1:] + $p["contents"])
    else
        .
    end
;

def add_drafts:
    if .contents | bool then
        . + (.contents | classify_draft | (.contents[]? |= add_drafts))
    else
        .
    end
;

# Resources are images and other things that should not be part of the table of
# contents - anything that is neither a directory nor has metadata.
def add_resources:
    if .contents | bool then
        . + (.contents | classify_resource | (.contents[]? |= add_resources))
    else
        .
    end;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    process_files
    | add_links
    | add_frontmatter
    | add_drafts
    | add_resources
;

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    (. == {} or . == [] or . == false or . == null or . == 0) | not
;

# Group the values of properties of an array of objects. For example, [{"a":1,
# "b":2}, {"a":3}] turns into {"a":[1, 3], "b":[3]}. This can be used to
# partition an array; try, for example:
# [1,2,3,4] | map({(if .%2==0 then "even" else "odd" end):.}) | group
def group:
    reduce ([.[] | to_entries[]] | group_by(.key))[] as $group
    ( {}
    ; .[ $group[0].key ] |= (. // []) + [ $group[].value ]
    )
;

# Insert a child element at a particular path.
# Like `setpath/2`, but instead of making objects like `{"a":{"b":{…}}}`, this
# makes objects like `{"contents":[{"name":"a", "contents":[{"name":"b",…}]}]}`
def insert($path; $child):
    if ($path | length) > 0 then
        if ([.contents[]? | select(.name == $path[0])] | length) > 0 then
            (.contents[] | select(.name == $path[0])) |= (. | insert($path[1:]; $child))
        else
            (.contents |= (. // []) + [ {} | .name = $path[0] | insert($path[1:]; $child)])
        end
    else
        . * $child
    end
;

# Merge together `filetree.json` and `*.meta.json` files, with behaviour
# depending on the filename of the object. Use with the `--null-input` switch.
def process_files:
    reduce inputs as $input
        (   {}
        ;   ($input | input_filename) as $f |
            if ($f | endswith(".meta.json")) then
                insert
                ( $f | ltrimstr($prefix) | rtrimstr(".meta.json") | split("/")
                ; {"meta": $input} )
            else
                reduce $input[] as $entry
                ( .
                ;   insert
                    ( $entry.path | split("/")
                    ; $entry )
                )
            end
        )
;

def classify_frontmatter:
    [ .[] | { ( if .name == "index.md" or .meta.frontmatter then "frontmatter" else "contents" end ) : . }]
    | group
    | .contents |= (. // [])
    | .frontmatter |= (. // [])
;

def classify_draft:
    [.[] | { (if .meta.draft then "drafts" else "contents" end ) : . }]
    | group
    | .contents |= (. // [])
    | .drafts |= (. // [])
;

def classify_resource:
    [ .[] | { ( if (.meta | bool) or (.contents | bool) then "contents" else "resources" end ) : . }]
    | group
    | .contents |= (. // [])
    | .resources |= (. // [])
;

def add_links:
    if (.path and (.path | endswith(".md")))
    then
        .link = (.path | rtrimstr(".md") | . + ".html")
    else
        .contents[]? |= add_links
    end
;

def add_frontmatter:
    if .contents | bool then
        . + (.contents | classify_frontmatter | (.contents[]? |= add_frontmatter) )
    else
        .
    end
;

def add_drafts:
    if .contents | bool then
        . + (.contents | classify_draft | (.contents[]? |= add_drafts))
    else
        .
    end
;

# Resources are images and other things that should not be part of the table of
# contents - anything that is neither a directory nor has metadata.
def add_resources:
    if .contents | bool then
        . + (.contents | classify_resource | (.contents[]? |= add_resources))
    else
        .
    end;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    process_files
    | add_links
    | add_frontmatter
    | add_drafts
    | add_resources
;


Indexer inserts files into correct paths directly.

Instead of making objects like `{"a": {"b": {…} }}` and then manually
reordering them into objects of the form `{"name":"a",
"contents":[{"name":"b",…}]}`, this is now done in one single step. This
should be clearer and will make it easier to shorten later.

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Select the descendant described by the array of names in `$path`.
def descend($path):
    if ($path | length) > 0
    then
        .contents[]? | select(.name == $path[0]) | descend($path[1:])
    else
        .
    end
;

# Insert a child element at a particular path.
# Like `setpath/2`, but instead of making objects like `{"a":{"b":{…}}}`, this
# makes objects like `{"contents":[{"name":"a", "contents":[{"name":"b",…}]}]}`
# I don't really like this function, because it's so verbose. Making a path
# expression for updating with the |= operator (see `descend/1`) doesn't seem
# to work when it is recursive?
def insert($path; $child):
    if ($path | length) > 0
    then
        if ([.contents[]? | select(.name == $path[0])] | length) > 0
        then
            (.contents[] | select(.name == $path[0])) |= (. | insert($path[1:]; $child))
        else
            (.contents = [.contents[]?] + [{"name":$path[0]} | insert($path[1:];$child)])
        end
    else
        . * $child
    end
;

# Given a stream of JSON objects of the form {"path":"a/b",…}, this function creates
# a single JSON object of the form {"path":"a", "a":{"path": "a/b"}}
def filetree:
    reduce inputs as $i
        ( []
        ; . + [$i]
        )
;

# Merge together `filetree.json` and `*.meta.json` files, as given in the
# format produced by `file_entries`.
def combine_files:
    reduce (to_entries | .[]) as $entry
        (   {}
        ;   . * (
            if ($entry.key | endswith(".meta.json"))
            then
                ( $entry.key
                    | ltrimstr($prefix)
                    | rtrimstr(".meta.json")
                    | split("/")
                ) as $path
                | insert($path; {"meta": $entry.value})
            else
                reduce $entry.value[] as $f
                ( .
                ; insert($f.path | split("/"); $f)
                )
            end
            )
        )
;

# Make an object out of a stream of files, associating the each filename with
# its contents. Use with the `--null-input` switch.
def file_entries:
    reduce inputs as $input
        ( {}
        ; . + { ($input | input_filename) : $input }
        )
;

# Adds links to each page object. The link should be the same as the path,
# except in the case of directories, which may either link to its index.md or
# have no link at all. If there is only one child to a page, perhaps it should
# collapse onto its parent.
def add_links:
    "work in progress"
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    file_entries
    | combine_files
;

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# This is a predicate that returns true if an entry produced by `to_entries` is
# to be considered a child file, and false if it is to be considered a property
# of the file or directory.
def pred:
    .key != "meta" and (.value | type == "object")
;

# Given a JSON object that contains keys representing both child files and file
# properties, this function leaves the file properties intact and moves the
# child files to a separate array at the "contents" key.
def move_children_to_array:
    if type == "object"
    then
        ( to_entries | map(select(pred) | .value | move_children_to_array)) as $sub
        | with_entries(select(pred | not))
        | .contents = ($sub | sort_by(.modified) | reverse)
    else
        .
    end
;

# Given a stream of JSON objects of the form {"path":"a/b",…}, this function creates
# a single JSON object of the form {"path":"a", "a":{"path": "a/b"}}
def filetree:
    reduce inputs as $i
        ( {}
        ; ($i.path | split("/")) as $p
        | setpath($p; getpath($p) + ($i | .name |= $p[-1:][0]))
        )
;

# Merge together `filetree.json` and `*.meta.json` files, as given in the
# format produced by `file_entries`.
def combine_filetree_metadata:
    reduce (to_entries | .[]) as $entry
        (   {}
        ;   . * (
            if ($entry.key | endswith(".meta.json"))
            then
                ( $entry.key
                    | ltrimstr($prefix)
                    | rtrimstr(".meta.json")
                    | split("/")
                ) as $path
                | {}
                | setpath($path + ["meta"]; $entry.value)
            else
                $entry.value
            end
            )
        )
;

# Select the descendant described by the array of names in `$path`.
def descend($path):
    if ($path | length) > 0
    then
        .contents[]? | select(.name == $path[0]) | descend($path[1:])
    else
        .
    end
;

# Insert a child element at a particular path.
# Like `setpath/2`, but instead of making objects like `{"a":{"b":{…}}}`, this
# makes objects like `{"contents":[{"name":"a", "contents":[{"name":"b",…}]}]}`
# I don't really like this function, because it's so verbose. Making a path
# expression for updating with the |= operator (see `descend/1`) doesn't seem
# to work when it is recursive?
def insert($path; $child):
    if ($path | length) > 0
    then
        if ([.contents[]? | select(.path == $path[0])] | length) > 0
        then
            (.contents[] | select(.name == $path[0])) |= (. | insert($path[1:]; $child))
        else
            (.contents = [.contents[]?] + [{"name":$path[0]} | insert($path[1:];$child)])
        end
    else
        . * $child
    end
;

# Make an object out of a stream of files, associating the each filename with
# its contents. Use with the `--null-input` switch.
def file_entries:
    reduce inputs as $input
        ( {}
        ; . + { ($input | input_filename) : $input }
        )
;

# Adds links to each page object. The link should be the same as the path,
# except in the case of directories, which may either link to its index.md or
# have no link at all. If there is only one child to a page, perhaps it should
# collapse onto its parent.
def add_links:
    "work in progress"
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index:
    file_entries
    | combine_filetree_metadata
    | move_children_to_array
;


removed unused module imports

removed unused module imports

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace credentials = "http://apps.28.io/credentials";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace random = "http://zorba.io/modules/random";

declare variable $newId as string := random:uuid();

declare %private function local:url($parameters as object?, $endpoint as string, $includeToken as boolean) as string
{
    "http://" || request:server-name() || ":" || request:server-port() ||
    "/v1/_queries/public/reports/" || $endpoint || ".jq?_method=POST&token=" || (if($includeToken) then $credentials:support-token else "{{token}}") || "&"||
    string-join(
        for $key in keys($parameters)
        for $value as string in (flatten($parameters.$key) ! string($$))
        return ($key||"="||$value),
        "&")
};

declare %an:sequential function local:export-facs-xbrlb() as item*
{
    let $parameters := {
        "_id": "supportFundamentalAccountingConcepts",
        "export": "true"
    }
    let $endpoint as string := "reports"
    let $url := local:url($parameters, $endpoint, true())
    let $request := http-client:get($url)
    return (local:url($parameters, $endpoint, false()),$request)
};

declare %an:sequential function local:import-facs-xbrlb($archive as xs:base64Binary) as item*
{
    let $parameters := {
        "import": "true",
        "_id": $newId,
        "label": encode-for-uri("Import test")
    }
    let $endpoint as string := "add-report"
    let $url := local:url($parameters, $endpoint, true())
    let $request := http-client:post($url, $archive)
    return (local:url($parameters, $endpoint, false()),$request)
};

declare %an:sequential function local:test-export-import() as item
{
  let $export := local:export-facs-xbrlb()
  let $export-url := $export[1]
  let $export-response := $export[2]
  let $status as integer := $export-response.status
  (:let $body as string := string:materialize($export-response.body.content):)
  return
      if($status ne 200)
      then
        {
            "url": $export-url,
            "status": $status,
            "reponse": $export-response
        }
      else
          let $fac-xbrlb as xs:base64Binary := xs:base64Binary($export-response.body.content)
          let $import := local:import-facs-xbrlb($fac-xbrlb)
          let $import-url := $import[1]
          let $import-response := $import[2]
          let $status as integer := $import-response.status
          let $body as object := parse-json($import-response.body.content)
          return
              if($status eq 200 and exists($body."Label") and $body.Label eq "Import test" and $body."_id" eq $newId)
              then true()
              else
                    {
                        "url": $import-url,
                        "status": $status,
                        "reponse": $body
                    }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    "export-import": local:test-export-import()
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";
import module namespace http-client = "http://zorba.io/modules/http-client";
import module namespace credentials = "http://apps.28.io/credentials";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace string = "http://zorba.io/modules/string";
import module namespace random = "http://zorba.io/modules/random";

declare variable $newId as string := random:uuid();

declare %private function local:url($parameters as object?, $endpoint as string, $includeToken as boolean) as string
{
    "http://" || request:server-name() || ":" || request:server-port() ||
    "/v1/_queries/public/reports/" || $endpoint || ".jq?_method=POST&token=" || (if($includeToken) then $credentials:support-token else "{{token}}") || "&"||
    string-join(
        for $key in keys($parameters)
        for $value as string in (flatten($parameters.$key) ! string($$))
        return ($key||"="||$value),
        "&")
};

declare %an:sequential function local:export-facs-xbrlb() as item*
{
    let $parameters := {
        "_id": "supportFundamentalAccountingConcepts",
        "export": "true"
    }
    let $endpoint as string := "reports"
    let $url := local:url($parameters, $endpoint, true())
    let $request := http-client:get($url)
    return (local:url($parameters, $endpoint, false()),$request)
};

declare %an:sequential function local:import-facs-xbrlb($archive as xs:base64Binary) as item*
{
    let $parameters := {
        "import": "true",
        "_id": $newId,
        "label": encode-for-uri("Import test")
    }
    let $endpoint as string := "add-report"
    let $url := local:url($parameters, $endpoint, true())
    let $request := http-client:post($url, $archive)
    return (local:url($parameters, $endpoint, false()),$request)
};

declare %an:sequential function local:test-export-import() as item
{
  let $export := local:export-facs-xbrlb()
  let $export-url := $export[1]
  let $export-response := $export[2]
  let $status as integer := $export-response.status
  (:let $body as string := string:materialize($export-response.body.content):)
  return
      if($status ne 200)
      then
        {
            "url": $export-url,
            "status": $status,
            "reponse": $export-response
        }
      else
          let $fac-xbrlb as xs:base64Binary := xs:base64Binary($export-response.body.content)
          let $import := local:import-facs-xbrlb($fac-xbrlb)
          let $import-url := $import[1]
          let $import-response := $import[2]
          let $status as integer := $import-response.status
          let $body as object := parse-json($import-response.body.content)
          return
              if($status eq 200 and exists($body."Label") and $body.Label eq "Import test" and $body."_id" eq $newId)
              then true()
              else
                    {
                        "url": $import-url,
                        "status": $status,
                        "reponse": $body
                    }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    "export-import": local:test-export-import()
})


jq: Improve filters [ci skip] (#1236)

jq: Improve filters [ci skip] (#1236)

- Include Chokidar in Local filter
- Introduce nouptodate & noproxy filters
- Document path filter caveat
- Find logs before/after some given time

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v);

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: clean | select(is_error);
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: clean | select(is_warn);
def warn_strict: clean | select(is_warn_strict);
def info_level: 30;
def is_info: .level >= info_level;
def info: clean | select(is_info);
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: clean | select(is_debug);

# Components:
def component(pattern): select(.component | test(pattern));
def App: component("^App$");
def Chokidar: component("^Chokidar|chokidar$");
def Fs: component("^Fs|FS$");
def LocalAnalysis: component("^LocalAnalysis|local/analysis$");
def LocalChange: component("^LocalChange|local/change$");
def LocalWatcher: component("^LocalWatcher$");
def LocalWriter: component("^LocalWriter$");
def Merge: component("^Merge$");
def Metadata: component("^Metadata$");
def Perfs: component("^Perfs|PerfReports$");
def Pouch: component("^Pouch$");
def Prep: component("^Prep$");
def RemoteCozy: component("^RemoteCozy$");
def RemoteUserActionRequired: component("^RemoteUserActionRequired|remote/user_action_required$");
def RemoteWarningPoller: component("^RemoteWarningPoller$");
def RemoteWatcher: component("^RemoteWatcher$");
def RemoteWriter: component("^RemoteWriter$");
def Sync: component("^Sync$");

def Local: select(.component | test("^local|chokidar"; "i"));
def Remote: select(.component | test("^remote"; "i"));

# Exclude "up-to-date" messages
#
#     yarn jq nouptodate path/to/logs*
#
def nouptodate: select(.msg | test("up[- ]to[- ]date") | not);

# Exclude Proxy stuff:
#
#     yarn jq noproxy path/to/logs*
#
def noproxy: select(.component | test("GUI:proxy") | not);

# Find conflicts:
#
#     yarn jq -c conflicts path/to/logs*
#
def is_conflict: .msg == "resolveConflictAsync";
def conflicts: clean | select(is_conflict) | {time,path};

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
# FIXME: Should not output the same line twice when both path & oldpath match.
def path(pattern):
  clean | select(
    (
      (.path // (.doc | .path) // (.change | .doc | .path) // (.idConflict | .newDoc | .path)),
      (.oldpath // (.was | .path) // (.idConflict | .existingDoc | .path)),
      ""
    )
      | strings
      | test(pattern)
  );

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#    yarn jq 'before("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#    yarn jq 'after("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#
def time(pattern):
  select(.time | test(pattern));
def before(time): select(.time <= time);
def after(time): select(.time >= time);

# Drop time properties
def notime: del(.time);

# Filter inode number(s):
#
#    yarn jq 'ino(7852458)' path/to/logs*
#    yarn jq 'ino(7852458; 464672)' path/to/logs*
#
def ino(n): select((.ino // (.doc|.ino) // (.change|.doc|.ino)) as $ino | [n] | flatten | map(. == $ino) | any);

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path,oldpath}
    | if .oldpath then . else del(.oldpath) end
    ;

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v);

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: clean | select(is_error);
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: clean | select(is_warn);
def warn_strict: clean | select(is_warn_strict);
def info_level: 30;
def is_info: .level >= info_level;
def info: clean | select(is_info);
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: clean | select(is_debug);

# Components:
def component(pattern): select(.component | test(pattern));
def App: component("^App$");
def Chokidar: component("^Chokidar|chokidar$");
def Fs: component("^Fs|FS$");
def LocalAnalysis: component("^LocalAnalysis|local/analysis$");
def LocalChange: component("^LocalChange|local/change$");
def LocalWatcher: component("^LocalWatcher$");
def LocalWriter: component("^LocalWriter$");
def Merge: component("^Merge$");
def Metadata: component("^Metadata$");
def Perfs: component("^Perfs|PerfReports$");
def Pouch: component("^Pouch$");
def Prep: component("^Prep$");
def RemoteCozy: component("^RemoteCozy$");
def RemoteUserActionRequired: component("^RemoteUserActionRequired|remote/user_action_required$");
def RemoteWarningPoller: component("^RemoteWarningPoller$");
def RemoteWatcher: component("^RemoteWatcher$");
def RemoteWriter: component("^RemoteWriter$");
def Sync: component("^Sync$");

def Local: select(.component | test("^local"; "i"));
def Remote: select(.component | test("^remote"; "i"));

# Find conflicts:
#
#     yarn jq -c conflicts path/to/logs*
#
def is_conflict: .msg == "resolveConflictAsync";
def conflicts: clean | select(is_conflict) | {time,path};

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
def path(pattern):
  clean | select(
    (
      (.path // (.doc | .path) // (.change | .doc | .path) // (.idConflict | .newDoc | .path)),
      (.oldpath // (.was | .path) // (.idConflict | .existingDoc | .path)),
      ""
    )
      | strings
      | test(pattern)
  );

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#
def time(pattern):
  select(.time | test(pattern));

# Drop time properties
def notime: del(.time);

# Filter inode number(s):
#
#    yarn jq 'ino(7852458)' path/to/logs*
#    yarn jq 'ino(7852458; 464672)' path/to/logs*
#
def ino(n): select((.ino // (.doc|.ino) // (.change|.doc|.ino)) as $ino | [n] | flatten | map(. == $ino) | any);

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path,oldpath}
    | if .oldpath then . else del(.oldpath) end
    ;

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );


:green_heart: Fix warning.

:green_heart: Fix warning.

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/backend";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec-filings = "http://28.io/modules/xbrl/profiles/sec/filings";
import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

declare option rest:response "first-item";

declare function local:links($component as object) as object
{
  {
    Components: backend:url("components", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    profile-name: $profile-name
                    }),
    FactTable: backend:url("facttable-for-component", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    profile-name: $profile-name,
                    labels: "true"
                    }),
    SpreadSheet: backend:url("spreadsheet-for-component", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    profile-name: $profile-name,
                    eliminate: "true",
                    validate: "true"
                }),
    ReportElements: backend:url("report-elements", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    profile-name: $profile-name
                }),
    ModelStructure: backend:url("modelstructure-for-component", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    profile-name: $profile-name,
                    indent: "true"
                })
  }
};

declare function local:statistics($component as object) as object
{
  {
    NumRules: size($component.Rules),
    NumNetworks: components:num-networks($component),
    NumReportElements: components:num-report-elements($component, $components:ALL-HYPERCUBES),
    NumHypercubes: components:num-report-elements(
        $component,
        $components:ALL-HYPERCUBES,
        "Hypercube"),
    NumDimensions: components:num-report-elements(
        $component,
        $components:ALL-HYPERCUBES,
        "Dimension"),
    NumMembers: components:num-report-elements(
        $component,
        $components:ALL-HYPERCUBES,
        "Member"),
    NumLineItems: components:num-report-elements(
        $component,
        $components:ALL-HYPERCUBES,
        "LineItems"),
    NumAbstracts: components:num-report-elements(
        $component,
        $components:ALL-HYPERCUBES,
        "Abstract"),
    NumConcepts: components:num-report-elements(
        $component,
        $components:ALL-HYPERCUBES,
        "Concept")
  }
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token              as string? external;
declare  %rest:env                              variable $request-uri        as string  external;
declare  %rest:case-insensitive                 variable $format             as string  external := "json";
declare  %rest:case-insensitive %rest:distinct  variable $cik                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode   as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker             as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear         as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod       as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $filingKind         as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $eid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $networkIdentifier  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $role               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $hypercube          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $reportElement      as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $concept            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $label              as string* external;
declare  %rest:case-insensitive                 variable $profile-name       as string  external := $config:profile-name;
declare  %rest:case-insensitive                 variable $count              as boolean external := false;

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $reportElement := ($reportElement, $concept)
let $networkIdentifier := distinct-values(($networkIdentifier, $role))

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic,
  (),
  (),
  (),
  ())

let $entities-not-found as boolean :=
  exists(($eid, $cik, $tag, $ticker, $sic)) and empty($entities)

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $filingKind,
  $aid)

let $archives-not-found as boolean :=
  exists(($entities, $fiscalPeriod, $fiscalYear, $filingKind, $aid)) and empty($archives)

let $entities as object*  := entities:entities($archives.Entity)
let $components as object* :=
    multiplexer:components(
      $profile-name,
      $archives,
      $cid,
      $reportElement,
      $disclosure,
      $networkIdentifier,
      $label[$profile-name ne "sec"],
      $label[$profile-name eq "sec"]
    )

let $res as object* :=
    for $component in $components
    let $disclosure := switch($profile-name)
                       case "sec" return sec-networks:disclosures($component)
                       default return $component.Profiles.FSA.Disclosure
    where not $disclosure eq "DefaultComponent"
    order by $component.Archive, $component.Label
    let $archive as object? := $archives[archives:aid($$) eq $component.Archive]
    let $entity as object? := $entities[entities:eid($$) = $archive.Entity]
    where empty($hypercube) or components:hypercubes($component) = $hypercube
    return {|
      {
        Archive: $component.Archive,
        Network: $component.Label
      },
      local:links($component),
      {
        NetworkURI: $component.Role
      },
      sec-networks:summary($component)[$profile-name eq "sec"],
      local:statistics($component),
      { Disclosure: $disclosure }[$profile-name eq "japan"],
      (: DeprecatedPastThisPoint" :)
      project($component, ("Role", "Label")),
      {
          AccessionNumber: $component.Archive,
          NetworkIdentifier : $component.Role,
          NetworkLabel: $component.Label,
          EntityRegistrantName : $entity.Profiles.SEC.CompanyName,
          CIK : entities:eid($entity),
          FiscalYear :$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
          FiscalPeriod :$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
          AcceptanceDatetime : sec-filings:acceptance-dateTimes($archive),
          FormType : $archive.Profiles.SEC.FormType
      }[$profile-name eq "sec"]
    |}

let $result := { Networks : [ if($count) then () else $res ] }
let $comment :=
 {
    NumNetworks : count($res),
    TotalNumNetworks: session:num-components(),
    TotalNumArchives: session:num-archives()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <Components>{
          api:json-to-xml($res.Networks[], "Networks")
        }</Components>
    },
    to-csv : function($res as object) as string {
        api:json-to-csv($res.Networks[])
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "networks")
return switch(true)
       case $entities-not-found return api:not-found("entity")
       case $archives-not-found return api:not-found("archive")
       default return api:check-and-return-results($token, $results, $format)

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/backend";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace labels = "http://28.io/modules/xbrl/labels";

import module namespace sec-filings = "http://28.io/modules/xbrl/profiles/sec/filings";
import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

declare option rest:response "first-item";

declare function local:links($component as object) as object
{
  {
    Components: backend:url("components", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    profile-name: $profile-name
                    }),
    FactTable: backend:url("facttable-for-component", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    profile-name: $profile-name,
                    labels: "true"
                    }),
    SpreadSheet: backend:url("spreadsheet-for-component", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    profile-name: $profile-name,
                    eliminate: "true",
                    validate: "true"
                }),
    ReportElements: backend:url("report-elements", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    profile-name: $profile-name
                }),
    ModelStructure: backend:url("modelstructure-for-component", {
                    token: $token,
                    aid: $component.Archive,
                    format: $format,
                    role: $component.Role,
                    profile-name: $profile-name,
                    indent: "true"
                })
  }
};

declare function local:statistics($component as object) as object
{
  {
    NumRules: size($component.Rules),
    NumNetworks: components:num-networks($component),
    NumReportElements: components:num-report-elements($component, $components:ALL-HYPERCUBES),
    NumHypercubes: components:num-report-elements(
        $component,
        $components:ALL-HYPERCUBES,
        "Hypercube"),
    NumDimensions: components:num-report-elements(
        $component,
        $components:ALL-HYPERCUBES,
        "Dimension"),
    NumMembers: components:num-report-elements(
        $component,
        $components:ALL-HYPERCUBES,
        "Member"),
    NumLineItems: components:num-report-elements(
        $component,
        $components:ALL-HYPERCUBES,
        "LineItems"),
    NumAbstracts: components:num-report-elements(
        $component,
        $components:ALL-HYPERCUBES,
        "Abstract"),
    NumConcepts: components:num-report-elements(
        $component,
        $components:ALL-HYPERCUBES,
        "Concept")
  }
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token              as string? external;
declare  %rest:env                              variable $request-uri        as string  external;
declare  %rest:case-insensitive                 variable $format             as string  external := "json";
declare  %rest:case-insensitive %rest:distinct  variable $cik                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode   as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker             as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear         as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod       as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $filingKind         as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $eid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $networkIdentifier  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $role               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $hypercube          as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $reportElement      as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $concept            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $label              as string* external;
declare  %rest:case-insensitive                 variable $profile-name       as string  external := $config:profile-name;
declare  %rest:case-insensitive                 variable $count              as boolean external := false;

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $reportElement := ($reportElement, $concept)
let $networkIdentifier := distinct-values(($networkIdentifier, $role))

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic,
  (),
  (),
  (),
  ())

let $entities-not-found as boolean :=
  exists(($eid, $cik, $tag, $ticker, $sic)) and empty($entities)

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $filingKind,
  $aid)

let $archives-not-found as boolean :=
  exists(($entities, $fiscalPeriod, $fiscalYear, $filingKind, $aid)) and empty($archives)

let $entities as object*  := entities:entities($archives.Entity)
let $components as object* :=
    multiplexer:components(
      $profile-name,
      $archives,
      $cid,
      $reportElement,
      $disclosure,
      $networkIdentifier,
      $label[$profile-name ne "sec"],
      $label[$profile-name eq "sec"]
    )

let $res as object* :=
    for $component in $components
    let $disclosure := switch($profile-name)
                       case "sec" return sec-networks:disclosures($component)
                       default return $component.Profiles.FSA.Disclosure
    where not $disclosure eq "DefaultComponent"
    order by $component.Archive, $component.Label
    let $archive as object? := $archives[archives:aid($$) eq $component.Archive]
    let $entity as object? := $entities[entities:eid($$) = $archive.Entity]
    where empty($hypercube) or components:hypercubes($component) = $hypercube
    return {|
      {
        Archive: $component.Archive,
        Network: $component.Label
      },
      local:links($component),
      {
        NetworkURI: $component.Role
      },
      sec-networks:summary($component)[$profile-name eq "sec"],
      local:statistics($component),
      { Disclosure: $disclosure }[$profile-name eq "japan"],
      (: DeprecatedPastThisPoint" :)
      project($component, ("Role", "Label")),
      {
          AccessionNumber: $component.Archive,
          NetworkIdentifier : $component.Role,
          NetworkLabel: $component.Label,
          EntityRegistrantName : $entity.Profiles.SEC.CompanyName,
          CIK : entities:eid($entity),
          FiscalYear :$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
          FiscalPeriod :$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
          AcceptanceDatetime : sec-filings:acceptance-dateTimes($archive),
          FormType : $archive.Profiles.SEC.FormType
      }[$profile-name eq "sec"]
    |}

let $result := { Networks : [ if($count) then () else $res ] }
let $comment :=
 {
    NumNetworks : count($res),
    TotalNumNetworks: session:num-components(),
    TotalNumArchives: session:num-archives()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <Components>{
          api:json-to-xml($res.Networks[], "Networks")
        }</Components>
    },
    to-csv : function($res as object) as string {
        api:json-to-csv($res.Networks[])
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "networks")
return switch(true)
       case $entities-not-found return api:not-found("entity")
       case $archives-not-found return api:not-found("archive")
       default return api:check-and-return-results($token, $results, $format)


:green_heart: Fix odata test.

:green_heart: Fix odata test.

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facts"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-404($params as object) as item
{
    let $endpoint := "facts"
    let $res as object := test:invoke-raw($endpoint, $params)
    return if($res.status eq 404)
           then true
           else {
              url: test:url($endpoint, $params),
              unexpectedResponse: $res
           }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw($endpoint, $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "Archive [Axis],Fiscal Period [Axis],Fiscal Year [Axis],Fiscal Period Type [Axis],Period [Axis],Reporting Entity [Axis],Concept [Axis],Unit [Axis],Legal Entity [Axis],Value,Decimals\r\n0000021344-14-000036,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),\"Cash and Cash Equivalents, at Carrying Value\",USD,Default Legal Entity [Member],11118000000,-6\r\n0000021344-13-000050,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),Assets,USD,Default Legal Entity [Member],89432000000,-6\r\n"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res,
        expected: $expectedLines
    }
};

declare %an:nondeterministic function local:test-labels-aids() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: "disc:AdvertisingCostsPolicyTextBlock",
        report: "Disclosures",
        format: "json",
        fiscalYear: 2014,
        fiscalPeriod: "FY",
        labels: true,
        aid: "0000858877-14-000029"
    }
    let $res := test:invoke($endpoint, $params)
    let $actual := [
            for $labels in $res[2].FactTable[].Labels
            return (keys($labels) ! $labels.$$)
        ]
    let $expected := [ "Archive [Axis]", "Fiscal Year [Axis]", "Non-Numeric [Member]", "Default Legal Entity [Member]", "Unit [Axis]", "Advertising Costs, Policy", "Fiscal Period [Axis]", "Concept [Axis]", "Reporting Entity [Axis]", "Legal Entity [Axis]", "Fiscal Period Type [Axis]", "Period [Axis]", "From July 28, 2013 to July 26, 2014", "CISCO SYSTEMS, INC. (858877)" ]
    let $status as integer := $res[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

local:check({
    cocacola-latest:
    (: this test will fail and needs to be updated if a newer report has been filed.
       current latest filing: 2014 :)
    local:test-facttable(462, {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    cocacola-all: local:test-facttable(477, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-all-q3: local:test-facttable(341, {
        ticker:"ko",
        fiscalYear: 2014,
        fiscalPeriod: [ "Q3" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-instant: local:test-facttable(170, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant" ]
    }),
    cocacola-ytd: local:test-facttable(307, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-qtd: local:test-facttable(102, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "Q2" ],
        fiscalPeriodType: [ "QTD" ]
    }),
    cocacolaCSVLabels: local:test-labels(),
    ciscoLabelsByAid: local:test-labels-aids(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(30, { (: AT&T wrongly stamped a filing as FY, but it's now FIXED :-):)
        tag:"DOW30",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-404({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    }),
    quartersfact: local:test-facttable(0, { (: this fact is not reported in the raw XML filing :)
        aid: "0000051143-13-000007",
        concept: "disc:AssetRetirementObligationsPolicy",
        fiscalPeriod: "Q3",
        fiscalYear: "2013",
        map: "Disclosures"
    }),
    quartersfact2: local:test-facttable(1, {
        aid: "0001193125-12-207154",
        concept: "disc:CashFlowOperatingCapitalTableTextBlock",
        fiscalPeriod: "Q1",
        fiscalYear: "2012",
        map: "Disclosures"
    }),
    generic: local:test-facttable(2, {
        "xbrl:Entity":"http://www.sec.gov/CIK 0000021344",
        "sec:Accepted":"20140227132423",
        concept:"us-gaap:Assets",
        profile-name: "generic"
    }),
    count: test:invoke-and-assert-deep-equal(
          "facts",
          {
              "ticker" : "ko",
              "fiscalYear" : ["2012", "2013", "2014"],
              "fiscalPeriod" : "FY",
              "count" : "true"
          },
          function($res as item*) as item* { $res.Statistics.NumFacts },
          1444
        ),
    odata: test:invoke-and-assert-deep-equal(
      "facts",
      {
          "ticker" : "ko",
          "fiscalYear" : "ALL",
          "fiscalPeriod" : "FY",
          "concept" : [ "us-gaap:Assets", "us-gaap:LiabilitiesCurrent", "us-gaap:StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" ],
          "format" : "odata-atom"
      },
      function($res as item*) as item* { count($res/*:feed/*:entry) },
      18,
      { Format: "xml" }
    )
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facts"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-404($params as object) as item
{
    let $endpoint := "facts"
    let $res as object := test:invoke-raw($endpoint, $params)
    return if($res.status eq 404)
           then true
           else {
              url: test:url($endpoint, $params),
              unexpectedResponse: $res
           }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-labels() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: [ "fac:Assets", "us-gaap:CashAndCashEquivalentsAtCarryingValue" ],
        report: "FundamentalAccountingConcepts",
        format: "csv",
        ticker: "ko",
        fiscalYear: 2013,
        fiscalPeriod: "Q3",
        labels: true
    }
    let $res as object := test:invoke-raw($endpoint, $params)
    let $actual := $res.body.content
    let $expectedLines := (
        "Archive [Axis],Fiscal Period [Axis],Fiscal Year [Axis],Fiscal Period Type [Axis],Period [Axis],Reporting Entity [Axis],Concept [Axis],Unit [Axis],Legal Entity [Axis],Value,Decimals\r\n0000021344-14-000036,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),\"Cash and Cash Equivalents, at Carrying Value\",USD,Default Legal Entity [Member],11118000000,-6\r\n0000021344-13-000050,Q3,2013,instant,\"September 27, 2013\",COCA COLA CO (21344),Assets,USD,Default Legal Entity [Member],89432000000,-6\r\n"
    )
    return if($res.status eq 200 and (every $line in $expectedLines satisfies contains($actual,$line))) then true else {
        url: test:url($endpoint, $params),
        unexpectedResponse: $res,
        expected: $expectedLines
    }
};

declare %an:nondeterministic function local:test-labels-aids() as item
{
    let $endpoint := "facts"
    let $params := {
        concept: "disc:AdvertisingCostsPolicyTextBlock",
        report: "Disclosures",
        format: "json",
        fiscalYear: 2014,
        fiscalPeriod: "FY",
        labels: true,
        aid: "0000858877-14-000029"
    }
    let $res := test:invoke($endpoint, $params)
    let $actual := [
            for $labels in $res[2].FactTable[].Labels
            return (keys($labels) ! $labels.$$)
        ]
    let $expected := [ "Archive [Axis]", "Fiscal Year [Axis]", "Non-Numeric [Member]", "Default Legal Entity [Member]", "Unit [Axis]", "Advertising Costs, Policy", "Fiscal Period [Axis]", "Concept [Axis]", "Reporting Entity [Axis]", "Legal Entity [Axis]", "Fiscal Period Type [Axis]", "Period [Axis]", "From July 28, 2013 to July 26, 2014", "CISCO SYSTEMS, INC. (858877)" ]
    let $status as integer := $res[1]
    return test:assert-eq-array($expected, $actual, $status, test:url($endpoint, $params))
};

local:check({
    cocacola-latest:
    (: this test will fail and needs to be updated if a newer report has been filed.
       current latest filing: 2014 :)
    local:test-facttable(462, {
        ticker:"ko",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    cocacola-all: local:test-facttable(477, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-all-q3: local:test-facttable(341, {
        ticker:"ko",
        fiscalYear: 2014,
        fiscalPeriod: [ "Q3" ],
        fiscalPeriodType: [ "instant", "YTD", "QTD" ]
    }),
    cocacola-instant: local:test-facttable(170, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "instant" ]
    }),
    cocacola-ytd: local:test-facttable(307, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "FY" ],
        fiscalPeriodType: [ "YTD" ]
    }),
    cocacola-qtd: local:test-facttable(102, {
        ticker:"ko",
        fiscalYear: 2013,
        fiscalPeriod: [ "Q2" ],
        fiscalPeriodType: [ "QTD" ]
    }),
    cocacolaCSVLabels: local:test-labels(),
    ciscoLabelsByAid: local:test-labels-aids(),
    tickerconcept: local:test-facttable(1, {
        ticker:"ko",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tickerfyfprole: local:test-facttable(1, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    tagconcept: local:test-facttable(30, { (: AT&T wrongly stamped a filing as FY, but it's now FIXED :-):)
        tag:"DOW30",
        concept:"us-gaap:Assets",
        fiscalYear: "LATEST",
        fiscalPeriod: "FY"
    }),
    tagfyfprole: local:test-facttable(30, {
        tag:"DOW30",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        concept:"us-gaap:Assets"
    }),
    testNoEntity4Sic: local:test-404({
        concept:"fac:Assets",
        map:"FundamentalAccountingConcepts",
        format:"csv",
        sic:"5052",
        fiscalYear:"2010",
        fiscalPeriod:"Q3"
    }),
    quartersfact: local:test-facttable(0, { (: this fact is not reported in the raw XML filing :)
        aid: "0000051143-13-000007",
        concept: "disc:AssetRetirementObligationsPolicy",
        fiscalPeriod: "Q3",
        fiscalYear: "2013",
        map: "Disclosures"
    }),
    quartersfact2: local:test-facttable(1, {
        aid: "0001193125-12-207154",
        concept: "disc:CashFlowOperatingCapitalTableTextBlock",
        fiscalPeriod: "Q1",
        fiscalYear: "2012",
        map: "Disclosures"
    }),
    generic: local:test-facttable(2, {
        "xbrl:Entity":"http://www.sec.gov/CIK 0000021344",
        "sec:Accepted":"20140227132423",
        concept:"us-gaap:Assets",
        profile-name: "generic"
    }),
    count: test:invoke-and-assert-deep-equal(
          "facts",
          {
              "ticker" : "ko",
              "fiscalYear" : ["2012", "2013", "2014"],
              "fiscalPeriod" : "FY",
              "count" : "true"
          },
          function($res as item*) as item* { $res.Statistics.NumFacts },
          1444
        ),
    odata: test:invoke-and-assert-deep-equal(
      "facts",
      {
          "ticker" : "ko",
          "fiscalYear" : "ALL",
          "fiscalPeriod" : "FY",
          "concept" : [ "us-gaap:Assets", "us-gaap:LiabilitiesCurrent", "us-gaap:StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" ],
          "format" : "odata-atom"
      },
      function($res as item*) as item* { count($res/*:feed/*:entry) },
      6,
      { Format: "xml" }
    )
})


Bug fix.

Bug fix.

"Previous" page links would always be present.

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    [. == (null,false,0,{},[])] | any | not
;

# Is a given array at least length $n?
def at_least($n):
    (. | length) >= $n
;

# Generate this object and all its descendants.
def entries:
    ., recurse(.contents?[]?)
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`. We use `name` and `contents` to be compatible
# with the output of the `tree` program on the shell.
def tree($path):
    if $path | at_least(2)
    then { "contents": [ tree($path[1:]) ] }
    else .
    end | .name = $path[0]
;

# Merge an array of page objects into a single page object.
def merge:
    map(to_entries) | add | group_by(.key) | map(
        { "key": (.[0].key)
        , "value": (
            if .[0].key == "contents"
            then [ .[].value ] | add | group_by(.name) | map(merge)
            elif [ .[].value == .[0].value ] | all
            then .[0].value
            else error("can't merge incompatible values for key '\(.[0].key)'")
            end
        ) }
    ) | from_entries
;

# Add paths to each object, that is, the names of the ancestors.
def directory:
    def f($d): .dir=($d|join("/")) | .name as $n | .contents[]?|=f($d+[$n]);
    f([])
;

# Every leaf node gets a basename, eg name without extension.
def basename:
    (entries | select(has("contents") | not)) |= (
        .basename = (.name | sub(".([A-z]*)$";""))
    )
;

# Any page gets its target formats.
def formats($all_formats):
    entries |= (
        .formats = [
            (.make // empty) |
            if . == "null" then empty else . end | # fix for wrong YAML parse
            if (. | type) == "array" then .[] else . end |
            tostring | ascii_downcase |
            if [. == $all_formats[]] | any then .
            elif . == "all" then $all_formats[]
            else error("unrecognized format: \(.)")
            end
        ]
    )
;

# Any page is linked to its first target format.
def link:
    (entries | select(.formats | at_least(1))) |= (
        .link = "\(.dir)/\(.basename).\(.formats[0])"
    )
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def frontmatter:
    (entries | select(has("contents"))) |= (
        (.frontmatter = [.contents[] | select(.frontmatter | bool)][0]) |
        (.contents = .contents - [.frontmatter])
    )
;

# Any page that has no link should be excluded from the index. It would be
# nicer if we could remove in one swoop with `entries | ... |= empty`. It only
# works sometimes, presumably since we'd change the objects we'd iterate over.
def clean:
    if has("contents") then .contents |= map(clean)
    elif has("link")   then .
    else empty end
;

# Add links to the next and previous page for every linked page.
def navigation:
    def f($paths; $i; $j; $key):
        setpath($paths[$i]+[$key];
            getpath($paths[$j]) | {link,title}
        )
    ;
    [ path(entries | select(.formats | at_least(1))) ] as $paths
    | ($paths | length) as $n
    | reduce range(0; $n-1) as $i (.; f($paths; $i; $i+1; "next"))
    | reduce range(1; $n)   as $i (.; f($paths; $i; $i-1; "prev"))
;

# Sort content first according to sort order in metadata.
def sorting:
    (entries | select(has("contents")) | .contents) |= (
        sort_by(.sort // .title // .name)
    )
;

# Add a note that tells us whether this has only children who have no more
# subchildren.
def annotate_leaves:
    (entries | select(has("contents"))) |= (
        .only_leaves = (.contents | all(has("contents") | not))
    )
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index($all_formats):
    merge
    | directory
    | basename
    | formats($all_formats)
    | link
    | clean
    | navigation
    | frontmatter
    | sorting
    | annotate_leaves
;

# Get target files and their dependencies as an enumeration of {"target":...,
# "dependencies":...} objects.
def targets($dest; $default_style):
    entries
    | ., (.frontmatter // empty)
    | .formats[] as $format
    | "\($dest)/\(.dir)/\(.basename).\($format)" as $doc
    | ["\($dest)/\(.dir)/\(.resources?[])"] as $external
    | ["\($dest)/\(.style // $default_style).css"] as $css
    | (if $format == "html" then ($css + $external) else [] end) as $linked
    | (if $format == "pdf"  then ($css + $external) else [] end) as $embedded
    | {"target": $format, "dependencies": ([$doc] + $linked) }
    , {"target": $doc, "dependencies": ($embedded) }
    , {"target": $doc, "dependencies": ["NEXT_LINK := \( .next.link // empty)"]}
    , {"target": $doc, "dependencies": ["PREV_LINK := \( .prev.link // empty)"]}
    , {"target": $doc, "dependencies": ["NEXT_TITLE := \( .next.title // empty)"]}
    , {"target": $doc, "dependencies": ["PREV_TITLE := \( .prev.title // empty)"]}
;

#!/usr/bin/env jq
# This is a collection of filters for `jq`, intended to manipulate JSON objects
# into JSON representing a website index, for further processing in a template.

# Convert "truthy" value to actual boolean.
def bool:
    [. == (null,false,0,{},[])] | any | not
;

# Is a given array at least length $n?
def at_least($n):
    (. | length) >= $n
;

# Generate this object and all its descendants.
def entries:
    ., recurse(.contents?[]?)
;

# Wrap an object in other objects so that the original object exists at a
# particular "path". Like `{} | setpath(["a","b"], …)`, but instead of making
# objects like `{"a":{"b":…}}`, this makes objects like `{"name":"a",
# "contents":[{"name":"b",…}]}`. We use `name` and `contents` to be compatible
# with the output of the `tree` program on the shell.
def tree($path):
    if $path | at_least(2)
    then { "contents": [ tree($path[1:]) ] }
    else .
    end | .name = $path[0]
;

# Merge an array of page objects into a single page object.
def merge:
    map(to_entries) | add | group_by(.key) | map(
        { "key": (.[0].key)
        , "value": (
            if .[0].key == "contents"
            then [ .[].value ] | add | group_by(.name) | map(merge)
            elif [ .[].value == .[0].value ] | all
            then .[0].value
            else error("can't merge incompatible values for key '\(.[0].key)'")
            end
        ) }
    ) | from_entries
;

# Add paths to each object, that is, the names of the ancestors.
def directory:
    def f($d): .dir=($d|join("/")) | .name as $n | .contents[]?|=f($d+[$n]);
    f([])
;

# Every leaf node gets a basename, eg name without extension.
def basename:
    (entries | select(has("contents") | not)) |= (
        .basename = (.name | sub(".([A-z]*)$";""))
    )
;

# Any page gets its target formats.
def formats($all_formats):
    entries |= (
        .formats = [
            (.make // empty) |
            if . == "null" then empty else . end | # fix for wrong YAML parse
            if (. | type) == "array" then .[] else . end |
            tostring | ascii_downcase |
            if [. == $all_formats[]] | any then .
            elif . == "all" then $all_formats[]
            else error("unrecognized format: \(.)")
            end
        ]
    )
;

# Any page is linked to its first target format.
def link:
    (entries | select(.formats | at_least(1))) |= (
        .link = "\(.dir)/\(.basename).\(.formats[0])"
    )
;

# Front matter is the page to be associated with the enclosing directory rather
# than itself.
def frontmatter:
    (entries | select(has("contents"))) |= (
        (.frontmatter = [.contents[] | select(.frontmatter | bool)][0]) |
        (.contents = .contents - [.frontmatter])
    )
;

# Any page that has no link should be excluded from the index. It would be
# nicer if we could remove in one swoop with `entries | ... |= empty`. It only
# works sometimes, presumably since we'd change the objects we'd iterate over.
def clean:
    if has("contents") then .contents |= map(clean)
    elif has("link")   then .
    else empty end
;

# Add links to the next and previous page for every linked page.
def navigation:
    def f($paths; $i; $j; $key):
        setpath($paths[$i]+[$key];
            getpath($paths[$j]) | {link,title}
        )
    ;
    [ path(entries | select(.formats | at_least(1))) ] as $paths
    | ($paths | length) as $n
    | reduce range(0; $n-1) as $i (.; f($paths; $i; $i+1; "next"))
    | reduce range(1, $n)   as $i (.; f($paths; $i; $i-1; "prev"))
;

# Sort content first according to sort order in metadata.
def sorting:
    (entries | select(has("contents")) | .contents) |= (
        sort_by(.sort // .title // .name)
    )
;

# Add a note that tells us whether this has only children who have no more
# subchildren.
def annotate_leaves:
    (entries | select(has("contents"))) |= (
        .only_leaves = (.contents | all(has("contents") | not))
    )
;

# Combines the given stream of JSON objects by merging them, and performs the
# given operations to turn it into a proper index.
def index($all_formats):
    merge
    | directory
    | basename
    | formats($all_formats)
    | link
    | clean
    | navigation
    | frontmatter
    | sorting
    | annotate_leaves
;

# Get target files and their dependencies as an enumeration of {"target":...,
# "dependencies":...} objects.
def targets($dest; $default_style):
    entries
    | ., (.frontmatter // empty)
    | .formats[] as $format
    | "\($dest)/\(.dir)/\(.basename).\($format)" as $doc
    | ["\($dest)/\(.dir)/\(.resources?[])"] as $external
    | ["\($dest)/\(.style // $default_style).css"] as $css
    | (if $format == "html" then ($css + $external) else [] end) as $linked
    | (if $format == "pdf"  then ($css + $external) else [] end) as $embedded
    | {"target": $format, "dependencies": ([$doc] + $linked) }
    , {"target": $doc, "dependencies": ($embedded) }
    , {"target": $doc, "dependencies": ["NEXT_LINK := \( .next.link // empty)"]}
    , {"target": $doc, "dependencies": ["PREV_LINK := \( .prev.link // empty)"]}
    , {"target": $doc, "dependencies": ["NEXT_TITLE := \( .next.title // empty)"]}
    , {"target": $doc, "dependencies": ["PREV_TITLE := \( .prev.title // empty)"]}
;


allow sec_pro users to use the report editor

import module namespace user = "http://apps.28.io/user";
import module namespace session = "http://apps.28.io/session";

if (is-available-collection($user:collection))
then ();
else create($user:collection);

if (is-available-collection($user:pictures))
then ();
else create($user:pictures);

if (is-available-collection($session:tokens))
then ();
else create($session:tokens);

if (is-available-collection($user:roles))
then truncate($user:roles);
else create($user:roles);

if (is-available-collection($user:rights))
then truncate($user:rights);
else create($user:rights);

if (is-available-collection("reportcache"))
then truncate("reportcache");
else create("reportcache");

if (is-available-collection("reportschemas"))
then ();
else create("reportschemas");

if (is-available-collection("reports"))
then ();
else create("reports");

if (is-available-collection("audit"))
then ();
else create("audit");

flush();

(: RIGHTS :)
user:new-right("rights", "Rights", (), 1);
user:new-right("settings", "Settings", "rights", 1);
user:new-right("users", "Users", "settings", 1);
user:new-right("roles", "Roles", "settings", 2);
user:new-right("reports", "Reports", "settings", 3);

(: RIGHTS: users :)
user:new-right("users_new", "Create New User", "users", 3);
user:new-right("users_list", "List All Users", "users", 2);
user:new-right("users_get", "View User", "users", 1);
user:new-right("users_get_self", "Get Profile", "users_get", 1);
user:new-right("users_edit", "Edit User", "users", 4);
user:new-right("users_edit_self", "Update Profile", "users_get_self", 1);
user:new-right("users_suspend", "Suspend User", "users_edit", 3);
user:new-right("users_activate", "Reactivate User", "users_edit", 2);
user:new-right("users_reset_password", "Reset Anybodys Password", "users", 4);
user:new-right("users_reset_password_self", "Reset Own Password", "users_get_self", 1);
user:new-right("users_upload_picture", "Upload User Picture", "users_edit", 4);
user:new-right("users_upload_picture_self", "Upload Profile Picture", "users_edit_self", 1);
user:new-right("users_remove_picture", "Remove User Picture", "users_edit", 4);
user:new-right("users_remove_picture_self", "Remove Profile Picture", "users_edit_self", 2);

(: RIGHTS: roles :)
user:new-right("users_assign_role","Assign Role to User","roles", 4);
user:new-right("users_close_assignment","Close Role Assignment of User","roles", 5);
user:new-right("roles_list","List All Roles and Rights","roles", 1);
user:new-right("roles_new","Create New Role","roles", 2);
user:new-right("roles_change_permissions","Change Permissions for Roles","roles", 3);

(: RIGHTS: reports :)
user:new-right("reports_get", "View Report", "reports", 1);
user:new-right("reports_validate", "Validate a Report", "reports", 2);
user:new-right("reports_edit", "Edit Report", "reports", 3);
user:new-right("reports_create", "Create a new Report", "reports_edit", 1);
user:new-right("reports_remove", "Remove Report", "reports_edit", 2);

(: RIGHTS: Data :)
user:new-right("data", "Data Access", (), 1);
user:new-right("data_sec", "SEC data access", "data", 1);
user:new-right("data_sec_dow30", "SEC data access to DOW30", "data_sec", 2);

(: ROLES :)
user:new-role("admin", "Administrator");
user:new-role("anybody", "All Users");
user:new-role("reports-admin", "Users Allowed to edit reports");
user:new-role("sec_pro", "Paid users (SEC Pro)");
user:new-role("sec_enterprise", "Paid users (SEC Enterprise)");

flush();

(: assign ROLES -> RIGHTS :)
user:allow("admin", "rights");
user:allow("admin", "data");
user:allow("anybody", "users_get_self");
user:allow("anybody", "users_reset_password_self");
user:allow("anybody", "users_upload_picture_self");
user:allow("anybody", "users_edit_self");
user:allow("anybody", "users_remove_picture_self");
user:allow("reports-admin", "reports");
user:allow("sec_pro", "data_sec");
user:allow("sec_pro", "reports");
user:allow("sec_enterprise", "data_sec");
user:allow("anybody", "data_sec_dow30");

(: Pro user :)
variable $user := user:get-by-email("support@28.io");
if(empty($user))
then {
    $user := user:new("support@28.io", "System", "Administrator", "foobar", {| |});
    user:assign-role($user, "sec_pro", (), (), $user);
  }
else ();

(: Admin user :)
variable $admin := user:get-by-email("admin@28.io");
if(empty($admin))
then {
    $admin := user:new("admin@28.io", "System", "Administrator", "12345", {| |});
    user:assign-role($admin, "admin", (), (), $admin);
  }
else ();

import module namespace user = "http://apps.28.io/user";
import module namespace session = "http://apps.28.io/session";

if (is-available-collection($user:collection))
then ();
else create($user:collection);

if (is-available-collection($user:pictures))
then ();
else create($user:pictures);

if (is-available-collection($session:tokens))
then ();
else create($session:tokens);

if (is-available-collection($user:roles))
then truncate($user:roles);
else create($user:roles);

if (is-available-collection($user:rights))
then truncate($user:rights);
else create($user:rights);

if (is-available-collection("reportcache"))
then truncate("reportcache");
else create("reportcache");

if (is-available-collection("reportschemas"))
then ();
else create("reportschemas");

if (is-available-collection("reports"))
then ();
else create("reports");

if (is-available-collection("audit"))
then ();
else create("audit");

flush();

(: RIGHTS :)
user:new-right("rights", "Rights", (), 1);
user:new-right("settings", "Settings", "rights", 1);
user:new-right("users", "Users", "settings", 1);
user:new-right("roles", "Roles", "settings", 2);
user:new-right("reports", "Reports", "settings", 3);

(: RIGHTS: users :)
user:new-right("users_new", "Create New User", "users", 3);
user:new-right("users_list", "List All Users", "users", 2);
user:new-right("users_get", "View User", "users", 1);
user:new-right("users_get_self", "Get Profile", "users_get", 1);
user:new-right("users_edit", "Edit User", "users", 4);
user:new-right("users_edit_self", "Update Profile", "users_get_self", 1);
user:new-right("users_suspend", "Suspend User", "users_edit", 3);
user:new-right("users_activate", "Reactivate User", "users_edit", 2);
user:new-right("users_reset_password", "Reset Anybodys Password", "users", 4);
user:new-right("users_reset_password_self", "Reset Own Password", "users_get_self", 1);
user:new-right("users_upload_picture", "Upload User Picture", "users_edit", 4);
user:new-right("users_upload_picture_self", "Upload Profile Picture", "users_edit_self", 1);
user:new-right("users_remove_picture", "Remove User Picture", "users_edit", 4);
user:new-right("users_remove_picture_self", "Remove Profile Picture", "users_edit_self", 2);

(: RIGHTS: roles :)
user:new-right("users_assign_role","Assign Role to User","roles", 4);
user:new-right("users_close_assignment","Close Role Assignment of User","roles", 5);
user:new-right("roles_list","List All Roles and Rights","roles", 1);
user:new-right("roles_new","Create New Role","roles", 2);
user:new-right("roles_change_permissions","Change Permissions for Roles","roles", 3);

(: RIGHTS: reports :)
user:new-right("reports_get", "View Report", "reports", 1);
user:new-right("reports_validate", "Validate a Report", "reports", 2);
user:new-right("reports_edit", "Edit Report", "reports", 3);
user:new-right("reports_create", "Create a new Report", "reports_edit", 1);
user:new-right("reports_remove", "Remove Report", "reports_edit", 2);

(: RIGHTS: Data :)
user:new-right("data", "Data Access", (), 1);
user:new-right("data_sec", "SEC data access", "data", 1);
user:new-right("data_sec_dow30", "SEC data access to DOW30", "data_sec", 2);

(: ROLES :)
user:new-role("admin", "Administrator");
user:new-role("anybody", "All Users");
user:new-role("reports-admin", "Users Allowed to edit reports");
user:new-role("sec_pro", "Paid users (SEC Pro)");
user:new-role("sec_enterprise", "Paid users (SEC Enterprise)");

flush();

(: assign ROLES -> RIGHTS :)
user:allow("admin", "rights");
user:allow("admin", "data");
user:allow("anybody", "users_get_self");
user:allow("anybody", "users_reset_password_self");
user:allow("anybody", "users_upload_picture_self");
user:allow("anybody", "users_edit_self");
user:allow("anybody", "users_remove_picture_self");
user:allow("reports-admin", "reports");
user:allow("sec_pro", "data_sec");
user:allow("sec_enterprise", "data_sec");
user:allow("anybody", "data_sec_dow30");

(: Pro user :)
variable $user := user:get-by-email("support@28.io");
if(empty($user))
then {
    $user := user:new("support@28.io", "System", "Administrator", "foobar", {| |});
    user:assign-role($user, "sec_pro", (), (), $user);
  }
else ();

(: Admin user :)
variable $admin := user:get-by-email("admin@28.io");
if(empty($admin))
then {
    $admin := user:new("admin@28.io", "System", "Administrator", "12345", {| |});
    user:assign-role($admin, "admin", (), (), $admin);
  }
else ();