jq: Add doc filter [ci skip]

jq: Add doc filter [ci skip]

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v);

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: clean | select(is_error);
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: clean | select(is_warn);
def warn_strict: clean | select(is_warn_strict);
def info_level: 30;
def is_info: .level >= info_level;
def info: clean | select(is_info);
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: clean | select(is_debug);

# Components:
def chokidar: select(.component == "Chokidar");
def LocalChange: select(.component == "local/change");
def LocalWatcher: select(.component == "LocalWatcher");
def Merge: select(.component == "Merge");
def Metadata: select(.component == "Metadata");
def Pouch: select(.component == "Pouch");
def Sync: select(.component == "Sync");

# Find conflicts:
#
#     yarn jq -c conflicts path/to/logs*
#
def is_conflict: .msg == "resolveConflictAsync";
def conflicts: clean | select(is_conflict) | {time,path};

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
def path(pattern): clean | select((.path,.oldpath,"") | strings | test(pattern));

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#
def time(pattern):
    select(.time | test(pattern));

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path};

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v);

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: clean | select(is_error);
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: clean | select(is_warn);
def warn_strict: clean | select(is_warn_strict);
def info_level: 30;
def is_info: .level >= info_level;
def info: clean | select(is_info);
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: clean | select(is_debug);

# Components:
def chokidar: select(.component == "Chokidar");
def LocalChange: select(.component == "local/change");
def LocalWatcher: select(.component == "LocalWatcher");
def Merge: select(.component == "Merge");
def Metadata: select(.component == "Metadata");
def Pouch: select(.component == "Pouch");
def Sync: select(.component == "Sync");

# Find conflicts:
#
#     yarn jq -c conflicts path/to/logs*
#
def is_conflict: .msg == "resolveConflictAsync";
def conflicts: clean | select(is_conflict) | {time,path};

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
def path(pattern): clean | select((.path,.oldpath,"") | strings | test(pattern));

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#
def time(pattern):
    select(.time | test(pattern));

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path};

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );


added audit trails to facts.jq

added audit trails to facts.jq

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace sec-fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";

import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-csv($o as object*) as string?
{
    if (exists($o)) (: bug in csv:serialize :)
    then
    string-join(
        csv:serialize(
            for $o in $o
            let $a := $o.Aspects
            return {|
                for $k in keys($a)
                return { $k : $a.$k },
                { "Value" : $o.Value },
                if (exists($o."Unit"))
                then { "Unit" :  $o."Unit" }
                else (),
                if (exists($o.Decimals))
                then { "Decimals" : $o.Decimals }
                else(),
                { "EntityRegistrantName" : $o.EntityRegistrantName },
                if (exists($o.ReportedConcept))
                then { "ReportedConcept" : $o.ReportedConcept }
                else ()
            |},
            { serialize-null-as : "" }
        )
    )
    else ()
};

declare function local:audittrail-to-xml($audit as item) as element()
{
    <AuditTrails>{
        for $a in flatten($audit)
        return (
            <Type>{$a.Type}</Type>,
            <Label>{$a.Label}</Label>,
            <Message>{$a.Message}</Message>,
            <Data>{
                if (exists($a.Data.OriginalConcept))
                then <OriginalConcept>{$a.Data.OriginalConcept}</OriginalConcept>
                else (),
                if (exists($a.Data.OutputConcept))
                then <OutputConcept>{$a.Data.OutputConcept}</OutputConcept>
                else (),
                if (exists($a.Data.AuditTrails))
                then local:audittrail-to-xml($a.Data.AuditTrails)
                else ()
            }</Data>
        )
    }</AuditTrails>
};

declare function local:to-xml($o as object*) as node()*
{
    (session:comment("xml", {
        NumFacts: count($o),
        TotalNumFacts: session:num-facts(),
        TotalNumArchives: session:num-archives(),
        TotalNumEntities: session:num-entities()
    }),
    <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
        for $o in $o
        let $a := $o.Aspects
        return
            <Fact>{
                <Aspects>{
                    for $k in keys($a)
                    return
                        <Aspect>
                            <Name>{$k}</Name>
                            <Value>{$a.$k}</Value>
                        </Aspect>
                }</Aspects>,
                <Value>{$o.Value}</Value>,
                <Type>{$o.Type}</Type>,
                if (exists($o.Unit))
                then <Unit>{$o.Unit}</Unit>
                else(),
                if (exists($o.Decimals))
                then <Decimals>{$o.Decimals}</Decimals>
                else (),
                <EntityRegistrantName>{$o.EntityRegistrantName}</EntityRegistrantName>,
                if (exists($o.AuditTrails))
                then local:audittrail-to-xml($o."AuditTrails")
                else ()
            }</Fact>
    }</FactTable>)
};

declare function local:hypercube() as object
{
    hypercubes:user-defined-hypercube({|
        let $concepts := (request:param-values("concept"), request:param-values("xbrl:Concept"))
        return
            if (exists($concepts))
            then { "xbrl:Concept" : { Domain : [ $concepts ] } }
            else (),

        if (not(request:param-names() = "dei:LegalEntityAxis"))
        then {
                "dei:LegalEntityAxis" : {
                    "Domain" : [ "sec:DefaultLegalEntity" ],
                    "Default" : "sec:DefaultLegalEntity"
                }
        } else (),

        for $p in request:param-names()
        where contains($p, ":") and not(starts-with($p, "xbrl:Concept"))
        group by $dimension-name := if (ends-with(lower-case($p), "::default"))
                                    then substring-before($p, "::default")
                                    else if (ends-with(lower-case($p), ":default"))
                                    then substring-before($p, ":default")
                                    else $p
        let $all := (request:param-values($dimension-name) ! upper-case($$)) = "ALL"
        return
        {
            $dimension-name : {|
                let $v := request:param-values($dimension-name)
                return
                    if (exists($v) and not($all))
                    then { "Domain" : [ $v ] }
                    else (),

            let $predefined-default := collection("defaultaxis")[$$.axis eq $dimension-name]
            let $is-default := ($p = $dimension-name || "::default") or ($p = $dimension-name || ":default")
            return
                if (exists($predefined-default) and not($is-default))
                then { "Default" : $predefined-default.default }
                else if ($is-default)
                then {
                        "Default" : (
                                request:param-values($dimension-name || "::default"),
                                request:param-values($dimension-name || ":default")
                            )[1]
                    }
                else ()
            |}
        }
    |})
};

declare function local:facts(
    $archives as object*,
    $map as string?,
    $rules as string?) as object*
{
    for $fact in
        for $f in hypercubes:facts-for-hypercube(local:hypercube(), $archives,
            {|
                { Filter : {
                        "Profiles.SEC.Fiscal": { "$exists" : true }
                    }
                },
                if (exists($map)) then { "concept-maps" : $map } else (),
                if (exists($rules)) then { "Rules" : $rules } else ()
            |}
        )
        group by $f.Aspects."xbrl:Entity",
                 $f.Profiles.SEC.Fiscal.Year,
                 $f.Profiles.SEC.Fiscal.Period,
                 $f.Aspects."xbrl:Concept"
        let $latest-accepted := max(distinct-values($f.Profiles.SEC.Accepted))
        return if (empty($latest-accepted))
               then $f
               else $f[$$.Profiles.SEC.Accepted eq $latest-accepted]
    return {|
        { Aspects : {|
            for $a in keys($fact.Aspects)
            where $a ne "xbrl:Unit"
            return { $a : $fact.Aspects.$a },
            { "bizql:FiscalPeriod" : $fact.Profiles.SEC.Fiscal.Period },
            { "bizql:FiscalYear" : $fact.Profiles.SEC.Fiscal.Year }
        |} },
        { Type: $fact.Type },
        if (exists($fact.Aspects."xbrl:Unit"))
        then { Unit: $fact.Aspects."xbrl:Unit" }
        else  (),
        if (exists($fact.Decimals))
        then { Decimals: $fact.Decimals }
        else (),
        { Value: $fact.Value },
        { "EntityRegistrantName" : companies:companies($fact.Aspects."xbrl:Entity").Profiles.SEC.CompanyName},
        if (exists(flatten($fact.AuditTrails[])))
        then { "AuditTrails" : $fact.AuditTrails }
        else ()
    |}
};

declare function local:filings(
    $ciks as string*,
    $tags as string*,
    $tickers as string*,
    $sics as string*,
    $fp as string*,
    $fy as string*) as object*
{
    let $entities := if ($tags = "ALL") then companies:companies()
                                        else (
                                            companies:companies($ciks),
                                            companies:companies-for-tags($tags),
                                            companies:companies-for-tickers($tickers),
                                            companies:companies-for-SIC($sics)
                                        )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then sec-fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else sec-fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                        case "ALL" return  $sec-fiscal:ALL_FISCAL_YEARS
                    default return $fy
                )
    for $fp in $fp
    return sec-fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy cast as integer)
};

session:audit-call();

let $format      := lower-case(request:param-values("format")[1])

(: choose
    1. entities (using ciks, tags, tickers, sics), FY, and FP ) or
    2. accession numbers
:)
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $sec-fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids     := request:param-values("aid")
let $map      := request:param-values("map")[1]
let $rules    := request:param-values("rules")[1]
let $archives := (
                    local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                    archives:archives($aids)
                 )
let $entities := companies:companies($archives.Entity)
return
    switch(true)
    case empty($archives) return {
        response:status-code(400);
        session:error("entities or archives not found (valid parameters: cik, ticker, tag, sic, aid)", $format)
    }
    case not (session:only-dow30($entities) or session:valid()) return {
        response:status-code(401);
        session:error("accessing facts of an entity that is not in the DOW30", $format)
    }
    default return
        let $facts := local:facts($archives, $map, $rules)
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($facts)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=facts.csv");
                local:to-csv($facts)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=fact.csv");
                local:to-csv($facts)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { NetworkIdentifier : "http://bizql.io/facts" },
                    { TableName : "xbrl:Facts" },
                    { FactTable : [ $facts ] },
                    session:comment("json", {
                            NumFacts: count($facts),
                            TotalNumFacts: session:num-facts(),
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })
                |}
            }

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace sec-fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";

import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-csv($o as object*) as string?
{
    if (exists($o)) (: bug in csv:serialize :)
    then
    string-join(
        csv:serialize(
            for $o in $o
            let $a := $o.Aspects
            return {|
                for $k in keys($a)
                return { $k : $a.$k },
                { "Value" : $o.Value },
                if (exists($o."Unit"))
                then { "Unit" :  $o."Unit" }
                else (),
                if (exists($o.Decimals))
                then { "Decimals" : $o.Decimals }
                else(),
                { "EntityRegistrantName" : $o.EntityRegistrantName },
                if (exists($o.ReportedConcept))
                then { "ReportedConcept" : $o.ReportedConcept }
                else ()
            |},
            { serialize-null-as : "" }
        )
    )
    else ()
};

declare function local:to-xml($o as object*) as node()*
{
    (session:comment("xml", {
        NumFacts: count($o),
        TotalNumFacts: session:num-facts(),
        TotalNumArchives: session:num-archives(),
        TotalNumEntities: session:num-entities()
    }),
    <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
        for $o in $o
        let $a := $o.Aspects
        return
            <Fact>{
                <Aspects>{
                    for $k in keys($a)
                    return
                        <Aspect>
                            <Name>{$k}</Name>
                            <Value>{$a.$k}</Value>
                        </Aspect>
                }</Aspects>,
                <Value>{$o.Value}</Value>,
                <Type>{$o.Type}</Type>,
                if (exists($o.Unit))
                then <Unit>{$o.Unit}</Unit>
                else(),
                if (exists($o.Decimals))
                then <Decimals>{$o.Decimals}</Decimals>
                else (),
                <EntityRegistrantName>{$o.EntityRegistrantName}</EntityRegistrantName>,
                if (exists($o.ReportedConcept))
                then <ReportedConcept>{$o."ReportedConcept"}</ReportedConcept>
                else ()
            }</Fact>
    }</FactTable>)
};

declare function local:hypercube() as object
{
    hypercubes:user-defined-hypercube({|
        let $concepts := (request:param-values("concept"), request:param-values("xbrl:Concept"))
        return
            if (exists($concepts))
            then { "xbrl:Concept" : { Domain : [ $concepts ] } }
            else (),

        if (not(request:param-names() = "dei:LegalEntityAxis"))
        then {
                "dei:LegalEntityAxis" : {
                    "Domain" : [ "sec:DefaultLegalEntity" ],
                    "Default" : "sec:DefaultLegalEntity"
                }
        } else (),

        for $p in request:param-names()
        where contains($p, ":") and not(starts-with($p, "xbrl:Concept"))
        group by $dimension-name := if (ends-with(lower-case($p), "::default"))
                                    then substring-before($p, "::default")
                                    else if (ends-with(lower-case($p), ":default"))
                                    then substring-before($p, ":default")
                                    else $p
        let $all := (request:param-values($dimension-name) ! upper-case($$)) = "ALL"
        return
        {
            $dimension-name : {|
                let $v := request:param-values($dimension-name)
                return
                    if (exists($v) and not($all))
                    then { "Domain" : [ $v ] }
                    else (),

            let $predefined-default := collection("defaultaxis")[$$.axis eq $dimension-name]
            let $is-default := ($p = $dimension-name || "::default") or ($p = $dimension-name || ":default")
            return
                if (exists($predefined-default) and not($is-default))
                then { "Default" : $predefined-default.default }
                else if ($is-default)
                then {
                        "Default" : (
                                request:param-values($dimension-name || "::default"),
                                request:param-values($dimension-name || ":default")
                            )[1]
                    }
                else ()
            |}
        }
    |})
};

declare function local:facts(
    $archives as object*,
    $map as string?,
    $rules as string?) as object*
{
    for $fact in
        for $f in hypercubes:facts-for-hypercube(local:hypercube(), $archives,
            {|
                { Filter : {
                        "Profiles.SEC.Fiscal": { "$exists" : true }
                    }
                },
                if (exists($map)) then { "concept-maps" : $map } else (),
                if (exists($rules)) then { "Rules" : $rules } else ()
            |}
        )
        group by $f.Aspects."xbrl:Entity",
                 $f.Profiles.SEC.Fiscal.Year,
                 $f.Profiles.SEC.Fiscal.Period,
                 $f.Aspects."xbrl:Concept"
        let $latest-accepted := max(distinct-values($f.Profiles.SEC.Accepted))
        return if (empty($latest-accepted))
               then $f
               else $f[$$.Profiles.SEC.Accepted eq $latest-accepted]
    return {|
        { Aspects : {|
            for $a in keys($fact.Aspects)
            where $a ne "xbrl:Unit"
            return { $a : $fact.Aspects.$a },
            { "bizql:FiscalPeriod" : $fact.Profiles.SEC.Fiscal.Period },
            { "bizql:FiscalYear" : $fact.Profiles.SEC.Fiscal.Year }
        |} },
        { Type: $fact.Type },
        if (exists($fact.Aspects."xbrl:Unit"))
        then { Unit: $fact.Aspects."xbrl:Unit" }
        else  (),
        if (exists($fact.Decimals))
        then { Decimals: $fact.Decimals }
        else (),
        { Value: $fact.Value },
        { "EntityRegistrantName" : companies:companies($fact.Aspects."xbrl:Entity").Profiles.SEC.CompanyName},
        if (exists($map) and ($fact."xbrl28:Type" eq "xbrl28:concept-maps"))
        then { "ReportedConcept" : $fact.AuditTrails[][$$.Type eq "xbrl28:concept-maps"].Data.OriginalConcept[1] }
        else ()
    |}
};

declare function local:filings(
    $ciks as string*,
    $tags as string*,
    $tickers as string*,
    $sics as string*,
    $fp as string*,
    $fy as string*) as object*
{
    let $entities := if ($tags = "ALL") then companies:companies()
                                        else (
                                            companies:companies($ciks),
                                            companies:companies-for-tags($tags),
                                            companies:companies-for-tickers($tickers),
                                            companies:companies-for-SIC($sics)
                                        )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then sec-fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else sec-fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                        case "ALL" return  $sec-fiscal:ALL_FISCAL_YEARS
                    default return $fy
                )
    for $fp in $fp
    return sec-fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy cast as integer)
};

session:audit-call();

let $format      := lower-case(request:param-values("format")[1])

(: choose
    1. entities (using ciks, tags, tickers, sics), FY, and FP ) or
    2. accession numbers
:)
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $sec-fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids     := request:param-values("aid")
let $map      := request:param-values("map")[1]
let $rules    := request:param-values("rules")[1]
let $archives := (
                    local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                    archives:archives($aids)
                 )
let $entities := companies:companies($archives.Entity)
return
    switch(true)
    case empty($archives) return {
        response:status-code(400);
        session:error("entities or archives not found (valid parameters: cik, ticker, tag, sic, aid)", $format)
    }
    case not (session:only-dow30($entities) or session:valid()) return {
        response:status-code(401);
        session:error("accessing facts of an entity that is not in the DOW30", $format)
    }
    default return
        let $facts := local:facts($archives, $map, $rules)
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($facts)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=facts.csv");
                local:to-csv($facts)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=fact.csv");
                local:to-csv($facts)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { NetworkIdentifier : "http://bizql.io/facts" },
                    { TableName : "xbrl:Facts" },
                    { FactTable : [ $facts ] },
                    session:comment("json", {
                            NumFacts: count($facts),
                            TotalNumFacts: session:num-facts(),
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })
                |}
            }


Update facts.jq

Update facts.jq

import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace rules = "http://28.io/modules/xbrl/rules";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace request = "http://www.28msec.com/modules/http-request";

declare function local:param-values($name as string) as string*
{
    switch(true)
     case $name eq "xbrl:Concept"
        return (request:param-values("concept"), request:param-values("xbrl:Concept"))

     case $name eq "sec:FiscalYear" and $profile-name eq "sec"
        return ($fiscalYear, request:param-values("sec:FiscalYear"))[$$ ne "LATEST"]
     case $name eq "sec:FiscalYear::type" and $profile-name eq "sec"
        return "integer"
     case $name eq "sec:FiscalPeriod" and $profile-name eq "sec"
        return ($fiscalPeriod, request:param-values("sec:FiscalPeriod"))
     case $name eq "dei:LegalEntityAxis" and $profile-name eq "sec"
        return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis")
     case $name eq "dei:LegalEntityAxis::default" return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis::default")
     case $name eq "xbrl:Entity" and $profile-name eq "sec" return (
         let $companies := companies:companies(
            $cik,
            api:preprocess-tags($tag),
            $ticker,
            $sic)._id
        return if(empty(($cik,$tag,$ticker,$sic)) or exists($companies))
               then $companies
               else "dummy",
        request:param-values("xbrl:Entity"))
     case $name eq "sec:Archive" and $profile-name eq "sec" return (
            let $fiscalYears := ($fiscalYear, request:param-values("sec:FiscalYear"))
            let $fiscalPeriods := local:param-values("sec:FiscalPeriod")
            let $entities := entities:entities(local:param-values("xbrl:Entity"))
            return
                if($fiscalYears = "LATEST")
                then fiscal-core:latest-filings($entities, $fiscalPeriods)._id
                else (),
            $aid,
            request:param-values("sec:Archive")
        )
     default return request:param-values($name)
};

declare function local:param-names() as string*
{
    let $names := request:param-names()
    return distinct-values((
        $names[contains($$, ":")],
        "xbrl:Concept"[$names = "concept"],

        "sec:Accepted"[$profile-name eq "sec"],
        "sec:FiscalPeriod"[$profile-name eq "sec"],
        "sec:FiscalYear"[$profile-name eq "sec"],
        "xbrl:Entity"[$profile-name eq "sec" and $names = ("cik", "tag", "ticker", "sic")],
        "dei:LegalEntityAxis"[$profile-name eq "sec"],
        "dei:LegalEntityAxis::default"[$profile-name eq "sec"],
        "sec:Archive"[$profile-name eq "sec"]))
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube() as object
{
    let $hypercube-spec :=
    {|
        for $parameter in local:param-names()
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (local:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $type as string? :=
            (local:param-values($dimension-name || "::type"), local:param-values($dimension-name || ":type"))[1]

        let $values := local:param-values($dimension-name)
        let $typed-values := if (exists($type)) then local:cast-sequence($values, $type) else $values

        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (local:param-values($dimension-name || "::default"), local:param-values($dimension-name || ":default"))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value

        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                { "Domain" : [ $typed-values ] }[(exists($typed-values) and not($all))],
                { "Default" : $typed-default-value }[$has-default]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($hypercube-spec)
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token             as string? external;
declare  %rest:case-insensitive                 variable $profile-name      as string  external := "sec";
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $validate          as boolean external := false;
declare  %rest:case-insensitive                 variable $labels            as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic)
let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $hypercube := local:hypercube()

let $facts :=
    let $options := {|
      {
        Hypercube : $hypercube,
        Validate: $validate
      },
      { "ConceptMaps" : $map }[exists($map)],
      { "Rules" : [ $rule ] }[exists($rule)]
    |}
    return if($profile-name eq "sec")
    then sec:facts-for($options)
    else facts:facts-for($options)

let $facts :=
  if(not $labels)
  then $facts
  else
    let $archives as string* := distinct-values($facts.Aspects."sec:Archive")
    let $concept-names as string* := distinct-values($facts.Aspects."xbrl:Concept")
    let $concepts as object* :=
      (
          concepts:concepts($concept-names, $archives, $concepts:ANY_COMPONENT_LINK_ROLE),
          (reports:concepts($report))[$$.Name = $concept-names]
      )
    let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $concepts:AMERICAN_ENGLISH )[1]
    let $roles as string* := ( $report.Role, $concepts:ANY_COMPONENT_LINK_ROLE )
    for $fact as object in $facts
    let $entityName as string := $entities[$$._id eq $fact.Aspects."xbrl:Entity"].Profiles.SEC.CompanyName
    return
    {|
      $fact,
      let $concept-labels as object? := facts:labels($fact, $roles, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
      let $standard-labels as object := conversion:get-standard-labels($fact, $entityName)
      return { Labels : {| $concept-labels, $standard-labels |} }
    |}

let $facts :=
  if($profile-name eq "sec")
  then api:normalize-facts($facts)
  else $facts

let $result := {
    NetworkIdentifier : "http://bizql.io/facts",
    TableName : "xbrl:Facts",
    FactTable : [ $facts ]
}
let $comment :=
{
    NumFacts: count($facts),
    TotalNumFacts: session:num-facts(),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "facts")
return api:check-and-return-results($token, $results, $format)

import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace facts = "http://28.io/modules/xbrl/facts";
import module namespace rules = "http://28.io/modules/xbrl/rules";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace request = "http://www.28msec.com/modules/http-request";

declare function local:param-values($name as string) as string*
{
    switch(true)
     case $name eq "xbrl:Concept"
        return (request:param-values("concept"), request:param-values("xbrl:Concept"))

     case $name eq "sec:FiscalYear" and $profile-name eq "sec"
        return ($fiscalYear, request:param-values("sec:FiscalYear"))[$$ ne "LATEST"]
     case $name eq "sec:FiscalYear::type" and $profile-name eq "sec"
        return "integer"
     case $name eq "sec:FiscalPeriod" and $profile-name eq "sec"
        return ($fiscalPeriod, request:param-values("sec:FiscalPeriod"))
     case $name eq "dei:LegalEntityAxis" and $profile-name eq "sec"
        return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis")
     case $name eq "dei:LegalEntityAxis::default" return
         if(empty((request:param-values("sec:LegalEntityAxis"), request:param-values("sec:LegalEntityAxis::default"))))
         then "sec:DefaultLegalEntity"
         else request:param-values("sec:LegalEntityAxis::default")
     case $name eq "xbrl:Entity" and $profile-name eq "sec" return (
         let $companies := companies:companies(
            $cik,
            api:preprocess-tags($tag),
            $ticker,
            $sic)._id
        return if(empty(($cik,$tag,$ticker,$sic)) or exists($companies))
               then $companies
               else "dummy",
        request:param-values("xbrl:Entity"))
     case $name eq "sec:Archive" and $profile-name eq "sec" return (
            let $fiscalYears := ($fiscalYear, request:param-values("sec:FiscalYear"))
            let $fiscalPeriods := local:param-values("sec:FiscalPeriod")
            let $entities := entities:entities(local:param-values("xbrl:Entity"))
            return
                if($fiscalYears = "LATEST")
                then fiscal-core:latest-filings($entities, $fiscalPeriods)._id
                else (),
            $aid,
            request:param-values("sec:Archive")
        )
     default return request:param-values($name)
};

declare function local:param-names() as string*
{
    let $names := request:param-names()
    return distinct-values((
        $names[contains($$, ":")],
        "xbrl:Concept"[$names = "concept"],

        "sec:Accepted"[$profile-name eq "sec"],
        "sec:FiscalPeriod"[$profile-name eq "sec"],
        "sec:FiscalYear"[$profile-name eq "sec"],
        "xbrl:Entity"[$profile-name eq "sec" and $names = ("cik", "tag", "ticker", "sic")],
        "dei:LegalEntityAxis"[$profile-name eq "sec"],
        "dei:LegalEntityAxis::default"[$profile-name eq "sec"],
        "sec:Archive"[$profile-name eq "sec"]))
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube() as object
{
    let $hypercube-spec :=
    {|
        for $parameter in local:param-names()
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (local:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $type as string? :=
            (local:param-values($dimension-name || "::type"), local:param-values($dimension-name || ":type"))[1]

        let $values := local:param-values($dimension-name)
        let $typed-values := if (exists($type)) then local:cast-sequence($values, $type) else $values

        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (local:param-values($dimension-name || "::default"), local:param-values($dimension-name || ":default"))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value

        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                { "Domain" : [ $typed-values ] }[(exists($typed-values) and not($all))],
                { "Default" : $typed-default-value }[$has-default]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($hypercube-spec)
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token             as string? external;
declare  %rest:case-insensitive                 variable $profile-name      as string  external := "sec";
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $validate          as boolean external := false;
declare  %rest:case-insensitive                 variable $labels            as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic)
let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $hypercube := local:hypercube()

let $facts :=
    let $options := {|
      {
        Hypercube : $hypercube,
        Validate: $validate
      },
      { "ConceptMaps" : $map }[exists($map)],
      { "Rules" : [ $rule ] }[exists($rule)]
    |}
    return if($profile-name eq "sec")
    then sec:facts-for($options)
    else facts:facts-for($options)

let $facts :=
  let $archives as string* := distinct-values($facts.Aspects."sec:Archive")
  let $concept-names as string* := distinct-values($facts.Aspects."xbrl:Concept")
  let $concepts as object* :=
      (
          concepts:concepts($concept-names, $archives, $concepts:ANY_COMPONENT_LINK_ROLE),
          (reports:concepts($report))[$$.Name = $concept-names]
      )
  let $language as string := ( $report.$components:DEFAULT-LANGUAGE , $concepts:AMERICAN_ENGLISH )[1]
  let $roles as string* := ( $report.Role, $concepts:ANY_COMPONENT_LINK_ROLE )
  for $fact as object in $facts
  let $entityName as string := $entities[$$._id eq $fact.Aspects."xbrl:Entity"].Profiles.SEC.CompanyName
  return
    if(not $labels)
    then $fact
    else {|
      $fact,
      let $concept-labels as object? := facts:labels($fact, $roles, $concepts:STANDARD_LABEL_ROLE, $language, $concepts, ())
      let $standard-labels as object := conversion:get-standard-labels($fact, $entityName)
      return { Labels : {| $concept-labels, $standard-labels |} }
    |}

let $facts := if($profile-name eq "sec") then api:normalize-facts($facts) else $facts

let $result := {
    NetworkIdentifier : "http://bizql.io/facts",
    TableName : "xbrl:Facts",
    FactTable : [ $facts ]
}
let $comment :=
{
    NumFacts: count($facts),
    TotalNumFacts: session:num-facts(),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "facts")
return api:check-and-return-results($token, $results, $format)


Implement type reference. Fix for Juan's schema.

jsoniq version "1.0";

(:
 : Copyright 2006-2013 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

(:~
 : JSound simple validator
 : This is a JSONiq implemenation of the JSound (the schema for JSON) validator.
 :
 : @author Cezar Andrei
 :)

module namespace jsv = "http://jsound.io/modules/validate";

import module namespace jn = "http://jsoniq.org/functions";
import module namespace map = "http://jsound.io/modules/validate/map";

declare namespace an = "http://zorba.io/annotations";

declare %an:sequential function
jsv:jsd-valid($jsd as object, $name as string, $ns as string,
  $instance as item) as boolean
{
  variable $jstypes := { "foo" : 5 };

  if( $jsd instance of object )
  then
    if ( exists($jsd("$namespace")) and exists($jsd("$types")) )
    then
      if ( $jsd("$namespace") instance of string )
      then
        if ( $jsd("$namespace") eq $ns )
        then
          if ( $jsd("$types") instance of array )
          then
            jsv:check-types( $jstypes, $jsd("$types") );
          else
            fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: value of '$types' must be an array.");
        else
          fn:error(QName("jsv", "Invalid"),
            "Requested type namespace not matching schema document namespace.");
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: value of '$namespace' must be a string.");
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: must contain '$types' and '$namespace' keys.");
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "The specified JSON document is not a valid JSound schema: top item not an object.");

  (: todo: validate all type refs :)

  jsv:validate-instance($jstypes, $name, $instance)
};

(: The check-xxx functions just checks if the JSound schema definitions are valid. :)

declare %an:sequential function
jsv:check-types($jstypes as object, $types as array) as boolean
{
  every $i in
    for $type in jn:members($types)
    return
      jsv:check-type($jstypes, $type )
  satisfies $i eq fn:true()
};

declare %an:sequential function
jsv:check-type($jstypes as object, $type as item) as boolean
{
  if( $type instance of object )
  then
    if ( exists($type("$kind")) and $type("$kind") instance of string )
    then
      switch( $type("$kind") )
      case "atomic" return jsv:check-atomic-type($jstypes, $type)
      case "object" return jsv:check-object-type($jstypes, $type)
      case "array"  return jsv:check-array-type ($jstypes, $type)
      case "union"  return jsv:check-union-type ($jstypes, $type)

      default return
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: type $kind unknown:" | $type("$kind") )

    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: type definition must contain '$kind' key with string value.")
  else
    if( $type instance of string )
    then
      jsv:check-ref-type($jstypes, $type)
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: type item not an object."  )
};

declare %an:sequential function
jsv:check-atomic-type($jstypes as object, $type as object) as boolean
{
  fn:trace( $type("$name"), "check-atomic-type");

  if( exists($type("$baseType")) and $type("$baseType") instance of string )
  then
    jsv:save-type($jstypes, $type)
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: atomic type definition must contain '$baseType' key with string value.")

  (: todo: add facets checks: length, min/max Inclusive/Exclusive, etc. :)
};

declare %an:sequential function
jsv:check-object-type($jstypes as object, $type as object) as boolean
{
  (: todo: check the reast of "object" type :)
  jsv:save-type($jstypes, $type);
  fn:trace( $type("$name"), "check-object-type");

  if( exists($type("$content")) )
  then
    if( $type("$content") instance of object )
    then
      every $i in
        for $k in jn:keys($type("$content"))
        return
          if( $type("$content")( fn:trace($k, "Key")) instance of object )
          then
            if( exists( $type("$content")($k)("$type")) )
            then
              jsv:check-type($jstypes, $type("$content")($k)("$type") )
            else
              if( exists( $type("$content")($k)("$kind")) )
              then
                jsv:check-type($jstypes, $type("$content")($k) )
              else
                fn:error(QName("jsv", "BadJSoundFormat"),
                  "Not a valid JSound doc: Key  does not contain a $type definition.")
          else
            fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: Key " | $k | " in $content must have an object value.")

      satisfies $i eq fn:true()
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: $content definion of object type must be an object.")
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
             "Not a valid JSound doc: Object type does not contain $content.")
};

declare %an:sequential function
jsv:check-array-type($jstypes as object, $type as object) as boolean
{
  fn:trace( $type("$name"), "check-array-type");

  (: check all the contents :)
  (: let $t := fn:trace($type, "Array type:") :)
  let $content := $type("$content")
  return
    if( empty($content) )
    then
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: $kind array must have a $content key.")
    else
      if( $content instance of array and jn:size($content) eq 1 )
      then
      {
        jsv:save-type($jstypes, $type);

        typeswitch( $content(1) )
        case string return
          fn:true()
        case object return
            jsv:check-type($jstypes, $content(1))
        default return
          fn:error(QName("jsv", "BadJSoundFormat"),
            "Not a valid JSound doc: for $kind array, $content[1] must be string or array.")
      }
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $kind array, $content must be an array of size 1.")

  (: Todo: check array facets :)
};

declare %an:sequential function
jsv:check-union-type($jstypes as object, $type as object) as boolean
{
  fn:trace( $type("$name"), "check-union-type");

  let $content := $type("$content")
  return
    if( empty($content) )
    then
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: $kind union must have a $content key.")
    else
      if( $content instance of array )
      then
      {
        jsv:save-type($jstypes, $type);

        every $i in
          for $c in jn:members($content)
          return
            typeswitch( $c )
            case string return
              fn:true()
            case object return
                jsv:check-type($jstypes, $c)
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind union, $content items must be strings or objects.")
        satisfies $i eq fn:true()
      }
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $kind union, $content must be an array.")
};

declare function
jsv:check-ref-type($jstypes as object, $type as string) as boolean
{
  switch( $type )
  case "string" return fn:true()
  case "integer" return fn:true()
  case "boolean" return fn:true()
  case "anyURI"  return fn:true()
  default return
  {
    if( map:has-key($jstypes, $type) )
    then
      fn:true()
    else
     (: todo: save all refs, check them at the end to see if they exist
     fn:trace($type, "NYI: type ref not yet suported");
     fn:error(QName("jsv", "NYI"), "Type ref: not yet implemented.") :)
     fn:true()
  }

  (: todo: apply resolution rules :)
};

declare %an:sequential function
jsv:save-type($jstypes as object, $type as item) as boolean
{
  if( fn:exists( $type("$name") ) )
  then
    if( $type("$name") instance of string )
    then
      map:set-if-empty($jstypes, $type("$name"), $type)
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $type, $name must be a string.")
  else
    fn:false()
};

(: The validate-xxx functions actualy do the validation of the instance :)

declare function
jsv:validate-instance($jstypes as object, $name as string, $instance as item)
 as boolean
{
  let $type := map:get($jstypes, $name)
  return
    if( exists($type) )
    then
      jsv:validate-type($jstypes, $type, $instance)
    else
      fn:error(QName("jsv", "Invalid"),
             "Requested type name not present in schema: " | $name)
};

declare function
jsv:validate-type($jstypes as object, $type as item, $instance as item) as boolean
{
  switch( $type("$kind") )
  case "atomic" return jsv:validate-atomic-type($jstypes, $type, $instance)
  case "object" return jsv:validate-object-type($jstypes, $type, $instance)
  case "array"  return jsv:validate-array-type($jstypes, $type, $instance)
  case "union"  return jsv:validate-union-type($jstypes, $type, $instance)

  default return
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: type $kind unknown:" | $type("$kind") )
};

declare function
jsv:validate-type-ref($jstypes as object, $type as string, $instance as item) as boolean
{
  (: todo: check first between the defined types in $jstypes and then the build-in ones, this is per spec :)

  if( map:has-key($jstypes, $type) )
  then
    jsv:validate-type($jstypes, map:get($jstypes, $type), $instance)
  else
    switch( $type )
    case "string" return
      if ( $instance instance of string )
      then
        fn:true()
      else
        fn:error(QName("jsv", "Invalid"), "Expected string value.")
    case "integer" return
      if ( $instance instance of integer or
           $instance instance of int )
      then
        fn:true()
      else
        fn:error(QName("jsv", "Invalid"), "Expected integer value.")
    case "anyURI" return
      if ( $instance instance of anyURI )
      then
        fn:true()
      else
        fn:error(QName("jsv", "Invalid"), "Expected anyURI value.")
    case "boolean" return
      if ( $instance instance of boolean )
      then
        fn:true()
      else
        fn:error(QName("jsv", "Invalid"), "Expected boolean value.")

    default return
        fn:error(QName("jsv", "Invalid"),
          fn:concat("Type ref not defined or a built-in type: ", $type) )
};

declare function
jsv:validate-atomic-type($jstypes as object, $type as object, $instance as item) as boolean
{
  fn:trace( $type("$name"), "validate-atomic-type");
  (: todo: check first between the defined types in $jstypes and then the build-in ones, this is per spec :)

  if ( exists($type("$baseType")) and $type("$baseType") instance of string )
  then
    switch( $type("$baseType") )
    case "integer" return
      if ( $instance instance of integer or $instance instance of int )
      then
        fn:true() and jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected integer or int value")

    case "string" return
      if ( $instance instance of string )
      then
        fn:true() and jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error( QName("jsv", "Invalid"), "Expected string value")

    case "anyURI" return
      if ( $instance instance of anyURI )
      then
        fn:true() and jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected anyURI value")

    case "boolean" return
      if ( $instance instance of boolean )
      then
        fn:true() and jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected boolean value")

    default return
      fn:error(QName("jsv", "NYI"),
        "$baseType: not yet implemented" | $type("$baseType") )
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: atomic type definition must contain '$baseType' key with string value.")

  (: todo: add facets checks: length, min/max Inclusive/Exclusive, etc. :)
};

declare function
jsv:validate-enumeration($jstypes as object, $type as object, $instance as item) as boolean
{
  variable $enum := $type("$enumeration");
  if( exists($enum) )
  then
    if( $enum instance of array )
    then
      if( some $v in jn:members($enum) satisfies $v eq $instance )
      then
        fn:true()
      else
        fn:error(QName("jsv", "Invalid"), "Instance value not in the enumeration list.")
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: the value of '$enumeration' must be an array.")
  else
    fn:true()
};

declare function
jsv:validate-object-type($jstypes as object, $type as object, $instance as item) as boolean
{
  fn:trace( $type("$name"), "validate-object-type");

  variable $open := $type("$open");

  if ( $instance instance of object )
  then
    if( (not exists($open)) or (exists($open) and $open eq true()) )
    then
      (: object is open: i.e. extra keys alowed but all defined keys must be in
         the instance except the default and optional ones :)
      (
        every $k in jn:keys($instance)
        satisfies
          let $ctype := $type("$content")($k)("$type")
          return
            typeswitch( $ctype )
            case string return
              jsv:validate-type-ref($jstypes, $ctype, $instance($k) )
            case object return
              jsv:validate-type($jstypes, $ctype, $instance($k) )
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind object, $content $type must be string or array.")
      ) and
      (
        every $k in
          jsv:value-except(jn:keys($type("$content")), jn:keys($instance))
        satisfies
          let $optional := $type("$content")($k)("$optional")
          return
            if( exists($optional) and $optional eq fn:true() )
            then fn:true()
            else
              fn:error(QName("jsv", "Invalid"),
                fn:concat("Required key not present in instance: ", $k) )
      )
    else
      (: object is closed: i.e. all defined keys must be in the instance except
         the default ones, no extra keys alowed :)
      (
        every $k in jn:keys($instance)
        satisfies
          let $ctype := $type("$content")($k)("$type")
          return
            typeswitch( $ctype )
            case string return
              jsv:validate-type-ref($jstypes, $ctype, $instance($k) )
            case object return
              jsv:validate-type($jstypes, $ctype, $instance($k) )
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind object, $content $type must be string or array.")
      ) and (
        every $k in (: jn:keys($type("$content")) except jn:keys($instance) :)
          jsv:value-except(jn:keys($type("$content")), jn:keys($instance))
        satisfies
          let $optional := $type("$content")($k)("$optional")
          return
            if( exists($optional) and $optional eq fn:true() )
            then fn:true()
            else
              fn:error(QName("jsv", "Invalid"),
                fn:concat("Required key not present in instance: ", $k) )
      )
  else
    fn:error( QName("jsv", "Invalid"),
      fn:concat("Instance not an object specified by type: ", jsv:get-type-name($type)) )
};

declare function
jsv:validate-array-type($jstypes as object, $type as object, $instance as item) as boolean
{
  fn:trace( $type("$name"), "validate-array-type");

  if( $instance instance of array )
  then
    (: check all the contents :)
    (: let $t := fn:trace($type, "Array type:") :)
    let $content := $type("$content")
    return
        typeswitch( $content(1) )
        case string return
          every $i in jn:members($instance) satisfies jsv:validate-type-ref($jstypes, $content(1), $i)
        case object return
          every $i in jn:members($instance) satisfies jsv:validate-type($jstypes, $content(1), $i)
        default return
          fn:error(QName("jsv", "BadJSoundFormat"),
            "Not a valid JSound doc: for $kind array, $content[1] must be string or array.")
  else
    fn:error(QName("jsv", "Invalid"),
      fn:concat("Instance not an array for type: ", jsv:get-type-name($type)) )

  (: check array facets :)
};

declare function
jsv:validate-union-type($jstypes as object, $type as object, $instance as item) as boolean
{
  let $content := $type("$content")
  return
    if(
      some $i in
        for $c in jn:members($content)
        return
          try
          {
            typeswitch( $c )
            case string return
              fn:true()
            case object return
                jsv:validate-type($jstypes, $c, $instance)
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind union, $content items must be strings or objects.")
          }
          catch Invalid
          {
             fn:false()
          }
      satisfies $i eq fn:true()
    )
    then fn:true()
    else
      fn:error(QName("jsv", "Invalid"),
        fn:concat("Instance not valid for union type: ", jsv:get-type-name($type)) )
};

(:---- Util functions ---------:)

declare function
jsv:get-type-name($type as object) as string
{
  let $name := $type("$name")
  return
    if ( exists($name) and $name instance of string )
    then
      $name
    else
      "N/A"
};

declare function
jsv:value-except($arg1 as xs:anyAtomicType*, $arg2 as xs:anyAtomicType*)
  as xs:anyAtomicType*
{
  (: fn:distinct-values( $arg1[ not($$ eq $arg2) ] ) :)
  fn:distinct-values(
    for $i in $arg1
    where not( some $x in $arg2 satisfies $i eq $x )
    return $i
  )
};

jsoniq version "1.0";

(:
 : Copyright 2006-2013 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

(:~
 : JSound simple validator
 : This is a JSONiq implemenation of the JSound (the schema for JSON) validator.
 :
 : @author Cezar Andrei
 :)

module namespace jsv = "http://jsound.io/modules/validate";

import module namespace jn = "http://jsoniq.org/functions";
import module namespace map = "http://jsound.io/modules/validate/map";

declare namespace an = "http://zorba.io/annotations";

declare %an:sequential function
jsv:jsd-valid($jsd as object, $name as string, $ns as string,
  $instance as item) as boolean
{
  variable $jstypes := { "foo" : 5 };

  if( $jsd instance of object )
  then
    if ( exists($jsd("$namespace")) and exists($jsd("$types")) )
    then
      if ( $jsd("$namespace") instance of string )
      then
        if ( $jsd("$namespace") eq $ns )
        then
          if ( $jsd("$types") instance of array )
          then
            jsv:check-types( $jstypes, $jsd("$types") );
          else
            fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: value of '$types' must be an array.");
        else
          fn:error(QName("jsv", "Invalid"),
            "Requested type namespace not matching schema document namespace.");
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: value of '$namespace' must be a string.");
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: must contain '$types' and '$namespace' keys.");
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "The specified JSON document is not a valid JSound schema: top item not an object.");

  (: todo: validate all type refs :)

  jsv:validate-instance($jstypes, $name, $instance)
};

(: The check-xxx functions just checks if the JSound schema definitions are valid. :)

declare %an:sequential function
jsv:check-types($jstypes as object, $types as array) as boolean
{
  every $i in
    for $type in jn:members($types)
    return
      jsv:check-type($jstypes, $type )
  satisfies $i eq fn:true()
};

declare %an:sequential function
jsv:check-type($jstypes as object, $type as item) as boolean
{
  if( $type instance of object )
  then
    if ( exists($type("$kind")) and $type("$kind") instance of string )
    then
      switch( $type("$kind") )
      case "atomic" return jsv:check-atomic-type($jstypes, $type)
      case "object" return jsv:check-object-type($jstypes, $type)
      case "array"  return jsv:check-array-type ($jstypes, $type)
      case "union"  return jsv:check-union-type ($jstypes, $type)

      default return
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: type $kind unknown:" | $type("$kind") )

    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: type definition must contain '$kind' key with string value.")
  else
    if( $type instance of string )
    then
      jsv:check-ref-type($jstypes, $type)
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: type item not an object."  )
};

declare %an:sequential function
jsv:check-atomic-type($jstypes as object, $type as object) as boolean
{
  fn:trace( $type("$name"), "check-atomic-type");

  if( exists($type("$baseType")) and $type("$baseType") instance of string )
  then
    jsv:save-type($jstypes, $type)
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: atomic type definition must contain '$baseType' key with string value.")

  (: todo: add facets checks: length, min/max Inclusive/Exclusive, etc. :)
};

declare %an:sequential function
jsv:check-object-type($jstypes as object, $type as object) as boolean
{
  (: todo: check the reast of "object" type :)
  jsv:save-type($jstypes, $type);
  fn:trace( $type("$name"), "check-object-type");

  if( exists($type("$content")) )
  then
    if( $type("$content") instance of object )
    then
      every $i in
        for $k in jn:keys($type("$content"))
        return
          if( $type("$content")( fn:trace($k, "Key")) instance of object )
          then
            if( exists( $type("$content")($k)("$type")) )
            then
              jsv:check-type($jstypes, $type("$content")($k)("$type") )
            else
              if( exists( $type("$content")($k)("$kind")) )
              then
                jsv:check-type($jstypes, $type("$content")($k) )
              else
                fn:error(QName("jsv", "BadJSoundFormat"),
                  "Not a valid JSound doc: Key  does not contain a $type definition.")
          else
            fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: Key " | $k | " in $content must have an object value.")

      satisfies $i eq fn:true()
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: $content definion of object type must be an object.")
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
             "Not a valid JSound doc: Object type does not contain $content.")
};

declare %an:sequential function
jsv:check-array-type($jstypes as object, $type as object) as boolean
{
  fn:trace( $type("$name"), "check-array-type");

  (: check all the contents :)
  (: let $t := fn:trace($type, "Array type:") :)
  let $content := $type("$content")
  return
    if( empty($content) )
    then
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: $kind array must have a $content key.")
    else
      if( $content instance of array and jn:size($content) eq 1 )
      then
      {
        jsv:save-type($jstypes, $type);

        typeswitch( $content(1) )
        case string return
          fn:true()
        case object return
            jsv:check-type($jstypes, $content(1))
        default return
          fn:error(QName("jsv", "BadJSoundFormat"),
            "Not a valid JSound doc: for $kind array, $content[1] must be string or array.")
      }
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $kind array, $content must be an array of size 1.")

  (: Todo: check array facets :)
};

declare %an:sequential function
jsv:check-union-type($jstypes as object, $type as object) as boolean
{
  fn:trace( $type("$name"), "check-union-type");

  let $content := $type("$content")
  return
    if( empty($content) )
    then
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: $kind union must have a $content key.")
    else
      if( $content instance of array )
      then
      {
        jsv:save-type($jstypes, $type);

        every $i in
          for $c in jn:members($content)
          return
            typeswitch( $c )
            case string return
              fn:true()
            case object return
                jsv:check-type($jstypes, $c)
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind union, $content items must be strings or objects.")
        satisfies $i eq fn:true()
      }
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $kind union, $content must be an array.")
};

declare function
jsv:check-ref-type($jstypes as object, $type as string) as boolean
{
  switch( $type )
  case "string" return fn:true()
  case "integer" return fn:true()
  case "boolean" return fn:true()
  case "anyURI"  return fn:true()
  default return
  {
    if( map:has-key($jstypes, $type) )
    then
      fn:true()
    else
     (: todo: save all refs, check them at the end to see if they exist
     fn:trace($type, "NYI: type ref not yet suported");
     fn:error(QName("jsv", "NYI"), "Type ref: not yet implemented.") :)
     fn:true()
  }

  (: todo: apply resolution rules :)
};

declare %an:sequential function
jsv:save-type($jstypes as object, $type as item) as boolean
{
  if( fn:exists( $type("$name") ) )
  then
    if( $type("$name") instance of string )
    then
      map:set-if-empty($jstypes, $type("$name"), $type)
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $type, $name must be a string.")
  else
    fn:false()
};

(: The validate-xxx functions actualy do the validation of the instance :)

declare function
jsv:validate-instance($jstypes as object, $name as string, $instance as item)
 as boolean
{
  let $type := map:get($jstypes, $name)
  return
    if( exists($type) )
    then
      jsv:validate-type($jstypes, $type, $instance)
    else
      fn:error(QName("jsv", "Invalid"),
             "Requested type name not present in schema: " | $name)
};

declare function
jsv:validate-type($jstypes as object, $type as item, $instance as item) as boolean
{
  switch( $type("$kind") )
  case "atomic" return jsv:validate-atomic-type($jstypes, $type, $instance)
  case "object" return jsv:validate-object-type($jstypes, $type, $instance)
  case "array"  return jsv:validate-array-type($jstypes, $type, $instance)
  case "union"  return jsv:validate-union-type($jstypes, $type, $instance)

  default return
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: type $kind unknown:" | $type("$kind") )
};

declare function
jsv:validate-type-ref($jstypes as object, $type as string, $instance as item) as boolean
{
  (: todo: check first between the defined types in $jstypes and then the build-in ones, this is per spec :)

  switch( $type )
  case "string" return
    if ( $instance instance of string )
    then
      fn:true()
    else
      fn:error(QName("jsv", "Invalid"), "Expected string value.")
  case "integer" return
    if ( $instance instance of integer or
         $instance instance of int )
    then
      fn:true()
    else
      fn:error(QName("jsv", "Invalid"), "Expected integer value.")
  case "anyURI" return
    if ( $instance instance of anyURI )
    then
      fn:true()
    else
      fn:error(QName("jsv", "Invalid"), "Expected anyURI value.")
  case "boolean" return
    if ( $instance instance of boolean )
    then
      fn:true()
    else
      fn:error(QName("jsv", "Invalid"), "Expected boolean value.")

  default return
    fn:error(QName("jsv", "NYI"),
      "Type ref: not yet implemented: " | $type )
};

declare function
jsv:validate-atomic-type($jstypes as object, $type as object, $instance as item) as boolean
{
  fn:trace( $type("$name"), "validate-atomic-type");
  (: todo: check first between the defined types in $jstypes and then the build-in ones, this is per spec :)

  if ( exists($type("$baseType")) and $type("$baseType") instance of string )
  then
    switch( $type("$baseType") )
    case "integer" return
      if ( $instance instance of integer or $instance instance of int )
      then
        fn:true() and jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected integer or int value")

    case "string" return
      if ( $instance instance of string )
      then
        fn:true() and jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error( QName("jsv", "Invalid"), "Expected string value")

    case "anyURI" return
      if ( $instance instance of anyURI )
      then
        fn:true() and jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected anyURI value")

    case "boolean" return
      if ( $instance instance of boolean )
      then
        fn:true() and jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected boolean value")

    default return
      fn:error(QName("jsv", "NYI"),
        "$baseType: not yet implemented" | $type("$baseType") )
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: atomic type definition must contain '$baseType' key with string value.")

  (: todo: add facets checks: length, min/max Inclusive/Exclusive, etc. :)
};

declare function
jsv:validate-enumeration($jstypes as object, $type as object, $instance as item) as boolean
{
  variable $enum := $type("$enumeration");
  if( exists($enum) )
  then
    if( $enum instance of array )
    then
      if( some $v in jn:members($enum) satisfies $v eq $instance )
      then
        fn:true()
      else
        fn:error(QName("jsv", "Invalid"), "Instance value not in the enumeration list.")
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: the value of '$enumeration' must be an array.")
  else
    fn:true()
};

declare function
jsv:validate-object-type($jstypes as object, $type as object, $instance as item) as boolean
{
  fn:trace( $type("$name"), "validate-object-type");

  variable $open := $type("$open");

  if ( $instance instance of object )
  then
    if( (not exists($open)) or (exists($open) and $open eq true()) )
    then
      (: object is open: i.e. extra keys alowed but all defined keys must be in
         the instance except the default and optional ones :)
      (
        every $k in jn:keys($instance)
        satisfies
          let $ctype := $type("$content")($k)("$type")
          return
            typeswitch( $ctype )
            case string return
              jsv:validate-type-ref($jstypes, $ctype, $instance($k) )
            case object return
              jsv:validate-type($jstypes, $ctype, $instance($k) )
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind object, $content $type must be string or array.")
      ) and
      (
        every $k in
          jsv:value-except(jn:keys($type("$content")), jn:keys($instance))
        satisfies
          let $optional := $type("$content")($k)("$optional")
          return
            if( exists($optional) and $optional eq fn:true() )
            then fn:true()
            else
              fn:error(QName("jsv", "Invalid"),
                fn:concat("Required key not present in instance: ", $k) )
      )
    else
      (: object is closed: i.e. all defined keys must be in the instance except
         the default ones, no extra keys alowed :)
      (
        every $k in jn:keys($instance)
        satisfies
          let $ctype := $type("$content")($k)("$type")
          return
            typeswitch( $ctype )
            case string return
              jsv:validate-type-ref($jstypes, $ctype, $instance($k) )
            case object return
              jsv:validate-type($jstypes, $ctype, $instance($k) )
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind object, $content $type must be string or array.")
      ) and (
        every $k in (: jn:keys($type("$content")) except jn:keys($instance) :)
          jsv:value-except(jn:keys($type("$content")), jn:keys($instance))
        satisfies
          let $optional := $type("$content")($k)("$optional")
          return
            if( exists($optional) and $optional eq fn:true() )
            then fn:true()
            else
              fn:error(QName("jsv", "Invalid"),
                fn:concat("Required key not present in instance: ", $k) )
      )
  else
    fn:error( QName("jsv", "Invalid"),
      fn:concat("Instance not an object specified by type: ", jsv:get-type-name($type)) )
};

declare function
jsv:validate-array-type($jstypes as object, $type as object, $instance as item) as boolean
{
  fn:trace( $type("$name"), "validate-array-type");

  if( $instance instance of array )
  then
    (: check all the contents :)
    (: let $t := fn:trace($type, "Array type:") :)
    let $content := $type("$content")
    return
        typeswitch( $content(1) )
        case string return
          every $i in jn:members($instance) satisfies jsv:validate-type-ref($jstypes, $content(1), $i)
        case object return
          every $i in jn:members($instance) satisfies jsv:validate-type($jstypes, $content(1), $i)
        default return
          fn:error(QName("jsv", "BadJSoundFormat"),
            "Not a valid JSound doc: for $kind array, $content[1] must be string or array.")
  else
    fn:error(QName("jsv", "Invalid"),
      fn:concat("Instance not an array for type: ", jsv:get-type-name($type)) )

  (: check array facets :)
};

declare function
jsv:validate-union-type($jstypes as object, $type as object, $instance as item) as boolean
{
  let $content := $type("$content")
  return
    if(
      some $i in
        for $c in jn:members($content)
        return
          try
          {
            typeswitch( $c )
            case string return
              fn:true()
            case object return
                jsv:validate-type($jstypes, $c, $instance)
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind union, $content items must be strings or objects.")
          }
          catch Invalid
          {
             fn:false()
          }
      satisfies $i eq fn:true()
    )
    then fn:true()
    else
      fn:error(QName("jsv", "Invalid"),
        fn:concat("Instance not valid for union type: ", jsv:get-type-name($type)) )
};

(:---- Util functions ---------:)

declare function
jsv:get-type-name($type as object) as string
{
  let $name := $type("$name")
  return
    if ( exists($name) and $name instance of string )
    then
      $name
    else
      "N/A"
};

declare function
jsv:value-except($arg1 as xs:anyAtomicType*, $arg2 as xs:anyAtomicType*)
  as xs:anyAtomicType*
{
  (: fn:distinct-values( $arg1[ not($$ eq $arg2) ] ) :)
  fn:distinct-values(
    for $i in $arg1
    where not( some $x in $arg2 satisfies $i eq $x )
    return $i
  )
};