fixed bug for xml format in modelstructure-for-component

jsoniq version "1.0";

import module namespace components = "http://xbrl.io/modules/bizql/components";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace filings = "http://xbrl.io/modules/bizql/profiles/sec/filings";
import module namespace fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

import module namespace sec-networks = "http://xbrl.io/modules/bizql/profiles/sec/networks";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-xml-rec($o as object, $level as integer) as element()
{
    for $o in $o
    return
        element { if ($o.Kind eq "Domain") then "Member" else $o.Kind } {
            attribute { "name" } { $o.Name },
            attribute { "label" } { $o.Label },
            if (exists($o.Balance)) then attribute { "balance" } { $o.Balance } else (),
            if (exists($o.DataType)) then attribute { "dataType" } { $o.DataType } else (),
            if (exists($o.BaseType)) then attribute { "baseDataType" } { $o.BaseType } else (),
            if (exists($o.IsAbstract)) then attribute { "abstract" } { $o.IsAbstract } else (),
            if (exists($o.PeriodType)) then attribute { "periodType" } { $o.PeriodType } else (),
            attribute { "level" } { $level },
            local:to-xml-rec($o.Children[], $level + 1)
        }
};

declare function local:to-xml($model as object) as node()*
{
    ((session:comment("xml", {
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })),
    <Component>
        <Network entityRegistrantName="{$model.EntityRegistrantName}"
                 accessionNumber="{$model.AccessionNumber}"
                 tableName="{$model.TableName}"
                 cik="{$model.CIK}"
                 label="{$model.Label}"
                 networkIdentifier="{$model.NetworkIdentifier}"
                 formType="{$model.FormType}"
                 fiscalPeriod="{$model.FiscalPeriod}"
                 fiscalYear="{$model.FiscalYear}"
                 acceptanceDatetime="{$model.AcceptanceDatetime}"
                 disclosure="{$model.Disclosure}"
                 >{
            local:to-xml-rec($model.ModelStructure, 0)
        }</Network>
    </Component>)

};

declare function local:to-csv-rec($objects as object*, $level as integer) as object*
{
    for $o in $objects
    let $object := {
        "Label" : $o.Label,
        "Name" : $o.Name,
        "ObjectClass" : if ($o.Kind eq "Domain") then "Member" else $o.Kind,
        "DataType" : $o.DataType,
        "BaseDataType" : $o.BaseType,
        "Balance" : $o.Balance,
        "Abstract" : $o.IsAbstract,
        "PeriodType" : $o.PeriodType,
        "Level" : $level
    }
    return (
        $object,
        local:to-csv-rec($o.Children[], $level + 1)
    )
};

declare function local:to-csv($model as object) as string
{
    let $lines := local:to-csv-rec($model.ModelStructure, 0)
    return
        if (exists($lines))
        then string-join(csv:serialize($lines, { serialize-null-as : "" }))
        else ""
};

declare function local:enrich-json-rec($objects as object*, $level as integer) as object*
{
    for $object in $objects
    return
        copy $o := $object
        modify (
            if (exists($o.Children)) then delete json $o("Children") else (),
            insert json { Level : $level } into $o,
            if ($o.Kind eq "Domain") then replace value of json $o("Kind") with "Member" else ()
        )
        return {|
            $o,
            let $children := local:enrich-json-rec($object.Children[], $level + 1)
            return if (exists($children)) then { Children : [ $children ] } else ()
        |}
};

declare function local:enrich-json($component as object) as object
{
    {
        ModelStructure : [ local:enrich-json-rec($component.ModelStructure, 0) ] ,
        CIK : $component.CIK,
        EntityRegistrantName : $component.EntityRegistrantName,
        Label : $component.Label,
        AccessionNumber : $component.AccessionNumber,
        TableName : $component.TableName,
        FormType : $component.FormType,
        FiscalPeriod : $component.FiscalPeriod,
        FiscalYear : $component.FiscalYear,
        AcceptanceDatetime : $component.AcceptanceDatetime,
        NetworkIdentifier: $component.NetworkIdentifier,
        Disclosure : $component.Disclosure
    }
};
declare function local:components-by-disclosures($disclosures as string*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    for $aid in $aids
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: $aid,
            "Profiles.SEC.Disclosure": { "$in" : [ $disclosures ] }
        })
};

declare function local:components-by-roles($roles as string*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: { "$in" : [ $aids ] },
            "Role": { "$in" : [ $roles ] }
        })
};

declare function local:components-by-concepts($concepts as object*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    let $ids := mongo:find($conn, "concepts",
        {|
            (
                { "Name" : { "$in" : [ $concepts ] } },
                { "Archive" : { "$in" : [ $aids ] } }
            )
        |}).Component
    return components:components($ids)
};

declare function local:filings(
    $ciks as string*,
    $tags as string*,
    $tickers as string*,
    $sics as string*,
    $fp as string*,
    $fy as string*) as object*
{
    let $entities := (
        companies:companies($ciks),
        companies:companies-for-tags($tags),
        companies:companies-for-tickers($tickers),
        companies:companies-for-SIC($sics)
    )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                        case "ALL" return $fiscal:ALL_FISCAL_YEARS
                    default return $fy cast as integer
                )
    for $fp in $fp
    return fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy)
};

let $format      := lower-case((request:param-values("format"), substring-after(request:path(), ".jq."))[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids        := archives:aid(request:param-values("aid"))
let $roles       := request:param-values("networkIdentifier")
let $archives    := (
                        local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                        archives:archives($aids)
                    )
let $cid         := request:param-values("cid")
let $concepts    := distinct-values(request:param-values("concept"))
let $disclosures := request:param-values("disclosure")
let $components  := (if (exists($cid))
                    then components:components($cid)
                    else (),
                    if (exists($concepts))
                    then local:components-by-concepts($concepts, $archives._id)
                    else if (exists($disclosures))
                    then local:components-by-disclosures($disclosures, $archives._id)
                    else if (exists($roles))
                    then local:components-by-roles($roles, $archives._id)
                    else components:components-for-archives($archives._id))
let $component := $components[1] (: only one for know :)
let $archive   := archives:archives($component.Archive)
let $entity    := entities:entities($archive.Entity)
return
     if (session:only-dow30($entity) or session:valid())
     then {
        let $model := {|
                    { CIK : $entity._id },
                    { EntityRegistrantName : $entity.Profiles.SEC.CompanyName },
                    { ModelStructure : sec-networks:model-structures($component) },
                    { TableName : sec-networks:tables($component, {IncludeImpliedTable: true}).Name },
                    { Label : $component.Label },
                    { AccessionNumber : $component.Archive },
                    { FormType : $archive.Profiles.SEC.FormType },
                    { FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus },
                    { FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus },
                    { AcceptanceDatetime : filings:acceptance-dateTimes($archive) },
                    { NetworkIdentifier: $component.Role },
                    { Disclosure : $component.Profiles.SEC.Disclosure }
        |}
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($model)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=modelstructure-" || $cid || ".csv");
                local:to-csv($model)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=modelstructure-" || $cid || ".csv");
                local:to-csv($model)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    local:enrich-json($model),
                    session:comment("json", {
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })
                |}
            }
     } else {
        response:status-code(401);
        session:error("accessing fact table for an entity that is not in the DOW30", $format)
     }

jsoniq version "1.0";

import module namespace components = "http://xbrl.io/modules/bizql/components";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace filings = "http://xbrl.io/modules/bizql/profiles/sec/filings";
import module namespace fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

import module namespace sec-networks = "http://xbrl.io/modules/bizql/profiles/sec/networks";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-xml-rec($o as object, $level as integer) as element()
{
    for $o in $o
    return
        element { if ($o.Kind eq "Domain") then "Member" else $o.Kind } {
            attribute { "name" } { $o.Name },
            attribute { "label" } { $o.Label },
            if (exists($o.Balance)) then attribute { "balance" } { $o.Balance } else (),
            if (exists($o.DataType)) then attribute { "dataType" } { $o.DataType } else (),
            if (exists($o.BaseType)) then attribute { "baseDataType" } { $o.BaseType } else (),
            if (exists($o.IsAbstract)) then attribute { "abstract" } { $o.IsAbstract } else (),
            if (exists($o.PeriodType)) then attribute { "periodType" } { $o.PeriodType } else (),
            attribute { "level" } { $level },
            local:to-xml-rec($o.Children[], $level + 1)
        }
};

declare function local:to-xml($model as object) as element()
{
    ((session:comment("xml", {
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })),
    <Component>
        <Network entityRegistrantName="{$model.EntityRegistrantName}"
                 accessionNumber="{$model.AccessionNumber}"
                 tableName="{$model.TableName}"
                 cik="{$model.CIK}"
                 label="{$model.Label}"
                 networkIdentifier="{$model.NetworkIdentifier}"
                 formType="{$model.FormType}"
                 fiscalPeriod="{$model.FiscalPeriod}"
                 fiscalYear="{$model.FiscalYear}"
                 acceptanceDatetime="{$model.AcceptanceDatetime}"
                 disclosure="{$model.Disclosure}"
                 >{
            local:to-xml-rec($model.ModelStructure, 0)
        }</Network>
    </Component>)

};

declare function local:to-csv-rec($objects as object*, $level as integer) as object*
{
    for $o in $objects
    let $object := {
        "Label" : $o.Label,
        "Name" : $o.Name,
        "ObjectClass" : if ($o.Kind eq "Domain") then "Member" else $o.Kind,
        "DataType" : $o.DataType,
        "BaseDataType" : $o.BaseType,
        "Balance" : $o.Balance,
        "Abstract" : $o.IsAbstract,
        "PeriodType" : $o.PeriodType,
        "Level" : $level
    }
    return (
        $object,
        local:to-csv-rec($o.Children[], $level + 1)
    )
};

declare function local:to-csv($model as object) as string
{
    let $lines := local:to-csv-rec($model.ModelStructure, 0)
    return
        if (exists($lines))
        then string-join(csv:serialize($lines, { serialize-null-as : "" }))
        else ""
};

declare function local:enrich-json-rec($objects as object*, $level as integer) as object*
{
    for $object in $objects
    return
        copy $o := $object
        modify (
            if (exists($o.Children)) then delete json $o("Children") else (),
            insert json { Level : $level } into $o,
            if ($o.Kind eq "Domain") then replace value of json $o("Kind") with "Member" else ()
        )
        return {|
            $o,
            let $children := local:enrich-json-rec($object.Children[], $level + 1)
            return if (exists($children)) then { Children : [ $children ] } else ()
        |}
};

declare function local:enrich-json($component as object) as object
{
    {
        ModelStructure : [ local:enrich-json-rec($component.ModelStructure, 0) ] ,
        CIK : $component.CIK,
        EntityRegistrantName : $component.EntityRegistrantName,
        Label : $component.Label,
        AccessionNumber : $component.AccessionNumber,
        TableName : $component.TableName,
        FormType : $component.FormType,
        FiscalPeriod : $component.FiscalPeriod,
        FiscalYear : $component.FiscalYear,
        AcceptanceDatetime : $component.AcceptanceDatetime,
        NetworkIdentifier: $component.NetworkIdentifier,
        Disclosure : $component.Disclosure
    }
};
declare function local:components-by-disclosures($disclosures as string*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    for $aid in $aids
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: $aid,
            "Profiles.SEC.Disclosure": { "$in" : [ $disclosures ] }
        })
};

declare function local:components-by-roles($roles as string*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    return
        mongo:find($conn, "components",
        {
            $components:ARCHIVE: { "$in" : [ $aids ] },
            "Role": { "$in" : [ $roles ] }
        })
};

declare function local:components-by-concepts($concepts as object*, $aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("components:CONNECTION-FAILED"), $err:description)
        }
    let $ids := mongo:find($conn, "concepts",
        {|
            (
                { "Name" : { "$in" : [ $concepts ] } },
                { "Archive" : { "$in" : [ $aids ] } }
            )
        |}).Component
    return components:components($ids)
};

declare function local:filings(
    $ciks as string*,
    $tags as string*,
    $tickers as string*,
    $sics as string*,
    $fp as string*,
    $fy as string*) as object*
{
    let $entities := (
        companies:companies($ciks),
        companies:companies-for-tags($tags),
        companies:companies-for-tickers($tickers),
        companies:companies-for-SIC($sics)
    )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                        case "ALL" return $fiscal:ALL_FISCAL_YEARS
                    default return $fy cast as integer
                )
    for $fp in $fp
    return fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy)
};

let $format      := lower-case((request:param-values("format"), substring-after(request:path(), ".jq."))[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids        := archives:aid(request:param-values("aid"))
let $roles       := request:param-values("networkIdentifier")
let $archives    := (
                        local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                        archives:archives($aids)
                    )
let $cid         := request:param-values("cid")
let $concepts    := distinct-values(request:param-values("concept"))
let $disclosures := request:param-values("disclosure")
let $components  := (if (exists($cid))
                    then components:components($cid)
                    else (),
                    if (exists($concepts))
                    then local:components-by-concepts($concepts, $archives._id)
                    else if (exists($disclosures))
                    then local:components-by-disclosures($disclosures, $archives._id)
                    else if (exists($roles))
                    then local:components-by-roles($roles, $archives._id)
                    else components:components-for-archives($archives._id))
let $component := $components[1] (: only one for know :)
let $archive   := archives:archives($component.Archive)
let $entity    := entities:entities($archive.Entity)
return
     if (session:only-dow30($entity) or session:valid())
     then {
        let $model := {|
                    { CIK : $entity._id },
                    { EntityRegistrantName : $entity.Profiles.SEC.CompanyName },
                    { ModelStructure : sec-networks:model-structures($component) },
                    { TableName : sec-networks:tables($component, {IncludeImpliedTable: true}).Name },
                    { Label : $component.Label },
                    { AccessionNumber : $component.Archive },
                    { FormType : $archive.Profiles.SEC.FormType },
                    { FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus },
                    { FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus },
                    { AcceptanceDatetime : filings:acceptance-dateTimes($archive) },
                    { NetworkIdentifier: $component.Role },
                    { Disclosure : $component.Profiles.SEC.Disclosure }
        |}
        return
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($model)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=modelstructure-" || $cid || ".csv");
                local:to-csv($model)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=modelstructure-" || $cid || ".csv");
                local:to-csv($model)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    local:enrich-json($model),
                    session:comment("json", {
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })
                |}
            }
     } else {
        response:status-code(401);
        session:error("accessing fact table for an entity that is not in the DOW30", $format)
     }


fixed failing test.

fixed failing test.

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace schema = "http://zorba.io/modules/schema";
import module namespace test = "http://apps.28.io/test";
import module namespace http-client = "http://zorba.io/modules/http-client";

declare function local:expected-OtherOperatingIncomeExpenses-ATnT-2013() as object {
    {
        "Aspects": {
            "xbrl28:Archive": "0000732717-14-000010",
            "xbrl:Concept": "fac:OtherOperatingIncomeExpenses",
            "xbrl:Entity": "http://www.sec.gov/CIK 0000732717",
            "xbrl:Period": "2013-01-01/2013-12-31",
            "sec:FiscalPeriod": "FY",
            "sec:FiscalPeriodType": "YTD",
            "sec:FiscalYear": 2013,
            "sec:Accepted": "20140221162554",
            "dei:LegalEntityAxis": "sec:DefaultLegalEntity",
            "sec:Archive" : "0000732717-14-000010"
        },
        "KeyAspects": [
            "xbrl:Concept",
            "xbrl:Entity",
            "xbrl:Period",
            "xbrl:Unit",
            "sec:Accepted"
        ],
        "Type": "NumericValue",
        "Value": -145082000000,
        "Decimals": -6,
        "Concept": {
            "Balance": "CREDIT",
            "PeriodType": "duration",
            "DataType": "xbrli:monetaryItemType"
        },
        "AuditTrails": [
            {
                "Id": "bf2d1587-491c-4492-b587-ddd00d762f2d",
                "Type": "xbrl28:formula",
                "Label": "Other Operating Income (Expenses)",
                "Message": "fac:OtherOperatingIncomeExpenses[-145,082,000,000 USD] = fac:OperatingIncomeLoss[30,479,000,000 USD] - fac:GrossProfit[77,288,000,000 USD] - fac:OperatingExpenses[98,273,000,000 USD]",
                "Data": {
                    "OutputConcept": "fac:OtherOperatingIncomeExpenses"
                }
            }
        ],
        "xbrl28:Type": "xbrl28:formula",
        "EntityRegistrantName": "AT&T INC.",
        "Unit": "iso4217:USD"
    }
};

declare function local:compare-audit-trails($expected as array?, $actual as array?) as object* {
    for $exp as object at $pos in $expected[]
    let $act as object? := $actual[][$pos]
    return
        if (exists($exp.Data.ValidatedFacts))
        then
            let $expM := copy $e := $exp modify ( delete json $e.Data.ValidatedFacts ) return $e
            let $actM := copy $a := $act modify ( delete json $a.Data.ValidatedFacts ) return $a
            return  (
                if(deep-equal($expM, $actM))
                then ()
                else
                    {
                        type: "unexpected-audittrails-value",
                        expected: $expM,
                        actual: $actM
                    },
                if(count($exp.Data.ValidatedFacts[]) eq count($act.Data.ValidatedFacts[]))
                then ()
                else
                    {
                        type: "number-of-validated-facts-differs",
                        expected: $exp,
                        actual: $act
                    }
            )
        else
            if(deep-equal($exp, $act))
            then ()
            else
                {
                    type: "unexpected-audittrails-value",
                    expected: $exp,
                    actual: $act
                }
};

declare function local:diff-facts($fact-expected as object, $fact-actual as object?) as object* {
    if(empty($fact-actual))
    then
        {
            type: "missing-fact-in-result"
        }
    else
        (
            for $key in distinct-values((keys($fact-expected), keys($fact-actual)))
            where not($key = ("Aspects", "KeyAspects", "Profiles", "AuditTrails"))
            let $exp-value := $fact-expected.$key
            let $act-value := $fact-actual.$key
            return
                if(deep-equal($exp-value, $act-value) or
                   (schema:schema-type($exp-value) eq schema:schema-type($act-value) and schema:schema-type($exp-value) eq xs:QName("xs:decimal") and fn:format-number(xs:decimal($exp-value),"#,##0.0000000000000") eq fn:format-number(xs:decimal($act-value),"#,##0.0000000000000"))
                )
                then ()
                else
                    {
                        type: "unexpected-fact-field-value",
                        field: $key,
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare aspects :)
            for $key in distinct-values((keys($fact-expected.Aspects), keys($fact-actual.Aspects)))
            let $exp-value := $fact-expected.Aspects.$key
            let $act-value := $fact-actual.Aspects.$key
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-aspect-value",
                        aspect: $key,
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare keyaspects :)
            let $exp-value := $fact-expected.KeyAspects
            let $act-value := $fact-actual.KeyAspects
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-keyaspects-value",
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare Profiles :)
            let $exp-value := $fact-expected.Profiles
            let $act-value := $fact-actual.Profiles
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-profiles-value",
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare Audittrails :)
            let $exp-value := $fact-expected.AuditTrails
            let $act-value := $fact-actual.AuditTrails
            return
                local:compare-audit-trails($exp-value, $act-value)
        )
};

declare function local:compare-fact-tables($fact-table-expected as object, $fact-table-actual as object) as object*{
    let $actual-facts as object+ := $fact-table-actual.FactTable[]
    let $expected-facts as object+ := $fact-table-expected.FactTable[]
    let $errors :=
        (
            for $exp in $expected-facts
            let $act := $actual-facts[$$.Aspects."xbrl:Concept" eq $exp.Aspects."xbrl:Concept"
                                      and $$.Aspects."xbrl:Entity" eq $exp.Aspects."xbrl:Entity"
                                      and $$.Aspects."xbrl:Period" eq $exp.Aspects."xbrl:Period"
                                      and $$.Aspects."xbrl28:Archive" eq $exp.Aspects."xbrl28:Archive"]
            let $errors as object* := local:diff-facts($exp, $act)
            return
                if(empty($errors))
                then ()
                else {
                    expectedFact: $exp,
                    actualFact: $act,
                    errors: [$errors]
                },

            for $act in $actual-facts
            let $exp := $expected-facts[$$.Aspects."xbrl:Concept" eq $act.Aspects."xbrl:Concept"
                                      and $$.Aspects."xbrl:Entity" eq $act.Aspects."xbrl:Entity"
                                      and $$.Aspects."xbrl:Period" eq $act.Aspects."xbrl:Period"
                                      and $$.Aspects."xbrl28:Archive" eq $act.Aspects."xbrl28:Archive"]
            where empty($exp)
            return
                {
                    unexpectedFact: $act
                }
        )
    return
        if(empty($errors))
        then ()
        else (
                {
                    expectedNumberOfFacts: count($expected-facts),
                    actualNumberOfFacts: count($actual-facts)
                },
                $errors
            )
};

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-facttable-fact($concept as string, $expected as object, $params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $facts as object* := $request[2].FactTable[]
    let $actual := $facts[$$.Aspects."xbrl:Concept" eq $concept]
    let $diff :=
      for $f in $actual return local:diff-facts($expected, $f)
    return if (empty($diff)) then true else { url: test:url($endpoint, $params), factDiffErrors: [ $diff ], expectedFact: $expected, actualFact: $actual }
};

declare %an:nondeterministic function local:test-report-does-not-exist($params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $status as integer := $request[1]
    return if ($status eq 404) then true else { url: test:url($endpoint, $params), unexpectedResponse: $request[2] }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-values() as item*
{
    let $endpoint := "facttable-for-report"
    let $params := {ticker:"ko",fiscalYear:"2013",fiscalPeriod:"FY",fiscalPeriodType: ["instant", "YTD", "QTD"],report:"FundamentalAccountingConcepts"}
    let $request := test:invoke($endpoint, $params)
    let $actual as object := $request[2]
    let $expected := parse-json(
        http-client:get("http://" || request:server-name() || ":" || request:server-port() ||
                        "/test/sec/facttable-for-report-expected1.jq").body.content)
    let $diff := local:compare-fact-tables($expected, $actual)
    return if (empty($diff)) then true else { url: test:url($endpoint, $params), factTableDiff: [ ({ params : $params },$diff) ], expectedFactTable: $expected, actualFactTable: $actual }
};

let $dow30 := test:is-dow30()
return local:check({
    cocacola: local:test-facttable(95, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2013",
        fiscalPeriod:"Q1"}),
    aid: local:test-facttable(if($dow30) then 0 else 95, {
        aid:"0001193125-14-157120",
        report:"FundamentalAccountingConcepts"}),
    aid2: local:test-facttable(
        (:
          its only 18 here because they filed a 10-Q with a fiscalPeriod focus of FY
          http://www.sec.gov/cgi-bin/viewer?action=view&cik=732717&accession_number=0000732717-14-000022&xbrl_type=v
        :)
        18, {
        aid:"0000732717-14-000022",
        report:"FundamentalAccountingConcepts"}),
    aid2-qtd: local:test-facttable(96, {
        aid:"0000732717-14-000022",
        report:"FundamentalAccountingConcepts",
        fiscalPeriodType: [ "instant", "QTD" ]
    }),
    tickerrole: local:test-facttable(95, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1"}),
    tickerfyfprole: local:test-facttable(247, {
        report:"FundamentalAccountingConcepts",
        ticker:["ko","wmt"],
        fiscalYear:"2013",
        fiscalPeriod:"FY",
        fiscalPeriodType:["instant", "QTD", "YTD"]
    }),
    bfiXOMUnits: local:test-facttable(401, {
        report:"supportBasicFinancialInformation",
        ticker:["xom"],
        fiscalYear:"ALL",
        fiscalPeriod:"FY",
        fiscalPeriodType:["instant", "YTD"]
    }),
    allvalues : local:test-values(),
    otheroperatingincometest: local:test-facttable-fact("fac:OtherOperatingIncomeExpenses",
                                                        local:expected-OtherOperatingIncomeExpenses-ATnT-2013(),
                                                        {
        report:"FundamentalAccountingConcepts",
        ticker:"t",
        fiscalYear:"2013",
        fiscalPeriod:["FY"]}),
    reportDoesntExist: local:test-report-does-not-exist({
        report:"report-not-found",
        ticker:"MSFT",
        fiscalYear:"ALL",
        fiscalPeriod:"FY"})
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace schema = "http://zorba.io/modules/schema";
import module namespace test = "http://apps.28.io/test";
import module namespace http-client = "http://zorba.io/modules/http-client";

declare function local:expected-OtherOperatingIncomeExpenses-ATnT-2013() as object {
    {
        "Aspects": {
            "xbrl28:Archive": "0000732717-14-000010",
            "xbrl:Concept": "fac:OtherOperatingIncomeExpenses",
            "xbrl:Entity": "http://www.sec.gov/CIK 0000732717",
            "xbrl:Period": "2013-01-01/2013-12-31",
            "sec:FiscalPeriod": "FY",
            "sec:FiscalPeriodType": "YTD",
            "sec:FiscalYear": 2013,
            "sec:Accepted": "20140221162554",
            "dei:LegalEntityAxis": "sec:DefaultLegalEntity",
            "sec:Archive" : "0000732717-14-000010"
        },
        "KeyAspects": [
            "xbrl:Concept",
            "xbrl:Entity",
            "xbrl:Period",
            "xbrl:Unit",
            "sec:Accepted"
        ],
        "Type": "NumericValue",
        "Value": -145082000000,
        "Decimals": -6,
        "Concept": {
            "Balance": "CREDIT",
            "PeriodType": "duration",
            "DataType": "xbrli:monetaryItemType"
        },
        "AuditTrails": [
            {
                "Id": "bf2d1587-491c-4492-b587-ddd00d762f2d",
                "Type": "xbrl28:formula",
                "Label": "Other Operating Income (Expenses)",
                "Message": "fac:OtherOperatingIncomeExpenses[-145,082,000,000 USD] = fac:OperatingIncomeLoss[30,479,000,000 USD] - fac:GrossProfit[77,288,000,000 USD] - fac:OperatingExpenses[98,273,000,000 USD]",
                "Data": {
                    "OutputConcept": "fac:OtherOperatingIncomeExpenses"
                }
            }
        ],
        "xbrl28:Type": "xbrl28:formula",
        "EntityRegistrantName": "AT&T INC.",
        "Unit": "iso4217:USD"
    }
};

declare function local:compare-audit-trails($expected as array?, $actual as array?) as object* {
    for $exp as object at $pos in $expected[]
    let $act as object? := $actual[][$pos]
    return
        if (exists($exp.Data.ValidatedFacts))
        then
            let $expM := copy $e := $exp modify ( delete json $e.Data.ValidatedFacts ) return $e
            let $actM := copy $a := $act modify ( delete json $a.Data.ValidatedFacts ) return $a
            return  (
                if(deep-equal($expM, $actM))
                then ()
                else
                    {
                        type: "unexpected-audittrails-value",
                        expected: $expM,
                        actual: $actM
                    },
                if(count($exp.Data.ValidatedFacts[]) eq count($act.Data.ValidatedFacts[]))
                then ()
                else
                    {
                        type: "number-of-validated-facts-differs",
                        expected: $exp,
                        actual: $act
                    }
            )
        else
            if(deep-equal($exp, $act))
            then ()
            else
                {
                    type: "unexpected-audittrails-value",
                    expected: $exp,
                    actual: $act
                }
};

declare function local:diff-facts($fact-expected as object, $fact-actual as object?) as object* {
    if(empty($fact-actual))
    then
        {
            type: "missing-fact-in-result"
        }
    else
        (
            for $key in distinct-values((keys($fact-expected), keys($fact-actual)))
            where not($key = ("Aspects", "KeyAspects", "Profiles", "AuditTrails"))
            let $exp-value := $fact-expected.$key
            let $act-value := $fact-actual.$key
            return
                if(deep-equal($exp-value, $act-value) or
                   (schema:schema-type($exp-value) eq schema:schema-type($act-value) and schema:schema-type($exp-value) eq xs:QName("xs:decimal") and fn:format-number(xs:decimal($exp-value),"#,##0.0000000000000") eq fn:format-number(xs:decimal($act-value),"#,##0.0000000000000"))
                )
                then ()
                else
                    {
                        type: "unexpected-fact-field-value",
                        field: $key,
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare aspects :)
            for $key in distinct-values((keys($fact-expected.Aspects), keys($fact-actual.Aspects)))
            let $exp-value := $fact-expected.Aspects.$key
            let $act-value := $fact-actual.Aspects.$key
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-aspect-value",
                        aspect: $key,
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare keyaspects :)
            let $exp-value := $fact-expected.KeyAspects
            let $act-value := $fact-actual.KeyAspects
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-keyaspects-value",
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare Profiles :)
            let $exp-value := $fact-expected.Profiles
            let $act-value := $fact-actual.Profiles
            return
                if(deep-equal($exp-value, $act-value))
                then ()
                else
                    {
                        type: "unexpected-profiles-value",
                        expected: $exp-value,
                        actual: $act-value
                    },
            (: compare Audittrails :)
            let $exp-value := $fact-expected.AuditTrails
            let $act-value := $fact-actual.AuditTrails
            return
                local:compare-audit-trails($exp-value, $act-value)
        )
};

declare function local:compare-fact-tables($fact-table-expected as object, $fact-table-actual as object) as object*{
    let $actual-facts as object+ := $fact-table-actual.FactTable[]
    let $expected-facts as object+ := $fact-table-expected.FactTable[]
    let $errors :=
        (
            for $exp in $expected-facts
            let $act := $actual-facts[$$.Aspects."xbrl:Concept" eq $exp.Aspects."xbrl:Concept"
                                      and $$.Aspects."xbrl:Entity" eq $exp.Aspects."xbrl:Entity"
                                      and $$.Aspects."xbrl:Period" eq $exp.Aspects."xbrl:Period"
                                      and $$.Aspects."xbrl28:Archive" eq $exp.Aspects."xbrl28:Archive"]
            let $errors as object* := local:diff-facts($exp, $act)
            return
                if(empty($errors))
                then ()
                else {
                    expectedFact: $exp,
                    actualFact: $act,
                    errors: [$errors]
                },

            for $act in $actual-facts
            let $exp := $expected-facts[$$.Aspects."xbrl:Concept" eq $act.Aspects."xbrl:Concept"
                                      and $$.Aspects."xbrl:Entity" eq $act.Aspects."xbrl:Entity"
                                      and $$.Aspects."xbrl:Period" eq $act.Aspects."xbrl:Period"
                                      and $$.Aspects."xbrl28:Archive" eq $act.Aspects."xbrl28:Archive"]
            where empty($exp)
            return
                {
                    unexpectedFact: $act
                }
        )
    return
        if(empty($errors))
        then ()
        else (
                {
                    expectedNumberOfFacts: count($expected-facts),
                    actualNumberOfFacts: count($actual-facts)
                },
                $errors
            )
};

declare %an:nondeterministic function local:test-facttable($expected as integer, $params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].FactTable[])
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-facttable-fact($concept as string, $expected as object, $params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $facts as object* := $request[2].FactTable[]
    let $actual := $facts[$$.Aspects."xbrl:Concept" eq $concept]
    let $diff :=
      for $f in $actual return local:diff-facts($expected, $f)
    return if (empty($diff)) then true else { url: test:url($endpoint, $params), factDiffErrors: [ $diff ], expectedFact: $expected, actualFact: $actual }
};

declare %an:nondeterministic function local:test-report-does-not-exist($params as object) as item
{
    let $endpoint := "facttable-for-report"
    let $request := test:invoke($endpoint, $params)
    let $status as integer := $request[1]
    return if ($status eq 404) then true else { url: test:url($endpoint, $params), unexpectedResponse: $request[2] }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

declare %an:nondeterministic function local:test-values() as item*
{
    let $endpoint := "facttable-for-report"
    let $params := {ticker:"ko",fiscalYear:"2013",fiscalPeriod:"FY",fiscalPeriodType: ["instant", "YTD", "QTD"],report:"FundamentalAccountingConcepts"}
    let $request := test:invoke($endpoint, $params)
    let $actual as object := $request[2]
    let $expected := parse-json(
        http-client:get("http://" || request:server-name() || ":" || request:server-port() ||
                        "/test/facttable-for-report-expected1.jq").body.content)
    let $diff := local:compare-fact-tables($expected, $actual)
    return if (empty($diff)) then true else { url: test:url($endpoint, $params), factTableDiff: [ ({ params : $params },$diff) ], expectedFactTable: $expected, actualFactTable: $actual }
};

let $dow30 := test:is-dow30()
return local:check({
    cocacola: local:test-facttable(95, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2013",
        fiscalPeriod:"Q1"}),
    aid: local:test-facttable(if($dow30) then 0 else 95, {
        aid:"0001193125-14-157120",
        report:"FundamentalAccountingConcepts"}),
    aid2: local:test-facttable(
        (:
          its only 18 here because they filed a 10-Q with a fiscalPeriod focus of FY
          http://www.sec.gov/cgi-bin/viewer?action=view&cik=732717&accession_number=0000732717-14-000022&xbrl_type=v
        :)
        18, {
        aid:"0000732717-14-000022",
        report:"FundamentalAccountingConcepts"}),
    aid2-qtd: local:test-facttable(96, {
        aid:"0000732717-14-000022",
        report:"FundamentalAccountingConcepts",
        fiscalPeriodType: [ "instant", "QTD" ]
    }),
    tickerrole: local:test-facttable(95, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1"}),
    tickerfyfprole: local:test-facttable(247, {
        report:"FundamentalAccountingConcepts",
        ticker:["ko","wmt"],
        fiscalYear:"2013",
        fiscalPeriod:"FY",
        fiscalPeriodType:["instant", "QTD", "YTD"]
    }),
    bfiXOMUnits: local:test-facttable(401, {
        report:"supportBasicFinancialInformation",
        ticker:["xom"],
        fiscalYear:"ALL",
        fiscalPeriod:"FY",
        fiscalPeriodType:["instant", "YTD"]
    }),
    allvalues : local:test-values(),
    otheroperatingincometest: local:test-facttable-fact("fac:OtherOperatingIncomeExpenses",
                                                        local:expected-OtherOperatingIncomeExpenses-ATnT-2013(),
                                                        {
        report:"FundamentalAccountingConcepts",
        ticker:"t",
        fiscalYear:"2013",
        fiscalPeriod:["FY"]}),
    reportDoesntExist: local:test-report-does-not-exist({
        report:"report-not-found",
        ticker:"MSFT",
        fiscalYear:"ALL",
        fiscalPeriod:"FY"})
})