:pencil: Add parameter to labels.jq for eliminating duplicate report elements.

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token              as string? external;
declare  %rest:env                              variable $request-uri        as string  external;
declare  %rest:case-insensitive                 variable $format             as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker             as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear         as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod       as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $filingKind         as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $eid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $networkIdentifier  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $role               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $reportElement      as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $concept            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure         as string* external;
declare  %rest:case-insensitive %rest:distinct  %rest:encoding("UTF-8") variable $label as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $language           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $labelRole          as string* external;
declare  %rest:case-insensitive                 variable $profile-name       as string  external := $config:profile-name;
declare  %rest:case-insensitive                 variable $onlyTextBlocks     as boolean? external := ();
declare  %rest:case-insensitive                 variable $kind               as string*  external := ();
declare  %rest:case-insensitive                 variable $eliminateReportElementDuplicates as boolean external := false;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $reportElement := ($reportElement, $concept)
let $networkIdentifier := distinct-values(($networkIdentifier, $role))

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic, ())

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $filingKind,
  $aid)

let $concepts as object* :=
    multiplexer:concepts(
      $profile-name,
      $archives,
      $reportElement,
      $disclosure,
      $networkIdentifier,
      $label[$profile-name ne "sec"],
      $label[$profile-name eq "sec"],
      (),
      false)[
    (empty($onlyTextBlocks) or $$.IsTextBlock eq $onlyTextBlocks) and
    (empty($kind) or $$.Kind = $kind)
  ]

let $res as object* :=
  if($eliminateReportElementDuplicates)
  then
    for $concept in $concepts
    group by $concept.Name
    let $concept := $concept[1]
    where empty($reportElement) or $concept.Name = $reportElement
    for $found-label in $concept.Labels[]
    where (empty($label) or $found-label.Value = $label) and
          (empty($language) or $found-label.Language = $language) and
          (empty($labelRole) or $found-label.Role = $labelRole)
    return {|
      { Archive: $concept.Archive },
      { ComponentRole: $concept.Role },
      { Concept: $concept.Name },
      $found-label
    |}
  else
    for $concept in $concepts
    where empty($reportElement) or $concept.Name = $reportElement
    for $found-label in $concept.Labels[]
    where (empty($label) or $found-label.Value = $label) and
          (empty($language) or $found-label.Language = $language) and
          (empty($labelRole) or $found-label.Role = $labelRole)
    return {|
      { Archive: $concept.Archive },
      { ComponentRole: $concept.Role },
      { Concept: $concept.Name },
      $found-label
    |}

let $result := { Labels: [ $res ] }
let $comment :=
 {
    NumLabels : count($res),
    TotalNumComponents: session:num-components(),
    TotalNumArchives: session:num-archives()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <Labels>{
          api:json-to-xml($res.Labels[], "Label")
        }</Labels>
    },
    to-csv : function($res as object) as string {
        api:json-to-csv($res.Labels[])
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "labels")
return api:check-and-return-results($token, $results, $format)

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token              as string? external;
declare  %rest:env                              variable $request-uri        as string  external;
declare  %rest:case-insensitive                 variable $format             as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker             as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear         as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod       as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $filingKind         as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $eid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $networkIdentifier  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $role               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $reportElement      as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $concept            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure         as string* external;
declare  %rest:case-insensitive %rest:distinct  %rest:encoding("UTF-8") variable $label as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $language           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $labelRole          as string* external;
declare  %rest:case-insensitive                 variable $profile-name       as string  external := $config:profile-name;
declare  %rest:case-insensitive                 variable $onlyTextBlocks     as boolean? external := ();
declare  %rest:case-insensitive                 variable $kind               as string*  external := ();

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $reportElement := ($reportElement, $concept)
let $networkIdentifier := distinct-values(($networkIdentifier, $role))

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic, ())

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $filingKind,
  $aid)

let $concepts as object* :=
    multiplexer:concepts(
      $profile-name,
      $archives,
      $reportElement,
      $disclosure,
      $networkIdentifier,
      $label[$profile-name ne "sec"],
      $label[$profile-name eq "sec"],
      (),
      false)[
    (empty($onlyTextBlocks) or $$.IsTextBlock eq $onlyTextBlocks) and
    (empty($kind) or $$.Kind = $kind)
  ]

let $res as object* :=
  for $concept in $concepts
  where empty($reportElement) or $concept.Name = $reportElement
  for $found-label in $concept.Labels[]
  where (empty($label) or $found-label.Value = $label) and
        (empty($language) or $found-label.Language = $language) and
        (empty($labelRole) or $found-label.Role = $labelRole)
  return {|
    { Archive: $concept.Archive },
    { ComponentRole: $concept.Role },
    { Concept: $concept.Name },
    $found-label
  |}

let $result := { Labels: [ $res ] }
let $comment :=
 {
    NumLabels : count($res),
    TotalNumComponents: session:num-components(),
    TotalNumArchives: session:num-archives()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <Labels>{
          api:json-to-xml($res.Labels[], "Label")
        }</Labels>
    },
    to-csv : function($res as object) as string {
        api:json-to-csv($res.Labels[])
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "labels")
return api:check-and-return-results($token, $results, $format)


use flatten for an array

use flatten for an array

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace sec-fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace core = "http://xbrl.io/modules/bizql/profiles/sec/core";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-csv($o as object*) as string?
{
    if (exists($o)) (: bug in csv:serialize :)
    then
    string-join(
        csv:serialize(
            for $o in $o
            let $a := $o.Aspects
            return {|
                for $k in keys($a)
                return { $k : $a.$k },
                { "Value" : $o.Value },
                if (exists($o."Unit"))
                then { "Unit" :  $o."Unit" }
                else (),
                if (exists($o.Decimals))
                then { "Decimals" : $o.Decimals }
                else(),
                { "EntityRegistrantName" : $o.EntityRegistrantName },
                if (exists($o.ReportedConcept))
                then { "ReportedConcept" : $o.ReportedConcept }
                else ()
            |},
            { serialize-null-as : "" }
        )
    )
    else ()
};

declare function local:audittrail-to-xml($audit as item) as element()
{
    <AuditTrails>{
        for $a in flatten($audit)
        return (
            <Type>{$a.Type}</Type>,
            <Label>{$a.Label}</Label>,
            <Message>{$a.Label}</Message>,
            <Data>{
                if (exists($a.Data.OriginalConcept))
                then <OriginalConcept>{$a.Data.OriginalConcept}</OriginalConcept>
                else (),
                if (exists($a.Data.OutputConcept))
                then <OutputConcept>{$a.Data.OutputConcept}</OutputConcept>
                else (),
                if (exists($a.Data.AuditTrails))
                then local:audittrail-to-xml($a.Data.AuditTrails)
                else ()
            }</Data>
        )
    }</AuditTrails>
};

declare function local:to-xml($o as object*) as node()*
{
    (session:comment("xml", {
                            NumFacts : count($o),
                            TotalNumFacts: session:num-facts(),
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        }),
    <FactTable NetworkIdentifier="http://bizql.io/facttable-for-report"
            TableName="xbrl:FactTableForReport">{
        for $o in $o
        let $a := $o.Aspects
        return
            <Fact>{
                <Aspects>{
                    for $k in keys($a)
                    return
                        <Aspect>
                            <Name>{$k}</Name>
                            <Value>{$a.$k}</Value>
                        </Aspect>
                }</Aspects>,
                <Value>{$o.Value}</Value>,
                <Type>{$o.Type}</Type>,
                if (exists($o.Unit))
                then <Unit>{$o.Unit}</Unit>
                else(),
                if (exists($o.Decimals))
                then <Decimals>{$o.Decimals}</Decimals>
                else (),
                <EntityRegistrantName>{$o.EntityRegistrantName}</EntityRegistrantName>,
                if (exists($o.AuditTrails))
                then local:audittrail-to-xml($o.AuditTrails)
                else ()
            }</Fact>
    }</FactTable>)
};

declare function local:filings(
    $ciks as string*,
    $tags as string*,
    $tickers as string*,
    $sics as string*,
    $fp as string*,
    $fy as string*) as object*
{
    let $entities := (
        companies:companies($ciks),
        companies:companies-for-tags($tags),
        companies:companies-for-tickers($tickers),
        companies:companies-for-SIC($sics)
    )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then sec-fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else sec-fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                    case "ALL" return  $sec-fiscal:ALL_FISCAL_YEARS
                    default return $fy
                )
    for $fp in $fp
    return sec-fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy ! ($$ cast as integer))
};

let $format      := lower-case((request:param-values("format"), substring-after(request:path(), ".jq."))[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $sec-fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids        := archives:aid(request:param-values("aid"))
let $archives    := (
                        local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                        archives:archives($aids)
                    )
let $entities    := entities:entities($archives.Entity)
let $report := request:param-values("report")
let $facts :=
    for $fact in core:facts-for-schema($report, $archives)
    group by $archive := $fact.Archive
    let $archive := archives:archives($archive)
    let $entity    := entities:entities($archive.Entity)
    let $fp := $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
    let $fy := $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus
    for $fact in $fact
    return
    {|
        { Aspects : {|
            for $a in keys($fact.Aspects)
            return { $a : $fact.Aspects.$a },
            { "secxbrl:FiscalPeriod" : $fp },
            { "secxbrl:FiscalYear" : $fy }
        |} },
        { Type: $fact.Type },
        if (exists($fact.Aspects."xbrl:Unit"))
        then { Unit: $fact.Aspects."xbrl:Unit" }
        else  (),
        if (exists($fact.Decimals))
        then { Decimals: $fact.Decimals }
        else (),
        { Value: $fact.Value },
        { "EntityRegistrantName" : $entity.Profiles.SEC.CompanyName },
        { "AuditTrails" : $fact.AuditTrails[] }

    |}
return
    if (session:only-dow30($entities) or session:valid())
        then {
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($facts)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=facts.csv");
                local:to-csv($facts)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=fact.csv");
                local:to-csv($facts)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { NetworkIdentifier : "http://secxbrl.info/facts" },
                    { TableName : "xbrl:Facts" },
                    { FactTable : [ $facts ] },
                    session:comment("json", {
                            NumFacts : count($facts),
                            TotalNumFacts: session:num-facts(),
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })
                |}
            }
        }
        else {
            response:status-code(401);
            session:error("accessing filings of an entity that is not in the DOW30", $format)
        }

import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace sec-fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace core = "http://xbrl.io/modules/bizql/profiles/sec/core";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-csv($o as object*) as string?
{
    if (exists($o)) (: bug in csv:serialize :)
    then
    string-join(
        csv:serialize(
            for $o in $o
            let $a := $o.Aspects
            return {|
                for $k in keys($a)
                return { $k : $a.$k },
                { "Value" : $o.Value },
                if (exists($o."Unit"))
                then { "Unit" :  $o."Unit" }
                else (),
                if (exists($o.Decimals))
                then { "Decimals" : $o.Decimals }
                else(),
                { "EntityRegistrantName" : $o.EntityRegistrantName },
                if (exists($o.ReportedConcept))
                then { "ReportedConcept" : $o.ReportedConcept }
                else ()
            |},
            { serialize-null-as : "" }
        )
    )
    else ()
};

declare function local:audittrail-to-xml($audit as item) as element()
{
    <AuditTrails>{
        for $a in if ($audit instance of array) then $audit[] else $audit
        return (
            <Type>{$a.Type}</Type>,
            <Label>{$a.Label}</Label>,
            <Message>{$a.Label}</Message>,
            <Data>{
                if (exists($a.Data.OriginalConcept))
                then <OriginalConcept>{$a.Data.OriginalConcept}</OriginalConcept>
                else (),
                if (exists($a.Data.OutputConcept))
                then <OutputConcept>{$a.Data.OutputConcept}</OutputConcept>
                else (),
                if (exists($a.Data.AuditTrails))
                then local:audittrail-to-xml($a.Data.AuditTrails)
                else ()
            }</Data>
        )
    }</AuditTrails>
};

declare function local:to-xml($o as object*) as node()*
{
    (session:comment("xml", {
                            NumFacts : count($o),
                            TotalNumFacts: session:num-facts(),
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        }),
    <FactTable NetworkIdentifier="http://bizql.io/facttable-for-report"
            TableName="xbrl:FactTableForReport">{
        for $o in $o
        let $a := $o.Aspects
        return
            <Fact>{
                <Aspects>{
                    for $k in keys($a)
                    return
                        <Aspect>
                            <Name>{$k}</Name>
                            <Value>{$a.$k}</Value>
                        </Aspect>
                }</Aspects>,
                <Value>{$o.Value}</Value>,
                <Type>{$o.Type}</Type>,
                if (exists($o.Unit))
                then <Unit>{$o.Unit}</Unit>
                else(),
                if (exists($o.Decimals))
                then <Decimals>{$o.Decimals}</Decimals>
                else (),
                <EntityRegistrantName>{$o.EntityRegistrantName}</EntityRegistrantName>,
                if (exists($o.AuditTrails))
                then local:audittrail-to-xml($o.AuditTrails)
                else ()
            }</Fact>
    }</FactTable>)
};

declare function local:filings(
    $ciks as string*,
    $tags as string*,
    $tickers as string*,
    $sics as string*,
    $fp as string*,
    $fy as string*) as object*
{
    let $entities := (
        companies:companies($ciks),
        companies:companies-for-tags($tags),
        companies:companies-for-tickers($tickers),
        companies:companies-for-SIC($sics)
    )
    for $entity in $entities
    for $fy in distinct-values(
                for $fy in $fy
                return
                    switch ($fy)
                    case "LATEST" return
                        for $p in $fp
                        return
                            if ($p eq "FY")
                            then sec-fiscal:latest-reported-fiscal-period($entity, "10-K").year
                            else sec-fiscal:latest-reported-fiscal-period($entity, "10-Q").year
                    case "ALL" return  $sec-fiscal:ALL_FISCAL_YEARS
                    default return $fy
                )
    for $fp in $fp
    return sec-fiscal:filings-for-entities-and-fiscal-periods-and-years($entity, $fp, $fy ! ($$ cast as integer))
};

let $format      := lower-case((request:param-values("format"), substring-after(request:path(), ".jq."))[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalYears := distinct-values(
                        for $y in request:param-values("fiscalYear", "LATEST")
                        return
                            if ($y eq "LATEST" or $y eq "ALL")
                            then $y
                            else if ($y castable as integer)
                            then $y
                            else ()
                    )
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $sec-fiscal:ALL_FISCAL_PERIODS
                        else $fp)
let $aids        := archives:aid(request:param-values("aid"))
let $archives    := (
                        local:filings($ciks, $tags, $tickers, $sics, $fiscalPeriods, $fiscalYears),
                        archives:archives($aids)
                    )
let $entities    := entities:entities($archives.Entity)
let $report := request:param-values("report")
let $facts :=
    for $fact in core:facts-for-schema($report, $archives)
    group by $archive := $fact.Archive
    let $archive := archives:archives($archive)
    let $entity    := entities:entities($archive.Entity)
    let $fp := $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
    let $fy := $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus
    for $fact in $fact
    return
    {|
        { Aspects : {|
            for $a in keys($fact.Aspects)
            return { $a : $fact.Aspects.$a },
            { "secxbrl:FiscalPeriod" : $fp },
            { "secxbrl:FiscalYear" : $fy }
        |} },
        { Type: $fact.Type },
        if (exists($fact.Aspects."xbrl:Unit"))
        then { Unit: $fact.Aspects."xbrl:Unit" }
        else  (),
        if (exists($fact.Decimals))
        then { Decimals: $fact.Decimals }
        else (),
        { Value: $fact.Value },
        { "EntityRegistrantName" : $entity.Profiles.SEC.CompanyName },
        { "AuditTrails" : $fact.AuditTrails[] }

    |}
return
    if (session:only-dow30($entities) or session:valid())
        then {
            switch ($format)
            case "xml" return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                local:to-xml($facts)
            }
            case "text" case "csv" return {
                response:content-type("text/csv");
                response:header("Content-Disposition", "attachment; filename=facts.csv");
                local:to-csv($facts)
            }
            case "excel" return {
                response:content-type("application/vnd.ms-excel");
                response:header("Content-Disposition", "attachment; filename=fact.csv");
                local:to-csv($facts)
            }
            default return {
                response:content-type("application/json");
                response:serialization-parameters({"indent" : true});
                {|
                    { NetworkIdentifier : "http://secxbrl.info/facts" },
                    { TableName : "xbrl:Facts" },
                    { FactTable : [ $facts ] },
                    session:comment("json", {
                            NumFacts : count($facts),
                            TotalNumFacts: session:num-facts(),
                            TotalNumArchives: session:num-archives(),
                            TotalNumEntities: session:num-entities()
                        })
                |}
            }
        }
        else {
            response:status-code(401);
            session:error("accessing filings of an entity that is not in the DOW30", $format)
        }


:pencil: Fix filings endpoint.

:pencil: Fix filings endpoint.

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/backend";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace filings = "http://28.io/modules/xbrl/profiles/sec/filings";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";
import module namespace xbrl-export = "http://28.io/modules/xbrl/export";

declare option rest:response "first-item";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token         as string? external;
declare  %rest:env                              variable $request-uri   as string  external;
declare  %rest:case-insensitive                 variable $format        as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $eid           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cik           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode    as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker        as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear    as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod  as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $filingKind    as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $aid           as string* external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := $config:profile-name;
declare  %rest:case-insensitive                 variable $count         as boolean external := false;

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic,
  (),
  (),
  (),
  ())

let $entities-not-found as boolean :=
  exists(($eid, $cik, $tag, $ticker, $sic)) and empty($entities)

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $filingKind,
  $aid)

let $summaries :=
    switch($profile-name)
    case "sec" return
        for $f in filings:summaries($archives)
        order by $f.Accepted descending
        return $f
    case "japan" return
      for $a in $archives
      order by $a.Profiles.FSA.SubmissionDate descending
      return project($a, ("_id", "Entity", "Profiles", "Statistics"))
    default return $archives

let $summaries :=
    for $archive in $summaries
    return {|
      switch($profile-name)
      case "sec" return project($archive, "AccessionNumber")
      default return { AID: $archive._id },
      {
        Components: backend:url("components",
          {
              token: $token,
              aid: encode-for-uri(switch($profile-name)
                                  case "sec" return $archive.AccessionNumber
                                  default return archives:aid($archive)),
              format: $format,
              profile-name: $profile-name
          }),
        Sections: backend:url("sections",
          {
              token: $token,
              aid: encode-for-uri(switch($profile-name)
                                  case "sec" return $archive.AccessionNumber
                                  default return archives:aid($archive)),
              format: $format,
              profile-name: $profile-name
          })
      },
      trim($archive, ("_id", "Statistics", "AccessionNumber")),
      {
          NumSections: archives:num-components($archive),
          NumFacts: archives:num-facts($archive),
          NumFootnotes: archives:num-footnotes($archive),
          NumReportElements: archives:num-report-elements($archive),
          NumHypercubes: archives:num-hypercubes($archive),
          NumDimensions: archives:num-dimensions($archive),
          NumMembers: archives:num-members($archive),
          NumLineItems: archives:num-line-items($archive),
          NumAbstracts: archives:num-abstracts($archive),
          NumConcepts: archives:num-concepts($archive)
      }
    |}

let $result := { "Archives" : [ if($count) then () else $summaries ] }
let $comment :=
{
    NumArchives: count($summaries),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        switch($profile-name)
        case "sec" return
          <Filings>{
            filings:summaries-to-xml($res.Archives[])
          }</Filings>
        default return
          <Archives>{
            api:json-to-xml($res.Archives[], "Archive")
          }</Archives>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec" return
            string-join(filings:summaries-to-csv($res.Archives[]))
        default return api:json-to-csv($res.Archives[])
    }
}

let $results :=
  if($format eq "xbrl")
  then switch(count($archives))
     case 1 return xbrl-export:archive($archives)
     case 0 return api:not-found("archive")
     default return (
       { status: 400, content-type: "application/json" },
       session:error("Cannot export more than one archive at a time.", "json")
     )
  else api:serialize($result, $comment, $serializers, $format, "filings")
return if($entities-not-found)
       then api:not-found("entity")
       else api:check-and-return-results($token, $results, $format)

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/backend";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace filings = "http://28.io/modules/xbrl/profiles/sec/filings";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";
import module namespace xbrl-export = "http://28.io/modules/xbrl/export";

declare option rest:response "first-item";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token         as string? external;
declare  %rest:env                              variable $request-uri   as string  external;
declare  %rest:case-insensitive                 variable $format        as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $eid           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cik           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode    as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker        as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic           as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear    as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod  as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $filingKind    as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $aid           as string* external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := $config:profile-name;
declare  %rest:case-insensitive                 variable $count         as boolean external := false;

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic,
  (),
  (),
  (),
  ())

let $entities-not-found as boolean :=
  exists(($eid, $cik, $tag, $ticker, $sic)) and empty($entities)

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $filingKind,
  $aid)

let $summaries :=
    switch($profile-name)
    case "sec" return
        for $f in filings:summaries($archives)
        order by $f.Accepted descending
        return $f
    case "japan" return
      for $a in $archives
      order by $a.Profiles.FSA.SubmissionDate descending
      return project($a, ("_id", "Entity", "Profiles", "Statistics"))
    default return $archives

let $summaries :=
    for $archive in $summaries
    return {|
      switch($profile-name)
      case "sec" return project($archive, "AccessionNumber")
      default return { AID: $archive._id },
      {
        Components: backend:url("components",
          {
              token: $token,
              aid: encode-for-uri(switch($profile-name)
                                  case "sec" return $archive.AccessionNumber
                                  default return archives:aid($archive)),
              format: $format,
              profile-name: $profile-name
          }),
        Networks: backend:url("networks",
          {
              token: $token,
              aid: encode-for-uri(switch($profile-name)
                                  case "sec" return $archive.AccessionNumber
                                  default return archives:aid($archive)),
              format: $format,
              profile-name: $profile-name
          })
      },
      trim($archive, ("_id", "Statistics", "AccessionNumber")),
      {
          NumComponents: archives:num-components($archive),
          NumFacts: archives:num-facts($archive),
          NumFootnotes: archives:num-footnotes($archive),
          NumReportElements: archives:num-report-elements($archive),
          NumHypercubes: archives:num-hypercubes($archive),
          NumDimensions: archives:num-dimensions($archive),
          NumMembers: archives:num-members($archive),
          NumLineItems: archives:num-line-items($archive),
          NumAbstracts: archives:num-abstracts($archive),
          NumConcepts: archives:num-concepts($archive)
      }
    |}

let $result := { "Archives" : [ if($count) then () else $summaries ] }
let $comment :=
{
    NumArchives: count($summaries),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        switch($profile-name)
        case "sec" return
          <Filings>{
            filings:summaries-to-xml($res.Archives[])
          }</Filings>
        default return
          <Archives>{
            api:json-to-xml($res.Archives[], "Archive")
          }</Archives>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec" return
            string-join(filings:summaries-to-csv($res.Archives[]))
        default return api:json-to-csv($res.Archives[])
    }
}

let $results :=
  if($format eq "xbrl")
  then switch(count($archives))
     case 1 return xbrl-export:archive($archives)
     case 0 return api:not-found("archive")
     default return (
       { status: 400, content-type: "application/json" },
       session:error("Cannot export more than one archive at a time.", "json")
     )
  else api:serialize($result, $comment, $serializers, $format, "filings")
return if($entities-not-found)
       then api:not-found("entity")
       else api:check-and-return-results($token, $results, $format)


Restored grouping in components API.

Restored grouping in components API.

import module namespace util = "http://secxbrl.info/modules/util";
import module namespace session = "http://apps.28.io/session";

import module namespace entities = "http://xbrl.io/modules/bizql/entities";

import module namespace sec-networks = "http://xbrl.io/modules/bizql/profiles/sec/networks";
import module namespace sec-networks2 = "http://xbrl.io/modules/bizql/profiles/sec/networks2";
import module namespace companies2 = "http://xbrl.io/modules/bizql/profiles/sec/companies2";
import module namespace fiscal-core = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace fiscal-core2 = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core2";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-csv($res as object*) as string*
{
    csv:serialize(
        for $a in $res
        for $c in $a.Components[]
        return {
            AcessionNumber : $a.AccessionNumber,
            EntityRegistrantName : $a.EntityRegistrantName,
            CIK : $a.CIK,
            FiscalYear : $a.FiscalYear,
            FiscalPeriod : $a.FiscalPeriod,
            AcceptanceDateTime : $a.AcceptanceDatetime,
            FormType : $a.FormType,
            NetworkLabel : $c.NetworkLabel,
            NetworkIdentifier : $c.NetworkRole,
            Category : $c.Category,
            SubCategory : $c.SubCategory,
            Table : flatten($c.Table),
            Disclosure : $c.Disclosure,
            ReportElements : $c.ReportElements,
            Tables : $c.Tables,
            Axis : $c.Axis,
            Members : $c.Members,
            LineItems : $c.LineItems,
            Concepts : $c.Concepts,
            Abstracts : $c.Abstracts
        },
    { serialize-null-as : "" })
};

session:audit-call();

(: Query parameters :)
let $format as string?         := request:param-values("format")
let $ciks as string*           := distinct-values(request:param-values("cik"))
let $tags as string*           := distinct-values(request:param-values("tag"))
let $tickers as string*        := distinct-values(request:param-values("ticker"))
let $sics as string*           := distinct-values(request:param-values("sic"))
let $fiscalYears as string*    := distinct-values(request:param-values("fiscalYear", "LATEST"))
let $fiscalPeriods as string*  := distinct-values(request:param-values("fiscalPeriod", "FY"))
let $aids as string*           := distinct-values(request:param-values("aid"))
let $roles as string*          := request:param-values("networkIdentifier")
let $cids as string*           := request:param-values("cid")
let $reportElements as string* := distinct-values(request:param-values("reportElement"))
let $concepts as string*       := distinct-values(request:param-values("concept"))
let $disclosures as string*    := request:param-values("disclosure")
let $search as string*         := request:param-values("label")

(: Post-processing :)
let $format as string? := (: backwards compatibility, to be deprecated  :)
    lower-case(($format, substring-after(request:path(), ".jq."))[1])
let $tags as string* := (: backwards compatibility, to be deprecated :)
    distinct-values($tags ! upper-case($$))
let $fiscalYears as integer* :=
    for $fy in $fiscalYears ! upper-case($$)
    return switch($fy)
           case "LATEST" return $fiscal-core2:LATEST_FISCAL_YEAR
           case "ALL" return $fiscal-core:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
let $fiscalPeriods as string* :=
    for $fp in $fiscalPeriods ! upper-case($$)
    return switch($fp)
           case "ALL" return $fiscal-core:ALL_FISCAL_PERIODS
           default return $fp
let $reportElements := ($reportElements, $concepts)

(: Object resolution :)
let $entities :=
    companies2:companies(
        $ciks,
        $tags,
        $tickers,
        $sics)
let $archives as object* := fiscal-core2:filings(
    $entities,
    $fiscalPeriods,
    $fiscalYears,
    $aids)
let $entities    := entities:entities($archives.Entity)
let $components  := sec-networks2:components(
    $archives,
    $cids,
    $reportElements,
    $disclosures,
    $roles,
    $search)
let $res         :=
    for $r in $components
    let $disclosure := sec-networks:disclosures($r)
    where $disclosure ne "DefaultComponent"
    where exists(sec-networks:model-structures($r))
    order by $r.Label
    group by $archive := $r.Archive
    let $archive := $archives[$$._id eq $archive]
    let $e := $entities[$$._id eq $archive.Entity]
    return
        {
           AccessionNumber : $archive._id,
           EntityRegistrantName : $e.Profiles.SEC.CompanyName,
           CIK : $e._id,
           FiscalYear :$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
           FiscalPeriod :$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
           AcceptanceDatetime : $archive.Profiles.SEC.AcceptanceDatetime,
           FormType : $archive.Profiles.SEC.FormType,
           Components : [
               sec-networks:summaries($r)
           ]
       }

let $result := { Archives: [ $res ] }
let $comment :=
 {
    NumComponents : count($components),
    TotalNumComponents: session:num-components(),
    TotalNumArchives: session:num-archives()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <Archives>{
                  for $r in flatten($res.Archives)
                  return
                      <Archive id="{$r.AccessionNumber}">
                         <EntityRegistrantName>{$r.EntityRegistrantName}</EntityRegistrantName>
                         <CIK>{$r.CIK}</CIK>
                         <FiscalYear>{$r.FiscalYear}</FiscalYear>
                         <FiscalPeriod>{$r.FiscalPeriod}</FiscalPeriod>
                         <AcceptanceDatetime>{$r.AcceptanceDatetime}</AcceptanceDatetime>
                         <FormType>{$r.FormType}</FormType>
                         <Components>{
                             sec-networks:summaries-to-xml(flatten($r.Components))
                         }</Components>
                     </Archive>
             }</Archives>
    },
    to-csv : function($res as object) as string {
        string-join(local:to-csv($res.Archives[]), "")
    }
}

let $results := util:serialize($result, $comment, $serializers, $format, "components")
return
    util:check-and-return-results($entities, $results, $format)

import module namespace util = "http://secxbrl.info/modules/util";
import module namespace session = "http://apps.28.io/session";

import module namespace entities = "http://xbrl.io/modules/bizql/entities";

import module namespace sec-networks = "http://xbrl.io/modules/bizql/profiles/sec/networks";
import module namespace sec-networks2 = "http://xbrl.io/modules/bizql/profiles/sec/networks2";
import module namespace companies2 = "http://xbrl.io/modules/bizql/profiles/sec/companies2";
import module namespace fiscal-core = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace fiscal-core2 = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core2";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace csv = "http://zorba.io/modules/json-csv";

declare function local:to-csv($res as object*) as string*
{
    csv:serialize(
        for $c in $res
        return {
            NetworkLabel : $c.NetworkLabel,
            NetworkIdentifier : $c.NetworkRole,
            Category : $c.Category,
            SubCategory : $c.SubCategory,
            Table : flatten($c.Table),
            Disclosure : $c.Disclosure,
            ReportElements : $c.ReportElements,
            Tables : $c.Tables,
            Axis : $c.Axis,
            Members : $c.Members,
            LineItems : $c.LineItems,
            Concepts : $c.Concepts,
            Abstracts : $c.Abstracts
        },
    { serialize-null-as : "" })
};

session:audit-call();

(: Query parameters :)
let $format as string?         := request:param-values("format")
let $ciks as string*           := distinct-values(request:param-values("cik"))
let $tags as string*           := distinct-values(request:param-values("tag"))
let $tickers as string*        := distinct-values(request:param-values("ticker"))
let $sics as string*           := distinct-values(request:param-values("sic"))
let $fiscalYears as string*    := distinct-values(request:param-values("fiscalYear", "LATEST"))
let $fiscalPeriods as string*  := distinct-values(request:param-values("fiscalPeriod", "FY"))
let $aids as string*           := distinct-values(request:param-values("aid"))
let $roles as string*          := request:param-values("networkIdentifier")
let $cids as string*           := request:param-values("cid")
let $reportElements as string* := distinct-values(request:param-values("reportElement"))
let $concepts as string*       := distinct-values(request:param-values("concept"))
let $disclosures as string*    := request:param-values("disclosure")
let $search as string*         := request:param-values("label")

(: Post-processing :)
let $format as string? := (: backwards compatibility, to be deprecated  :)
    lower-case(($format, substring-after(request:path(), ".jq."))[1])
let $tags as string* := (: backwards compatibility, to be deprecated :)
    distinct-values($tags ! upper-case($$))
let $fiscalYears as integer* :=
    for $fy in $fiscalYears ! upper-case($$)
    return switch($fy)
           case "LATEST" return $fiscal-core2:LATEST_FISCAL_YEAR
           case "ALL" return $fiscal-core:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
let $fiscalPeriods as string* :=
    for $fp in $fiscalPeriods ! upper-case($$)
    return switch($fp)
           case "ALL" return $fiscal-core:ALL_FISCAL_PERIODS
           default return $fp
let $reportElements := ($reportElements, $concepts)

(: Object resolution :)
let $entities :=
    companies2:companies(
        $ciks,
        $tags,
        $tickers,
        $sics)
let $archives as object* := fiscal-core2:filings(
    $entities,
    $fiscalPeriods,
    $fiscalYears,
    $aids)
let $entities    := entities:entities($archives.Entity)
let $components  := sec-networks2:components(
    $archives,
    $cids,
    $reportElements,
    $disclosures,
    $roles,
    $search)
let $res         :=
    for $r in $components
    let $disclosure := sec-networks:disclosures($r)
    where $disclosure ne "DefaultComponent"
    where exists(sec-networks:model-structures($r))
    order by $r.Label
    group by $archive := $r.Archive
    return sec-networks:summaries($r)

let $result := { Components: [ $res ] }
let $comment :=
 {
    NumComponents : count($components),
    TotalNumComponents: session:num-components(),
    TotalNumArchives: session:num-archives()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <Components>{ $res.Components[] ! sec-networks:summaries-to-xml($$) }</Components>
    },
    to-csv : function($res as object) as string {
        string-join(local:to-csv($res.Components[]), "")
    }
}

let $results := util:serialize($result, $comment, $serializers, $format, "components")
return
    util:check-and-return-results($entities, $results, $format)


Update drugfilter.jq

Update drugfilter.jq

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv
.[]
#start from the .results array
|.results

#Creates new object with .safetyreportid string and .patient.drug array
|map(with_entries(select(.key == "safetyreportid")) + (.patient.drug[]))

# Flattens .openfda and turns openfda arrays into strings delimited with ;
|map(with_entries(select(.key != "openfda")) + (.openfda//{openfda:{"NA":"NA"}}|with_entries(.value = ([.value[]|tostring]|sort|join(";")) |.key |= "openfda_"  + .)))

#Flattens .activesubstance array
|map(with_entries(select(.key != "activesubstance")) + (.activesubstance))

#Grabs all keys, and builds CSV, to fix jagged array
|["safetyreportid",
"actiondrug",
"activesubstancename",
"drugadditional",
"drugadministrationroute",
"drugauthorizationnumb",
"drugbatchnumb",
"drugcharacterization",
"drugcumulativedosagenumb",
"drugcumulativedosageunit",
"drugdosageform",
"drugdosagetext",
"drugenddate",
"drugenddateformat",
"drugindication",
"drugintervaldosagedefinition",
"drugintervaldosageunitnumb",
"drugrecurreadministration",
"drugseparatedosagenumb",
"drugstartdate",
"drugstartdateformat",
"drugstructuredosagenumb",
"drugstructuredosageunit",
"drugtreatmentduration",
"drugtreatmentdurationunit",
"medicinalproduct",
"openfda_application_number",
"openfda_brand_name",
"openfda_generic_name",
"openfda_manufacturer_name",
"openfda_nui",
"openfda_package_ndc",
"openfda_pharm_class_cs",
"openfda_pharm_class_epc",
"openfda_pharm_class_moa",
"openfda_pharm_class_pe",
"openfda_product_ndc",
"openfda_product_type",
"openfda_route",
"openfda_rxcui",
"openfda_spl_id",
"openfda_spl_set_id",
"openfda_substance_name",
"openfda_unii"] as $keys
|$keys, (.[]|[.[$keys[]]])|@csv

# Require jq > 1.4
# Takes open.fda.gov drug event JSON and transforms into compact flat linear records suitable for Perl recs-tocsv
.[]
#start from the .results array
|.results

#Creates new object with .safetyreportid string and .patient.drug array
|map(with_entries(select(.key == "safetyreportid")) + (.patient.drug[]))

# Flattens .openfda and turns openfda arrays into strings delimited with ;
|map(with_entries(select(.key != "openfda")) + (.openfda//{openfda:{"NA":"NA"}}|with_entries(.value = ([.value[]|tostring]|sort|join(";")) |.key |= "openfda_"  + .)))

#Flattens .activesubstance array
|map(with_entries(select(.key != "activesubstance")) + (.activesubstance))

#Grabs all keys, and builds CSV, to fix jagged array
|["safetyreportid",
"actiondrug",
"activesubstancename",
"drugadditional",
"drugadministrationroute",
"drugauthorizationnumb",
"drugbatchnumb",
"drugcharacterization",
"drugcumulativedosagenumb",
"drugcumulativedosageunit",
"drugdosageform",
"drugdosagetext",
"drugenddate",
"drugenddateformat",
"drugindication",
"drugintervaldosagedefinition",
"drugintervaldosageunitnumb",
"drugrecurreadministration",
"drugseparatedosagenumb",
"drugstartdate",
"drugstartdateformat",
"drugstructuredosagenumb",
"drugstructuredosageunit",
"drugtreatmentduration",
"drugtreatmentdurationunit",
"medicinalproduct",
"openfda_application_number",
"openfda_brand_name",
"openfda_generic_name",
"openfda_manufacturer_name",
"openfda_nui",
"openfda_openfda",
"openfda_package_ndc",
"openfda_pharm_class_cs",
"openfda_pharm_class_epc",
"openfda_pharm_class_moa",
"openfda_pharm_class_pe",
"openfda_product_ndc",
"openfda_product_type",
"openfda_route",
"openfda_rxcui",
"openfda_spl_id",
"openfda_spl_set_id",
"openfda_substance_name",
"openfda_unii"] as $keys
|$keys, (.[]|[.[$keys[]]])|@csv


:pencil: Add links to report elements and validation errors in SEC profile.

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/backend";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec-filings = "http://28.io/modules/xbrl/profiles/sec/filings";
import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare option rest:response "first-item";

declare function local:to-csv($res as object*) as string*
{
    csv:serialize(
        for $a in $res
        for $c in $a.Components[]
        return {
            AcessionNumber : $a.AccessionNumber,
            NetworkIdentifier : $c.NetworkIdentifier,
            FactTable: $c.FactTable,
            SpreadSheet: $c.SpreadSheet,
            EntityRegistrantName : $a.EntityRegistrantName,
            CIK : $a.CIK,
            FiscalYear : $a.FiscalYear,
            FiscalPeriod : $a.FiscalPeriod,
            AcceptanceDateTime : $a.AcceptanceDatetime,
            FormType : $a.FormType,
            NetworkLabel : $c.NetworkLabel,
            Category : $c.Category,
            SubCategory : $c.SubCategory,
            Table : flatten($c.Table),
            Disclosure : $c.Disclosure,
            ReportElements : $c.ReportElements,
            Tables : $c.Tables,
            Axis : $c.Axis,
            Members : $c.Members,
            LineItems : $c.LineItems,
            Concepts : $c.Concepts,
            Abstracts : $c.Abstracts
        },
    { serialize-null-as : "" })
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token              as string? external;
declare  %rest:env                              variable $request-uri        as string  external;
declare  %rest:case-insensitive                 variable $format             as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode   as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker             as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear         as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod       as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $filingKind         as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $eid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $networkIdentifier  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $role               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $reportElement      as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $concept            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $label              as string* external;
declare  %rest:case-insensitive                 variable $profile-name       as string  external := $config:profile-name;

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $reportElement := ($reportElement, $concept)
let $networkIdentifier := distinct-values(($networkIdentifier, $role))

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic, ())

let $entities-not-found as boolean :=
  exists(($eid, $cik, $tag, $ticker, $sic)) and empty($entities)

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $filingKind,
  $aid)

let $archives-not-found as boolean :=
  exists(($entities, $fiscalPeriod, $fiscalYear, $filingKind, $aid)) and empty($archives)

let $entities as object*  := entities:entities($archives.Entity)
let $components as object* :=
    multiplexer:components(
      $profile-name,
      $archives,
      $cid,
      $reportElement,
      $disclosure,
      $networkIdentifier,
      $label[$profile-name ne "sec"],
      $label[$profile-name eq "sec"]
    )

let $res as object* :=
    switch($profile-name)
    case "sec" return
        for $r in $components
        let $disclosure := sec-networks:disclosures($r)
        where $disclosure ne "DefaultComponent"
        order by $r.Label
        group by $archive := $r.Archive
        let $archive := $archives[archives:aid($$) eq $archive]
        let $e := $entities[entities:eid($$) eq $archive.Entity]
        return
            {
               AccessionNumber : archives:aid($archive),
               EntityRegistrantName : $e.Profiles.SEC.CompanyName,
               CIK : entities:eid($e),
               FiscalYear :$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
               FiscalPeriod :$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
               AcceptanceDatetime : sec-filings:acceptance-dateTimes($archive),
               FormType : $archive.Profiles.SEC.FormType,
               Components : [
                    for $component in sec-networks:summaries($r)
                    return copy $c := $component
                    modify insert json {
                        FactTable: backend:url("facttable-for-component", {
                            token: $token,
                            aid: archives:aid($archive),
                            format: $format,
                            role: $component.NetworkIdentifier,
                            profile-name: $profile-name
                            }),
                        SpreadSheet: "http://rendering.secxbrl.info/#?url=" || encode-for-uri(
                            backend:url("spreadsheet-for-component", {
                            token: $token,
                            aid: archives:aid($archive),
                            format: $format,
                            role: $component.NetworkIdentifier,
                            profile-name: $profile-name
                            }),
                        ReportElements: backend:url("report-elements", {
                            token: $token,
                            aid: $r.Archive,
                            format: $format,
                            role: $r.Role,
                            profile-name: $profile-name
                            }),
                        ValidationErrors: [ components:validation-errors($r) ]
                    } into $c
                    return $c
               ]
           }
    default return
        for $r in $components
        return {|
          {
            Archive: $r.Archive,
            Label: $r.Label,
            Role: $r.Role,
            FactTable: backend:url("facttable-for-component", {
                            token: $token,
                            aid: $r.Archive,
                            format: $format,
                            role: $r.Role,
                            profile-name: $profile-name
                            }),
            SpreadSheet: "http://rendering.secxbrl.info/#?url=" || encode-for-uri(
                        backend:url("spreadsheet-for-component", {
                            token: $token,
                            aid: $r.Archive,
                            format: $format,
                            role: $r.Role,
                            profile-name: $profile-name,
                            eliminate: "true"
                        })),
            ReportElements: backend:url("report-elements", {
                            token: $token,
                            aid: $r.Archive,
                            format: $format,
                            role: $r.Role,
                            profile-name: $profile-name
                        }),
            NumRules: size($r.Rules),
            NumNetworks: components:num-networks($r),
            NumReportElements: components:num-report-elements($r),
            NumHypercubes: components:num-hypercubes($r),
            NumDimensions: components:num-dimensions($r),
            NumMembers: components:num-members($r),
            NumLineItems: components:num-line-items($r),
            NumAbstracts: components:num-abstracts($r),
            NumConcepts: components:num-concepts($r),
            Hypercubes: [ keys($r.Hypercubes) ],
            ValidationErrors: [ components:validation-errors($r) ]
          },
          { Disclosure: $r.Profiles.FSA.Disclosure}[$profile-name eq "japan"]
        |}
let $result := switch($profile-name)
               case "sec"
               return { Archives: [ $res ] }
               default return { Components : [ $res ] }
let $comment :=
 {
    NumComponents : count($components),
    TotalNumComponents: session:num-components(),
    TotalNumArchives: session:num-archives()
}
let $serializers := {
    to-xml : switch($profile-name)
        case "sec"
        return function($res as object) as node() {
        <Archives>{
                  for $r in flatten($res.Archives)
                  return
                      <Archive id="{$r.AccessionNumber}">
                         <EntityRegistrantName>{$r.EntityRegistrantName}</EntityRegistrantName>
                         <CIK>{$r.CIK}</CIK>
                         <FiscalYear>{$r.FiscalYear}</FiscalYear>
                         <FiscalPeriod>{$r.FiscalPeriod}</FiscalPeriod>
                         <AcceptanceDatetime>{$r.AcceptanceDatetime}</AcceptanceDatetime>
                         <FormType>{$r.FormType}</FormType>
                         <Components>{
                             sec-networks:summaries-to-xml(flatten($r.Components))
                         }</Components>
                     </Archive>
             }</Archives>
        }
        default return function($res as object) as node() {
        <Components>{
          api:json-to-xml($res.Components[], "Component")
        }</Components>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec" return string-join(local:to-csv($res.Archives[]), "")
        default return api:json-to-csv($res.Components[])
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "components")
return switch(true)
       case $entities-not-found return api:not-found("entity")
       case $archives-not-found return api:not-found("archive")
       default return api:check-and-return-results($token, $results, $format)

import module namespace config = "http://apps.28.io/config";
import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";
import module namespace backend = "http://apps.28.io/backend";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";

import module namespace sec-filings = "http://28.io/modules/xbrl/profiles/sec/filings";
import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";
import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

import module namespace csv = "http://zorba.io/modules/json-csv";

declare option rest:response "first-item";

declare function local:to-csv($res as object*) as string*
{
    csv:serialize(
        for $a in $res
        for $c in $a.Components[]
        return {
            AcessionNumber : $a.AccessionNumber,
            NetworkIdentifier : $c.NetworkIdentifier,
            FactTable: $c.FactTable,
            SpreadSheet: $c.SpreadSheet,
            EntityRegistrantName : $a.EntityRegistrantName,
            CIK : $a.CIK,
            FiscalYear : $a.FiscalYear,
            FiscalPeriod : $a.FiscalPeriod,
            AcceptanceDateTime : $a.AcceptanceDatetime,
            FormType : $a.FormType,
            NetworkLabel : $c.NetworkLabel,
            Category : $c.Category,
            SubCategory : $c.SubCategory,
            Table : flatten($c.Table),
            Disclosure : $c.Disclosure,
            ReportElements : $c.ReportElements,
            Tables : $c.Tables,
            Axis : $c.Axis,
            Members : $c.Members,
            LineItems : $c.LineItems,
            Concepts : $c.Concepts,
            Abstracts : $c.Abstracts
        },
    { serialize-null-as : "" })
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token              as string? external;
declare  %rest:env                              variable $request-uri        as string  external;
declare  %rest:case-insensitive                 variable $format             as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $edinetcode   as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker             as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear         as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod       as string* external := "ALL";
declare  %rest:case-insensitive %rest:distinct  variable $filingKind         as string* external := ();
declare  %rest:case-insensitive %rest:distinct  variable $eid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $aid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $networkIdentifier  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $role               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $reportElement      as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $concept            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $label              as string* external;
declare  %rest:case-insensitive                 variable $profile-name       as string  external := $config:profile-name;

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $reportElement := ($reportElement, $concept)
let $networkIdentifier := distinct-values(($networkIdentifier, $role))

let $cik as string* :=
    switch($profile-name)
    case "sec" return $cik
    case "japan" return $edinetcode
    default return ()

(: Entity resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic, ())

let $entities-not-found as boolean :=
  exists(($eid, $cik, $tag, $ticker, $sic)) and empty($entities)

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $filingKind,
  $aid)

let $archives-not-found as boolean :=
  exists(($entities, $fiscalPeriod, $fiscalYear, $filingKind, $aid)) and empty($archives)

let $entities as object*  := entities:entities($archives.Entity)
let $components as object* :=
    multiplexer:components(
      $profile-name,
      $archives,
      $cid,
      $reportElement,
      $disclosure,
      $networkIdentifier,
      $label[$profile-name ne "sec"],
      $label[$profile-name eq "sec"]
    )

let $res as object* :=
    switch($profile-name)
    case "sec" return
        for $r in $components
        let $disclosure := sec-networks:disclosures($r)
        where $disclosure ne "DefaultComponent"
        order by $r.Label
        group by $archive := $r.Archive
        let $archive := $archives[archives:aid($$) eq $archive]
        let $e := $entities[entities:eid($$) eq $archive.Entity]
        return
            {
               AccessionNumber : archives:aid($archive),
               EntityRegistrantName : $e.Profiles.SEC.CompanyName,
               CIK : entities:eid($e),
               FiscalYear :$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
               FiscalPeriod :$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
               AcceptanceDatetime : sec-filings:acceptance-dateTimes($archive),
               FormType : $archive.Profiles.SEC.FormType,
               Components : [
                    for $component in sec-networks:summaries($r)
                    return copy $c := $component
                    modify insert json {
                        FactTable: backend:url("facttable-for-component", {
                            token: $token,
                            aid: archives:aid($archive),
                            format: $format,
                            role: $component.NetworkIdentifier,
                            profile-name: $profile-name
                            }),
                        SpreadSheet: "http://rendering.secxbrl.info/#?url=" || encode-for-uri(
                            backend:url("spreadsheet-for-component", {
                            token: $token,
                            aid: archives:aid($archive),
                            format: $format,
                            role: $component.NetworkIdentifier,
                            profile-name: $profile-name
                            })
                        )
                    } into $c
                    return $c
               ]
           }
    default return
        for $r in $components
        return {|
          {
            Archive: $r.Archive,
            Label: $r.Label,
            Role: $r.Role,
            FactTable: backend:url("facttable-for-component", {
                            token: $token,
                            aid: $r.Archive,
                            format: $format,
                            role: $r.Role,
                            profile-name: $profile-name
                            }),
            SpreadSheet: "http://rendering.secxbrl.info/#?url=" || encode-for-uri(
                        backend:url("spreadsheet-for-component", {
                            token: $token,
                            aid: $r.Archive,
                            format: $format,
                            role: $r.Role,
                            profile-name: $profile-name,
                            eliminate: "true"
                        })),
            ReportElements: backend:url("report-elements", {
                            token: $token,
                            aid: $r.Archive,
                            format: $format,
                            role: $r.Role,
                            profile-name: $profile-name
                        }),
            NumRules: size($r.Rules),
            NumNetworks: components:num-networks($r),
            NumReportElements: components:num-report-elements($r),
            NumHypercubes: components:num-hypercubes($r),
            NumDimensions: components:num-dimensions($r),
            NumMembers: components:num-members($r),
            NumLineItems: components:num-line-items($r),
            NumAbstracts: components:num-abstracts($r),
            NumConcepts: components:num-concepts($r),
            Hypercubes: [ keys($r.Hypercubes) ],
            ValidationErrors: [ components:validation-errors($r) ]
          },
          { Disclosure: $r.Profiles.FSA.Disclosure}[$profile-name eq "japan"]
        |}
let $result := switch($profile-name)
               case "sec"
               return { Archives: [ $res ] }
               default return { Components : [ $res ] }
let $comment :=
 {
    NumComponents : count($components),
    TotalNumComponents: session:num-components(),
    TotalNumArchives: session:num-archives()
}
let $serializers := {
    to-xml : switch($profile-name)
        case "sec"
        return function($res as object) as node() {
        <Archives>{
                  for $r in flatten($res.Archives)
                  return
                      <Archive id="{$r.AccessionNumber}">
                         <EntityRegistrantName>{$r.EntityRegistrantName}</EntityRegistrantName>
                         <CIK>{$r.CIK}</CIK>
                         <FiscalYear>{$r.FiscalYear}</FiscalYear>
                         <FiscalPeriod>{$r.FiscalPeriod}</FiscalPeriod>
                         <AcceptanceDatetime>{$r.AcceptanceDatetime}</AcceptanceDatetime>
                         <FormType>{$r.FormType}</FormType>
                         <Components>{
                             sec-networks:summaries-to-xml(flatten($r.Components))
                         }</Components>
                     </Archive>
             }</Archives>
        }
        default return function($res as object) as node() {
        <Components>{
          api:json-to-xml($res.Components[], "Component")
        }</Components>
    },
    to-csv : function($res as object) as string {
        switch($profile-name)
        case "sec" return string-join(local:to-csv($res.Archives[]), "")
        default return api:json-to-csv($res.Components[])
    }
}

let $results := api:serialize($result, $comment, $serializers, $format, "components")
return switch(true)
       case $entities-not-found return api:not-found("entity")
       case $archives-not-found return api:not-found("archive")
       default return api:check-and-return-results($token, $results, $format)