Added $minLength and $maxLength facet validation for array types.

jsoniq version "1.0";

(:
 : Copyright 2006-2013 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

(:~
 : JSound simple validator
 : This is a JSONiq implemenation of the JSound (the schema for JSON) validator.
 :
 : @author Cezar Andrei
 :)

module namespace jsv = "http://jsound.io/modules/validate";

import module namespace jn = "http://jsoniq.org/functions";
import module namespace map = "http://jsound.io/modules/validate/map";

declare namespace an = "http://zorba.io/annotations";

declare %an:sequential function
jsv:jsd-valid($jsd as object, $name as string, $ns as string,
  $instance as item) as boolean
{
  variable $jstypes := { "foo" : 5 };

  if( $jsd instance of object )
  then
    if ( exists($jsd("$namespace")) and exists($jsd("$types")) )
    then
      if ( $jsd("$namespace") instance of string )
      then
        if ( $jsd("$namespace") eq $ns )
        then
          if ( $jsd("$types") instance of array )
          then
            jsv:check-types( $jstypes, $jsd("$types") );
          else
            fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: value of '$types' must be an array.");
        else
          fn:error(QName("jsv", "Invalid"),
            "Requested type namespace not matching schema document namespace.");
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: value of '$namespace' must be a string.");
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: must contain '$types' and '$namespace' keys.");
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "The specified JSON document is not a valid JSound schema: top item not an object.");

  (: todo: validate all type refs :)

  jsv:validate-instance($jstypes, $name, $instance)
};

(: The check-xxx functions just checks if the JSound schema definitions are valid. :)

declare %an:sequential function
jsv:check-types($jstypes as object, $types as array) as boolean
{
  every $i in
    for $type in jn:members($types)
    return
      jsv:check-type($jstypes, $type )
  satisfies $i eq fn:true()
};

declare %an:sequential function
jsv:check-type($jstypes as object, $type as item) as boolean
{
  if( $type instance of object )
  then
    if ( exists($type("$kind")) and $type("$kind") instance of string )
    then
      switch( $type("$kind") )
      case "atomic" return jsv:check-atomic-type($jstypes, $type)
      case "object" return jsv:check-object-type($jstypes, $type)
      case "array"  return jsv:check-array-type ($jstypes, $type)
      case "union"  return jsv:check-union-type ($jstypes, $type)

      default return
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: type $kind unknown:" | $type("$kind") )

    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: type definition must contain '$kind' key with string value.")
  else
    if( $type instance of string )
    then
      jsv:check-ref-type($jstypes, $type)
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: type item not an object."  )
};

declare %an:sequential function
jsv:check-atomic-type($jstypes as object, $type as object) as boolean
{
  fn:trace( $type("$name"), "check-atomic-type");

  if( exists($type("$baseType")) and $type("$baseType") instance of string )
  then
    jsv:save-type($jstypes, $type)
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: atomic type definition must contain '$baseType' key with string value.")

  (: todo: add facets checks: length, min/max Inclusive/Exclusive, etc. :)
};

declare %an:sequential function
jsv:check-object-type($jstypes as object, $type as object) as boolean
{
  (: todo: check the reast of "object" type :)
  jsv:save-type($jstypes, $type);
  fn:trace( $type("$name"), "check-object-type");

  if( exists($type("$content")) )
  then
    if( $type("$content") instance of object )
    then
      every $i in
        for $k in jn:keys($type("$content"))
        return
          if( $type("$content")( fn:trace($k, "Key")) instance of object )
          then
            if( exists( $type("$content")($k)("$type")) )
            then
              jsv:check-type($jstypes, $type("$content")($k)("$type") )
            else
              if( exists( $type("$content")($k)("$kind")) )
              then
                jsv:check-type($jstypes, $type("$content")($k) )
              else
                fn:error(QName("jsv", "BadJSoundFormat"),
                  "Not a valid JSound doc: Key  does not contain a $type definition.")
          else
            fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: Key " | $k | " in $content must have an object value.")

      satisfies $i eq fn:true()
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: $content definion of object type must be an object.")
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
             "Not a valid JSound doc: Object type does not contain $content.")
};

declare %an:sequential function
jsv:check-array-type($jstypes as object, $type as object) as boolean
{
  fn:trace( $type("$name"), "check-array-type");

  (: check all the contents :)
  (: let $t := fn:trace($type, "Array type:") :)
  let $content := $type("$content")
  let $checkContent :=
    if( empty($content) )
    then
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: $kind array must have a $content key.")
    else
      if( $content instance of array and jn:size($content) eq 1 )
      then
      {
        jsv:save-type($jstypes, $type);

        typeswitch( $content(1) )
        case string return
          fn:true()
        case object return
            jsv:check-type($jstypes, $content(1))
        default return
          fn:error(QName("jsv", "BadJSoundFormat"),
            "Not a valid JSound doc: for $kind array, $content[1] must be string or array.")
      }
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $kind array, $content must be an array of size 1.")

  let $checkMinLength :=
      if( fn:exists($type("$minLength")) )
      then
        if( $type("$minLength") instance of integer  )
        then fn:true()
        else fn:error(QName("jsv", "BadJSoundFormat"),
                      "Not a valid JSound doc: for $kind array, $minLength must be an integer.")
      else fn:true()

  let $checkMaxLength :=
      if( fn:exists($type("$maxLength")) )
      then
        if( $type("$maxLength") instance of integer  )
        then fn:true()
        else fn:error(QName("jsv", "BadJSoundFormat"),
                      "Not a valid JSound doc: for $kind array, $maxLength must be an integer.")
      else fn:true()

  return
    $checkContent and $checkMinLength and $checkMaxLength
};

declare %an:sequential function
jsv:check-union-type($jstypes as object, $type as object) as boolean
{
  fn:trace( $type("$name"), "check-union-type");

  let $content := $type("$content")
  return
    if( empty($content) )
    then
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: $kind union must have a $content key.")
    else
      if( $content instance of array )
      then
      {
        jsv:save-type($jstypes, $type);

        every $i in
          for $c in jn:members($content)
          return
            typeswitch( $c )
            case string return
              fn:true()
            case object return
                jsv:check-type($jstypes, $c)
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind union, $content items must be strings or objects.")
        satisfies $i eq fn:true()
      }
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $kind union, $content must be an array.")
};

declare function
jsv:check-ref-type($jstypes as object, $type as string) as boolean
{
  switch( $type )
  case "string" return fn:true()
  case "integer" return fn:true()
  case "boolean" return fn:true()
  case "anyURI"  return fn:true()
  default return
  {
    if( map:has-key($jstypes, $type) )
    then
      fn:true()
    else
     (: todo: save all refs, check them at the end to see if they exist
     fn:trace($type, "NYI: type ref not yet suported");
     fn:error(QName("jsv", "NYI"), "Type ref: not yet implemented.") :)
     fn:true()
  }

  (: todo: apply resolution rules :)
};

declare %an:sequential function
jsv:save-type($jstypes as object, $type as item) as boolean
{
  if( fn:exists( $type("$name") ) )
  then
    if( $type("$name") instance of string )
    then
      map:set-if-empty($jstypes, $type("$name"), $type)
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $type, $name must be a string.")
  else
    fn:false()
};

(: The validate-xxx functions actualy do the validation of the instance :)

declare function
jsv:validate-instance($jstypes as object, $name as string, $instance as item)
 as boolean
{
  let $type := map:get($jstypes, $name)
  return
    if( exists($type) )
    then
      jsv:validate-type($jstypes, $type, $instance)
    else
      fn:error(QName("jsv", "Invalid"),
             "Requested type name not present in schema: " | $name)
};

declare function
jsv:validate-type($jstypes as object, $type as item, $instance as item) as boolean
{
  switch( $type("$kind") )
  case "atomic" return jsv:validate-atomic-type($jstypes, $type, $instance)
  case "object" return jsv:validate-object-type($jstypes, $type, $instance)
  case "array"  return jsv:validate-array-type($jstypes, $type, $instance)
  case "union"  return jsv:validate-union-type($jstypes, $type, $instance)

  default return
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: type $kind unknown:" | $type("$kind") )
};

declare function
jsv:validate-type-ref($jstypes as object, $type as string, $instance as item) as boolean
{
  (: todo: check first between the defined types in $jstypes and then the build-in ones, this is per spec :)

  if( map:has-key($jstypes, $type) )
  then
    jsv:validate-type($jstypes, map:get($jstypes, $type), $instance)
  else
    switch( $type )
    case "string" return
      if ( $instance instance of string )
      then
        fn:true()
      else
        fn:error(QName("jsv", "Invalid"), "Expected string value.")
    case "integer" return
      if ( $instance instance of integer or
           $instance instance of int )
      then
        fn:true()
      else
        fn:error(QName("jsv", "Invalid"), "Expected integer value.")
    case "anyURI" return
      if ( $instance instance of anyURI )
      then
        fn:true()
      else
        fn:error(QName("jsv", "Invalid"), "Expected anyURI value.")
    case "boolean" return
      if ( $instance instance of boolean )
      then
        fn:true()
      else
        fn:error(QName("jsv", "Invalid"), "Expected boolean value.")

    default return
        fn:error(QName("jsv", "Invalid"),
          fn:concat("Type ref not defined or a built-in type: ", $type) )
};

declare function
jsv:validate-atomic-type($jstypes as object, $type as object, $instance as item) as boolean
{
  fn:trace( $type("$name"), "validate-atomic-type");
  (: todo: check first between the defined types in $jstypes and then the build-in ones, this is per spec :)

  if ( exists($type("$baseType")) and $type("$baseType") instance of string )
  then
    switch( $type("$baseType") )
    case "integer" return
      if ( $instance instance of integer or $instance instance of int )
      then
        fn:true() and jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected integer or int value")

    case "string" return
      if ( $instance instance of string )
      then
        fn:true() and jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error( QName("jsv", "Invalid"), "Expected string value")

    case "anyURI" return
      if ( $instance instance of anyURI )
      then
        fn:true() and jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected anyURI value")

    case "boolean" return
      if ( $instance instance of boolean )
      then
        fn:true() and jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected boolean value")

    default return
      fn:error(QName("jsv", "NYI"),
        "$baseType: not yet implemented" | $type("$baseType") )
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: atomic type definition must contain '$baseType' key with string value.")

  (: todo: add facets checks: length, min/max Inclusive/Exclusive, etc. :)
};

declare function
jsv:validate-enumeration($jstypes as object, $type as object, $instance as item) as boolean
{
  variable $enum := $type("$enumeration");
  if( exists($enum) )
  then
    if( $enum instance of array )
    then
      if( some $v in jn:members($enum) satisfies $v eq $instance )
      then
        fn:true()
      else
        fn:error(QName("jsv", "Invalid"), "Instance value not in the enumeration list.")
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: the value of '$enumeration' must be an array.")
  else
    fn:true()
};

declare function
jsv:validate-object-type($jstypes as object, $type as object, $instance as item) as boolean
{
  fn:trace( $type("$name"), "validate-object-type");

  variable $open := $type("$open");

  if ( $instance instance of object )
  then
    if( (not exists($open)) or (exists($open) and $open eq true()) )
    then
      (: object is open: i.e. extra keys alowed but all defined keys must be in
         the instance except the default and optional ones :)
      (
        every $k in jn:keys($instance)
        satisfies
          let $ctype := $type("$content")($k)("$type")
          return
            typeswitch( $ctype )
            case string return
              jsv:validate-type-ref($jstypes, $ctype, $instance($k) )
            case object return
              jsv:validate-type($jstypes, $ctype, $instance($k) )
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind object, $content $type must be string or array.")
      ) and
      (
        every $k in
          jsv:value-except(jn:keys($type("$content")), jn:keys($instance))
        satisfies
          let $optional := $type("$content")($k)("$optional")
          return
            if( exists($optional) and $optional eq fn:true() )
            then fn:true()
            else
              fn:error(QName("jsv", "Invalid"),
                fn:concat("Required key not present in instance: ", $k) )
      )
    else
      (: object is closed: i.e. all defined keys must be in the instance except
         the default ones, no extra keys alowed :)
      (
        every $k in jn:keys($instance)
        satisfies
          let $ctype := $type("$content")($k)("$type")
          return
            typeswitch( $ctype )
            case string return
              jsv:validate-type-ref($jstypes, $ctype, $instance($k) )
            case object return
              jsv:validate-type($jstypes, $ctype, $instance($k) )
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind object, $content $type must be string or array.")
      ) and (
        every $k in (: jn:keys($type("$content")) except jn:keys($instance) :)
          jsv:value-except(jn:keys($type("$content")), jn:keys($instance))
        satisfies
          let $optional := $type("$content")($k)("$optional")
          return
            if( exists($optional) and $optional eq fn:true() )
            then fn:true()
            else
              fn:error(QName("jsv", "Invalid"),
                fn:concat("Required key not present in instance: ", $k) )
      )
  else
    fn:error( QName("jsv", "Invalid"),
      fn:concat("Instance not an object specified by type: ", jsv:get-type-name($type)) )
};

declare function
jsv:validate-array-type($jstypes as object, $type as object, $instance as item) as boolean
{
  fn:trace( $type("$name"), "validate-array-type");

  if( $instance instance of array )
  then
    (: check all the contents :)
    let $content := $type("$content")
    let $validateContent :=
        typeswitch( $content(1) )
        case string return
          every $i in jn:members($instance) satisfies jsv:validate-type-ref($jstypes, $content(1), $i)
        case object return
          every $i in jn:members($instance) satisfies jsv:validate-type($jstypes, $content(1), $i)
        default return
          fn:error(QName("jsv", "BadJSoundFormat"),
            "Not a valid JSound doc: for $kind array, $content[1] must be string or array.")
    let $validateMinLength :=
      if( fn:exists($type("$minLength")) )
      then
        if ( $type("$minLength") le size($instance) )
        then fn:true()
        else fn:error(QName("jsv", "Invalid"),
                      "Invalid array size, smaller then $minLength facet constraint.")
      else fn:true()
    let $validateMaxLength :=
      if( fn:exists($type("$maxLength")) )
      then
        if( $type("$maxLength") ge size($instance) )
        then fn:true()
        else fn:error(QName("jsv", "Invalid"),
                      "Invalid array size, bigger than $maxLength facet constraint.")
      else fn:true()
    return
      $validateContent and $validateMinLength and $validateMaxLength
  else
    fn:error(QName("jsv", "Invalid"),
      fn:concat("Instance not an array for type: ", jsv:get-type-name($type)) )

  (: check array facets :)
};

declare function
jsv:validate-union-type($jstypes as object, $type as object, $instance as item) as boolean
{
  let $content := $type("$content")
  return
    if(
      some $i in
        for $c in jn:members($content)
        return
          try
          {
            typeswitch( $c )
            case string return
              fn:true()
            case object return
                jsv:validate-type($jstypes, $c, $instance)
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind union, $content items must be strings or objects.")
          }
          catch Invalid
          {
             fn:false()
          }
      satisfies $i eq fn:true()
    )
    then fn:true()
    else
      fn:error(QName("jsv", "Invalid"),
        fn:concat("Instance not valid for union type: ", jsv:get-type-name($type)) )
};

(:---- Util functions ---------:)

declare function
jsv:get-type-name($type as object) as string
{
  let $name := $type("$name")
  return
    if ( exists($name) and $name instance of string )
    then
      $name
    else
      "N/A"
};

declare function
jsv:value-except($arg1 as xs:anyAtomicType*, $arg2 as xs:anyAtomicType*)
  as xs:anyAtomicType*
{
  (: fn:distinct-values( $arg1[ not($$ eq $arg2) ] ) :)
  fn:distinct-values(
    for $i in $arg1
    where not( some $x in $arg2 satisfies $i eq $x )
    return $i
  )
};

jsoniq version "1.0";

(:
 : Copyright 2006-2013 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

(:~
 : JSound simple validator
 : This is a JSONiq implemenation of the JSound (the schema for JSON) validator.
 :
 : @author Cezar Andrei
 :)

module namespace jsv = "http://jsound.io/modules/validate";

import module namespace jn = "http://jsoniq.org/functions";
import module namespace map = "http://jsound.io/modules/validate/map";

declare namespace an = "http://zorba.io/annotations";

declare %an:sequential function
jsv:jsd-valid($jsd as object, $name as string, $ns as string,
  $instance as item) as boolean
{
  variable $jstypes := { "foo" : 5 };

  if( $jsd instance of object )
  then
    if ( exists($jsd("$namespace")) and exists($jsd("$types")) )
    then
      if ( $jsd("$namespace") instance of string )
      then
        if ( $jsd("$namespace") eq $ns )
        then
          if ( $jsd("$types") instance of array )
          then
            jsv:check-types( $jstypes, $jsd("$types") );
          else
            fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: value of '$types' must be an array.");
        else
          fn:error(QName("jsv", "Invalid"),
            "Requested type namespace not matching schema document namespace.");
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: value of '$namespace' must be a string.");
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: must contain '$types' and '$namespace' keys.");
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "The specified JSON document is not a valid JSound schema: top item not an object.");

  (: todo: validate all type refs :)

  jsv:validate-instance($jstypes, $name, $instance)
};

(: The check-xxx functions just checks if the JSound schema definitions are valid. :)

declare %an:sequential function
jsv:check-types($jstypes as object, $types as array) as boolean
{
  every $i in
    for $type in jn:members($types)
    return
      jsv:check-type($jstypes, $type )
  satisfies $i eq fn:true()
};

declare %an:sequential function
jsv:check-type($jstypes as object, $type as item) as boolean
{
  if( $type instance of object )
  then
    if ( exists($type("$kind")) and $type("$kind") instance of string )
    then
      switch( $type("$kind") )
      case "atomic" return jsv:check-atomic-type($jstypes, $type)
      case "object" return jsv:check-object-type($jstypes, $type)
      case "array"  return jsv:check-array-type ($jstypes, $type)
      case "union"  return jsv:check-union-type ($jstypes, $type)

      default return
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: type $kind unknown:" | $type("$kind") )

    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: type definition must contain '$kind' key with string value.")
  else
    if( $type instance of string )
    then
      jsv:check-ref-type($jstypes, $type)
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: type item not an object."  )
};

declare %an:sequential function
jsv:check-atomic-type($jstypes as object, $type as object) as boolean
{
  fn:trace( $type("$name"), "check-atomic-type");

  if( exists($type("$baseType")) and $type("$baseType") instance of string )
  then
    jsv:save-type($jstypes, $type)
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: atomic type definition must contain '$baseType' key with string value.")

  (: todo: add facets checks: length, min/max Inclusive/Exclusive, etc. :)
};

declare %an:sequential function
jsv:check-object-type($jstypes as object, $type as object) as boolean
{
  (: todo: check the reast of "object" type :)
  jsv:save-type($jstypes, $type);
  fn:trace( $type("$name"), "check-object-type");

  if( exists($type("$content")) )
  then
    if( $type("$content") instance of object )
    then
      every $i in
        for $k in jn:keys($type("$content"))
        return
          if( $type("$content")( fn:trace($k, "Key")) instance of object )
          then
            if( exists( $type("$content")($k)("$type")) )
            then
              jsv:check-type($jstypes, $type("$content")($k)("$type") )
            else
              if( exists( $type("$content")($k)("$kind")) )
              then
                jsv:check-type($jstypes, $type("$content")($k) )
              else
                fn:error(QName("jsv", "BadJSoundFormat"),
                  "Not a valid JSound doc: Key  does not contain a $type definition.")
          else
            fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: Key " | $k | " in $content must have an object value.")

      satisfies $i eq fn:true()
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: $content definion of object type must be an object.")
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
             "Not a valid JSound doc: Object type does not contain $content.")
};

declare %an:sequential function
jsv:check-array-type($jstypes as object, $type as object) as boolean
{
  fn:trace( $type("$name"), "check-array-type");

  (: check all the contents :)
  (: let $t := fn:trace($type, "Array type:") :)
  let $content := $type("$content")
  return
    if( empty($content) )
    then
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: $kind array must have a $content key.")
    else
      if( $content instance of array and jn:size($content) eq 1 )
      then
      {
        jsv:save-type($jstypes, $type);

        typeswitch( $content(1) )
        case string return
          fn:true()
        case object return
            jsv:check-type($jstypes, $content(1))
        default return
          fn:error(QName("jsv", "BadJSoundFormat"),
            "Not a valid JSound doc: for $kind array, $content[1] must be string or array.")
      }
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $kind array, $content must be an array of size 1.")

  (: Todo: check array facets :)
};

declare %an:sequential function
jsv:check-union-type($jstypes as object, $type as object) as boolean
{
  fn:trace( $type("$name"), "check-union-type");

  let $content := $type("$content")
  return
    if( empty($content) )
    then
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: $kind union must have a $content key.")
    else
      if( $content instance of array )
      then
      {
        jsv:save-type($jstypes, $type);

        every $i in
          for $c in jn:members($content)
          return
            typeswitch( $c )
            case string return
              fn:true()
            case object return
                jsv:check-type($jstypes, $c)
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind union, $content items must be strings or objects.")
        satisfies $i eq fn:true()
      }
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $kind union, $content must be an array.")
};

declare function
jsv:check-ref-type($jstypes as object, $type as string) as boolean
{
  switch( $type )
  case "string" return fn:true()
  case "integer" return fn:true()
  case "boolean" return fn:true()
  case "anyURI"  return fn:true()
  default return
  {
    if( map:has-key($jstypes, $type) )
    then
      fn:true()
    else
     (: todo: save all refs, check them at the end to see if they exist
     fn:trace($type, "NYI: type ref not yet suported");
     fn:error(QName("jsv", "NYI"), "Type ref: not yet implemented.") :)
     fn:true()
  }

  (: todo: apply resolution rules :)
};

declare %an:sequential function
jsv:save-type($jstypes as object, $type as item) as boolean
{
  if( fn:exists( $type("$name") ) )
  then
    if( $type("$name") instance of string )
    then
      map:set-if-empty($jstypes, $type("$name"), $type)
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $type, $name must be a string.")
  else
    fn:false()
};

(: The validate-xxx functions actualy do the validation of the instance :)

declare function
jsv:validate-instance($jstypes as object, $name as string, $instance as item)
 as boolean
{
  let $type := map:get($jstypes, $name)
  return
    if( exists($type) )
    then
      jsv:validate-type($jstypes, $type, $instance)
    else
      fn:error(QName("jsv", "Invalid"),
             "Requested type name not present in schema: " | $name)
};

declare function
jsv:validate-type($jstypes as object, $type as item, $instance as item) as boolean
{
  switch( $type("$kind") )
  case "atomic" return jsv:validate-atomic-type($jstypes, $type, $instance)
  case "object" return jsv:validate-object-type($jstypes, $type, $instance)
  case "array"  return jsv:validate-array-type($jstypes, $type, $instance)
  case "union"  return jsv:validate-union-type($jstypes, $type, $instance)

  default return
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: type $kind unknown:" | $type("$kind") )
};

declare function
jsv:validate-type-ref($jstypes as object, $type as string, $instance as item) as boolean
{
  (: todo: check first between the defined types in $jstypes and then the build-in ones, this is per spec :)

  if( map:has-key($jstypes, $type) )
  then
    jsv:validate-type($jstypes, map:get($jstypes, $type), $instance)
  else
    switch( $type )
    case "string" return
      if ( $instance instance of string )
      then
        fn:true()
      else
        fn:error(QName("jsv", "Invalid"), "Expected string value.")
    case "integer" return
      if ( $instance instance of integer or
           $instance instance of int )
      then
        fn:true()
      else
        fn:error(QName("jsv", "Invalid"), "Expected integer value.")
    case "anyURI" return
      if ( $instance instance of anyURI )
      then
        fn:true()
      else
        fn:error(QName("jsv", "Invalid"), "Expected anyURI value.")
    case "boolean" return
      if ( $instance instance of boolean )
      then
        fn:true()
      else
        fn:error(QName("jsv", "Invalid"), "Expected boolean value.")

    default return
        fn:error(QName("jsv", "Invalid"),
          fn:concat("Type ref not defined or a built-in type: ", $type) )
};

declare function
jsv:validate-atomic-type($jstypes as object, $type as object, $instance as item) as boolean
{
  fn:trace( $type("$name"), "validate-atomic-type");
  (: todo: check first between the defined types in $jstypes and then the build-in ones, this is per spec :)

  if ( exists($type("$baseType")) and $type("$baseType") instance of string )
  then
    switch( $type("$baseType") )
    case "integer" return
      if ( $instance instance of integer or $instance instance of int )
      then
        fn:true() and jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected integer or int value")

    case "string" return
      if ( $instance instance of string )
      then
        fn:true() and jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error( QName("jsv", "Invalid"), "Expected string value")

    case "anyURI" return
      if ( $instance instance of anyURI )
      then
        fn:true() and jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected anyURI value")

    case "boolean" return
      if ( $instance instance of boolean )
      then
        fn:true() and jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected boolean value")

    default return
      fn:error(QName("jsv", "NYI"),
        "$baseType: not yet implemented" | $type("$baseType") )
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: atomic type definition must contain '$baseType' key with string value.")

  (: todo: add facets checks: length, min/max Inclusive/Exclusive, etc. :)
};

declare function
jsv:validate-enumeration($jstypes as object, $type as object, $instance as item) as boolean
{
  variable $enum := $type("$enumeration");
  if( exists($enum) )
  then
    if( $enum instance of array )
    then
      if( some $v in jn:members($enum) satisfies $v eq $instance )
      then
        fn:true()
      else
        fn:error(QName("jsv", "Invalid"), "Instance value not in the enumeration list.")
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: the value of '$enumeration' must be an array.")
  else
    fn:true()
};

declare function
jsv:validate-object-type($jstypes as object, $type as object, $instance as item) as boolean
{
  fn:trace( $type("$name"), "validate-object-type");

  variable $open := $type("$open");

  if ( $instance instance of object )
  then
    if( (not exists($open)) or (exists($open) and $open eq true()) )
    then
      (: object is open: i.e. extra keys alowed but all defined keys must be in
         the instance except the default and optional ones :)
      (
        every $k in jn:keys($instance)
        satisfies
          let $ctype := $type("$content")($k)("$type")
          return
            typeswitch( $ctype )
            case string return
              jsv:validate-type-ref($jstypes, $ctype, $instance($k) )
            case object return
              jsv:validate-type($jstypes, $ctype, $instance($k) )
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind object, $content $type must be string or array.")
      ) and
      (
        every $k in
          jsv:value-except(jn:keys($type("$content")), jn:keys($instance))
        satisfies
          let $optional := $type("$content")($k)("$optional")
          return
            if( exists($optional) and $optional eq fn:true() )
            then fn:true()
            else
              fn:error(QName("jsv", "Invalid"),
                fn:concat("Required key not present in instance: ", $k) )
      )
    else
      (: object is closed: i.e. all defined keys must be in the instance except
         the default ones, no extra keys alowed :)
      (
        every $k in jn:keys($instance)
        satisfies
          let $ctype := $type("$content")($k)("$type")
          return
            typeswitch( $ctype )
            case string return
              jsv:validate-type-ref($jstypes, $ctype, $instance($k) )
            case object return
              jsv:validate-type($jstypes, $ctype, $instance($k) )
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind object, $content $type must be string or array.")
      ) and (
        every $k in (: jn:keys($type("$content")) except jn:keys($instance) :)
          jsv:value-except(jn:keys($type("$content")), jn:keys($instance))
        satisfies
          let $optional := $type("$content")($k)("$optional")
          return
            if( exists($optional) and $optional eq fn:true() )
            then fn:true()
            else
              fn:error(QName("jsv", "Invalid"),
                fn:concat("Required key not present in instance: ", $k) )
      )
  else
    fn:error( QName("jsv", "Invalid"),
      fn:concat("Instance not an object specified by type: ", jsv:get-type-name($type)) )
};

declare function
jsv:validate-array-type($jstypes as object, $type as object, $instance as item) as boolean
{
  fn:trace( $type("$name"), "validate-array-type");

  if( $instance instance of array )
  then
    (: check all the contents :)
    (: let $t := fn:trace($type, "Array type:") :)
    let $content := $type("$content")
    return
        typeswitch( $content(1) )
        case string return
          every $i in jn:members($instance) satisfies jsv:validate-type-ref($jstypes, $content(1), $i)
        case object return
          every $i in jn:members($instance) satisfies jsv:validate-type($jstypes, $content(1), $i)
        default return
          fn:error(QName("jsv", "BadJSoundFormat"),
            "Not a valid JSound doc: for $kind array, $content[1] must be string or array.")
  else
    fn:error(QName("jsv", "Invalid"),
      fn:concat("Instance not an array for type: ", jsv:get-type-name($type)) )

  (: check array facets :)
};

declare function
jsv:validate-union-type($jstypes as object, $type as object, $instance as item) as boolean
{
  let $content := $type("$content")
  return
    if(
      some $i in
        for $c in jn:members($content)
        return
          try
          {
            typeswitch( $c )
            case string return
              fn:true()
            case object return
                jsv:validate-type($jstypes, $c, $instance)
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind union, $content items must be strings or objects.")
          }
          catch Invalid
          {
             fn:false()
          }
      satisfies $i eq fn:true()
    )
    then fn:true()
    else
      fn:error(QName("jsv", "Invalid"),
        fn:concat("Instance not valid for union type: ", jsv:get-type-name($type)) )
};

(:---- Util functions ---------:)

declare function
jsv:get-type-name($type as object) as string
{
  let $name := $type("$name")
  return
    if ( exists($name) and $name instance of string )
    then
      $name
    else
      "N/A"
};

declare function
jsv:value-except($arg1 as xs:anyAtomicType*, $arg2 as xs:anyAtomicType*)
  as xs:anyAtomicType*
{
  (: fn:distinct-values( $arg1[ not($$ eq $arg2) ] ) :)
  fn:distinct-values(
    for $i in $arg1
    where not( some $x in $arg2 satisfies $i eq $x )
    return $i
  )
};


Update rvkadjust.jq to fix issue with notes

# Adjust JSKOS export of RVK data emitted by mc2skos

.notation[] |= gsub("-"; " - ") |
.uri |= gsub("_"; "%20") |
.uri |= gsub("(?<a>[^\/]*)-(?<b>[^\/]*)$"; (.a)+"%20-%20"+(.b)) |
if has("inScheme") then .inScheme[].uri = "http://uri.gbv.de/terminology/rvk/" else . end |
if has("topConceptOf") then .topConceptOf[].uri = "http://uri.gbv.de/terminology/rvk/" else . end |
if has("broader") then .broader[].uri |= gsub("_"; "%20") else . end |
if has("broader") then .broader[].uri |= gsub("(?<a>[^\/]*)-(?<b>[^\/]*)$"; (.a)+"%20-%20"+(.b)) else . end |
if .broader[0].uri == .uri then del(.broader[0]) else . end |
if .broader[1].uri == .uri then del(.broader[1]) else . end |
if .prefLabel.de|type == "array" then .prefLabel.de = (.prefLabel.de|join(" / ")) else  . end |
if .scopeNote.de|type == "string" then .scopeNote.de = [.scopeNote.de] else . end |
if .editorialNote.de|type == "string" then .editorialNote.de = [.editorialNote.de] else . end


Always asking for token in facttable-for-component.jq.

jsoniq version "1.0";

import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace filings = "http://28.io/modules/xbrl/profiles/sec/filings";
import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace networks = "http://28.io/modules/xbrl/networks";
import module namespace concept-maps = "http://28.io/modules/xbrl/concept-maps";
import module namespace rules = "http://28.io/modules/xbrl/rules";

import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";

import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace api = "http://apps.28.io/api";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token              as string? external;
declare  %rest:env                              variable $request-uri        as string  external;
declare  %rest:case-insensitive                 variable $format             as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker             as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear         as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod       as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $networkIdentifier  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $concept            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $reportElement      as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $label              as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $rollup             as string* external;
declare  %rest:case-insensitive                 variable $map                as string? external;
declare  %rest:case-insensitive                 variable $validate           as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules   as string? external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $reportElement := ($reportElement, $concept)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic)
let $archive as object? := fiscal-core:filings(
    $entities,
    $fiscalPeriod,
    $fiscalYear,
    $aid)
let $entity    := entities:entities($archive.Entity)
let $components  := sec-networks:components(
    $archive,
    $cid,
    $reportElement,
    $disclosure,
    $networkIdentifier,
    $label)
let $component as object? := $components[1] (: only one for know :)
let $cid as string? := components:cid($component)
let $rules as object* := if(exists($additional-rules)) then rules:rules($additional-rules) else ()

(: Fact resolution :)
let $facts :=
    if (exists($rollup))
         then
             let $calc-network := networks:networks-for-components-and-short-names($component, $networks:CALCULATION_NETWORK)
             let $hc := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube")
             let $hc := hypercubes:modify-hypercube($hc, {
                 "sec:FiscalYear" : { Type: "integer", Default: null },
                 "sec:FiscalPeriod" : { Type: "string", Default: null }
             })
             let $options as object? := if(exists($rules)) then { Rules: [ $rules ] } else ()
             let $p := hypercubes:populate-networks-with-facts($calc-network, $hc, $archive, $options)
             let $map := concept-maps:concept-maps($map)
             let $concepts :=
                if (not $map instance of null)
                then
                    for $d in $rollup[]
                    return
                        keys(descendant-objects($p)[$$.Name = keys($map.Trees($d).To)][1].To)
                else
                    for $d in $rollup[]
                    return ($d, keys(descendant-objects($p)[$$.Name eq $d].To))
             return sec:facts-for-archives-and-concepts($archive, $concepts, { Hypercube: $hc })
         else components:facts(
            $component,
            {|
                {
                    Validate: $validate,
                    FilterOverride : {
                        "sec:FiscalPeriod" : { Type: "string", Default: null },
                        "sec:FiscalYear" : { Type: "string", Default: null }
                    }
                },
                if(exists($rules))
                then { Rules : [ $rules ] }
                else ()
            |}
        )

let $facts := api:normalize-facts($facts)

let $results :=
    switch ($format)
    case "xml" return {
        response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
        (session:comment("xml", {
                    NumFacts : count($facts),
                    TotalNumFacts: session:num-facts(),
                    TotalNumArchives: session:num-archives(),
                    TotalNumEntities: session:num-entities()
                }),
        <FactTable entityRegistrantName="{$entity.Profiles.SEC.CompanyName}"
            cik="{$entity._id}"
            tableName="{sec-networks:tables($component, {IncludeImpliedTable: true}).Name}"
            label="{$component.Label}"
            accessionNumber="{$component.Archive}"
            networkIdentifier="{$component.Role}"
            formType="{$archive.Profiles.SEC.FormType}"
            fiscalPeriod="{$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus}"
            fiscalYear="{$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus}"
            acceptanceDatetime="{filings:acceptance-dateTimes($archive)}"
            disclosure="{$component.Profiles.SEC.Disclosure}"
            >{
            conversion:facts-to-xml($facts, { Caller: "Component" })
        }</FactTable>)
    }
    case "text" case "csv" return {
        response:content-type("text/csv");
        response:header("Content-Disposition", "attachment; filename=facttable-" || $cid || ".csv");
        conversion:facts-to-csv($facts, { Caller: "Component"})
    }
    case "excel" return {
        response:content-type("application/vnd.ms-excel");
        response:header("Content-Disposition", "attachment; filename=facttable-" || $cid || ".csv");
        conversion:facts-to-csv($facts, { Caller: "Component"})
    }
    default return {
        response:content-type("application/json");
        response:serialization-parameters({"indent" : true});
        {|
            {
                CIK : $entity._id,
                EntityRegistrantName : $entity.Profiles.SEC.CompanyName,
                TableName : sec-networks:tables($component, {IncludeImpliedTable: true}).Name,
                Label : $component.Label,
                AccessionNumber : $component.Archive,
                FormType : $archive.Profiles.SEC.FormType,
                FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
                FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                AcceptanceDatetime : filings:acceptance-dateTimes($archive),
                NetworkIdentifier: $component.Role,
                Disclosure : $component.Profiles.SEC.Disclosure,
                FactTable : [ $facts ]
            },
            session:comment(
                "json",
                {
                    NumFacts : count($facts),
                    TotalNumFacts: session:num-facts(),
                    TotalNumArchives: session:num-archives(),
                    TotalNumEntities: session:num-entities()
                }
            )
        |}
    }
return api:check-and-return-results($token, $results, $format)

jsoniq version "1.0";

import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace filings = "http://28.io/modules/xbrl/profiles/sec/filings";
import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace networks = "http://28.io/modules/xbrl/networks";
import module namespace concept-maps = "http://28.io/modules/xbrl/concept-maps";
import module namespace rules = "http://28.io/modules/xbrl/rules";

import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace sec-networks = "http://28.io/modules/xbrl/profiles/sec/networks";

import module namespace response = "http://www.28msec.com/modules/http-response";

import module namespace session = "http://apps.28.io/session";
import module namespace api = "http://apps.28.io/api";

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token              as string? external;
declare  %rest:env                              variable $request-uri        as string  external;
declare  %rest:case-insensitive                 variable $format             as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker             as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear         as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod       as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $networkIdentifier  as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $cid                as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $concept            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $reportElement      as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $disclosure         as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $label              as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $rollup             as string* external;
declare  %rest:case-insensitive                 variable $map                as string? external;
declare  %rest:case-insensitive                 variable $validate           as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules   as string? external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)
let $reportElement := ($reportElement, $concept)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic)
let $archive as object? := fiscal-core:filings(
    $entities,
    $fiscalPeriod,
    $fiscalYear,
    $aid)
let $entity    := entities:entities($archive.Entity)
let $components  := sec-networks:components(
    $archive,
    $cid,
    $reportElement,
    $disclosure,
    $networkIdentifier,
    $label)
let $component as object? := $components[1] (: only one for know :)
let $cid as string? := components:cid($component)
let $rules as object* := if(exists($additional-rules)) then rules:rules($additional-rules) else ()

(: Fact resolution :)
let $facts :=
    if (exists($rollup))
         then
             let $calc-network := networks:networks-for-components-and-short-names($component, $networks:CALCULATION_NETWORK)
             let $hc := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube")
             let $hc := hypercubes:modify-hypercube($hc, {
                 "sec:FiscalYear" : { Type: "integer", Default: null },
                 "sec:FiscalPeriod" : { Type: "string", Default: null }
             })
             let $options as object? := if(exists($rules)) then { Rules: [ $rules ] } else ()
             let $p := hypercubes:populate-networks-with-facts($calc-network, $hc, $archive, $options)
             let $map := concept-maps:concept-maps($map)
             let $concepts :=
                if (not $map instance of null)
                then
                    for $d in $rollup[]
                    return
                        keys(descendant-objects($p)[$$.Name = keys($map.Trees($d).To)][1].To)
                else
                    for $d in $rollup[]
                    return ($d, keys(descendant-objects($p)[$$.Name eq $d].To))
             return sec:facts-for-archives-and-concepts($archive, $concepts, { Hypercube: $hc })
         else components:facts(
            $component,
            {|
                {
                    Validate: $validate,
                    FilterOverride : {
                        "sec:FiscalPeriod" : { Type: "string", Default: null },
                        "sec:FiscalYear" : { Type: "string", Default: null }
                    }
                },
                if(exists($rules))
                then { Rules : [ $rules ] }
                else ()
            |}
        )

let $facts := api:normalize-facts($facts)

let $results :=
    switch ($format)
    case "xml" return {
        response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
        (session:comment("xml", {
                    NumFacts : count($facts),
                    TotalNumFacts: session:num-facts(),
                    TotalNumArchives: session:num-archives(),
                    TotalNumEntities: session:num-entities()
                }),
        <FactTable entityRegistrantName="{$entity.Profiles.SEC.CompanyName}"
            cik="{$entity._id}"
            tableName="{sec-networks:tables($component, {IncludeImpliedTable: true}).Name}"
            label="{$component.Label}"
            accessionNumber="{$component.Archive}"
            networkIdentifier="{$component.Role}"
            formType="{$archive.Profiles.SEC.FormType}"
            fiscalPeriod="{$archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus}"
            fiscalYear="{$archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus}"
            acceptanceDatetime="{filings:acceptance-dateTimes($archive)}"
            disclosure="{$component.Profiles.SEC.Disclosure}"
            >{
            conversion:facts-to-xml($facts, { Caller: "Component" })
        }</FactTable>)
    }
    case "text" case "csv" return {
        response:content-type("text/csv");
        response:header("Content-Disposition", "attachment; filename=facttable-" || $cid || ".csv");
        conversion:facts-to-csv($facts, { Caller: "Component"})
    }
    case "excel" return {
        response:content-type("application/vnd.ms-excel");
        response:header("Content-Disposition", "attachment; filename=facttable-" || $cid || ".csv");
        conversion:facts-to-csv($facts, { Caller: "Component"})
    }
    default return {
        response:content-type("application/json");
        response:serialization-parameters({"indent" : true});
        {|
            {
                CIK : $entity._id,
                EntityRegistrantName : $entity.Profiles.SEC.CompanyName,
                TableName : sec-networks:tables($component, {IncludeImpliedTable: true}).Name,
                Label : $component.Label,
                AccessionNumber : $component.Archive,
                FormType : $archive.Profiles.SEC.FormType,
                FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus,
                FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                AcceptanceDatetime : filings:acceptance-dateTimes($archive),
                NetworkIdentifier: $component.Role,
                Disclosure : $component.Profiles.SEC.Disclosure,
                FactTable : [ $facts ]
            },
            session:comment(
                "json",
                {
                    NumFacts : count($facts),
                    TotalNumFacts: session:num-facts(),
                    TotalNumArchives: session:num-archives(),
                    TotalNumEntities: session:num-entities()
                }
            )
        |}
    }
return api:check-and-return-results($token, $entity, $results, $format)


made migration profile dependent.

made migration profile dependent.

import module namespace config = "http://apps.28.io/config";

if($config:profile-name eq "sec")
then
    let $fiscalPeriodType as string := "sec:FiscalPeriodType"

    for $report in db:collection("reports")
    let $hypercubeAspects as object := $report.Hypercubes."xbrl:DefaultHypercube".Aspects
    let $filters as object? := $report.Filters
    let $record := find("reports", { "_id" : $report."_id" })
    return
        {
            if(not(empty($hypercubeAspects.$fiscalPeriodType)))
            then ();
            else insert json {
                $fiscalPeriodType: {
                  "Name": $fiscalPeriodType,
                  "Label": "Fiscal Period Type",
                  "Kind": "TypedDimension",
                  "Type": "string",
                  "DomainRestriction": {
                    "Name": "sec:FiscalPeriodTypeDomain",
                    "Label": "Fiscal Period Type Domain",
                    "Enumeration": [ "instant", "YTD" ]
                  }
                }
            } into $hypercubeAspects;

            if(empty($filters) or not(empty($filters.fiscalPeriodType)))
            then ();
            else insert json { "fiscalPeriodType" : [ "instant", "YTD" ] } into $filters;

            if(not(empty($hypercubeAspects."sec:Archive")) and empty($hypercubeAspects."xbrl28:Archive"))
            then
                let $newArchive :=
                  copy $new := $hypercubeAspects."sec:Archive"
                  modify (replace value of json $new.Name with "xbrl28:Archive")
                  return $new
                return
                    insert json {
                        "xbrl28:Archive": $newArchive
                    } into $hypercubeAspects;
            else ();

            if(not(empty($hypercubeAspects."sec:Archive")))
            then
                delete json $hypercubeAspects."sec:Archive";
            else ();

            db:edit($record, $report);
        }
else ()


Always asking for token in facts.jq.

Always asking for token in facts.jq.

import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace rules = "http://28.io/modules/xbrl/rules";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";

declare function local:contains-aspect($name as string) as boolean
{
  exists(request:param-names()[starts-with($$, $name)])
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube(
    $entities as object*,
    $fiscalPeriods as string*,
    $fiscalYears as integer*,
    $aids as string*) as object
{
    let $shortcut-hypercube-spec :=
    {|
        let $concepts := request:param-values("concept")
        return { "xbrl:Concept" : { Domain : [ $concepts ] } }[exists($concepts)],
        fiscal-core:filter-override(
            $entities,
            $fiscalYears,
            $fiscalPeriods,
            $aids)
    |}
    let $main-hypercube-spec :=
    {|
        {
            "dei:LegalEntityAxis" : {
                "Domain" : [ "sec:DefaultLegalEntity" ],
                "Default" : "sec:DefaultLegalEntity"
            }
        }[not local:contains-aspect("dei:LegalEntityAxis")],

        for $parameter in (
            request:param-names(),
            keys($shortcut-hypercube-spec),
            "sec:Accepted",
            "sec:FiscalPeriod",
            "sec:FiscalYear")
        where contains($parameter, ":")
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (request:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $user-defined-type as string? :=
            (request:param-values($dimension-name || "::type"), request:param-values($dimension-name || ":type"))[1]
        let $type as string? :=
            switch(true)
            case exists($user-defined-type)
                return $user-defined-type
            case exists($shortcut-hypercube-spec.$dimension-name.Type)
                return $shortcut-hypercube-spec.$dimension-name.Type
            default
                return ()
        let $values := (request:param-values($dimension-name), $shortcut-hypercube-spec.$dimension-name.Domain[])
        let $typed-values := if (exists($type)) then local:cast-sequence($values, $type) else $values
        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (request:param-values($dimension-name || "::default"), request:param-values($dimension-name || ":default"))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value
        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                { "Domain" : [ $typed-values ] }[(exists($typed-values) and not($all))],
                { "Default" : $typed-default-value }[$has-default]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($main-hypercube-spec)
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token             as string? external;
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $validate          as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic)
let $archives as object* := fiscal-core:filings(
    $entities,
    $fiscalPeriod,
    $fiscalYear,
    $aid)
let $entities := entities:entities($archives.Entity)
let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $hypercube := local:hypercube($entities, $fiscalPeriod, $fiscalYear, $aid)

let $facts :=
    if(empty($archives))
    then ()
    else
        for $fact in sec:facts-for(
            {|
                {
                    Hypercube : $hypercube,
                    Validate: $validate
                },
                { "ConceptMaps" : $map }[exists($map)],
                { "Rules" : [ $rule ] }[exists($rule)]
            |}
        )
        return {|
            $fact,
            { "EntityRegistrantName" : $entities[$$._id eq $fact.Aspects."xbrl:Entity"].Profiles.SEC.CompanyName}
        |}

let $facts := api:normalize-facts($facts)

let $result := {
    NetworkIdentifier : "http://bizql.io/facts",
    TableName : "xbrl:Facts",
    FactTable : [ $facts ]
}
let $comment :=
{
    NumFacts: count($facts),
    TotalNumFacts: session:num-facts(),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}

return
    if(empty($archives) and (not(empty($aid)) or not(empty($cik))))
    then
    {
        response:status-code(404);
        session:error("entities or archives not found (valid parameters: cik, ticker, tag, sic, aid)", $format)
    }
    else let $results := api:serialize($result, $comment, $serializers, $format, "facts")
         return api:check-and-return-results($token, $results, $format)

import module namespace api = "http://apps.28.io/api";
import module namespace session = "http://apps.28.io/session";

import module namespace conversion = "http://28.io/modules/xbrl/conversion";
import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace rules = "http://28.io/modules/xbrl/rules";

import module namespace sec = "http://28.io/modules/xbrl/profiles/sec/core";
import module namespace companies = "http://28.io/modules/xbrl/profiles/sec/companies";
import module namespace fiscal-core = "http://28.io/modules/xbrl/profiles/sec/fiscal/core";

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace request = "http://www.28msec.com/modules/http-request";

declare function local:contains-aspect($name as string) as boolean
{
  exists(request:param-names()[starts-with($$, $name)])
};

declare function local:cast-sequence($values as atomic*, $type as string) as atomic*
{
  for $value in $values
  return
      switch ($type)
      case "integer" return $value cast as integer
      case "string" return $value cast as string
      default return error(xs:QName("local:unsupported-type"), $type || ": unsupported type")
};

declare function local:hypercube(
    $entities as object*,
    $fiscalPeriods as string*,
    $fiscalYears as integer*,
    $aids as string*) as object
{
    let $shortcut-hypercube-spec :=
    {|
        let $concepts := request:param-values("concept")
        return { "xbrl:Concept" : { Domain : [ $concepts ] } }[exists($concepts)],
        fiscal-core:filter-override(
            $entities,
            $fiscalYears,
            $fiscalPeriods,
            $aids)
    |}
    let $main-hypercube-spec :=
    {|
        {
            "dei:LegalEntityAxis" : {
                "Domain" : [ "sec:DefaultLegalEntity" ],
                "Default" : "sec:DefaultLegalEntity"
            }
        }[not local:contains-aspect("dei:LegalEntityAxis")],

        for $parameter in (
            request:param-names(),
            keys($shortcut-hypercube-spec),
            "sec:Accepted",
            "sec:FiscalPeriod",
            "sec:FiscalYear")
        where contains($parameter, ":")
        group by $dimension-name :=
            switch(true)
            case ends-with(lower-case($parameter), "::default")
                return substring-before($parameter, "::default")
            case ends-with(lower-case($parameter), ":default")
                return substring-before($parameter, ":default")
            case ends-with(lower-case($parameter), "::type")
                return substring-before($parameter, "::type")
            default
                return $parameter
        let $all as boolean :=
            (request:param-values($dimension-name) ! upper-case($$)) = "ALL"
        let $user-defined-type as string? :=
            (request:param-values($dimension-name || "::type"), request:param-values($dimension-name || ":type"))[1]
        let $type as string? :=
            switch(true)
            case exists($user-defined-type)
                return $user-defined-type
            case exists($shortcut-hypercube-spec.$dimension-name.Type)
                return $shortcut-hypercube-spec.$dimension-name.Type
            default
                return ()
        let $values := (request:param-values($dimension-name), $shortcut-hypercube-spec.$dimension-name.Domain[])
        let $typed-values := if (exists($type)) then local:cast-sequence($values, $type) else $values
        let $has-default := ($parameter = $dimension-name || "::default") or ($parameter = $dimension-name || ":default")
        let $default-value := (request:param-values($dimension-name || "::default"), request:param-values($dimension-name || ":default"))[1]
        let $typed-default-value := if (exists($type)) then local:cast-sequence($default-value, $type) else $default-value
        return
        {
            $dimension-name : {|
                { "Type" : $type }[exists($type)],
                { "Domain" : [ $typed-values ] }[(exists($typed-values) and not($all))],
                { "Default" : $typed-default-value }[$has-default]
            |}
        }
    |}
    return hypercubes:user-defined-hypercube($main-hypercube-spec)
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token             as string? external;
declare  %rest:env                              variable $request-uri       as string  external;
declare  %rest:case-insensitive                 variable $format            as string? external;
declare  %rest:case-insensitive %rest:distinct  variable $cik               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $tag               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker            as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $sic               as string* external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear        as string* external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod      as string* external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid               as string* external;
declare  %rest:case-insensitive                 variable $map               as string? external;
declare  %rest:case-insensitive                 variable $rule              as string? external;
declare  %rest:case-insensitive                 variable $report            as string? external;
declare  %rest:case-insensitive                 variable $validate          as boolean external := false;
declare  %rest:case-insensitive                 variable $additional-rules  as string? external;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities as object* :=
    companies:companies(
        $cik,
        $tag,
        $ticker,
        $sic)
let $archives as object* := fiscal-core:filings(
    $entities,
    $fiscalPeriod,
    $fiscalYear,
    $aid)
let $entities := entities:entities($archives.Entity)
let $report as object? := reports:reports($report)
let $map as item* :=
    if(exists($report))
    then reports:concept-map($report)
    else $map
let $rule as item* :=
    (
        if(exists($report))
        then reports:rules($report)
        else rules:rules($rule),
        if(exists($additional-rules))
        then rules:rules($additional-rules)
        else ()
    )

let $hypercube := local:hypercube($entities, $fiscalPeriod, $fiscalYear, $aid)

let $facts :=
    if(empty($archives))
    then ()
    else
        for $fact in sec:facts-for(
            {|
                {
                    Hypercube : $hypercube,
                    Validate: $validate
                },
                { "ConceptMaps" : $map }[exists($map)],
                { "Rules" : [ $rule ] }[exists($rule)]
            |}
        )
        return {|
            $fact,
            { "EntityRegistrantName" : $entities[$$._id eq $fact.Aspects."xbrl:Entity"].Profiles.SEC.CompanyName}
        |}

let $facts := api:normalize-facts($facts)

let $result := {
    NetworkIdentifier : "http://bizql.io/facts",
    TableName : "xbrl:Facts",
    FactTable : [ $facts ]
}
let $comment :=
{
    NumFacts: count($facts),
    TotalNumFacts: session:num-facts(),
    TotalNumArchives: session:num-archives(),
    TotalNumEntities: session:num-entities()
}
let $serializers := {
    to-xml : function($res as object) as node()* {
           <FactTable NetworkIdentifier="http://bizql.io/facts"
            TableName="xbrl:Facts">{
                conversion:facts-to-xml($res.FactTable[], { Caller: "Report"})
        }</FactTable>
    },
    to-csv : function($res as object) as string {
        (conversion:facts-to-csv($res.FactTable[], { Caller: "Report" }), "")[1]
    }
}

return
    if(empty($archives) and (not(empty($aid)) or not(empty($cik))))
    then
    {
        response:status-code(404);
        session:error("entities or archives not found (valid parameters: cik, ticker, tag, sic, aid)", $format)
    }
    else let $results := api:serialize($result, $comment, $serializers, $format, "facts")
         return api:check-and-return-results($token, $entities, $results, $format)


Update report-elements.jq

Update report-elements.jq

import module namespace util = "http://secxbrl.info/modules/util";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";
import module namespace components = "http://xbrl.io/modules/bizql/components";
import module namespace components2 = "http://xbrl.io/modules/bizql/components2";
import module namespace concept-maps = "http://xbrl.io/modules/bizql/concept-maps";

import module namespace companies2 = "http://xbrl.io/modules/bizql/profiles/sec/companies2";
import module namespace fiscal-core = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace fiscal-core2 = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core2";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace seq = "http://zorba.io/modules/sequence";

declare namespace concepts = "http://www.28msec.com/modules/bizql/concepts";

declare function local:to-csv($concepts as item*, $onlyNames as boolean) as string
{
    if ($onlyNames)
    then
        string-join(("Name", $concepts), "
")
    else
        string-join(csv:serialize($concepts, { serialize-null-as : "" }))
};

declare function local:to-xml($concepts as item*, $onlyNames as boolean) as element()*
{
    for $c in $concepts
    return
        <ReportElement>{
             if ($onlyNames)
             then
                <Name>{$c}</Name>
             else (
                <Name>{$c.Name}</Name>,
                <Label>{$c.Label}</Label>,
                <IsNillable>{$c.IsNillable}</IsNillable>,
                <IsAbstract>{$c.IsAbstract}</IsAbstract>,
                <PeriodType>{$c.PeriodType}</PeriodType>,
                <Balance>{$c.Balance}</Balance>,
                <SubstitutionGroup>{$c.SubstitutionGroup}</SubstitutionGroup>,
                <DataType>{$c.DataType}</DataType>,
                <BaseType>{$c.BaseType}</BaseType>,
                <ClosestSchemaBuiltinType>{$c.ClosestSchemaBuiltinType}</ClosestSchemaBuiltinType>,
                <IsTextBlock>{$c.IsTextBlock}</IsTextBlock>,
                <ComponentRole>{$c.ComponentRole}</ComponentRole>,
                <ComponentLabel>{$c.ComponentLabel}</ComponentLabel>,
                <AccessionNumber>{$c.AccessionNumber}</AccessionNumber>,
                <CIK>{$c.CIK}</CIK>,
                <EntityRegistrantName>{$c.EntityRegistrantName}</EntityRegistrantName>,
                <FiscalYear>{$c.FiscalYear}</FiscalYear>,
                <FiscalPeriod>{$c.FiscalPeriod}</FiscalPeriod>
             )
        }</ReportElement>
};

declare function local:concepts-for-archives($aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("concepts:CONNECTION-FAILED"), $err:description)
        }
    return
        mongo:find($conn, "concepts",
        {
            "Archive": { "$in" : [ $aids ] }
        })
};

declare function local:concepts-for-archives($aids as string*, $names as string*, $map as string?) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("concepts:CONNECTION-FAILED"), $err:description)
        }
    let $map := if (exists($map)) then concept-maps:concept-maps($map) else ()

    let $all-concepts-computable-by-maps as string* := keys($map.Trees)

    let $concepts-computable-by-maps as object* :=
        for $c in $names[$$ = $all-concepts-computable-by-maps]
        return $map.Trees.$c

    let $mapped-names := (keys($concepts-computable-by-maps.To ), $concepts-computable-by-maps.To [].Name)

    let $concepts-not-computable-by-maps as string* := seq:value-except($names, $mapped-names)
    let $results-not-computed-by-maps := mongo:find($conn, "concepts",
        {
            "Name" : { "$in" : [ $concepts-not-computable-by-maps ] },
            "Archive": { "$in" : [ $aids ] }
        })
    let $results-computed-by-maps :=
        for $c in mongo:find($conn, "concepts",
            {
                "Name" : { "$in" : [ $mapped-names ] },
                "Archive": { "$in" : [ $aids ] }
            })
        group by $c.Component
        let $c := $c[1]
        let $map-concept := (for $candidate in $concepts-computable-by-maps
                            where $c.Name = (keys($candidate.To), $candidate.To[].Name)
                            return $candidate)[1]
        return
            copy $n := $concepts
            modify (
                replace value of json $n.Name with $candidate-concept.Name,
                insert json  { Origin : $concepts.Name } into $n)
            return $n
    return ($results-not-computed-by-maps, $results-computed-by-maps)
};

declare function local:concepts-for-archives-and-labels($aids as string*, $labels as string) as object*
{
    let $conn :=
        let $credentials := credentials:credentials("MongoDB", "xbrl")
        return
            try {
                mongo:connect($credentials)
            } catch mongo:* {
                error(QName("components:CONNECTION-FAILED"), $err:description)
            }
    return mongo:run-cmd-deterministic(
        $conn,
        {
            "text" : "concepts",
            "filter" : { "Archive" : { "$in" : [ $aids ] } },
            "search" : $labels,
            "limit" : 100,
            "score" : { "$meta" : "textScore" },
            "sort" : { score: { "$meta" : "textScore" } }
        }).results[].obj
};

session:audit-call();

(: Query parameters :)
let $format as string?         := request:param-values("format")
let $ciks as string*           := distinct-values(request:param-values("cik"))
let $tags as string*           := distinct-values(request:param-values("tag"))
let $tickers as string*        := distinct-values(request:param-values("ticker"))
let $sics as string*           := distinct-values(request:param-values("sic"))
let $fiscalYears as string*    := distinct-values(request:param-values("fiscalYear", "LATEST"))
let $fiscalPeriods as string*  := distinct-values(request:param-values("fiscalPeriod", "FY"))
let $aids as string*           := distinct-values(request:param-values("aid"))
let $labels as string*         := request:param-values("label")
let $map as string?            := request:param-values("map")
let $names as string*          := request:param-values("name")

(: Post-processing :)
let $format as string? := (: backwards compatibility, to be deprecated  :)
    lower-case(($format, substring-after(request:path(), ".jq."))[1])
let $tags as string* := (: backwards compatibility, to be deprecated :)
    distinct-values($tags ! upper-case($$))
let $fiscalYears as integer* :=
    for $fy in $fiscalYears ! upper-case($$)
    return switch($fy)
           case "LATEST" return $fiscal-core2:LATEST_FISCAL_YEAR
           case "ALL" return $fiscal-core:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
let $fiscalPeriods as string* :=
    for $fp in $fiscalPeriods ! upper-case($$)
    return switch($fp)
           case "ALL" return $fiscal-core:ALL_FISCAL_PERIODS
           default return $fp

(: Object resolution :)
let $entities :=
    companies2:companies(
        $ciks,
        $tags,
        $tickers,
        $sics)
let $archives as object* := fiscal-core2:filings(
    $entities,
    $fiscalPeriods,
    $fiscalYears,
    $aids)
let $entities    := entities:entities($archives.Entity)
let $onlyNames   := let $o := request:param-values("onlyNames")[1] return if (exists($o)) then ($o cast as boolean) else false

let $concepts := if (exists($names))
                  then local:concepts-for-archives($archives._id, $names, $map)
                  else if (exists($labels))
                  then local:concepts-for-archives-and-labels($archives._id, $labels[1])
                  else local:concepts-for-archives($archives._id)
let $result := {
    ReportElements : [
        if ($onlyNames)
        then distinct-values($concepts.Name)
        else
            let $all-aids := $concepts.Archive
            let $roles := $concepts.Role
            let $components := components2:components-for-archives-and-roles($all-aids, $roles)
            return
            for $concept in $concepts
            group by $archive := $concept.Archive,  $role := $concept.Role
            let $component := $components[$$.Archive eq $archive and $$.Role eq $role]
            let $default-hc := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube")
            let $members := $default-hc.Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain".Members
            let $archive := $archives[$$._id eq $archive]
            let $entity := $entities[$$._id eq $archive.Entity]
            let $metadata := {|
                { Origin : $concept.Origin }[exists($concept.Origin)],
                {
                    Name: $concept.Name,
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label,
                    AccessionNumber : $archive._id,
                    CIK : $entity._id,
                    EntityRegistrantName : $entity.Profiles.SEC.CompanyName,
                    FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                    FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
                }
            |}
            for $concept in $concept
            let $original-name := ($concept.Origin, $concept.Name)[1]
            return {|
                trim($members.$original-name, "Name"),
                $metadata
            |}
    ]
}
let $comment := {
    NumConcepts: count($concepts),
    TotalNumConcepts: session:num-concepts()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <ReportElements>{
                    local:to-xml($res.ReportElements[], $onlyNames)
                }</ReportElements>
    },
    to-csv : function($res as object) as string {
        local:to-csv($res.ReportElements[], $onlyNames)
    }
}
let $results := util:serialize($result, $comment, $serializers, $format, "report-elements")
return
    util:check-and-return-results($entities, $results, $format)

import module namespace util = "http://secxbrl.info/modules/util";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace session = "http://apps.28.io/session";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace entities = "http://xbrl.io/modules/bizql/entities";
import module namespace hypercubes = "http://xbrl.io/modules/bizql/hypercubes";
import module namespace components = "http://xbrl.io/modules/bizql/components";
import module namespace components2 = "http://xbrl.io/modules/bizql/components2";
import module namespace concept-maps = "http://xbrl.io/modules/bizql/concept-maps";

import module namespace companies2 = "http://xbrl.io/modules/bizql/profiles/sec/companies2";
import module namespace fiscal-core = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";
import module namespace fiscal-core2 = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core2";

import module namespace mongo = "http://www.28msec.com/modules/mongodb";
import module namespace credentials = "http://www.28msec.com/modules/credentials";

import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace seq = "http://zorba.io/modules/sequence";

declare namespace concepts = "http://www.28msec.com/modules/bizql/concepts";

declare function local:to-csv($concepts as item*, $onlyNames as boolean) as string
{
    if ($onlyNames)
    then
        string-join(("Name", $concepts), "
")
    else
        string-join(csv:serialize($concepts, { serialize-null-as : "" }))
};

declare function local:to-xml($concepts as item*, $onlyNames as boolean) as element()*
{
    for $c in $concepts
    return
        <ReportElement>{
             if ($onlyNames)
             then
                <Name>{$c}</Name>
             else (
                <Name>{$c.Name}</Name>,
                <Label>{$c.Label}</Label>,
                <IsNillable>{$c.IsNillable}</IsNillable>,
                <IsAbstract>{$c.IsAbstract}</IsAbstract>,
                <PeriodType>{$c.PeriodType}</PeriodType>,
                <Balance>{$c.Balance}</Balance>,
                <SubstitutionGroup>{$c.SubstitutionGroup}</SubstitutionGroup>,
                <DataType>{$c.DataType}</DataType>,
                <BaseType>{$c.BaseType}</BaseType>,
                <ClosestSchemaBuiltinType>{$c.ClosestSchemaBuiltinType}</ClosestSchemaBuiltinType>,
                <IsTextBlock>{$c.IsTextBlock}</IsTextBlock>,
                <ComponentRole>{$c.ComponentRole}</ComponentRole>,
                <ComponentLabel>{$c.ComponentLabel}</ComponentLabel>,
                <AccessionNumber>{$c.AccessionNumber}</AccessionNumber>,
                <CIK>{$c.CIK}</CIK>,
                <EntityRegistrantName>{$c.EntityRegistrantName}</EntityRegistrantName>,
                <FiscalYear>{$c.FiscalYear}</FiscalYear>,
                <FiscalPeriod>{$c.FiscalPeriod}</FiscalPeriod>
             )
        }</ReportElement>
};

declare function local:concepts-for-archives($aids as string*) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("concepts:CONNECTION-FAILED"), $err:description)
        }
    return
        mongo:find($conn, "concepts",
        {
            "Archive": { "$in" : [ $aids ] }
        })
};

declare function local:concepts-for-archives($aids as string*, $names as string*, $map as string?) as object*
{
    let $conn :=
      let $credentials := credentials:credentials("MongoDB", "xbrl")
      return
        try {
            mongo:connect($credentials)
        } catch mongo:* {
            error(QName("concepts:CONNECTION-FAILED"), $err:description)
        }
    let $map := if (exists($map)) then concept-maps:concept-maps($map) else ()

    let $all-concepts-computable-by-maps as string* := keys($map.Trees)

    let $concepts-computable-by-maps as object* :=
        for $c in $names[$$ = $all-concepts-computable-by-maps]
        return $map.Trees.$c

    let $mapped-names := (keys($concepts-computable-by-maps.To ), $concepts-computable-by-maps.To [].Name)

    let $concepts-not-computable-by-maps as string* := seq:value-except($names, $mapped-names)
    let $results-not-computed-by-maps := mongo:find($conn, "concepts",
        {
            "Name" : { "$in" : [ $concepts-not-computable-by-maps ] },
            "Archive": { "$in" : [ $aids ] }
        })
    let $results-computed-by-maps :=
        for $c in mongo:find($conn, "concepts",
            {
                "Name" : { "$in" : [ $mapped-names ] },
                "Archive": { "$in" : [ $aids ] }
            })
        group by $c.Component
        let $c := $c[1]
        let $map-concept := (for $candidate in $concepts-computable-by-maps
                             where $c.Name = (keys($candidate.To), $candidate.To[].Name)
                             return $candidate)[1]         for $candidate-concept in $concepts-computable-by-maps
        let $concepts := $c[$$.Name = (keys($candidate-concept.To), $candidate-concept.To[].Name)]
        return
            copy $n := $concepts
            modify (
                replace value of json $n.Name with $candidate-concept.Name,
                insert json  { Origin : $concepts.Name } into $n)
            return $n
    return ($results-not-computed-by-maps, $results-computed-by-maps)
};

declare function local:concepts-for-archives-and-labels($aids as string*, $labels as string) as object*
{
    let $conn :=
        let $credentials := credentials:credentials("MongoDB", "xbrl")
        return
            try {
                mongo:connect($credentials)
            } catch mongo:* {
                error(QName("components:CONNECTION-FAILED"), $err:description)
            }
    return mongo:run-cmd-deterministic(
        $conn,
        {
            "text" : "concepts",
            "filter" : { "Archive" : { "$in" : [ $aids ] } },
            "search" : $labels,
            "limit" : 100,
            "score" : { "$meta" : "textScore" },
            "sort" : { score: { "$meta" : "textScore" } }
        }).results[].obj
};

session:audit-call();

(: Query parameters :)
let $format as string?         := request:param-values("format")
let $ciks as string*           := distinct-values(request:param-values("cik"))
let $tags as string*           := distinct-values(request:param-values("tag"))
let $tickers as string*        := distinct-values(request:param-values("ticker"))
let $sics as string*           := distinct-values(request:param-values("sic"))
let $fiscalYears as string*    := distinct-values(request:param-values("fiscalYear", "LATEST"))
let $fiscalPeriods as string*  := distinct-values(request:param-values("fiscalPeriod", "FY"))
let $aids as string*           := distinct-values(request:param-values("aid"))
let $labels as string*         := request:param-values("label")
let $map as string?            := request:param-values("map")
let $names as string*          := request:param-values("name")

(: Post-processing :)
let $format as string? := (: backwards compatibility, to be deprecated  :)
    lower-case(($format, substring-after(request:path(), ".jq."))[1])
let $tags as string* := (: backwards compatibility, to be deprecated :)
    distinct-values($tags ! upper-case($$))
let $fiscalYears as integer* :=
    for $fy in $fiscalYears ! upper-case($$)
    return switch($fy)
           case "LATEST" return $fiscal-core2:LATEST_FISCAL_YEAR
           case "ALL" return $fiscal-core:ALL_FISCAL_YEARS
           default return if($fy castable as integer) then integer($fy) else ()
let $fiscalPeriods as string* :=
    for $fp in $fiscalPeriods ! upper-case($$)
    return switch($fp)
           case "ALL" return $fiscal-core:ALL_FISCAL_PERIODS
           default return $fp

(: Object resolution :)
let $entities :=
    companies2:companies(
        $ciks,
        $tags,
        $tickers,
        $sics)
let $archives as object* := fiscal-core2:filings(
    $entities,
    $fiscalPeriods,
    $fiscalYears,
    $aids)
let $entities    := entities:entities($archives.Entity)
let $onlyNames   := let $o := request:param-values("onlyNames")[1] return if (exists($o)) then ($o cast as boolean) else false

let $concepts := if (exists($names))
                  then local:concepts-for-archives($archives._id, $names, $map)
                  else if (exists($labels))
                  then local:concepts-for-archives-and-labels($archives._id, $labels[1])
                  else local:concepts-for-archives($archives._id)
let $result := {
    ReportElements : [
        if ($onlyNames)
        then distinct-values($concepts.Name)
        else
            let $all-aids := $concepts.Archive
            let $roles := $concepts.Role
            let $components := components2:components-for-archives-and-roles($all-aids, $roles)
            return
            for $concept in $concepts
            group by $archive := $concept.Archive,  $role := $concept.Role
            let $component := $components[$$.Archive eq $archive and $$.Role eq $role]
            let $default-hc := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube")
            let $members := $default-hc.Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain".Members
            let $archive := $archives[$$._id eq $archive]
            let $entity := $entities[$$._id eq $archive.Entity]
            let $metadata := {|
                { Origin : $concept.Origin }[exists($concept.Origin)],
                {
                    Name: $concept.Name,
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label,
                    AccessionNumber : $archive._id,
                    CIK : $entity._id,
                    EntityRegistrantName : $entity.Profiles.SEC.CompanyName,
                    FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                    FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
                }
            |}
            for $concept in $concept
            let $original-name := ($concept.Origin, $concept.Name)[1]
            return {|
                trim($members.$original-name, "Name"),
                $metadata
            |}
    ]
}
let $comment := {
    NumConcepts: count($concepts),
    TotalNumConcepts: session:num-concepts()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <ReportElements>{
                    local:to-xml($res.ReportElements[], $onlyNames)
                }</ReportElements>
    },
    to-csv : function($res as object) as string {
        local:to-csv($res.ReportElements[], $onlyNames)
    }
}
let $results := util:serialize($result, $comment, $serializers, $format, "report-elements")
return
    util:check-and-return-results($entities, $results, $format)