Use || fro string concatenation. Cleanup.

jsoniq version "1.0";

(:
 : Copyright 2006-2013 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

(:~
 : JSound simple validator
 : This is a JSONiq implemenation of the JSound (the schema for JSON) validator.
 :
 : @author Cezar Andrei
 :)

module namespace jsv = "http://jsound.io/modules/validate";

import module namespace jn = "http://jsoniq.org/functions";
import module namespace map = "http://jsound.io/modules/validate/map";

declare namespace an = "http://zorba.io/annotations";

declare %an:sequential function
jsv:jsd-valid($jsd as object, $name as string, $ns as string,
  $instance as item) as boolean
{
  variable $jstypes := { "foo" : 5 };

  switch( false )
  case  $jsd instance of object
  return
    fn:error(QName("jsv", "BadJSoundFormat"),
      "The specified JSON document is not a valid JSound schema: top item not an object.");

  case exists($jsd("$namespace")) and exists($jsd("$types"))
  return
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: must contain '$types' and '$namespace' keys.");

  case $jsd("$namespace") instance of string
  return
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: value of '$namespace' must be a string.");

  case $jsd("$namespace") eq $ns
  return
    fn:error(QName("jsv", "Invalid"),
            "Requested type namespace not matching schema document namespace.");

  case $jsd("$types") instance of array
  return
    fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: value of '$types' must be an array.");

  default
  return
    jsv:check-types( $jstypes, $jsd("$types") );

  (: todo: validate all type refs :)

  jsv:validate-instance($jstypes, $name, $instance)
};

(: The check-xxx functions just checks if the JSound schema definitions are valid. :)

declare %an:sequential function
jsv:check-types($jstypes as object, $types as array) as boolean
{
  every $i in
    for $type in jn:members($types)
    return
      jsv:check-type($jstypes, $type )
  satisfies $i eq true
};

declare %an:sequential function
jsv:check-type($jstypes as object, $type as item) as boolean
{
  if( $type instance of object )
  then
    if ( exists($type("$kind")) and $type("$kind") instance of string )
    then
      switch( $type("$kind") )
      case "atomic" return jsv:check-atomic-type($jstypes, $type)
      case "object" return jsv:check-object-type($jstypes, $type)
      case "array"  return jsv:check-array-type ($jstypes, $type)
      case "union"  return jsv:check-union-type ($jstypes, $type)

      default return
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: type $kind unknown:" || $type("$kind") )

    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: type definition must contain '$kind' key with string value.")
  else
    if( $type instance of string )
    then
      jsv:check-ref-type($jstypes, $type)
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: type item not an object."  )
};

declare %an:sequential function
jsv:check-atomic-type($jstypes as object, $type as object) as boolean
{
  (: fn:trace( $type("$name"), "check-atomic-type"); :)

  if( exists($type("$baseType")) and $type("$baseType") instance of string )
  then
    jsv:save-type($jstypes, $type)
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: atomic type definition must contain '$baseType' key with string value.")

  (: todo: add facets checks: length, min/max Inclusive/Exclusive, etc. :)
};

declare %an:sequential function
jsv:check-object-type($jstypes as object, $type as object) as boolean
{
  (: todo: check the reast of "object" type :)
  jsv:save-type($jstypes, $type);
  (: fn:trace( $type("$name"), "check-object-type"); :)

  if( exists($type("$content")) )
  then
    if( $type("$content") instance of object )
    then
      every $i in
        for $k in jn:keys($type("$content"))
        return
          if( $type("$content")($k) instance of object )
          then
            if( exists( $type("$content")($k)("$type")) )
            then
              jsv:check-type($jstypes, $type("$content")($k)("$type") )
            else
              if( exists( $type("$content")($k)("$kind")) )
              then
                jsv:check-type($jstypes, $type("$content")($k) )
              else
                fn:error(QName("jsv", "BadJSoundFormat"),
                  "Not a valid JSound doc: Key  does not contain a $type definition.")
          else
            fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: Key " || $k || " in $content must have an object value.")

      satisfies $i eq true
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: $content definion of object type must be an object.")
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
             "Not a valid JSound doc: Object type does not contain $content.")
};

declare %an:sequential function
jsv:check-array-type($jstypes as object, $type as object) as boolean
{
  (: fn:trace( $type("$name"), "check-array-type"); :)

  (: check all the contents :)
  let $content := $type("$content")
  let $checkContent :=
    if( empty($content) )
    then
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: $kind array must have a $content key.")
    else
      if( $content instance of array and jn:size($content) eq 1 )
      then
      {
        jsv:save-type($jstypes, $type);

        typeswitch( $content(1) )
        case string return
          true
        case object return
            jsv:check-type($jstypes, $content(1))
        default return
          fn:error(QName("jsv", "BadJSoundFormat"),
            "Not a valid JSound doc: for $kind array, $content[1] must be string or array.")
      }
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $kind array, $content must be an array of size 1.")

  let $checkMinLength :=
      if( fn:exists($type("$minLength")) )
      then
        if( $type("$minLength") instance of integer  )
        then true
        else fn:error(QName("jsv", "BadJSoundFormat"),
                      "Not a valid JSound doc: for $kind array, $minLength must be an integer.")
      else true

  let $checkMaxLength :=
      if( fn:exists($type("$maxLength")) )
      then
        if( $type("$maxLength") instance of integer  )
        then true
        else fn:error(QName("jsv", "BadJSoundFormat"),
                      "Not a valid JSound doc: for $kind array, $maxLength must be an integer.")
      else true

  return
    $checkContent and $checkMinLength and $checkMaxLength
};

declare %an:sequential function
jsv:check-union-type($jstypes as object, $type as object) as boolean
{
  (: fn:trace( $type("$name"), "check-union-type"); :)

  let $content := $type("$content")
  return
    if( empty($content) )
    then
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: $kind union must have a $content key.")
    else
      if( $content instance of array )
      then
      {
        jsv:save-type($jstypes, $type);

        every $i in
          for $c in jn:members($content)
          return
            typeswitch( $c )
            case string return
              true
            case object return
                jsv:check-type($jstypes, $c)
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind union, $content items must be strings or objects.")
        satisfies $i eq true
      }
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $kind union, $content must be an array.")
};

declare function
jsv:check-ref-type($jstypes as object, $type as string) as boolean
{
  switch( $type )
  case "string" return true
  case "integer" return true
  case "boolean" return true
  case "anyURI"  return true
  default return
  {
    if( map:has-key($jstypes, $type) )
    then
      true
    else
     (: todo: save all refs, check them at the end to see if they exist :)
     true
  }

  (: todo: apply resolution rules :)
};

declare %an:sequential function
jsv:save-type($jstypes as object, $type as item) as boolean
{
  switch( false )
  case fn:exists( $type("$name") )
    return false
  case $type("$name") instance of string
    return fn:error(QName("jsv", "BadJSoundFormat"),
                    "Not a valid JSound doc: for $type, $name must be a string.")
  case map:set-if-empty($jstypes, $type("$name"), $type)
    return fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: type '" || $type("$name") || "' already defined.")
  default return true()
};

(: The validate-xxx functions actualy do the validation of the instance :)

declare function
jsv:validate-instance($jstypes as object, $name as string, $instance as item)
 as boolean
{
  let $type := map:get($jstypes, $name)
  return
    if( exists($type) )
    then
      jsv:validate-type($jstypes, $type, $instance)
    else
      fn:error(QName("jsv", "Invalid"),
             "Requested type name not present in schema: " || $name)
};

declare function
jsv:validate-type($jstypes as object, $type as item, $instance as item) as boolean
{
  switch( $type("$kind") )
  case "atomic" return jsv:validate-atomic-type($jstypes, $type, $instance)
  case "object" return jsv:validate-object-type($jstypes, $type, $instance)
  case "array"  return jsv:validate-array-type($jstypes, $type, $instance)
  case "union"  return jsv:validate-union-type($jstypes, $type, $instance)

  default return
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: type $kind unknown:" || $type("$kind") )
};

declare function
jsv:validate-type-ref($jstypes as object, $type as string, $instance as item) as boolean
{
  (: todo: check first between the defined types in $jstypes and then the build-in ones, this is per spec :)

  if( map:has-key($jstypes, $type) )
  then
    jsv:validate-type($jstypes, map:get($jstypes, $type), $instance)
  else
    switch( $type )
    case "string" return
      if ( $instance instance of string )
      then
        true
      else
        fn:error(QName("jsv", "Invalid"), "Expected string value.")
    case "integer" return
      if ( $instance instance of integer or
           $instance instance of int )
      then
        true
      else
        fn:error(QName("jsv", "Invalid"), "Expected integer value.")
    case "anyURI" return
      if ( $instance instance of anyURI )
      then
        true
      else
        fn:error(QName("jsv", "Invalid"), "Expected anyURI value.")
    case "boolean" return
      if ( $instance instance of boolean )
      then
        true
      else
        fn:error(QName("jsv", "Invalid"), "Expected boolean value.")

    default return
        fn:error(QName("jsv", "Invalid"),
          fn:concat("Type ref not defined or a built-in type: ", $type) )
};

declare function
jsv:validate-atomic-type($jstypes as object, $type as object, $instance as item) as boolean
{
  (: fn:trace( $type("$name"), "validate-atomic-type"); :)
  (: todo: check first between the defined types in $jstypes and then the build-in ones, this is per spec :)

  if ( exists($type("$baseType")) and $type("$baseType") instance of string )
  then
    switch( $type("$baseType") )
    case "integer" return
      if ( $instance instance of integer or $instance instance of int )
      then
        jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected integer or int value")

    case "string" return
      if ( $instance instance of string )
      then
        jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error( QName("jsv", "Invalid"), "Expected string value")

    case "anyURI" return
      if ( $instance instance of anyURI )
      then
        jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected anyURI value")

    case "boolean" return
      if ( $instance instance of boolean )
      then
        jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected boolean value")

    default return
      fn:error(QName("jsv", "NYI"),
        "$baseType: not yet implemented" || $type("$baseType") )
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: atomic type definition must contain '$baseType' key with string value.")

  (: todo: add facets checks: length, min/max Inclusive/Exclusive, etc. :)
};

declare function
jsv:validate-enumeration($jstypes as object, $type as object, $instance as item) as boolean
{
  variable $enum := $type("$enumeration");
  if( exists($enum) )
  then
    if( $enum instance of array )
    then
      if( some $v in jn:members($enum) satisfies $v eq $instance )
      then
        true
      else
        fn:error(QName("jsv", "Invalid"), "Instance value not in the enumeration list.")
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: the value of '$enumeration' must be an array.")
  else
    true
};

declare function
jsv:validate-object-type($jstypes as object, $type as object, $instance as item) as boolean
{
  (: fn:trace( $type("$name"), "validate-object-type"); :)

  variable $open := $type("$open");

  if ( $instance instance of object )
  then
    if( (not exists($open)) or (exists($open) and $open eq true()) )
    then
      (: object is open: i.e. extra keys alowed but all defined keys must be in
         the instance except the default and optional ones :)
      (
        every $k in jn:keys($instance)
        satisfies
          let $ctype := $type("$content")($k)("$type")
          return
            typeswitch( $ctype )
            case string return
              jsv:validate-type-ref($jstypes, $ctype, $instance($k) )
            case object return
              jsv:validate-type($jstypes, $ctype, $instance($k) )
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind object, $content $type must be string or array.")
      ) and
      (
        every $k in
          jsv:value-except(jn:keys($type("$content")), jn:keys($instance))
        satisfies
          let $optional := $type("$content")($k)("$optional")
          return
            if( exists($optional) and $optional eq true )
            then true
            else
              fn:error(QName("jsv", "Invalid"),
                fn:concat("Required key not present in instance: ", $k) )
      )
    else
      (: object is closed: i.e. all defined keys must be in the instance except
         the default ones, no extra keys alowed :)
      (
        every $k in jn:keys($instance)
        satisfies
          let $ctype := $type("$content")($k)("$type")
          return
            typeswitch( $ctype )
            case string return
              jsv:validate-type-ref($jstypes, $ctype, $instance($k) )
            case object return
              jsv:validate-type($jstypes, $ctype, $instance($k) )
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind object, $content $type must be string or array.")
      ) and (
        every $k in (: jn:keys($type("$content")) except jn:keys($instance) :)
          jsv:value-except(jn:keys($type("$content")), jn:keys($instance))
        satisfies
          let $optional := $type("$content")($k)("$optional")
          return
            if( exists($optional) and $optional eq true )
            then true
            else
              fn:error(QName("jsv", "Invalid"),
                fn:concat("Required key not present in instance: ", $k) )
      )
  else
    fn:error( QName("jsv", "Invalid"),
      fn:concat("Instance not an object specified by type: ", jsv:get-type-name($type)) )
};

declare function
jsv:validate-array-type($jstypes as object, $type as object, $instance as item) as boolean
{
  (: fn:trace( $type("$name"), "validate-array-type"); :)

  if( $instance instance of array )
  then
    (: check all the contents :)
    let $content := $type("$content")
    let $validateContent :=
        typeswitch( $content(1) )
        case string return
          every $i in jn:members($instance) satisfies jsv:validate-type-ref($jstypes, $content(1), $i)
        case object return
          every $i in jn:members($instance) satisfies jsv:validate-type($jstypes, $content(1), $i)
        default return
          fn:error(QName("jsv", "BadJSoundFormat"),
            "Not a valid JSound doc: for $kind array, $content[1] must be string or array.")
    let $validateMinLength :=
      if( fn:exists($type("$minLength")) )
      then
        if ( $type("$minLength") le size($instance) )
        then true
        else fn:error(QName("jsv", "Invalid"),
                      "Invalid array size, smaller then $minLength facet constraint.")
      else true
    let $validateMaxLength :=
      if( fn:exists($type("$maxLength")) )
      then
        if( $type("$maxLength") ge size($instance) )
        then true
        else fn:error(QName("jsv", "Invalid"),
                      "Invalid array size, bigger than $maxLength facet constraint.")
      else true
    return
      $validateContent and $validateMinLength and $validateMaxLength
  else
    fn:error(QName("jsv", "Invalid"),
      fn:concat("Instance not an array for type: ", jsv:get-type-name($type)) )

  (: check array facets :)
};

declare function
jsv:validate-union-type($jstypes as object, $type as object, $instance as item) as boolean
{
  let $content := $type("$content")
  return
    if(
      some $i in
        for $c in jn:members($content)
        return
          try
          {
            typeswitch( $c )
            case string return
              true
            case object return
                jsv:validate-type($jstypes, $c, $instance)
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind union, $content items must be strings or objects.")
          }
          catch Invalid
          {
             false
          }
      satisfies $i eq true
    )
    then true
    else
      fn:error(QName("jsv", "Invalid"),
        fn:concat("Instance not valid for union type: ", jsv:get-type-name($type)) )
};

(:---- Util functions ---------:)

declare function
jsv:get-type-name($type as object) as string
{
  let $name := $type("$name")
  return
    if ( exists($name) and $name instance of string )
    then
      $name
    else
      "N/A"
};

declare function
jsv:value-except($arg1 as xs:anyAtomicType*, $arg2 as xs:anyAtomicType*)
  as xs:anyAtomicType*
{
  (: fn:distinct-values( $arg1[ not($$ eq $arg2) ] ) :)
  fn:distinct-values(
    for $i in $arg1
    where not( some $x in $arg2 satisfies $i eq $x )
    return $i
  )
};

jsoniq version "1.0";

(:
 : Copyright 2006-2013 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

(:~
 : JSound simple validator
 : This is a JSONiq implemenation of the JSound (the schema for JSON) validator.
 :
 : @author Cezar Andrei
 :)

module namespace jsv = "http://jsound.io/modules/validate";

import module namespace jn = "http://jsoniq.org/functions";
import module namespace map = "http://jsound.io/modules/validate/map";

declare namespace an = "http://zorba.io/annotations";

declare %an:sequential function
jsv:jsd-valid($jsd as object, $name as string, $ns as string,
  $instance as item) as boolean
{
  variable $jstypes := { "foo" : 5 };

(:
  if( $jsd instance of object )
  then
    if ( exists($jsd("$namespace")) and exists($jsd("$types")) )
    then
      if ( $jsd("$namespace") instance of string )
      then
        if ( $jsd("$namespace") eq $ns )
        then
          if ( $jsd("$types") instance of array )
          then
            jsv:check-types( $jstypes, $jsd("$types") );
          else
            fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: value of '$types' must be an array.");
        else
          fn:error(QName("jsv", "Invalid"),
            "Requested type namespace not matching schema document namespace.");
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: value of '$namespace' must be a string.");
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: must contain '$types' and '$namespace' keys.");
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "The specified JSON document is not a valid JSound schema: top item not an object.");
:)

  switch( false )
  case  $jsd instance of object
  return
    fn:error(QName("jsv", "BadJSoundFormat"),
      "The specified JSON document is not a valid JSound schema: top item not an object.");

  case exists($jsd("$namespace")) and exists($jsd("$types"))
  return
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: must contain '$types' and '$namespace' keys.");

  case $jsd("$namespace") instance of string
  return
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: value of '$namespace' must be a string.");

  case $jsd("$namespace") eq $ns
  return
    fn:error(QName("jsv", "Invalid"),
            "Requested type namespace not matching schema document namespace.");

  case $jsd("$types") instance of array
  return
    fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: value of '$types' must be an array.");

  default
  return
    jsv:check-types( $jstypes, $jsd("$types") );

  (: todo: validate all type refs :)

  jsv:validate-instance($jstypes, $name, $instance)
};

(: The check-xxx functions just checks if the JSound schema definitions are valid. :)

declare %an:sequential function
jsv:check-types($jstypes as object, $types as array) as boolean
{
  every $i in
    for $type in jn:members($types)
    return
      jsv:check-type($jstypes, $type )
  satisfies $i eq true
};

declare %an:sequential function
jsv:check-type($jstypes as object, $type as item) as boolean
{
  if( $type instance of object )
  then
    if ( exists($type("$kind")) and $type("$kind") instance of string )
    then
      switch( $type("$kind") )
      case "atomic" return jsv:check-atomic-type($jstypes, $type)
      case "object" return jsv:check-object-type($jstypes, $type)
      case "array"  return jsv:check-array-type ($jstypes, $type)
      case "union"  return jsv:check-union-type ($jstypes, $type)

      default return
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: type $kind unknown:" | $type("$kind") )

    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: type definition must contain '$kind' key with string value.")
  else
    if( $type instance of string )
    then
      jsv:check-ref-type($jstypes, $type)
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: type item not an object."  )
};

declare %an:sequential function
jsv:check-atomic-type($jstypes as object, $type as object) as boolean
{
  fn:trace( $type("$name"), "check-atomic-type");

  if( exists($type("$baseType")) and $type("$baseType") instance of string )
  then
    jsv:save-type($jstypes, $type)
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: atomic type definition must contain '$baseType' key with string value.")

  (: todo: add facets checks: length, min/max Inclusive/Exclusive, etc. :)
};

declare %an:sequential function
jsv:check-object-type($jstypes as object, $type as object) as boolean
{
  (: todo: check the reast of "object" type :)
  jsv:save-type($jstypes, $type);
  fn:trace( $type("$name"), "check-object-type");

  if( exists($type("$content")) )
  then
    if( $type("$content") instance of object )
    then
      every $i in
        for $k in jn:keys($type("$content"))
        return
          if( $type("$content")( fn:trace($k, "Key")) instance of object )
          then
            if( exists( $type("$content")($k)("$type")) )
            then
              jsv:check-type($jstypes, $type("$content")($k)("$type") )
            else
              if( exists( $type("$content")($k)("$kind")) )
              then
                jsv:check-type($jstypes, $type("$content")($k) )
              else
                fn:error(QName("jsv", "BadJSoundFormat"),
                  "Not a valid JSound doc: Key  does not contain a $type definition.")
          else
            fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: Key " | $k | " in $content must have an object value.")

      satisfies $i eq true
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
              "Not a valid JSound doc: $content definion of object type must be an object.")
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
             "Not a valid JSound doc: Object type does not contain $content.")
};

declare %an:sequential function
jsv:check-array-type($jstypes as object, $type as object) as boolean
{
  fn:trace( $type("$name"), "check-array-type");

  (: check all the contents :)
  (: let $t := fn:trace($type, "Array type:") :)
  let $content := $type("$content")
  let $checkContent :=
    if( empty($content) )
    then
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: $kind array must have a $content key.")
    else
      if( $content instance of array and jn:size($content) eq 1 )
      then
      {
        jsv:save-type($jstypes, $type);

        typeswitch( $content(1) )
        case string return
          true
        case object return
            jsv:check-type($jstypes, $content(1))
        default return
          fn:error(QName("jsv", "BadJSoundFormat"),
            "Not a valid JSound doc: for $kind array, $content[1] must be string or array.")
      }
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $kind array, $content must be an array of size 1.")

  let $checkMinLength :=
      if( fn:exists($type("$minLength")) )
      then
        if( $type("$minLength") instance of integer  )
        then true
        else fn:error(QName("jsv", "BadJSoundFormat"),
                      "Not a valid JSound doc: for $kind array, $minLength must be an integer.")
      else true

  let $checkMaxLength :=
      if( fn:exists($type("$maxLength")) )
      then
        if( $type("$maxLength") instance of integer  )
        then true
        else fn:error(QName("jsv", "BadJSoundFormat"),
                      "Not a valid JSound doc: for $kind array, $maxLength must be an integer.")
      else true

  return
    $checkContent and $checkMinLength and $checkMaxLength
};

declare %an:sequential function
jsv:check-union-type($jstypes as object, $type as object) as boolean
{
  fn:trace( $type("$name"), "check-union-type");

  let $content := $type("$content")
  return
    if( empty($content) )
    then
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: $kind union must have a $content key.")
    else
      if( $content instance of array )
      then
      {
        jsv:save-type($jstypes, $type);

        every $i in
          for $c in jn:members($content)
          return
            typeswitch( $c )
            case string return
              true
            case object return
                jsv:check-type($jstypes, $c)
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind union, $content items must be strings or objects.")
        satisfies $i eq true
      }
      else
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $kind union, $content must be an array.")
};

declare function
jsv:check-ref-type($jstypes as object, $type as string) as boolean
{
  switch( $type )
  case "string" return true
  case "integer" return true
  case "boolean" return true
  case "anyURI"  return true
  default return
  {
    if( map:has-key($jstypes, $type) )
    then
      true
    else
     (: todo: save all refs, check them at the end to see if they exist
     fn:trace($type, "NYI: type ref not yet suported");
     fn:error(QName("jsv", "NYI"), "Type ref: not yet implemented.") :)
     true
  }

  (: todo: apply resolution rules :)
};

declare %an:sequential function
jsv:save-type($jstypes as object, $type as item) as boolean
{
(:
  todo: check if "switch case" below is equivalent

  if( fn:exists( $type("$name") ) )
  then
    if( $type("$name") instance of string )
    then
      if( not( map:set-if-empty($jstypes, $type("$name"), $type) ) )
      then
        fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: type '" | $type("$name") | "' already defined.")
      else
        true()
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: for $type, $name must be a string.")
  else
    false
:)

  switch( false )
  case fn:exists( $type("$name") )
    return false
  case $type("$name") instance of string
    return fn:error(QName("jsv", "BadJSoundFormat"),
                    "Not a valid JSound doc: for $type, $name must be a string.")
  case map:set-if-empty($jstypes, $type("$name"), $type)
    return fn:error(QName("jsv", "BadJSoundFormat"),
          "Not a valid JSound doc: type '" || $type("$name") || "' already defined.")
  default return true()
};

(: The validate-xxx functions actualy do the validation of the instance :)

declare function
jsv:validate-instance($jstypes as object, $name as string, $instance as item)
 as boolean
{
  let $type := map:get($jstypes, $name)
  return
    if( exists($type) )
    then
      jsv:validate-type($jstypes, $type, $instance)
    else
      fn:error(QName("jsv", "Invalid"),
             "Requested type name not present in schema: " | $name)
};

declare function
jsv:validate-type($jstypes as object, $type as item, $instance as item) as boolean
{
  switch( $type("$kind") )
  case "atomic" return jsv:validate-atomic-type($jstypes, $type, $instance)
  case "object" return jsv:validate-object-type($jstypes, $type, $instance)
  case "array"  return jsv:validate-array-type($jstypes, $type, $instance)
  case "union"  return jsv:validate-union-type($jstypes, $type, $instance)

  default return
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: type $kind unknown:" | $type("$kind") )
};

declare function
jsv:validate-type-ref($jstypes as object, $type as string, $instance as item) as boolean
{
  (: todo: check first between the defined types in $jstypes and then the build-in ones, this is per spec :)

  if( map:has-key($jstypes, $type) )
  then
    jsv:validate-type($jstypes, map:get($jstypes, $type), $instance)
  else
    switch( $type )
    case "string" return
      if ( $instance instance of string )
      then
        true
      else
        fn:error(QName("jsv", "Invalid"), "Expected string value.")
    case "integer" return
      if ( $instance instance of integer or
           $instance instance of int )
      then
        true
      else
        fn:error(QName("jsv", "Invalid"), "Expected integer value.")
    case "anyURI" return
      if ( $instance instance of anyURI )
      then
        true
      else
        fn:error(QName("jsv", "Invalid"), "Expected anyURI value.")
    case "boolean" return
      if ( $instance instance of boolean )
      then
        true
      else
        fn:error(QName("jsv", "Invalid"), "Expected boolean value.")

    default return
        fn:error(QName("jsv", "Invalid"),
          fn:concat("Type ref not defined or a built-in type: ", $type) )
};

declare function
jsv:validate-atomic-type($jstypes as object, $type as object, $instance as item) as boolean
{
  fn:trace( $type("$name"), "validate-atomic-type");
  (: todo: check first between the defined types in $jstypes and then the build-in ones, this is per spec :)

  if ( exists($type("$baseType")) and $type("$baseType") instance of string )
  then
    switch( $type("$baseType") )
    case "integer" return
      if ( $instance instance of integer or $instance instance of int )
      then
        jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected integer or int value")

    case "string" return
      if ( $instance instance of string )
      then
        jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error( QName("jsv", "Invalid"), "Expected string value")

    case "anyURI" return
      if ( $instance instance of anyURI )
      then
        jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected anyURI value")

    case "boolean" return
      if ( $instance instance of boolean )
      then
        jsv:validate-enumeration($jstypes, $type, $instance)
      else
        fn:error(QName("jsv", "Invalid"), "Expected boolean value")

    default return
      fn:error(QName("jsv", "NYI"),
        "$baseType: not yet implemented" | $type("$baseType") )
  else
    fn:error(QName("jsv", "BadJSoundFormat"),
      "Not a valid JSound doc: atomic type definition must contain '$baseType' key with string value.")

  (: todo: add facets checks: length, min/max Inclusive/Exclusive, etc. :)
};

declare function
jsv:validate-enumeration($jstypes as object, $type as object, $instance as item) as boolean
{
  variable $enum := $type("$enumeration");
  if( exists($enum) )
  then
    if( $enum instance of array )
    then
      if( some $v in jn:members($enum) satisfies $v eq $instance )
      then
        true
      else
        fn:error(QName("jsv", "Invalid"), "Instance value not in the enumeration list.")
    else
      fn:error(QName("jsv", "BadJSoundFormat"),
        "Not a valid JSound doc: the value of '$enumeration' must be an array.")
  else
    true
};

declare function
jsv:validate-object-type($jstypes as object, $type as object, $instance as item) as boolean
{
  fn:trace( $type("$name"), "validate-object-type");

  variable $open := $type("$open");

  if ( $instance instance of object )
  then
    if( (not exists($open)) or (exists($open) and $open eq true()) )
    then
      (: object is open: i.e. extra keys alowed but all defined keys must be in
         the instance except the default and optional ones :)
      (
        every $k in jn:keys($instance)
        satisfies
          let $ctype := $type("$content")($k)("$type")
          return
            typeswitch( $ctype )
            case string return
              jsv:validate-type-ref($jstypes, $ctype, $instance($k) )
            case object return
              jsv:validate-type($jstypes, $ctype, $instance($k) )
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind object, $content $type must be string or array.")
      ) and
      (
        every $k in
          jsv:value-except(jn:keys($type("$content")), jn:keys($instance))
        satisfies
          let $optional := $type("$content")($k)("$optional")
          return
            if( exists($optional) and $optional eq true )
            then true
            else
              fn:error(QName("jsv", "Invalid"),
                fn:concat("Required key not present in instance: ", $k) )
      )
    else
      (: object is closed: i.e. all defined keys must be in the instance except
         the default ones, no extra keys alowed :)
      (
        every $k in jn:keys($instance)
        satisfies
          let $ctype := $type("$content")($k)("$type")
          return
            typeswitch( $ctype )
            case string return
              jsv:validate-type-ref($jstypes, $ctype, $instance($k) )
            case object return
              jsv:validate-type($jstypes, $ctype, $instance($k) )
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind object, $content $type must be string or array.")
      ) and (
        every $k in (: jn:keys($type("$content")) except jn:keys($instance) :)
          jsv:value-except(jn:keys($type("$content")), jn:keys($instance))
        satisfies
          let $optional := $type("$content")($k)("$optional")
          return
            if( exists($optional) and $optional eq true )
            then true
            else
              fn:error(QName("jsv", "Invalid"),
                fn:concat("Required key not present in instance: ", $k) )
      )
  else
    fn:error( QName("jsv", "Invalid"),
      fn:concat("Instance not an object specified by type: ", jsv:get-type-name($type)) )
};

declare function
jsv:validate-array-type($jstypes as object, $type as object, $instance as item) as boolean
{
  fn:trace( $type("$name"), "validate-array-type");

  if( $instance instance of array )
  then
    (: check all the contents :)
    let $content := $type("$content")
    let $validateContent :=
        typeswitch( $content(1) )
        case string return
          every $i in jn:members($instance) satisfies jsv:validate-type-ref($jstypes, $content(1), $i)
        case object return
          every $i in jn:members($instance) satisfies jsv:validate-type($jstypes, $content(1), $i)
        default return
          fn:error(QName("jsv", "BadJSoundFormat"),
            "Not a valid JSound doc: for $kind array, $content[1] must be string or array.")
    let $validateMinLength :=
      if( fn:exists($type("$minLength")) )
      then
        if ( $type("$minLength") le size($instance) )
        then true
        else fn:error(QName("jsv", "Invalid"),
                      "Invalid array size, smaller then $minLength facet constraint.")
      else true
    let $validateMaxLength :=
      if( fn:exists($type("$maxLength")) )
      then
        if( $type("$maxLength") ge size($instance) )
        then true
        else fn:error(QName("jsv", "Invalid"),
                      "Invalid array size, bigger than $maxLength facet constraint.")
      else true
    return
      $validateContent and $validateMinLength and $validateMaxLength
  else
    fn:error(QName("jsv", "Invalid"),
      fn:concat("Instance not an array for type: ", jsv:get-type-name($type)) )

  (: check array facets :)
};

declare function
jsv:validate-union-type($jstypes as object, $type as object, $instance as item) as boolean
{
  let $content := $type("$content")
  return
    if(
      some $i in
        for $c in jn:members($content)
        return
          try
          {
            typeswitch( $c )
            case string return
              true
            case object return
                jsv:validate-type($jstypes, $c, $instance)
            default return
              fn:error(QName("jsv", "BadJSoundFormat"),
                "Not a valid JSound doc: for $kind union, $content items must be strings or objects.")
          }
          catch Invalid
          {
             false
          }
      satisfies $i eq true
    )
    then true
    else
      fn:error(QName("jsv", "Invalid"),
        fn:concat("Instance not valid for union type: ", jsv:get-type-name($type)) )
};

(:---- Util functions ---------:)

declare function
jsv:get-type-name($type as object) as string
{
  let $name := $type("$name")
  return
    if ( exists($name) and $name instance of string )
    then
      $name
    else
      "N/A"
};

declare function
jsv:value-except($arg1 as xs:anyAtomicType*, $arg2 as xs:anyAtomicType*)
  as xs:anyAtomicType*
{
  (: fn:distinct-values( $arg1[ not($$ eq $arg2) ] ) :)
  fn:distinct-values(
    for $i in $arg1
    where not( some $x in $arg2 satisfies $i eq $x )
    return $i
  )
};


reverted back to old syntax for this test

declare namespace an = "http://www.zorba-xquery.com/annotations";

declare default function namespace "foo.com";

declare variable $input:=
  for $i in 1 to 1000
  return ({"a":3, "b" : "foo"}, {"a": { "d" : 4}, "c":[1,2,3]});

declare %an:sequential function collapse-general($x)
{
   variable $val := [], $obj := [], $arr := [];

   $x ! (typeswitch (.)
         case object() return  { append json . into $obj; }
         case array() return { append json . into $arr; }
         default return { append json . into $val; });

   if (size($val) eq 0)
   then ()
   else ["_VAL", members($val)],

   if (size($obj) eq 0)
   then ()
   else ["_OBJ", collapse-objects(members($obj))],

   if (size($arr) eq 0)
   then ()
   else  ["_ARR", collapse-general(members($arr) ! members(.)) ]
};

declare %an:sequential function collapse-objects($x as object()*)
{
  {| for $y in distinct-values($x ! (keys(.)))
         let $z := $x($y)
         return {$y : collapse-general($z)} |}
};

collapse-objects($input),
"
"

declare namespace an = "http://www.zorba-xquery.com/annotations";

declare default function namespace "foo.com";

declare variable $input:=
  for $i in 1 to 1000
  return ({"a":3, "b" : "foo"}, {"a": { "d" : 4}, "c":[1,2,3]});

declare %an:sequential function collapse-general($x)
{
   variable $val := [], $obj := [], $arr := [];

   $x ! (typeswitch (.)
         case object() return  { append json . into $obj; ()}
         case array() return { append json . into $arr; ()}
         default return { append json . into $val; ()});

   if (size($val) eq 0)
   then ()
   else ["_VAL", members($val)],

   if (size($obj) eq 0)
   then ()
   else ["_OBJ", collapse-objects(members($obj))],

   if (size($arr) eq 0)
   then ()
   else  ["_ARR", collapse-general(members($arr) ! members(.)) ]
};

declare %an:sequential function collapse-objects($x as object()*)
{
  {| for $y in distinct-values($x ! (keys(.)))
         let $z := $x($y)
         return {$y : collapse-general($z)} |}
};

collapse-objects($input),
"
"


:green_heart: Fix tests.

:green_heart: Fix tests.

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-spreadsheet($expected as integer, $params as object) as item
{
    let $endpoint := "spreadsheet-for-report"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].TableSet[].TableCells.Facts[][].Value)
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-report-does-not-exist($params as object) as item
{
    let $endpoint := "spreadsheet-for-report"
    let $request := test:invoke($endpoint, $params)
    let $status as integer := $request[1]
    return if ($status eq 404) then true else { url: test:url($endpoint, $params), unexpectedResponse: $request[2] }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    cocacola: local:test-spreadsheet(93, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2013",
        fiscalPeriod:"Q1"}),
    tickerrole: local:test-spreadsheet(93, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1"}),
    tickerconcept: local:test-spreadsheet(372, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"ALL",
        fiscalPeriodType:["instant", "YTD"]
    }),
    tickerconcept-qtd: local:test-spreadsheet(192, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"ALL",
        fiscalPeriodType:["instant", "QTD"]
    }),
    tickerfyfprole: local:test-spreadsheet(187, {
        report:"FundamentalAccountingConcepts",
        ticker:["ko","wmt"],
        fiscalYear:"2013",
        fiscalPeriod:"FY",
        fiscalPeriodType:["instant", "YTD"],
        eliminate:"true"}),
    reportDoesntExist: local:test-report-does-not-exist({
        report:"report-not-found",
        ticker:"MSFT",
        fiscalYear:"ALL",
        fiscalPeriod:"FY"}),
    GeneralInformation: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-GeneralInformation",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          360
        ),
    GeneralInformation-periods: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-GeneralInformation",
              ticker: "ko",
              fiscalYear:"ALL",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          66
        ),
    CashFlowStatement: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-CashFlowStatement",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          330
        ),
    NetCashFlowBreakdown: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-NetCashFlowBreakdown",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          120
        ),
    BalanceSheetClassified: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-BalanceSheetClassified",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          377
        ),
    IncomeStatementMultiStep: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-IncomeStatementMultiStep",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          450
        ),
    StatementOfComprehensiveIncome: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-StatementOfComprehensiveIncome",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          90
        ),
    ComprehensiveIncomeBreakdown: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-ComprehensiveIncomeBreakdown",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          90
        ),
    NetIncomeLossBreakdown: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-NetIncomeLossBreakdown",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          90
        ),
    NetIncomeLossAvailableToCommonBreakdown: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-NetIncomeLossAvailableToCommonBreakdown",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          90
        ),
    KeyRatios: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-KeyRatios",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          120
        )
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-spreadsheet($expected as integer, $params as object) as item
{
    let $endpoint := "spreadsheet-for-report"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].TableSet[].TableCells.Facts[][].Value)
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:nondeterministic function local:test-report-does-not-exist($params as object) as item
{
    let $endpoint := "spreadsheet-for-report"
    let $request := test:invoke($endpoint, $params)
    let $status as integer := $request[1]
    return if ($status eq 404) then true else { url: test:url($endpoint, $params), unexpectedResponse: $request[2] }
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    cocacola: local:test-spreadsheet(93, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2013",
        fiscalPeriod:"Q1"}),
    tickerrole: local:test-spreadsheet(93, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1"}),
    tickerconcept: local:test-spreadsheet(372, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"ALL",
        fiscalPeriodType:["instant", "YTD"]
    }),
    tickerconcept-qtd: local:test-spreadsheet(192, {
        report:"FundamentalAccountingConcepts",
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"ALL",
        fiscalPeriodType:["instant", "QTD"]
    }),
    tickerfyfprole: local:test-spreadsheet(187, {
        report:"FundamentalAccountingConcepts",
        ticker:["ko","wmt"],
        fiscalYear:"2013",
        fiscalPeriod:"FY",
        fiscalPeriodType:["instant", "YTD"],
        eliminate:"true"}),
    reportDoesntExist: local:test-report-does-not-exist({
        report:"report-not-found",
        ticker:"MSFT",
        fiscalYear:"ALL",
        fiscalPeriod:"FY"}),
    GeneralInformation: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-GeneralInformation",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          360
        ),
    GeneralInformation-periods: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-GeneralInformation",
              ticker: "ko",
              fiscalYear:"ALL",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          60
        ),
    CashFlowStatement: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-CashFlowStatement",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          330
        ),
    NetCashFlowBreakdown: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-NetCashFlowBreakdown",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          120
        ),
    BalanceSheetClassified: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-BalanceSheetClassified",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          377
        ),
    IncomeStatementMultiStep: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-IncomeStatementMultiStep",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          450
        ),
    StatementOfComprehensiveIncome: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-StatementOfComprehensiveIncome",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          90
        ),
    ComprehensiveIncomeBreakdown: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-ComprehensiveIncomeBreakdown",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          90
        ),
    NetIncomeLossBreakdown: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-NetIncomeLossBreakdown",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          90
        ),
    NetIncomeLossAvailableToCommonBreakdown: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-NetIncomeLossAvailableToCommonBreakdown",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          90
        ),
    KeyRatios: test:invoke-and-assert-deep-equal(
          "spreadsheet-for-report",
          {
              report: "COMID-BSC-CF1-ISM-IEMIX-OILY-KeyRatios",
              tag: "DOW30",
              fiscalYear:"2014",
              fiscalPeriod:"FY",
              validate:"true"
          },
          function($res as item*) as item* { count($res.TableSet[].TableCells.Facts[][].Value) },
          120
        )
})


Update spreadsheet-for-component.jq

Update spreadsheet-for-component.jq

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-spreadsheet($expected as integer, $params as object) as item
{
    let $endpoint := "spreadsheet-for-component"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].TableSet[].TableCells.Facts[][].Value)
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    cocacola: local:test-spreadsheet(1, {
        ticker:"ko"
    }),
    tickerrole: local:test-spreadsheet(60, {
        ticker:"ko",
        networkIdentifier:"http://www.thecocacolacompany.com/role/ConsolidatedStatementsOfIncome"
    }),
    tickerconcept: local:test-spreadsheet(76, {
        ticker:"ko",
        fiscalYear:"2013",
        concept:"us-gaap:Assets"
    }),
    tickerfyfprole: local:test-spreadsheet(70, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        networkIdentifier:"http://www.thecoca-colacompany.com/role/CondensedConsolidatedBalanceSheets"
    }),
    generic: local:test-spreadsheet(70, {
        profile-name: "generic",
        aid: "0000021344-14-000008",
        role:"http://www.thecocacolacompany.com/role/ConsolidatedBalanceSheets"
    })
})

import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace test = "http://apps.28.io/test";

declare %an:nondeterministic function local:test-spreadsheet($expected as integer, $params as object) as item
{
    let $endpoint := "spreadsheet-for-component"
    let $request := test:invoke($endpoint, $params)
    let $actual as integer := count($request[2].TableSet[].TableCells.Facts[][].Value)
    let $status as integer := $request[1]
    return test:assert-eq($expected, $actual, $status, test:url($endpoint, $params))
};

declare %an:sequential function local:check($o as object) as object
{
    if (not(every $k in (keys($o) ! $o.$$) satisfies ($k instance of boolean and $k)))
    then {
            response:status-code(500);
            $o
    } else
            $o
};

local:check({
    cocacola: local:test-spreadsheet(1, {
        ticker:"ko"
    }),
    tickerrole: local:test-spreadsheet(60, {
        ticker:"ko",
        networkIdentifier:"http://www.thecocacolacompany.com/role/ConsolidatedStatementsOfIncome"
    }),
    tickerconcept: local:test-spreadsheet(76, {
        ticker:"ko",
        fiscalYear:"2013",
        concept:"us-gaap:Assets"
    }),
    tickerfyfprole: local:test-spreadsheet(70, {
        ticker:"ko",
        fiscalYear:"2012",
        fiscalPeriod:"Q1",
        networkIdentifier:"http://www.thecoca-colacompany.com/role/CondensedConsolidatedBalanceSheets"
    })
})


jq: Avoid errors with path filter

jq: Avoid errors with path filter

  The `path` filter looks for the given pattern in a lot of different
  attributes which might not always be available which can result in
  errors poluting the results.
  By using the `try` function, we can make sure only succeeding calls to
  the filter are used in the results.

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v) |
  del(.level) |
  del(.local) |
  del(.remote) |
  {time,component,msg,path}+(del(.time)|del(.component)|del(.msg)|del(.path));

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: select(is_error) | clean;
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: select(is_warn) | clean;
def warn_strict: select(is_warn_strict) | clean;
def info_level: 30;
def is_info: .level >= info_level;
def info: select(is_info) | clean;
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: select(is_debug) | clean;

# Components:
def component(pattern): select(.component | test(pattern));
def App: component("^App$");
def Chokidar: component("^Chokidar|chokidar$");
def Fs: component("^Fs|FS$");
def LocalAnalysis: component("^LocalAnalysis|local/analysis$");
def LocalChange: component("^LocalChange|local/change$");
def LocalWatcher: component("^LocalWatcher$");
def LocalWriter: component("^LocalWriter$");
def Merge: component("^Merge$");
def Metadata: component("^Metadata$");
def Perfs: component("^Perfs|PerfReports$");
def Pouch: component("^Pouch$");
def Prep: component("^Prep$");
def RemoteCozy: component("^RemoteCozy$");
def RemoteUserActionRequired: component("^RemoteUserActionRequired|remote/user_action_required$");
def RemoteWarningPoller: component("^RemoteWarningPoller$");
def RemoteWatcher: component("^RemoteWatcher$");
def RemoteWriter: component("^RemoteWriter$");
def Sync: component("^Sync$");

def Local: select(.component | test("^local|chokidar"; "i"));
def Remote: select(.component | test("^remote"; "i"));

# Exclude "up-to-date" messages
#
#     yarn jq nouptodate path/to/logs*
#
def nouptodate: select(.msg | test("up[- ]to[- ]date") | not);

# Exclude initial scan messages
#
#     yarn jq noscan path/to/logs*
#
def noscan: select(.action != "scan");

# Exclude Proxy stuff:
#
#     yarn jq noproxy path/to/logs*
#
def noproxy: select(.component | test("GUI:proxy") | not);

# Find conflicts:
#
#     yarn jq -c conflict path/to/logs*
#
def is_conflict: .msg == "resolveConflictAsync";
def conflict: clean | select(is_conflict);

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
# FIXME: Should not output the same line twice when both path & oldpath match.
def path(pattern):
  clean | select(
    (
      try (.path // (.doc | .path) // (.change | .doc | .path) // (.idConflict | .newDoc | .path) // (.event | .path)),
      try (.oldpath // (.was | .path) // (.idConflict | .existingDoc | .path) // (.event | .oldPath)),
      ""
    )
      | strings
      | test(pattern)
  );

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#    yarn jq 'before("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#    yarn jq 'after("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#
def time(pattern):
  select(.time | test(pattern));
def before(time): select(.time <= time);
def after(time): select(.time >= time);

# Drop time properties
def notime: del(.time);

# Filter inode number(s):
#
#    yarn jq 'ino(7852458)' path/to/logs*
#    yarn jq 'ino(7852458; 464672)' path/to/logs*
#
def ino(n): select((.ino // (.doc|.ino) // (.change|.doc|.ino)) as $ino | [n] | flatten | map(. == $ino) | any);

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path,oldpath,event}
    | if .event then del(.event) + (.event|{path, oldpath: .oldPath, action}) else del(.event) end
    | if .oldpath then . else del(.oldpath) end
    ;

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );

# Custom jq filters for log analysis.
# See doc/developer/log_analysis.md to get started.

# Remove fields that are almost never used while debugging.
# To be used in other filters.
def clean:
  del(.hostname) |
  del(.name) |
  del(.pid) |
  del(.v) |
  del(.level) |
  del(.local) |
  del(.remote) |
  {time,component,msg,path}+(del(.time)|del(.component)|del(.msg)|del(.path));

# Filter by log level:
#
#     yarn jq error path/to/logs*
#     yarn jq warn path/to/logs*
#
# To get strictly warnings (without errors):
#
#     yarn jq warn_strict path/to/logs*
#
def error_level: 50;
def is_error: .level >= error_level;
def error: select(is_error) | clean;
def warn_level: 40;
def is_warn: .level >= warn_level;
def is_warn_strict: .level == warn_level;
def warn: select(is_warn) | clean;
def warn_strict: select(is_warn_strict) | clean;
def info_level: 30;
def is_info: .level >= info_level;
def info: select(is_info) | clean;
def debug_level: 20;
def is_debug: .level >= debug_level;
def debug: select(is_debug) | clean;

# Components:
def component(pattern): select(.component | test(pattern));
def App: component("^App$");
def Chokidar: component("^Chokidar|chokidar$");
def Fs: component("^Fs|FS$");
def LocalAnalysis: component("^LocalAnalysis|local/analysis$");
def LocalChange: component("^LocalChange|local/change$");
def LocalWatcher: component("^LocalWatcher$");
def LocalWriter: component("^LocalWriter$");
def Merge: component("^Merge$");
def Metadata: component("^Metadata$");
def Perfs: component("^Perfs|PerfReports$");
def Pouch: component("^Pouch$");
def Prep: component("^Prep$");
def RemoteCozy: component("^RemoteCozy$");
def RemoteUserActionRequired: component("^RemoteUserActionRequired|remote/user_action_required$");
def RemoteWarningPoller: component("^RemoteWarningPoller$");
def RemoteWatcher: component("^RemoteWatcher$");
def RemoteWriter: component("^RemoteWriter$");
def Sync: component("^Sync$");

def Local: select(.component | test("^local|chokidar"; "i"));
def Remote: select(.component | test("^remote"; "i"));

# Exclude "up-to-date" messages
#
#     yarn jq nouptodate path/to/logs*
#
def nouptodate: select(.msg | test("up[- ]to[- ]date") | not);

# Exclude initial scan messages
#
#     yarn jq noscan path/to/logs*
#
def noscan: select(.action != "scan");

# Exclude Proxy stuff:
#
#     yarn jq noproxy path/to/logs*
#
def noproxy: select(.component | test("GUI:proxy") | not);

# Find conflicts:
#
#     yarn jq -c conflict path/to/logs*
#
def is_conflict: .msg == "resolveConflictAsync";
def conflict: clean | select(is_conflict);

# Non-issue filters (so we can ignore them when looking for real issues):
def is_net_error: .msg | test("net::");
def is_maintenance_page: .msg | test("Maintenance en cours");
def is_seq_already_synced: .msg == "Seq was already synced!";
def is_pending_changes: .msg | test("Prepend [0-9]+ pending change");
def is_non_issue:
  (
    is_net_error or is_maintenance_page or is_seq_already_synced or
    is_pending_changes
  );

# Find main issues:
#
#     yarn jq issues path/to/logs*
#
def is_issue: (is_warn or is_conflict) and (is_non_issue | not);
def issues: clean | select(is_issue);

# Filter by path (should handle moves, conflict renaming...):
#
#     yarn jq 'path("foo/bar")' path/to/logs*
#     yarn jq 'path("foo\\\\bar")' path/to/logs*
#
# FIXME: Should not output the same line twice when both path & oldpath match.
def path(pattern):
  clean | select(
    (
      (.path // (.doc | .path) // (.change | .doc | .path) // (.idConflict | .newDoc | .path) // (.event | .path)),
      (.oldpath // (.was | .path) // (.idConflict | .existingDoc | .path) // (.event | .oldPath)),
      ""
    )
      | strings
      | test(pattern)
  );

# Include/exclude GUI stuff:
#
#    yarn jq gui path/to/logs*
#    yarn jq no_gui path/to/logs*
#
def is_gui: .component | test("GUI");
def gui: select(is_gui);
def no_gui: select(is_gui | not);

# To make `mocha` component messages more visible in test logs:
#
#    yarn jq -c 'debug|short|mocha' debug.log
#
# Or to completely ignore them:
#
#    yarn jq -c no_mocha debug.log
#
# Please note the `mocha` filter should always be the last one since it
# converts mocha log entries to strings.
def is_mocha: .component == "mocha";
def mocha: if is_mocha then .msg | gsub("\\n+"; "") else . end;
def no_mocha: select(is_mocha | not);

# Find the OS / app versions:
#
#     yarn jq client path/to/logs*
#
def client:
  select(.appVersion)
    |del(.name)
    |del(.level)
    |del(.v)
    ;

# Filter by file extension:
#
#    yarn jq 'ext("png")' path/to/logs*
#
# You can use/define additional filters for common extensions:
#
#    yarn jq xls path/to/logs*
#
def is_ext(x): .path | test("\\." + x);
def ext(x): select(is_ext(x));
def no_ext(x): select(is_ext(x) | not);
def xls: ext("xls");
def no_xls: no_ext("xls");
def txt: ext("txt");
def no_txt: no_ext("txt");

# Filter by msg pattern:
#
#    yarn jq 'msg("422")' path/to/logs*
#    yarn jq 'msg("offline$")' path/to/logs*
#
def msg(pattern):
  select(.msg | test(pattern));

# Filter by time pattern:
#
#    yarn jq 'time("2018-04-14T22:34:25.453Z")' path/to/logs*
#    yarn jq 'time("2018-04-14")' path/to/logs*
#    yarn jq 'time("T22:34")' path/to/logs*
#    yarn jq 'before("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#    yarn jq 'after("2018-04-14T22:34:25.453Z")' path/to/logs* // inclusive
#
def time(pattern):
  select(.time | test(pattern));
def before(time): select(.time <= time);
def after(time): select(.time >= time);

# Drop time properties
def notime: del(.time);

# Filter inode number(s):
#
#    yarn jq 'ino(7852458)' path/to/logs*
#    yarn jq 'ino(7852458; 464672)' path/to/logs*
#
def ino(n): select((.ino // (.doc|.ino) // (.change|.doc|.ino)) as $ino | [n] | flatten | map(. == $ino) | any);

# Filter things with an attached doc and return it:
#
#    yarn jq 'Pouch|doc' path/to/logs*
#
def doc:
  select(.doc) | .doc;

# Get a global overview of another filter:
#
#    yarn -s jq -c 'select(...)|short' path/to/logs*
#
def short:
  {time,component,msg,path,oldpath,event}
    | if .event then del(.event) + (.event|{path, oldpath: .oldPath, action}) else del(.event) end
    | if .oldpath then . else del(.oldpath) end
    ;

# FIXME: Find a way to make aggregation work, e.g.:
#
#     yarn jq 'issues|.msg' | yarn jq:frequencies
#
# We should be able to use --slurp in another aggregation script since the lines
# amount should have been reduced by the initial fitering.
def frequencies:
  reduce .[] as $x ({};
    . as $counts | $counts + {($x): (1 + (($counts|getpath([$x])) // 0))}
  );


removed accidental tabs

removed accidental tabs

import module namespace api = "http://apps.28.io/api";

import module namespace session = "http://apps.28.io/session";

import module namespace entities = "http://28.io/modules/xbrl/entities";
import module namespace hypercubes = "http://28.io/modules/xbrl/hypercubes";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace concepts = "http://28.io/modules/xbrl/concepts";
import module namespace reports = "http://28.io/modules/xbrl/reports";
import module namespace concept-maps = "http://28.io/modules/xbrl/concept-maps";
import module namespace config = "http://apps.28.io/config";

import module namespace multiplexer = "http://28.io/modules/xbrl/profiles/multiplexer";

import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

import module namespace csv = "http://zorba.io/modules/json-csv";
import module namespace seq = "http://zorba.io/modules/sequence";

declare function local:to-csv($concepts as item*, $onlyNames as boolean) as string
{
    if ($onlyNames)
    then
        string-join(("Name", $concepts), "
")
    else
        string-join(csv:serialize(trim($concepts, "Labels"), { serialize-null-as : "" }))
};

declare function local:to-xml($concepts as item*, $onlyNames as boolean) as element()*
{
    for $c in $concepts
    return
        <ReportElement>{
             if ($onlyNames)
             then
                <Name>{$c}</Name>
             else (
                <Name>{$c.Name}</Name>,
                <Label>{$c.Label}</Label>,
                <IsNillable>{$c.IsNillable}</IsNillable>,
                <IsAbstract>{$c.IsAbstract}</IsAbstract>,
                <PeriodType>{$c.PeriodType}</PeriodType>,
                <Balance>{$c.Balance}</Balance>,
                <SubstitutionGroup>{$c.SubstitutionGroup}</SubstitutionGroup>,
                <DataType>{$c.DataType}</DataType>,
                <BaseType>{$c.BaseType}</BaseType>,
                <ClosestSchemaBuiltinType>{$c.ClosestSchemaBuiltinType}</ClosestSchemaBuiltinType>,
                <IsTextBlock>{$c.IsTextBlock}</IsTextBlock>,
                <ComponentRole>{$c.ComponentRole}</ComponentRole>,
                <ComponentLabel>{$c.ComponentLabel}</ComponentLabel>,
                <AccessionNumber>{$c.AccessionNumber}</AccessionNumber>,
                <CIK>{$c.CIK}</CIK>,
                <EntityRegistrantName>{$c.EntityRegistrantName}</EntityRegistrantName>,
                <FiscalYear>{$c.FiscalYear}</FiscalYear>,
                <FiscalPeriod>{$c.FiscalPeriod}</FiscalPeriod>
             )
        }</ReportElement>
};

declare function local:concepts-for-archives(
    $aids as string*,
    $names as string*,
    $map as object?,
    $options as object?) as object*
{
    let $projection as object := if($options.OnlyNames eq true) then { Name: 1 } else {}
    let $concepts-computable-by-maps as object* :=
        switch(true)
            case not exists($map) return ()
            case not exists($names) return values($map.Trees)
            default return
                let $keys as string* := keys($map.Trees)
                for $concept as string in $names[$$ = $keys]
                return $map.Trees.$concept
    let $mapped-names as string* := (keys($concepts-computable-by-maps.To ), $concepts-computable-by-maps.To [].Name)
    let $concepts-not-computable-by-maps as string* := seq:value-except($names, $mapped-names)

    let $all-results as object* := mw:find($concepts:col,
        {
            "Archive": { "$in" : [ $aids ] }
        },
        $projection)
    let $results-not-computed-by-maps as object* := mw:find($concepts:col,
        {
            "Name" : { "$in" : [ $concepts-not-computable-by-maps ] },
            "Archive": { "$in" : [ $aids ] }
        },
        $projection)
    let $results-computed-by-maps as object* :=
        let $all-results as object* := mw:find($concepts:col,
            {
                "Name" : { "$in" : [ $mapped-names ] },
                "Archive": { "$in" : [ $aids ] }
            },
            $projection)
        for $concept as object in $concepts-computable-by-maps
        for $result as object in
            for $candidate-concept in (keys($concept.To), $concept[].Name)
            let $facts := $all-results[$$.Name = $candidate-concept]
            where exists($facts)
            count $c
            where $c eq 1
            return $facts
        let $map-concept := (for $candidate in $concepts-computable-by-maps
                            where $result.Name = (keys($candidate.To), $candidate.To[].Name)
                            return $candidate)[1]
        return
            copy $n := $result
            modify (
                replace value of json $n.Name with $map-concept.Name,
                insert json  { Origin : $result.Name } into $n)
            return $n
    return if(empty($map) and empty($names))
           then $all-results
           else ($results-not-computed-by-maps, $results-computed-by-maps)
};

declare function local:concepts-for-archives-and-labels($aids as string*, $labels as string) as object*
{
   mw:run-cmd-deterministic(
      {
        "text" : "concepts",
        "filter" : { "Archive" : { "$in" : [ $aids ] } },
        "search" : $labels,
        "limit" : 100,
        "score" : { "$meta" : "textScore" },
        "sort" : { score: { "$meta" : "textScore" } }
      }).results[].obj
};

(: Query parameters :)
declare  %rest:case-insensitive                 variable $token         as string?  external;
declare  %rest:case-insensitive                 variable $profile-name  as string  external := $config:profile-name;
declare  %rest:env                              variable $request-uri   as string   external;
declare  %rest:case-insensitive                 variable $format        as string?  external;
declare  %rest:case-insensitive %rest:distinct  variable $cik           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $tag           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $ticker        as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $sic           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $fiscalYear    as string*  external := "LATEST";
declare  %rest:case-insensitive %rest:distinct  variable $fiscalPeriod  as string*  external := "FY";
declare  %rest:case-insensitive %rest:distinct  variable $aid           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $eid           as string*  external;
declare  %rest:case-insensitive %rest:distinct  variable $label         as string*  external;
declare  %rest:case-insensitive                 variable $map           as string?  external;
declare  %rest:case-insensitive                 variable $report        as string?  external;
declare  %rest:case-insensitive %rest:distinct  variable $name          as string*  external;
declare  %rest:case-insensitive                 variable $onlyNames     as boolean? external := false;

session:audit-call($token);

(: Post-processing :)
let $format as string? := api:preprocess-format($format, $request-uri)
let $fiscalYear as integer* := api:preprocess-fiscal-years($fiscalYear)
let $fiscalPeriod as string* := api:preprocess-fiscal-periods($fiscalPeriod)
let $tag as string* := api:preprocess-tags($tag)

(: Object resolution :)
let $entities := multiplexer:entities(
  $profile-name,
  $eid,
  $cik,
  $tag,
  $ticker,
  $sic)

let $archives as object* := multiplexer:filings(
  $profile-name,
  $entities,
  $fiscalPeriod,
  $fiscalYear,
  $aid)

let $entities as object* :=
    ($entities[$$._id = $archives.Entity],
    let $not-found := $archives.Entity[not $entities._id = $$]
    where exists($not-found)
    return entities:entities($not-found))
let $map as object? :=
    if(exists($report))
    then reports:concept-map($report)
    else concept-maps:concept-maps($map)
let $concepts as object* :=
    if (exists($label))
        then local:concepts-for-archives-and-labels($archives._id, $label[1])
        else local:concepts-for-archives($archives._id, $name, $map, { OnlyNames: $onlyNames })

let $result :=
    if($profile-name eq "sec")
    then {
        ReportElements : [
            if ($onlyNames)
            then distinct-values($concepts.Name)
            else
                let $all-aids := $concepts.Archive
                let $roles := $concepts.Role
                let $components := components:components-for-archives-and-roles($all-aids, $roles)
                return
                for $concept in $concepts
                group by $archive := $concept.Archive,  $role := $concept.Role
                let $component as object := $components[$$.Archive eq $archive and $$.Role eq $role]
                let $default-hc as object := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube")
                let $members as object* := $default-hc.Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain".Members
                let $archive as object := $archives[$$._id eq $archive]
                let $entity as object := $entities[$$._id eq $archive.Entity]
                let $metadata := {
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label,
                    AccessionNumber : $archive._id,
                    CIK : $entity._id,
                    EntityRegistrantName : $entity.Profiles.SEC.CompanyName,
                    FiscalYear : $archive.Profiles.SEC.Fiscal.DocumentFiscalYearFocus,
                    FiscalPeriod : $archive.Profiles.SEC.Fiscal.DocumentFiscalPeriodFocus
                }
                for $concept in $concept
                let $original-name := ($concept.Origin, $concept.Name)[1]
                return {|
                    project($concept, ("Name", "Origin")),
                    {
                      Labels: [
                        for $labelRole in keys($concept.Labels)
                        for $language in keys($concept.Labels.$labelRole)
                        return {
                          Role: $labelRole,
                          Language: $language,
                          Value: $concept.Labels.$labelRole.$language
                        }
                      ]
                    },
                    trim($members.$original-name, ("Name", "Labels")),
                    $metadata
                |}
        ]
    }
    else {
        ReportElements : [
            if ($onlyNames)
            then distinct-values($concepts.Name)
            else
                let $all-aids := $concepts.Archive
                let $roles := $concepts.Role
                let $components := components:components-for-archives-and-roles($all-aids, $roles)
                return
                for $concept in $concepts
                group by $archive := $concept.Archive,  $role := $concept.Role
                let $component as object := $components[$$.Archive eq $archive and $$.Role eq $role]
                let $default-hc as object := hypercubes:hypercubes-for-components($component, "xbrl:DefaultHypercube")
                let $members as object* := $default-hc.Aspects."xbrl:Concept".Domains."xbrl:ConceptDomain".Members
                let $metadata := {
                    ComponentRole : $component.Role,
                    ComponentLabel : $component.Label,
                    Archive : $archive
                }
                for $concept in $concept
                let $original-name := ($concept.Origin, $concept.Name)[1]
                return {|
                    project($concept, ("Name", "Origin")),
                    {
                      Labels: [
                        for $labelRole in keys($concept.Labels)
                        for $language in keys($concept.Labels.$labelRole)
                        return {
                          Role: $labelRole,
                          Language: $language,
                          Value: $concept.Labels.$labelRole.$language
                        }
                      ]
                    },
                    trim($members.$original-name, ("Name", "Labels")),
                    $metadata
                |}
        ]
    }
let $comment := {
    NumConcepts: count($concepts),
    TotalNumConcepts: session:num-concepts()
}
let $serializers := {
    to-xml : function($res as object) as node() {
        <ReportElements>{
                    local:to-xml($res.ReportElements[], $onlyNames)
                }</ReportElements>
    },
    to-csv : function($res as object) as string {
        local:to-csv($res.ReportElements[], $onlyNames)
    }
}
let $results := api:serialize($result, $comment, $serializers, $format, "report-elements")
return api:check-and-return-results($token, $results, $format)


:books: Fix documentation in concepts module.

jsoniq version "1.0";
(:
 : Copyright 2014 28msec Inc.
 :)

(:~
 :
This module provides functionality for retrieving concepts
 : and labels.

 :
 : @author Federico Cavalieri
 :)
module namespace concepts = "http://28.io/modules/xbrl/concepts";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Name of the collection the concepts are stored in.
 :)
declare variable $concepts:col as xs:string := "concepts";

(:~
 : Name of the field pointing to the archive.
 :)
declare variable $concepts:ARCHIVE as xs:string := "Archive";

(:~
 : Name of the field pointing to the role.
 :)
declare variable $concepts:ROLE as xs:string := "Role";

(:~
 : Name of the field pointing to the name.
 :)
declare variable $concepts:NAME as xs:string := "Name";

(:~
 : Name of the field pointing to the labels.
 :)
declare variable $concepts:LABELS as xs:string := "Labels";

(:~
 : The default component link role.
 :)
declare variable $concepts:DEFAULT_COMPONENT_LINK_ROLE as xs:string :=
    "http://www.xbrl.org/2003/role/link";

(:~
 : Allow any component link role
 :)
declare variable $concepts:ANY_COMPONENT_LINK_ROLE as xs:string :=
    ":any component:";

(:~
 : A joker to ask for all concept names.
 :)
declare variable $concepts:ALL_CONCEPT_NAMES as xs:string := ":any concept:";

(:~
 : A joker to ask for all concept labels.
 :)
declare variable $concepts:ALL_CONCEPT_LABELS as xs:string := ":any concept label:";

(:~
 :
Retrieves all concepts.

 :
 : @return all concepts.
 :)
declare function concepts:concepts() as object*
{
  mw:find($concepts:col,{})
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives-or-ids the archives or their IDs.
 : @param $component-roles the component roles.
 : @param $labels labels for exact matches.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives-or-ids as item*,
    $component-roles as string*,
    $labels as string*
  ) as object*
{
  concepts:concepts($concept-names, $archives-or-ids, $component-roles, $labels, ())
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives-or-ids the archives or their IDs.
 : @param $component-roles the component roles.
 : @param $labels labels for exact matches.
 : @param $options one option OnlyNames to only output concepts with the Name field.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives-or-ids as item*,
    $component-roles as string*,
    $labels as string*,
    $options as object?
  ) as object*
{
  let $projection as object := if($options.OnlyNames eq true) then { Name: 1 } else {}
  let $archives := archives:aid($archives-or-ids)
  return
  if (exists($archives))
  then mw:find($concepts:col,
    {|
      {
        $concepts:ARCHIVE : { "$in" : [ $archives ] }
      },
      {
        $concepts:ROLE : { "$in" :
          [ $component-roles ]
        }
      }[not $component-roles = $concepts:ANY_COMPONENT_LINK_ROLE],
      {
        $concepts:NAME : { "$in" : [ $concept-names ] }
      }[not $concept-names = $concepts:ALL_CONCEPT_NAMES],
      {
        "Labels.Value" : { "$in" : [ $labels ] }
      }[not $labels = $concepts:ALL_CONCEPT_LABELS]
    |},
    $projection
  )
  else ()
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives-or-ids the archives or their IDs.
 : @param $component-roles the component roles.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives-or-ids as item*,
    $component-roles as string*
  ) as object*
{
  concepts:concepts(
    $concept-names,
    $archives-or-ids,
    $component-roles,
    $concepts:ALL_CONCEPT_LABELS
  )
};

(:~
 :
Retrieves the concepts which:
 :  - concept name matches a given one;
 :  - archive number matches that of a given component;
 :  - component role matches that of a given component or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $component-or-ids the CIDs or the components themselves.
 : @param $labels labels for exact matches.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts-for-components(
    $concept-names as string*,
    $labels as string*,
    $component-or-ids as item*,
    $options as object?) as object*
{
  for $component in components:components($component-or-ids)
  return concepts:concepts($concept-names,
                           $component.Archive[not $$ instance of null],
                           $component.Role,
                           $labels,
                           $options)
};

jsoniq version "1.0";
(:
 : Copyright 2014 28msec Inc.
 :)

(:~
 :
This module provides functionality for retrieving concepts
 : and labels.

 :
 : @author Federico Cavalieri
 :)
module namespace concepts = "http://28.io/modules/xbrl/concepts";

import module namespace archives = "http://28.io/modules/xbrl/archives";
import module namespace components = "http://28.io/modules/xbrl/components";
import module namespace mw = "http://28.io/modules/xbrl/mongo-wrapper";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:~
 : Name of the collection the concepts are stored in.
 :)
declare variable $concepts:col as xs:string := "concepts";

(:~
 : Name of the field pointing to the archive.
 :)
declare variable $concepts:ARCHIVE as xs:string := "Archive";

(:~
 : Name of the field pointing to the role.
 :)
declare variable $concepts:ROLE as xs:string := "Role";

(:~
 : Name of the field pointing to the name.
 :)
declare variable $concepts:NAME as xs:string := "Name";

(:~
 : Name of the field pointing to the labels.
 :)
declare variable $concepts:LABELS as xs:string := "Labels";

(:~
 : The default component link role.
 :)
declare variable $concepts:DEFAULT_COMPONENT_LINK_ROLE as xs:string :=
    "http://www.xbrl.org/2003/role/link";

(:~
 : Allow any component link role
 :)
declare variable $concepts:ANY_COMPONENT_LINK_ROLE as xs:string :=
    ":any component:";

(:~
 : A joker to ask for all concept names.
 :)
declare variable $concepts:ALL_CONCEPT_NAMES as xs:string := ":any concept:";

(:~
 : A joker to ask for all concept labels.
 :)
declare variable $concepts:ALL_CONCEPT_LABELS as xs:string := ":any concept label:";

(:~
 :
Retrieves all concepts.

 :
 : @return all concepts.
 :)
declare function concepts:concepts() as object*
{
  mw:find($concepts:col,{})
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives the archive numbers.
 : @param $component-roles the component roles.
 : @param $labels labels for exact matches.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archive-or-ids as item*,
    $component-roles as string*,
    $labels as string*
  ) as object*
{
  concepts:concepts($concept-names, $archive-or-ids, $component-roles, $labels, ())
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives the archive numbers.
 : @param $component-roles the component roles.
 : @param $labels labels for exact matches.
 : @param $options one option OnlyNames to only output concepts with the Name field.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archive-or-ids as item*,
    $component-roles as string*,
    $labels as string*,
    $options as object?
  ) as object*
{
  let $projection as object := if($options.OnlyNames eq true) then { Name: 1 } else {}
  let $archives := archives:aid($archive-or-ids)
  return
  if (exists($archives))
  then mw:find($concepts:col,
    {|
      {
        $concepts:ARCHIVE : { "$in" : [ $archives ] }
      },
      {
        $concepts:ROLE : { "$in" :
          [ $component-roles ]
        }
      }[not $component-roles = $concepts:ANY_COMPONENT_LINK_ROLE],
      {
        $concepts:NAME : { "$in" : [ $concept-names ] }
      }[not $concept-names = $concepts:ALL_CONCEPT_NAMES],
      {
        "Labels.Value" : { "$in" : [ $labels ] }
      }[not $labels = $concepts:ALL_CONCEPT_LABELS]
    |},
    $projection
  )
  else ()
};

(:~
 :
Retrieves the concepts which:
 :  - concept name and archive number match a given one;
 :  - component role matches a given one or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $archives the archive numbers.
 : @param $component-roles the component roles.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts(
    $concept-names as string*,
    $archives as string*,
    $component-roles as string*
  ) as object*
{
  concepts:concepts(
    $concept-names,
    $archives,
    $component-roles,
    $concepts:ALL_CONCEPT_LABELS
  )
};

(:~
 :
Retrieves the concepts which:
 :  - concept name matches a given one;
 :  - archive number matches that of a given component;
 :  - component role matches that of a given component or is the default
 :    component role.
 :

 :
 : @param $concept-names the concepts names.
 : @param $component-or-ids the CIDs or the components themselves.
 : @param $labels labels for exact matches.
 :
 : @return the matching concepts.
 :)
declare function concepts:concepts-for-components(
    $concept-names as string*,
    $labels as string*,
    $component-or-ids as item*,
    $options as object?) as object*
{
  for $component in components:components($component-or-ids)
  return concepts:concepts($concept-names,
                           $component.Archive[not $$ instance of null],
                           $component.Role,
                           $labels,
                           $options)
};