files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  """Backend interface.
  
  To be used as a modeling and inference backend in a :class:`bayeslite.BayesDB`
  handle, a backends must first be registered with
  :func:`bayesdb_register_backend`.
  
  The CGPM backend is registered by default, but we can suppress
  that for illustration::
  
     import bayeslite
     from bayeslite.backends.cgpm_backend import CGPM_Backend
  
     bdb = bayeslite.bayesdb_open(pathname='foo.bdb', builtin_backends=False)
     backend = CGPM_Backend(cgpm_registry=dict(), multiprocess=False)
     bayeslite.bayesdb_register_backend(bdb, backend)
  
  Then you can model a table and query the probable implications of the data in
  the table::
  
     bdb.execute('create population p for t with schema(guess stattypes of (*))')
     bdb.execute('create generator p_cc for t using cgpm;')
     bdb.execute('initialize 10 models for t_cc')
     bdb.execute('analyze t_cc for 10 iterations')
     for x in bdb.execute('estimate pairwise dependence probability from t_cc'):
         print x
  """
  
  from bayeslite.util import cursor_value
  
  builtin_backends = []
  builtin_backend_names = set()
  
  def bayesdb_builtin_backend(backend):
      name = backend.name()
      assert name not in builtin_backend_names
      builtin_backends.append(backend)
      builtin_backend_names.add(name)
  
  def bayesdb_register_builtin_backends(bdb):
      """Register all builtin backends in `bdb`."""
      for backend in builtin_backends:
          bayesdb_register_backend(bdb, backend)
  
  def bayesdb_register_backend(bdb, backend):
      """Register `backend` in `bdb`, creating any necessary tables.
  
      `backend` must not already be registered in any BayesDB, nor any
      backend by the same name.
      """
      name = backend.name()
      if name in bdb.backends:
          raise ValueError('Backend already registered: %s' % (name,))
      with bdb.savepoint():
          backend.register(bdb)
          bdb.backends[name] = backend
  
  def bayesdb_deregister_backend(bdb, backend):
      """Deregister `backend`, which must have been registered in `bdb`."""
      name = backend.name()
      assert name in bdb.backends
      assert bdb.backends[name] == backend
      del bdb.backends[name]
  
  def bayesdb_backend_version(bdb, mm_name):
      cursor = bdb.sql_execute('''
          SELECT version FROM bayesdb_backend WHERE name = ?
      ''', (mm_name,))
      return cursor_value(cursor, nullok=True)
  
  class BayesDB_Backend(object):
      """BayesDB backend interface.
  
      Subclasses of :class:`BayesDB_Backend` implement the
      functionality needed by probabilistic BQL queries to sample from
      and inquire about the posterior distribution of a generative model
      conditioned on data in a table.  Instances of subclasses of
      `BayesDB_Backend` contain any in-memory state associated with
      the backend in the database.
      """
  
      def name(self):
          """Return the name of the backend as a str."""
          raise NotImplementedError
  
      def register(self, bdb):
          """Install any state needed for the backend in `bdb`.
  
          Called by :func:`bayeslite.bayesdb_register_backend`.
  
          Normally this will create SQL tables if necessary.
          """
          raise NotImplementedError
  
      def set_multiprocess(self, switch):
          """Switch between multiprocessing and single processing.
  
          The boolean variable `switch` toggles between single (`False`) and multi
          (`True`) processing, if the choice is available, and otherwise ignores
          the request.
          """
          raise NotImplementedError
  
      def create_generator(self, bdb, table, schema, **kwargs):
          """Create a generator for a table with the given schema.
  
          Called when executing ``CREATE GENERATOR``.
  
          Must parse `schema
  
  [... the rest is omitted ...]
files: src/backend.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  """Backend interface.
  
  To be used as a modeling and inference backend in a :class:`bayeslite.BayesDB`
  handle, a backends must first be registered with
  :func:`bayesdb_register_backend`.
  
  The CGPM backend is registered by default, but we can suppress
  that for illustration::
  
     import bayeslite
     from bayeslite.backends.cgpm_backend import CGPM_Backend
  
     bdb = bayeslite.bayesdb_open(pathname='foo.bdb', builtin_backends=False)
     backend = CGPM_Backend(cgpm_registry=dict(), multiprocess=False)
     bayeslite.bayesdb_register_backend(bdb, backend)
  
  Then you can model a table and query the probable implications of the data in
  the table::
  
     bdb.execute('create population p for t with schema(guess stattypes for (*))')
     bdb.execute('create generator p_cc for t using cgpm;')
     bdb.execute('initialize 10 models for t_cc')
     bdb.execute('analyze t_cc for 10 iterations')
     for x in bdb.execute('estimate pairwise dependence probability from t_cc'):
         print x
  """
  
  from bayeslite.util import cursor_value
  
  builtin_backends = []
  builtin_backend_names = set()
  
  def bayesdb_builtin_backend(backend):
      name = backend.name()
      assert name not in builtin_backend_names
      builtin_backends.append(backend)
      builtin_backend_names.add(name)
  
  def bayesdb_register_builtin_backends(bdb):
      """Register all builtin backends in `bdb`."""
      for backend in builtin_backends:
          bayesdb_register_backend(bdb, backend)
  
  def bayesdb_register_backend(bdb, backend):
      """Register `backend` in `bdb`, creating any necessary tables.
  
      `backend` must not already be registered in any BayesDB, nor any
      backend by the same name.
      """
      name = backend.name()
      if name in bdb.backends:
          raise ValueError('Backend already registered: %s' % (name,))
      with bdb.savepoint():
          backend.register(bdb)
          bdb.backends[name] = backend
  
  def bayesdb_deregister_backend(bdb, backend):
      """Deregister `backend`, which must have been registered in `bdb`."""
      name = backend.name()
      assert name in bdb.backends
      assert bdb.backends[name] == backend
      del bdb.backends[name]
  
  def bayesdb_backend_version(bdb, mm_name):
      cursor = bdb.sql_execute('''
          SELECT version FROM bayesdb_backend WHERE name = ?
      ''', (mm_name,))
      return cursor_value(cursor, nullok=True)
  
  class BayesDB_Backend(object):
      """BayesDB backend interface.
  
      Subclasses of :class:`BayesDB_Backend` implement the
      functionality needed by probabilistic BQL queries to sample from
      and inquire about the posterior distribution of a generative model
      conditioned on data in a table.  Instances of subclasses of
      `BayesDB_Backend` contain any in-memory state associated with
      the backend in the database.
      """
  
      def name(self):
          """Return the name of the backend as a str."""
          raise NotImplementedError
  
      def register(self, bdb):
          """Install any state needed for the backend in `bdb`.
  
          Called by :func:`bayeslite.bayesdb_register_backend`.
  
          Normally this will create SQL tables if necessary.
          """
          raise NotImplementedError
  
      def set_multiprocess(self, switch):
          """Switch between multiprocessing and single processing.
  
          The boolean variable `switch` toggles between single (`False`) and multi
          (`True`) processing, if the choice is available, and otherwise ignores
          the request.
          """
          raise NotImplementedError
  
      def create_generator(self, bdb, table, schema, **kwargs):
          """Create a generator for a table with the given schema.
  
          Called when executing ``CREATE GENERATOR``.
  
          Must parse `schema
  
  [... the rest is omitted ...]
files: src/backend.py
files: /*
   *  Copyright (c) 2010-2016, MIT Probabilistic Computing Project
   *
   *  Licensed under the Apache License, Version 2.0 (the "License");
   *  you may not use this file except in compliance with the License.
   *  You may obtain a copy of the License at
   *
   *      http://www.apache.org/licenses/LICENSE-2.0
   *
   *  Unless required by applicable law or agreed to in writing, software
   *  distributed under the License is distributed on an "AS IS" BASIS,
   *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   *  See the License for the specific language governing permissions and
   *  limitations under the License.
   */
  
  bql(start)              ::= phrases(phrases).
  
  phrases(none)           ::= .
  phrases(some)           ::= phrases(phrases) phrase_opt(phrase) T_SEMI.
  phrase_opt(none)        ::= .
  phrase_opt(some)        ::= phrase(phrase).
  phrase(command)         ::= command(c).
  phrase(query)           ::= query(q).
  
  /*
   * Transactions
   */
  command(begin)          ::= K_BEGIN.
  command(rollback)       ::= K_ROLLBACK.
  command(commit)         ::= K_COMMIT.
  
  /* XXX Need database names.      */
  
  /*
   * SQL Data Definition Language subset
   */
  command(createtab_as)   ::= K_CREATE temp_opt(temp) K_TABLE
                                  ifnotexists(ifnotexists)
                                  table_name(name) K_AS query(query).
  command(createtab_csv)   ::= K_CREATE temp_opt(temp) K_TABLE
                                  ifnotexists(ifnotexists)
                                  table_name(name) K_FROM pathname(csv).
  command(droptab)        ::= K_DROP K_TABLE ifexists(ifexists) table_name(name).
  command(altertab)       ::= K_ALTER K_TABLE table_name(table)
                                  altertab_cmds(cmds).
  
  altertab_cmds(one)      ::= altertab_cmd(cmd).
  altertab_cmds(many)     ::= altertab_cmds(cmds) T_COMMA altertab_cmd(cmd).
  
  altertab_cmd(renametab) ::= K_RENAME K_TO table_name(name).
  altertab_cmd(renamecol) ::= K_RENAME k_column_opt column_name(old)
                                  K_TO column_name(new).
  
  k_column_opt            ::= .
  k_column_opt            ::= K_COLUMN.
  
  pathname(p)             ::= L_STRING(name).
  
  /*
   * BQL Model Definition Language
   */
  command(guess_schema)   ::= K_GUESS K_SCHEMA K_FOR table_name(table).
  
  command(create_pop)     ::= K_CREATE K_POPULATION ifnotexists(ifnotexists)
                                  population_name(name)
                                  K_FOR table_name(table)
                                  with_schema_opt
                                  T_LROUND|T_LCURLY pop_schema(schema)
                                          T_RROUND|T_RCURLY.
  command(drop_pop)       ::= K_DROP K_POPULATION ifexists(ifexists)
                                  population_name(name).
  
  with_schema_opt ::= .
  with_schema_opt ::= K_WITH K_SCHEMA.
  
  /*
   * XXX alter population xyz
   *      rename to pqr
   *      add variable ...
   *      remove variable ...
   *      set statistical type ...
   */
  
  command(alterpop)  ::= K_ALTER K_POPULATION
                                  population_name(population) alterpop_cmds(cmds).
  
  alterpop_cmds(one)      ::= alterpop_cmd(cmd).
  alterpop_cmds(many)     ::= alterpop_cmds(cmds) T_COMMA alterpop_cmd(cmd).
  
  alterpop_cmd(stattype)  ::= K_SET K_STATTYPES|K_STATTYPE
                                  K_FOR|K_OF pop_columns(cols)
                                  K_TO stattype(stattype).
  
  alterpop_cmd(addvar)    ::= K_ADD K_VARIABLE column_name(col) stattype_opt(st).
  
  pop_schema(one)         ::= pop_clause(cl).
  pop_schema(many)        ::= pop_schema(schema) T_SEMI pop_clause(cl).
  
  pop_clause(empty)       ::= .
  pop_clause(column)      ::= column_name(col) stattype(st).
  pop_clause(stattype)    ::= K_SET K_STATTYPES|K_STATTYPE
                                  K_OF pop_columns(cols)
                                  K_TO stattype(st).
  pop_clause(ignore)      ::= K_IGNORE pop_columns(cols).
  pop_clause(guess)       ::= K_GUESS stattypes_of_opt pop_columns_guess(cols).
  
  stattype_opt(none)      ::= .
  stattype_opt(one
  
  [... the rest is omitted ...]
files: src/grammar.y
files: /*
   *  Copyright (c) 2010-2016, MIT Probabilistic Computing Project
   *
   *  Licensed under the Apache License, Version 2.0 (the "License");
   *  you may not use this file except in compliance with the License.
   *  You may obtain a copy of the License at
   *
   *      http://www.apache.org/licenses/LICENSE-2.0
   *
   *  Unless required by applicable law or agreed to in writing, software
   *  distributed under the License is distributed on an "AS IS" BASIS,
   *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   *  See the License for the specific language governing permissions and
   *  limitations under the License.
   */
  
  bql(start)              ::= phrases(phrases).
  
  phrases(none)           ::= .
  phrases(some)           ::= phrases(phrases) phrase_opt(phrase) T_SEMI.
  phrase_opt(none)        ::= .
  phrase_opt(some)        ::= phrase(phrase).
  phrase(command)         ::= command(c).
  phrase(query)           ::= query(q).
  
  /*
   * Transactions
   */
  command(begin)          ::= K_BEGIN.
  command(rollback)       ::= K_ROLLBACK.
  command(commit)         ::= K_COMMIT.
  
  /* XXX Need database names.      */
  
  /*
   * SQL Data Definition Language subset
   */
  command(createtab_as)   ::= K_CREATE temp_opt(temp) K_TABLE
                                  ifnotexists(ifnotexists)
                                  table_name(name) K_AS query(query).
  command(createtab_csv)   ::= K_CREATE temp_opt(temp) K_TABLE
                                  ifnotexists(ifnotexists)
                                  table_name(name) K_FROM pathname(csv).
  command(droptab)        ::= K_DROP K_TABLE ifexists(ifexists) table_name(name).
  command(altertab)       ::= K_ALTER K_TABLE table_name(table)
                                  altertab_cmds(cmds).
  
  altertab_cmds(one)      ::= altertab_cmd(cmd).
  altertab_cmds(many)     ::= altertab_cmds(cmds) T_COMMA altertab_cmd(cmd).
  
  altertab_cmd(renametab) ::= K_RENAME K_TO table_name(name).
  altertab_cmd(renamecol) ::= K_RENAME k_column_opt column_name(old)
                                  K_TO column_name(new).
  
  k_column_opt            ::= .
  k_column_opt            ::= K_COLUMN.
  
  pathname(p)             ::= L_STRING(name).
  
  /*
   * BQL Model Definition Language
   */
  command(guess_schema)   ::= K_GUESS K_SCHEMA K_FOR table_name(table).
  
  command(create_pop)     ::= K_CREATE K_POPULATION ifnotexists(ifnotexists)
                                  population_name(name)
                                  K_FOR table_name(table)
                                  with_schema_opt
                                  T_LROUND|T_LCURLY pop_schema(schema)
                                          T_RROUND|T_RCURLY.
  command(drop_pop)       ::= K_DROP K_POPULATION ifexists(ifexists)
                                  population_name(name).
  
  with_schema_opt ::= .
  with_schema_opt ::= K_WITH K_SCHEMA.
  
  /*
   * XXX alter population xyz
   *      rename to pqr
   *      add variable ...
   *      remove variable ...
   *      set statistical type ...
   */
  
  command(alterpop)  ::= K_ALTER K_POPULATION
                                  population_name(population) alterpop_cmds(cmds).
  
  alterpop_cmds(one)      ::= alterpop_cmd(cmd).
  alterpop_cmds(many)     ::= alterpop_cmds(cmds) T_COMMA alterpop_cmd(cmd).
  
  alterpop_cmd(stattype)  ::= K_SET K_STATTYPES|K_STATTYPE
                                  K_FOR|K_OF pop_columns(cols)
                                  K_TO stattype(stattype).
  
  alterpop_cmd(addvar)    ::= K_ADD K_VARIABLE column_name(col) stattype_opt(st).
  
  pop_schema(one)         ::= pop_clause(cl).
  pop_schema(many)        ::= pop_schema(schema) T_SEMI pop_clause(cl).
  
  pop_clause(empty)       ::= .
  pop_clause(column)      ::= column_name(col) stattype(st).
  pop_clause(model)       ::= K_MODEL pop_columns(cols) K_AS stattype(st).
  pop_clause(ignore)      ::= K_IGNORE pop_columns(cols).
  pop_clause(guess)       ::= K_GUESS stattypes_for_opt pop_columns_guess(cols).
  
  stattype_opt(none)      ::= .
  stattype_opt(one)       ::= stattype(st).
  
  stattype(st)            ::
  
  [... the rest is omitted ...]
files: src/grammar.y
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  """BQL parser front end."""
  
  import StringIO
  
  import bayeslite.ast as ast
  import bayeslite.grammar as grammar
  import bayeslite.scan as scan
  from bayeslite.exception import BQLParseError
  
  def parse_bql_phrases(scanner):
      semantics = BQLSemantics()
      parser = grammar.Parser(semantics)
      while not semantics.failed:
          token = scanner.read()
          semantics.context.append(token)
          if token[0] == -1:      # error
              semantics.syntax_error(token)
          else:
              if token[0] == 0:   # EOF
                  # Implicit ; at EOF.
                  parser.feed((grammar.T_SEMI, ''))
              parser.feed(token)
          if semantics.phrase is not None:
              phrase = semantics.phrase
              semantics.phrase = None
              if 0 < len(semantics.errors):
                  # Keep parsing in order to detect more errors, but
                  # don't yield any broken phrases in case the caller
                  # will try to process them before we finish parsing
                  # the whole thing.
                  continue
              if 0 < scanner.n_numpar:
                  n_numpar = scanner.n_numpar
                  nampar_map = scanner.nampar_map
                  yield ast.Parametrized(phrase, n_numpar, nampar_map)
              else:
                  yield phrase
          if token[0] == 0:       # EOF
              break
      if 0 < len(semantics.errors):
          raise BQLParseError(semantics.errors)
      if semantics.failed:
          raise BQLParseError(['parse failed mysteriously!'])
  
  def parse_bql_string_pos(string):
      """Yield ``(phrase, pos)`` for each BQL phrase in `string`.
  
      `phrase` is the parsed AST.  `pos` is zero-based index of the code
      point at which `phrase` starts.
      """
      scanner = scan.BQLScanner(StringIO.StringIO(string), '(string)')
      phrases = parse_bql_phrases(scanner)
      # XXX Don't dig out internals of scanner: fix plex to have a
      # public API for finding the current position.
      return ((phrase, scanner.cur_pos) for phrase in phrases)
  
  def parse_bql_string_pos_1(string):
      """Return ``(phrase, pos)`` for the first BQL phrase in `string`.
  
      May not report parse errors afterward.
      """
      for phrase, pos in parse_bql_string_pos(string):
          return (phrase, pos)
      return None
  
  def parse_bql_string(string):
      """Yield each parsed BQL phrase AST in `string`."""
      return (phrase for phrase, _pos in parse_bql_string_pos(string))
  
  def bql_string_complete_p(string):
      """True if `string` has at least one complete BQL phrase or error.
  
      False if empty or if the last BQL phrase is incomplete.
      """
      scanner = scan.BQLScanner(StringIO.StringIO(string), '(string)')
      semantics = BQLSemantics()
      parser = grammar.Parser(semantics)
      nonsemi = False
      while not semantics.failed:
          token = scanner.read()
          if token[0] == -1:      # error
              # Say it's complete so the caller will try to parse it and
              # choke on the error.
              return True
          elif token[0] == 0:
              # EOF.  Hope we have a complete phrase.
              break
          elif token[0] != grammar.T_SEMI:
              # Got a non-semicolon token.  Clear any previous phrase,
              # if we had one.
              nonsemi = True
              semantics.phrase = None
          parser.feed(token)
      if 0 < len(semantics.errors):
          return True
      if semantics.failed:
          return True
      return (not nonsemi) or (semantics.phrase is not None)
  
  class BQLSemantics(object):
      def __init__(self):
          self.phrase = None
          self.context = []
          self.errors = []
          self.failed = False
  
      def accept(self):
          pass
      def parse_failed(self):
         
  
  [... the rest is omitted ...]
files: src/parse.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  """BQL parser front end."""
  
  import StringIO
  
  import bayeslite.ast as ast
  import bayeslite.grammar as grammar
  import bayeslite.scan as scan
  from bayeslite.exception import BQLParseError
  
  def parse_bql_phrases(scanner):
      semantics = BQLSemantics()
      parser = grammar.Parser(semantics)
      while not semantics.failed:
          token = scanner.read()
          semantics.context.append(token)
          if token[0] == -1:      # error
              semantics.syntax_error(token)
          else:
              if token[0] == 0:   # EOF
                  # Implicit ; at EOF.
                  parser.feed((grammar.T_SEMI, ''))
              parser.feed(token)
          if semantics.phrase is not None:
              phrase = semantics.phrase
              semantics.phrase = None
              if 0 < len(semantics.errors):
                  # Keep parsing in order to detect more errors, but
                  # don't yield any broken phrases in case the caller
                  # will try to process them before we finish parsing
                  # the whole thing.
                  continue
              if 0 < scanner.n_numpar:
                  n_numpar = scanner.n_numpar
                  nampar_map = scanner.nampar_map
                  yield ast.Parametrized(phrase, n_numpar, nampar_map)
              else:
                  yield phrase
          if token[0] == 0:       # EOF
              break
      if 0 < len(semantics.errors):
          raise BQLParseError(semantics.errors)
      if semantics.failed:
          raise BQLParseError(['parse failed mysteriously!'])
  
  def parse_bql_string_pos(string):
      """Yield ``(phrase, pos)`` for each BQL phrase in `string`.
  
      `phrase` is the parsed AST.  `pos` is zero-based index of the code
      point at which `phrase` starts.
      """
      scanner = scan.BQLScanner(StringIO.StringIO(string), '(string)')
      phrases = parse_bql_phrases(scanner)
      # XXX Don't dig out internals of scanner: fix plex to have a
      # public API for finding the current position.
      return ((phrase, scanner.cur_pos) for phrase in phrases)
  
  def parse_bql_string_pos_1(string):
      """Return ``(phrase, pos)`` for the first BQL phrase in `string`.
  
      May not report parse errors afterward.
      """
      for phrase, pos in parse_bql_string_pos(string):
          return (phrase, pos)
      return None
  
  def parse_bql_string(string):
      """Yield each parsed BQL phrase AST in `string`."""
      return (phrase for phrase, _pos in parse_bql_string_pos(string))
  
  def bql_string_complete_p(string):
      """True if `string` has at least one complete BQL phrase or error.
  
      False if empty or if the last BQL phrase is incomplete.
      """
      scanner = scan.BQLScanner(StringIO.StringIO(string), '(string)')
      semantics = BQLSemantics()
      parser = grammar.Parser(semantics)
      nonsemi = False
      while not semantics.failed:
          token = scanner.read()
          if token[0] == -1:      # error
              # Say it's complete so the caller will try to parse it and
              # choke on the error.
              return True
          elif token[0] == 0:
              # EOF.  Hope we have a complete phrase.
              break
          elif token[0] != grammar.T_SEMI:
              # Got a non-semicolon token.  Clear any previous phrase,
              # if we had one.
              nonsemi = True
              semantics.phrase = None
          parser.feed(token)
      if 0 < len(semantics.errors):
          return True
      if semantics.failed:
          return True
      return (not nonsemi) or (semantics.phrase is not None)
  
  class BQLSemantics(object):
      def __init__(self):
          self.phrase = None
          self.context = []
          self.errors = []
          self.failed = False
  
      def accept(self):
          pass
      def parse_failed(self):
         
  
  [... the rest is omitted ...]
files: src/parse.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import StringIO
  import apsw
  import pytest
  import struct
  
  import bayeslite
  import bayeslite.ast as ast
  import bayeslite.compiler as compiler
  import bayeslite.core as core
  import bayeslite.guess as guess
  import bayeslite.backends.troll_rng as troll
  import bayeslite.parse as parse
  
  from bayeslite.exception import BQLError
  from bayeslite.math_util import relerr
  from bayeslite.backends.cgpm_backend import CGPM_Backend
  from bayeslite.util import cursor_value
  
  import test_core
  import test_csv
  
  from stochastic import stochastic
  
  def bql2sql(string, setup=None):
      with bayeslite.bayesdb_open(':memory:') as bdb:
          test_core.t1_schema(bdb)
          test_core.t1_data(bdb)
          bdb.execute('''
              create population p1 for t1 (
                  id ignore;
                  label categorical;
                  age numerical;
                  weight numerical
              )
          ''')
          if setup is not None:
              setup(bdb)
          phrases = parse.parse_bql_string(string)
          out = compiler.Output(0, {}, ())
          for phrase in phrases:
              assert ast.is_query(phrase)
              compiler.compile_query(bdb, phrase, out)
              out.write(';')
          return out.getvalue()
  
  # XXX Kludgey mess.  Please reorganize.
  def bql2sqlparam(string):
      with bayeslite.bayesdb_open(':memory:') as bdb:
          test_core.t1_schema(bdb)
          test_core.t1_data(bdb)
          bdb.execute('''
              create population p1 for t1 (
                  id ignore;
                  label categorical;
                  age numerical;
                  weight numerical
              )
          ''')
          phrases = parse.parse_bql_string(string)
          out0 = StringIO.StringIO()
          for phrase in phrases:
              out = None
              if isinstance(phrase, ast.Parametrized):
                  bindings = (None,) * phrase.n_numpar
                  out = compiler.Output(phrase.n_numpar, phrase.nampar_map,
                      bindings)
                  phrase = phrase.phrase
              else:
                  out = StringIO.StringIO()
              assert ast.is_query(phrase)
              compiler.compile_query(bdb, phrase, out)
              # XXX Do something about the parameters.
              out0.write(out.getvalue())
              out0.write(';')
          return out0.getvalue()
  
  def bql_execute(bdb, string, bindings=()):
      return map(tuple, bdb.execute(string, bindings))
  
  def empty(cursor):
      assert cursor is not None
      assert cursor.description is not None
      assert len(cursor.description) == 0
      with pytest.raises(StopIteration):
          cursor.next()
  
  def test_trivial_population():
      with test_csv.bayesdb_csv_file(test_csv.csv_data) as (bdb, fname):
          with open(fname, 'rU') as f:
              bayeslite.bayesdb_read_csv(bdb, 't', f, header=True, create=True)
          # XXX if (not) exists
          bdb.execute('''
              create population p for t (
                  guess stattypes of (*);
                  age numerical
              )
          ''')
          bdb.execute('drop population p')
  
  def test_population_invalid_numerical():
      with test_csv.bayesdb_csv_file(test_csv.csv_data) as (bdb, fname):
          with open(fname, 'rU') as f:
              bayeslite.bayesdb_read_csv(bdb, 't', f, header=True, create=True)
          with pytest.raises(BQLError):
              bdb.execute('''
                  create population p for t (
                      guess stattypes of (*);
                      gender numerical
                  )
              ''')
  
  def test_population_invalid_numerical_alterpop_addvar():
      with test_csv.bayesdb_csv_file(test_csv.csv_data) as (bdb, fname):
          with open(fname, 'rU') as f:
              bayeslite.bayesdb_read_csv(bdb, 't', f,
  
  [... the rest is omitted ...]
files: tests/test_bql.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import StringIO
  import apsw
  import pytest
  import struct
  
  import bayeslite
  import bayeslite.ast as ast
  import bayeslite.compiler as compiler
  import bayeslite.core as core
  import bayeslite.guess as guess
  import bayeslite.backends.troll_rng as troll
  import bayeslite.parse as parse
  
  from bayeslite.exception import BQLError
  from bayeslite.math_util import relerr
  from bayeslite.backends.cgpm_backend import CGPM_Backend
  from bayeslite.util import cursor_value
  
  import test_core
  import test_csv
  
  from stochastic import stochastic
  
  def bql2sql(string, setup=None):
      with bayeslite.bayesdb_open(':memory:') as bdb:
          test_core.t1_schema(bdb)
          test_core.t1_data(bdb)
          bdb.execute('''
              create population p1 for t1 (
                  id ignore;
                  label categorical;
                  age numerical;
                  weight numerical
              )
          ''')
          if setup is not None:
              setup(bdb)
          phrases = parse.parse_bql_string(string)
          out = compiler.Output(0, {}, ())
          for phrase in phrases:
              assert ast.is_query(phrase)
              compiler.compile_query(bdb, phrase, out)
              out.write(';')
          return out.getvalue()
  
  # XXX Kludgey mess.  Please reorganize.
  def bql2sqlparam(string):
      with bayeslite.bayesdb_open(':memory:') as bdb:
          test_core.t1_schema(bdb)
          test_core.t1_data(bdb)
          bdb.execute('''
              create population p1 for t1 (
                  id ignore;
                  label categorical;
                  age numerical;
                  weight numerical
              )
          ''')
          phrases = parse.parse_bql_string(string)
          out0 = StringIO.StringIO()
          for phrase in phrases:
              out = None
              if isinstance(phrase, ast.Parametrized):
                  bindings = (None,) * phrase.n_numpar
                  out = compiler.Output(phrase.n_numpar, phrase.nampar_map,
                      bindings)
                  phrase = phrase.phrase
              else:
                  out = StringIO.StringIO()
              assert ast.is_query(phrase)
              compiler.compile_query(bdb, phrase, out)
              # XXX Do something about the parameters.
              out0.write(out.getvalue())
              out0.write(';')
          return out0.getvalue()
  
  def bql_execute(bdb, string, bindings=()):
      return map(tuple, bdb.execute(string, bindings))
  
  def empty(cursor):
      assert cursor is not None
      assert cursor.description is not None
      assert len(cursor.description) == 0
      with pytest.raises(StopIteration):
          cursor.next()
  
  def test_trivial_population():
      with test_csv.bayesdb_csv_file(test_csv.csv_data) as (bdb, fname):
          with open(fname, 'rU') as f:
              bayeslite.bayesdb_read_csv(bdb, 't', f, header=True, create=True)
          # XXX if (not) exists
          bdb.execute('''
              create population p for t (
                  guess stattypes for (*);
                  age numerical
              )
          ''')
          bdb.execute('drop population p')
  
  def test_population_invalid_numerical():
      with test_csv.bayesdb_csv_file(test_csv.csv_data) as (bdb, fname):
          with open(fname, 'rU') as f:
              bayeslite.bayesdb_read_csv(bdb, 't', f, header=True, create=True)
          with pytest.raises(BQLError):
              bdb.execute('''
                  create population p for t (
                      guess stattypes for (*);
                      gender numerical
                  )
              ''')
  
  def test_population_invalid_numerical_alterpop_addvar():
      with test_csv.bayesdb_csv_file(test_csv.csv_data) as (bdb, fname):
          with open(fname, 'rU') as f:
              bayeslite.bayesdb_read_csv(bdb, 't', f,
  
  [... the rest is omitted ...]
files: tests/test_bql.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  from __future__ import division
  
  import StringIO
  import contextlib
  import math
  import time
  
  import numpy as np
  import pytest
  
  from cgpm.cgpm import CGpm
  from cgpm.dummy.barebones import BareBonesCGpm
  from cgpm.dummy.piecewise import PieceWise
  from cgpm.dummy.trollnormal import TrollNormal
  from cgpm.factor.factor import FactorAnalysis
  from cgpm.regressions.linreg import LinearRegression
  from cgpm.utils import general as gu
  
  from bayeslite import bayesdb_nullify
  from bayeslite import bayesdb_open
  from bayeslite import bayesdb_read_csv
  from bayeslite import bayesdb_register_backend
  from bayeslite.core import bayesdb_get_generator
  from bayeslite.core import bayesdb_get_population
  from bayeslite.exception import BQLError
  from bayeslite.backends.cgpm_backend import CGPM_Backend
  from bayeslite.util import cursor_value
  
  import test_csv
  
  @contextlib.contextmanager
  def cgpm_smoke_bdb():
      with bayesdb_open(':memory:', builtin_backends=False) as bdb:
          registry = {
              'piecewise': PieceWise,
          }
          bayesdb_register_backend(
              bdb, CGPM_Backend(registry, multiprocess=0))
  
          bdb.sql_execute('CREATE TABLE t (Output, cat, Input)')
          for i in xrange(3):
              for j in xrange(3):
                  for k in xrange(3):
                      output = i + j/(k + 1)
                      cat = -1 if (i + j*k) % 2 else +1
                      input = (i*j - k)**2
                      if i % 2:
                          output = None
                      if j % 2:
                          cat = None
                      if k % 2:
                          input = None
                      bdb.sql_execute('''
                          INSERT INTO t (output, cat, input) VALUES (?, ?, ?)
                      ''', (output, cat, input))
  
          bdb.execute('''
              CREATE POPULATION p FOR t WITH SCHEMA(
                  output  NUMERICAL;
                  input   NUMERICAL;
                  cat     NOMINAL;
              )
          ''')
  
          yield bdb
  
  @contextlib.contextmanager
  def cgpm_dummy_satellites_bdb():
      with bayesdb_open(':memory:', builtin_backends=False) as bdb:
          bdb.sql_execute('''
              CREATE TABLE satellites_ucs (
                  apogee,
                  class_of_orbit,
                  country_of_operator,
                  launch_mass,
                  perigee,
                  period
          )''')
          for l, f in [
              ('geo', lambda x, y: x + y**2),
              ('leo', lambda x, y: math.sin(x + y)),
              (None, lambda x, y: x + y**2),
              (None, lambda x, y: math.sin(x + y)),
          ]:
              for x in xrange(5):
                  for y in xrange(5):
                      countries = ['US', 'Russia', 'China', 'Bulgaria']
                      country = countries[bdb._np_prng.randint(0, len(countries))]
                      mass = bdb._np_prng.normal(1000, 50)
                      bdb.sql_execute('''
                          INSERT INTO satellites_ucs
                              (country_of_operator, launch_mass, class_of_orbit,
                                  apogee, perigee, period)
                              VALUES (?,?,?,?,?,?)
                      ''', (country, mass, l, x, y, f(x, y)))
          yield bdb
  
  def test_cgpm_no_empty_categories():
      with cgpm_smoke_bdb() as bdb:
          bdb.sql_execute('CREATE TABLE f (a, b, c);')
          rows = [['', '\'\'', 'nan'], [1.1, 3, ''], ['""""', 1, 1]]
          for row in rows:
              b
  
  [... the rest is omitted ...]
files: tests/test_cgpm.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  from __future__ import division
  
  import StringIO
  import contextlib
  import math
  import time
  
  import numpy as np
  import pytest
  
  from cgpm.cgpm import CGpm
  from cgpm.dummy.barebones import BareBonesCGpm
  from cgpm.dummy.piecewise import PieceWise
  from cgpm.dummy.trollnormal import TrollNormal
  from cgpm.factor.factor import FactorAnalysis
  from cgpm.regressions.linreg import LinearRegression
  from cgpm.utils import general as gu
  
  from bayeslite import bayesdb_nullify
  from bayeslite import bayesdb_open
  from bayeslite import bayesdb_read_csv
  from bayeslite import bayesdb_register_backend
  from bayeslite.core import bayesdb_get_generator
  from bayeslite.core import bayesdb_get_population
  from bayeslite.exception import BQLError
  from bayeslite.backends.cgpm_backend import CGPM_Backend
  from bayeslite.util import cursor_value
  
  import test_csv
  
  @contextlib.contextmanager
  def cgpm_smoke_bdb():
      with bayesdb_open(':memory:', builtin_backends=False) as bdb:
          registry = {
              'piecewise': PieceWise,
          }
          bayesdb_register_backend(
              bdb, CGPM_Backend(registry, multiprocess=0))
  
          bdb.sql_execute('CREATE TABLE t (Output, cat, Input)')
          for i in xrange(3):
              for j in xrange(3):
                  for k in xrange(3):
                      output = i + j/(k + 1)
                      cat = -1 if (i + j*k) % 2 else +1
                      input = (i*j - k)**2
                      if i % 2:
                          output = None
                      if j % 2:
                          cat = None
                      if k % 2:
                          input = None
                      bdb.sql_execute('''
                          INSERT INTO t (output, cat, input) VALUES (?, ?, ?)
                      ''', (output, cat, input))
  
          bdb.execute('''
              CREATE POPULATION p FOR t WITH SCHEMA(
                  MODEL output, input AS NUMERICAL;
                  MODEL cat AS CATEGORICAL
              )
          ''')
  
          yield bdb
  
  @contextlib.contextmanager
  def cgpm_dummy_satellites_bdb():
      with bayesdb_open(':memory:', builtin_backends=False) as bdb:
          bdb.sql_execute('''
              CREATE TABLE satellites_ucs (
                  apogee,
                  class_of_orbit,
                  country_of_operator,
                  launch_mass,
                  perigee,
                  period
          )''')
          for l, f in [
              ('geo', lambda x, y: x + y**2),
              ('leo', lambda x, y: math.sin(x + y)),
              (None, lambda x, y: x + y**2),
              (None, lambda x, y: math.sin(x + y)),
          ]:
              for x in xrange(5):
                  for y in xrange(5):
                      countries = ['US', 'Russia', 'China', 'Bulgaria']
                      country = countries[bdb._np_prng.randint(0, len(countries))]
                      mass = bdb._np_prng.normal(1000, 50)
                      bdb.sql_execute('''
                          INSERT INTO satellites_ucs
                              (country_of_operator, launch_mass, class_of_orbit,
                                  apogee, perigee, period)
                              VALUES (?,?,?,?,?,?)
                      ''', (country, mass, l, x, y, f(x, y)))
          yield bdb
  
  def test_cgpm_no_empty_categories():
      with cgpm_smoke_bdb() as bdb:
          bdb.sql_execute('CREATE TABLE f (a, b, c);')
          rows = [['', '\'\'', 'nan'], [1.1, 3, ''], ['""""', 1, 1]]
          for row in rows:
              bdb.sql
  
  [... the rest is omitted ...]
files: tests/test_cgpm.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import contextlib
  import pytest
  
  import bayeslite.core
  
  from bayeslite import bayesdb_register_backend
  from bayeslite.exception import BQLError
  from bayeslite.math_util import abserr
  from bayeslite.backends.cgpm_backend import CGPM_Backend
  
  from test_cgpm import cgpm_dummy_satellites_bdb
  
  
  @contextlib.contextmanager
  def cgpm_dummy_satellites_pop_bdb():
      with cgpm_dummy_satellites_bdb() as bdb:
          bdb.execute('''
              create population satellites for satellites_ucs with schema(
                  apogee numerical;
                  class_of_orbit categorical;
                  country_of_operator categorical;
                  launch_mass numerical;
                  perigee numerical;
                  period numerical
              )
          ''')
          backend = CGPM_Backend(dict(), multiprocess=0)
          bayesdb_register_backend(bdb, backend)
          yield bdb
  
  
  def test_cgpm_alter_basic():
      with cgpm_dummy_satellites_pop_bdb() as bdb:
          bdb.execute('''
              create generator g0 for satellites using cgpm(
                  subsample 10
              );
          ''')
          bdb.execute('initialize 4 models for g0')
  
          population_id = bayeslite.core.bayesdb_get_population(
              bdb, 'satellites')
          generator_id = bayeslite.core.bayesdb_get_generator(
              bdb, population_id, 'g0')
  
          # Make all variables independent in models 0-1.
          bdb.execute('''
              alter generator g0 models (0,1)
                  ensure variables * independent;
          ''')
          dependencies = bdb.execute('''
              estimate dependence probability from pairwise variables of
              satellites modeled by g0 using models 0-1;
          ''').fetchall()
          for _, var0, var1, value in dependencies:
              assert value == int(var0 == var1)
  
          # Make all variables dependent in all models.
          bdb.execute('''
              alter generator g0
                  ensure variables * dependent;
          ''')
          dependencies = bdb.execute('''
              estimate dependence probability from pairwise variables of
              satellites
          ''').fetchall()
          for _, _, _, value in dependencies:
              assert value == 1
  
          # Make all variables independent again.
          bdb.execute('''
              alter generator g0
                  ensure variables * independent;
          ''')
  
          # Move apogee, perigee->view(period), launch_mass->view(class_of_orbit)
          # All other variables should remain independent.
          bdb.execute('''
              alter generator g0
                  ensure variables (apogee, period, perigee) in context of period,
                  ensure variable launch_mass in context of class_of_orbit;
          ''')
          dependencies = bdb.execute('''
              estimate dependence probability from pairwise variables of
              satellites
          ''').fetchall()
          blocks = [
              ['apogee','perigee','period'],
              ['launch_mass', 'class_of_orbit']
          ]
          for _, var0, var1, value in dependencies:
              if var0 in blocks[0] and var1 in blocks[0]:
                  assert value == 1.
              elif var0 in blocks[1] and var1 in blocks[1]:
                  assert value == 1.
              elif var0 == var1:
                  assert value == 1.
              else:
                  assert value == 0.
  
          # Move period to a singleton in model 3.
          bdb.execute('''
              alter generator g0 model (3)
                  ensure variable period in singleton view
          ''')
          dependencies = bdb.execute('''
              estimate dependence probability with period from variables of
              satellites modeled by g0 using models 0,1,2
          ''')
          for _, var0, value in dependencies:
              if var0 in blocks[0]:
                  assert value == 1
              else:
                  assert value == 0
          dependencies = bdb.execute('''
              estimate dependence probability with period from variables of
              satellites
  
  [... the rest is omitted ...]
files: tests/test_cgpm_alter.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import contextlib
  import pytest
  
  import bayeslite.core
  
  from bayeslite import bayesdb_register_backend
  from bayeslite.exception import BQLError
  from bayeslite.math_util import abserr
  from bayeslite.backends.cgpm_backend import CGPM_Backend
  
  from test_cgpm import cgpm_dummy_satellites_bdb
  
  
  @contextlib.contextmanager
  def cgpm_dummy_satellites_pop_bdb():
      with cgpm_dummy_satellites_bdb() as bdb:
          bdb.execute('''
              create population satellites for satellites_ucs with schema(
                  model apogee as numerical;
                  model class_of_orbit as categorical;
                  model country_of_operator as categorical;
                  model launch_mass as numerical;
                  model perigee as numerical;
                  model period as numerical
              )
          ''')
          backend = CGPM_Backend(dict(), multiprocess=0)
          bayesdb_register_backend(bdb, backend)
          yield bdb
  
  
  def test_cgpm_alter_basic():
      with cgpm_dummy_satellites_pop_bdb() as bdb:
          bdb.execute('''
              create generator g0 for satellites using cgpm(
                  subsample 10
              );
          ''')
          bdb.execute('initialize 4 models for g0')
  
          population_id = bayeslite.core.bayesdb_get_population(
              bdb, 'satellites')
          generator_id = bayeslite.core.bayesdb_get_generator(
              bdb, population_id, 'g0')
  
          # Make all variables independent in models 0-1.
          bdb.execute('''
              alter generator g0 models (0,1)
                  ensure variables * independent;
          ''')
          dependencies = bdb.execute('''
              estimate dependence probability from pairwise variables of
              satellites modeled by g0 using models 0-1;
          ''').fetchall()
          for _, var0, var1, value in dependencies:
              assert value == int(var0 == var1)
  
          # Make all variables dependent in all models.
          bdb.execute('''
              alter generator g0
                  ensure variables * dependent;
          ''')
          dependencies = bdb.execute('''
              estimate dependence probability from pairwise variables of
              satellites
          ''').fetchall()
          for _, _, _, value in dependencies:
              assert value == 1
  
          # Make all variables independent again.
          bdb.execute('''
              alter generator g0
                  ensure variables * independent;
          ''')
  
          # Move apogee, perigee->view(period), launch_mass->view(class_of_orbit)
          # All other variables should remain independent.
          bdb.execute('''
              alter generator g0
                  ensure variables (apogee, period, perigee) in context of period,
                  ensure variable launch_mass in context of class_of_orbit;
          ''')
          dependencies = bdb.execute('''
              estimate dependence probability from pairwise variables of
              satellites
          ''').fetchall()
          blocks = [
              ['apogee','perigee','period'],
              ['launch_mass', 'class_of_orbit']
          ]
          for _, var0, var1, value in dependencies:
              if var0 in blocks[0] and var1 in blocks[0]:
                  assert value == 1.
              elif var0 in blocks[1] and var1 in blocks[1]:
                  assert value == 1.
              elif var0 == var1:
                  assert value == 1.
              else:
                  assert value == 0.
  
          # Move period to a singleton in model 3.
          bdb.execute('''
              alter generator g0 model (3)
                  ensure variable period in singleton view
          ''')
          dependencies = bdb.execute('''
              estimate dependence probability with period from variables of
              satellites modeled by g0 using models 0,1,2
          ''')
          for _, var0, value in dependencies:
              if var0 in blocks[0]:
                  assert value == 1
              else:
                  assert value == 0
          dependencies = bdb.execute('''
  
  
  [... the rest is omitted ...]
files: tests/test_cgpm_alter.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import pytest
  
  import bayeslite.core
  
  from bayeslite import bayesdb_register_backend
  from bayeslite.exception import BQLError
  from bayeslite.backends.cgpm_backend import CGPM_Backend
  
  from test_cgpm import cgpm_dummy_satellites_bdb
  
  # This test suite needs expanding to confirm the subproblems are compiled to the
  # right inference kernels in cgpm_backend, but the information is quite nested
  # so it is not trivial to retrieve this information. Considering refactoring the
  # interpreter to make unit testing easier.
  
  
  def test_analysis_subproblems_basic():
      with cgpm_dummy_satellites_bdb() as bdb:
          bdb.execute('''
              CREATE POPULATION satellites FOR satellites_ucs WITH SCHEMA(
                  SET STATTYPE OF class_of_orbit TO CATEGORICAL;
                  SET STATTYPE OF country_of_operator TO CATEGORICAL;
                  SET STATTYPE OF launch_mass TO NUMERICAL;
                  SET STATTYPE OF perigee TO NUMERICAL;
                  SET STATTYPE OF period TO NUMERICAL
              )
          ''')
          bayesdb_register_backend(bdb, CGPM_Backend(dict(), multiprocess=0))
          bdb.execute('''
              CREATE GENERATOR g0 FOR satellites USING cgpm(
                  SUBSAMPLE 10
              );
          ''')
          bdb.execute('INITIALIZE 4 MODELS FOR g0')
  
          # Test each subproblem individually except for variable hyperparameters.
          for optimized in ['', 'OPTIMIZED;',]:
              for subproblem in [
                  'variable clustering',
                  'variable clustering concentration',
                  'row clustering',
                  'row clustering concentration',
              ]:
                  bdb.execute('''
                      ANALYZE g0 MODELS 0,1 FOR 4 ITERATION(
                          SUBPROBLEM %s;
                          %s
                      );
                  ''' % (subproblem, optimized))
  
          # Test variable hyperparameters.
          bdb.execute('''
              ANALYZE g0 FOR 1 ITERATION (
                  VARIABLES period, launch_mass;
                  SUBPROBLEM variable hyperparameters;
              )
          ''')
          with pytest.raises(BQLError):
              # OPTIMIZED backend does not support variable hyperparameters.
              bdb.execute('''
                  ANALYZE g0 FOR 1 SECONDS (
                      SUBPROBLEM variable hyperparameters;
                      OPTIMIZED;
                  )
              ''')
  
          # Test rows.
          generator_id = bayeslite.core.bayesdb_get_generator(bdb, None, 'g0')
          cursor = bdb.execute('''
              SELECT table_rowid FROM  bayesdb_cgpm_individual
              WHERE generator_id = ?
          ''', (generator_id,))
          subsample_rows = [c[0] for c in cursor]
          bad_rows = [i for i in xrange(20) if i not in subsample_rows]
          for optimized in ['', 'OPTIMIZED;']:
              bdb.execute('''
                  ANALYZE g0 MODEL 3 FOR 1 ITERATION (
                      VARIABLES class_of_orbit;
                      ROWS %s;
                      SUBPROBLEMS (
                          row clustering,
                          row clustering concentration
                      );
                      %s
              )
              ''' % (','.join(map(str, subsample_rows)), optimized))
              with pytest.raises(BQLError):
                  # Fail on rows not in the population or subsample.
                  bdb.execute('''
                      ANALYZE g0 MODEL 3 FOR 1 ITERATION (
                          VARIABLES class_of_orbit;
                          ROWS %s;
                          SUBPROBLEMS (
                              row clustering,
                              row clustering concentration
                          );
                          %s
                  )
                  ''' % (','.join(map(str, bad_rows)), optimized))
  
  
  [... the rest is omitted ...]
files: tests/test_cgpm_analysis.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import pytest
  
  import bayeslite.core
  
  from bayeslite import bayesdb_register_backend
  from bayeslite.exception import BQLError
  from bayeslite.backends.cgpm_backend import CGPM_Backend
  
  from test_cgpm import cgpm_dummy_satellites_bdb
  
  # This test suite needs expanding to confirm the subproblems are compiled to the
  # right inference kernels in cgpm_backend, but the information is quite nested
  # so it is not trivial to retrieve this information. Considering refactoring the
  # interpreter to make unit testing easier.
  
  
  def test_analysis_subproblems_basic():
      with cgpm_dummy_satellites_bdb() as bdb:
          bdb.execute('''
              CREATE POPULATION satellites FOR satellites_ucs WITH SCHEMA(
                  MODEL apogee AS NUMERICAL;
                  MODEL class_of_orbit AS CATEGORICAL;
                  MODEL country_of_operator AS CATEGORICAL;
                  MODEL launch_mass AS NUMERICAL;
                  MODEL perigee AS NUMERICAL;
                  MODEL period AS NUMERICAL
              )
          ''')
          bayesdb_register_backend(bdb, CGPM_Backend(dict(), multiprocess=0))
          bdb.execute('''
              CREATE GENERATOR g0 FOR satellites USING cgpm(
                  SUBSAMPLE 10
              );
          ''')
          bdb.execute('INITIALIZE 4 MODELS FOR g0')
  
          # Test each subproblem individually except for variable hyperparameters.
          for optimized in ['', 'OPTIMIZED;',]:
              for subproblem in [
                  'variable clustering',
                  'variable clustering concentration',
                  'row clustering',
                  'row clustering concentration',
              ]:
                  bdb.execute('''
                      ANALYZE g0 MODELS 0,1 FOR 4 ITERATION(
                          SUBPROBLEM %s;
                          %s
                      );
                  ''' % (subproblem, optimized))
  
          # Test variable hyperparameters.
          bdb.execute('''
              ANALYZE g0 FOR 1 ITERATION (
                  VARIABLES period, launch_mass;
                  SUBPROBLEM variable hyperparameters;
              )
          ''')
          with pytest.raises(BQLError):
              # OPTIMIZED backend does not support variable hyperparameters.
              bdb.execute('''
                  ANALYZE g0 FOR 1 SECONDS (
                      SUBPROBLEM variable hyperparameters;
                      OPTIMIZED;
                  )
              ''')
  
          # Test rows.
          generator_id = bayeslite.core.bayesdb_get_generator(bdb, None, 'g0')
          cursor = bdb.execute('''
              SELECT table_rowid FROM  bayesdb_cgpm_individual
              WHERE generator_id = ?
          ''', (generator_id,))
          subsample_rows = [c[0] for c in cursor]
          bad_rows = [i for i in xrange(20) if i not in subsample_rows]
          for optimized in ['', 'OPTIMIZED;']:
              bdb.execute('''
                  ANALYZE g0 MODEL 3 FOR 1 ITERATION (
                      VARIABLES class_of_orbit;
                      ROWS %s;
                      SUBPROBLEMS (
                          row clustering,
                          row clustering concentration
                      );
                      %s
              )
              ''' % (','.join(map(str, subsample_rows)), optimized))
              with pytest.raises(BQLError):
                  # Fail on rows not in the population or subsample.
                  bdb.execute('''
                      ANALYZE g0 MODEL 3 FOR 1 ITERATION (
                          VARIABLES class_of_orbit;
                          ROWS %s;
                          SUBPROBLEMS (
                              row clustering,
                              row clustering concentration
                          );
                          %s
                  )
                  ''' % (','.join(map(str, bad_rows)), optimized))
  
  
  [... the rest is omitted ...]
files: tests/test_cgpm_analysis.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import contextlib
  import itertools
  import pytest
  
  from bayeslite import bayesdb_open
  from bayeslite.exception import BQLError
  from bayeslite.exception import BQLParseError
  
  
  '''This test provides coverage for ESTIMATE and SIMULATE of conditional mutual
  information queries for univariate and multivariate targets and conditioning
  statements.'''
  
  
  @contextlib.contextmanager
  def smoke_bdb():
      with bayesdb_open(':memory:') as bdb:
          bdb.sql_execute('CREATE TABLE t (a, b, c, d, e)')
  
          for a, b, c, d, e in itertools.product(*([range(2)]*4+[['x','y']])):
              # XXX Insert synthetic data generator here.
              bdb.sql_execute('''
                  INSERT INTO t (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)
              ''', (a, b, c, d, e))
  
          bdb.execute('''
              CREATE POPULATION p FOR t WITH SCHEMA (
                  SET STATTYPES OF a, b, c, d TO NUMERICAL;
                  SET STATTYPES OF e TO NOMINAL
              )
          ''')
  
          bdb.execute('CREATE GENERATOR m1 FOR p;')
          bdb.execute('INITIALIZE 10 MODELS FOR m1;')
  
          bdb.execute('CREATE GENERATOR m2 FOR p;')
          bdb.execute('INITIALIZE 10 MODELS FOR m2;')
          yield bdb
  
  
  def test_estimate_cmi_basic__ci_slow():
      with smoke_bdb() as bdb:
          bql = '''
              ESTIMATE MUTUAL INFORMATION OF a WITH b
              BY p
              MODELED BY m1
          '''
          result = bdb.execute(bql).fetchall()
          assert len(result) == 1
  
  
  def test_estimate_cmi_no_condition():
      with smoke_bdb() as bdb:
          # Univariate targets.
          bql = '''
              ESTIMATE MUTUAL INFORMATION OF a WITH b USING 10 SAMPLES
              BY p
          '''
          result = bdb.execute(bql).fetchall()
          assert len(result) == 1
          # Multivariate targets.
          bql = '''
              ESTIMATE MUTUAL INFORMATION OF (a, e) WITH b USING 10 SAMPLES
              BY p
          '''
          result = bdb.execute(bql).fetchall()
          assert len(result) == 1
  
  
  def test_estimate_cmi_equality_condition():
      with smoke_bdb() as bdb:
          # Univariate targets.
          bql = '''
              ESTIMATE
                  MUTUAL INFORMATION OF a WITH b GIVEN
                      (c = 1, e = 'x') USING 10 SAMPLES
              BY p;
          '''
          result = bdb.execute(bql).fetchall()
          assert len(result) == 1
          # Multivariate targets.
          bql = '''
              ESTIMATE
                  MUTUAL INFORMATION OF (a, d) WITH b GIVEN
                      (c = 1, e = 'x') USING 10 SAMPLES
              BY p;
          '''
          result = bdb.execute(bql).fetchall()
          assert len(result) == 1
  
  
  def test_estimate_cmi_marginal_condition__ci_slow():
      with smoke_bdb() as bdb:
          # Univariate targets.
          bql = '''
              ESTIMATE
                  MUTUAL INFORMATION OF a WITH b GIVEN (d) USING 10 SAMPLES
              BY p;
          '''
          result = bdb.execute(bql).fetchall()
          assert len(result) == 1
          # Multivariate targets.
          bql = '''
              ESTIMATE
                  MUTUAL INFORMATION OF (a, b) WITH (c) GIVEN (d=1, e)
                  USING 10 SAMPLES
              BY p;
          '''
          result = bdb.execute(bql).fetchall()
          assert len(result) == 1
  
  
  def test_estimate_cmi_equality_marginal_condition__ci_slow():
      with smoke_bdb() as b
  
  [... the rest is omitted ...]
files: tests/test_cmi.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import contextlib
  import itertools
  import pytest
  
  from bayeslite import bayesdb_open
  from bayeslite.exception import BQLError
  from bayeslite.exception import BQLParseError
  
  
  '''This test provides coverage for ESTIMATE and SIMULATE of conditional mutual
  information queries for univariate and multivariate targets and conditioning
  statements.'''
  
  
  @contextlib.contextmanager
  def smoke_bdb():
      with bayesdb_open(':memory:') as bdb:
          bdb.sql_execute('CREATE TABLE t (a, b, c, d, e)')
  
          for a, b, c, d, e in itertools.product(*([range(2)]*4+[['x','y']])):
              # XXX Insert synthetic data generator here.
              bdb.sql_execute('''
                  INSERT INTO t (a, b, c, d, e) VALUES (?, ?, ?, ?, ?)
              ''', (a, b, c, d, e))
  
          bdb.execute('''
              CREATE POPULATION p FOR t WITH SCHEMA (
                  MODEL a, b, c, d AS NUMERICAL;
                  MODEL e AS NOMINAL
              )
          ''')
  
          bdb.execute('CREATE GENERATOR m1 FOR p;')
          bdb.execute('INITIALIZE 10 MODELS FOR m1;')
  
          bdb.execute('CREATE GENERATOR m2 FOR p;')
          bdb.execute('INITIALIZE 10 MODELS FOR m2;')
          yield bdb
  
  
  def test_estimate_cmi_basic__ci_slow():
      with smoke_bdb() as bdb:
          bql = '''
              ESTIMATE MUTUAL INFORMATION OF a WITH b
              BY p
              MODELED BY m1
          '''
          result = bdb.execute(bql).fetchall()
          assert len(result) == 1
  
  
  def test_estimate_cmi_no_condition():
      with smoke_bdb() as bdb:
          # Univariate targets.
          bql = '''
              ESTIMATE MUTUAL INFORMATION OF a WITH b USING 10 SAMPLES
              BY p
          '''
          result = bdb.execute(bql).fetchall()
          assert len(result) == 1
          # Multivariate targets.
          bql = '''
              ESTIMATE MUTUAL INFORMATION OF (a, e) WITH b USING 10 SAMPLES
              BY p
          '''
          result = bdb.execute(bql).fetchall()
          assert len(result) == 1
  
  
  def test_estimate_cmi_equality_condition():
      with smoke_bdb() as bdb:
          # Univariate targets.
          bql = '''
              ESTIMATE
                  MUTUAL INFORMATION OF a WITH b GIVEN
                      (c = 1, e = 'x') USING 10 SAMPLES
              BY p;
          '''
          result = bdb.execute(bql).fetchall()
          assert len(result) == 1
          # Multivariate targets.
          bql = '''
              ESTIMATE
                  MUTUAL INFORMATION OF (a, d) WITH b GIVEN
                      (c = 1, e = 'x') USING 10 SAMPLES
              BY p;
          '''
          result = bdb.execute(bql).fetchall()
          assert len(result) == 1
  
  
  def test_estimate_cmi_marginal_condition__ci_slow():
      with smoke_bdb() as bdb:
          # Univariate targets.
          bql = '''
              ESTIMATE
                  MUTUAL INFORMATION OF a WITH b GIVEN (d) USING 10 SAMPLES
              BY p;
          '''
          result = bdb.execute(bql).fetchall()
          assert len(result) == 1
          # Multivariate targets.
          bql = '''
              ESTIMATE
                  MUTUAL INFORMATION OF (a, b) WITH (c) GIVEN (d=1, e)
                  USING 10 SAMPLES
              BY p;
          '''
          result = bdb.execute(bql).fetchall()
          assert len(result) == 1
  
  
  def test_estimate_cmi_equality_marginal_condition__ci_slow():
      with smoke_bdb() as bdb:
          # Univariate targets.
  
  
  [... the rest is omitted ...]
files: tests/test_cmi.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import bayeslite
  
  def test_conditional_probability_simple_inferences():
      data = [
          ['x', 'a'], ['x', 'a'], ['x', 'a'],
          ['y', 'b'], ['y', 'b'], ['y', 'b'],
      ]
      with bayeslite.bayesdb_open() as bdb:
          bdb.sql_execute('create table t(foo, bar)')
          for row in data:
              bdb.sql_execute('insert into t values (?, ?)', row)
          bdb.execute('''
              create population p for t (
                  foo categorical;
                  bar categorical;
              )
          ''')
          bdb.execute('create generator p_cc for p using cgpm;')
          bdb.execute('initialize 10 models for p_cc')
          bdb.execute('analyze p_cc for 100 iterations')
          cursor = bdb.execute('''
              estimate
                  probability density of foo = 'x',
                  probability density of foo = 'x' given (bar = 'a'),
                  probability density of foo = 'x' given (bar = 'b'),
                  probability density of foo = 'y',
                  probability density of foo = 'y' given (bar = 'a'),
                  probability density of foo = 'y' given (bar = 'b')
  
              by p
          ''').fetchall()
          px, pxa, pxb, py, pya, pyb = cursor[0]
          # Inferences on x.
          assert px < pxa
          assert pxb < px
          # Inferences on y.
          assert py < pyb
          assert pya < py
  
  
  [... the rest is omitted ...]
files: tests/test_condprob.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import bayeslite
  
  def test_conditional_probability_simple_inferences():
      data = [
          ['x', 'a'], ['x', 'a'], ['x', 'a'],
          ['y', 'b'], ['y', 'b'], ['y', 'b'],
      ]
      with bayeslite.bayesdb_open() as bdb:
          bdb.sql_execute('create table t(foo, bar)')
          for row in data:
              bdb.sql_execute('insert into t values (?, ?)', row)
          bdb.execute('''
              create population p for t (
                  model foo, bar as categorical
              )
          ''')
          bdb.execute('create generator p_cc for p using cgpm;')
          bdb.execute('initialize 10 models for p_cc')
          bdb.execute('analyze p_cc for 100 iterations')
          cursor = bdb.execute('''
              estimate
                  probability density of foo = 'x',
                  probability density of foo = 'x' given (bar = 'a'),
                  probability density of foo = 'x' given (bar = 'b'),
                  probability density of foo = 'y',
                  probability density of foo = 'y' given (bar = 'a'),
                  probability density of foo = 'y' given (bar = 'b')
  
              by p
          ''').fetchall()
          px, pxa, pxb, py, pya, pyb = cursor[0]
          # Inferences on x.
          assert px < pxa
          assert pxb < px
          # Inferences on y.
          assert py < pyb
          assert pya < py
  
  
  [... the rest is omitted ...]
files: tests/test_condprob.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import apsw
  import contextlib
  import itertools
  import json
  import pytest
  import tempfile
  
  import bayeslite
  import bayeslite.backend
  import bayeslite.bqlfn as bqlfn
  import bayeslite.core as core
  
  from bayeslite.backends.cgpm_backend import CGPM_Backend
  
  from bayeslite import bql_quote_name
  from bayeslite.sqlite3_util import sqlite3_connection
  from bayeslite.util import cursor_value
  
  def powerset(s):
      s = list(s)
      combinations = (itertools.combinations(s, r) for r in range(len(s) + 1))
      return itertools.chain.from_iterable(combinations)
  
  @contextlib.contextmanager
  def bayesdb(backend=None, **kwargs):
      if backend is None:
          backend = CGPM_Backend(cgpm_registry={}, multiprocess=False)
      bdb = bayeslite.bayesdb_open(builtin_backends=False, **kwargs)
      bayeslite.bayesdb_register_backend(bdb, backend)
      try:
          yield bdb
      finally:
          bdb.close()
  
  def test_bayesdb_instantiation():
      # Use bayesdb_open -- don't instantiate directly.
      with pytest.raises(TypeError):
          bayeslite.BayesDB()  # pylint: disable=no-value-for-parameter
      with pytest.raises(ValueError):
          bayeslite.BayesDB(':memory:', 0xdeadbeef)
  
  def test_prng_determinism():
      bdb = bayeslite.bayesdb_open(builtin_backends=False)
      assert bdb._prng.weakrandom_uniform(1000) == 303
      assert bdb.py_prng.uniform(0, 1) == 0.6156331606142532
      assert bdb.np_prng.uniform(0, 1) == 0.28348770982811367
  
  def test_openclose():
      with bayesdb():
          pass
  
  def test_bad_db_application_id():
      with tempfile.NamedTemporaryFile(prefix='bayeslite') as f:
          with sqlite3_connection(f.name) as db:
              db.cursor().execute('PRAGMA application_id = 42')
              db.cursor().execute('PRAGMA user_version = 3')
          with pytest.raises(IOError):
              with bayesdb(pathname=f.name):
                  pass
  
  def test_bad_db_user_version():
      # XXX Would be nice to avoid a named temporary file here.  Pass
      # the sqlite3 database connection in?
      with tempfile.NamedTemporaryFile(prefix='bayeslite') as f:
          with sqlite3_connection(f.name) as db:
              db.cursor().execute('PRAGMA application_id = 1113146434')
              db.cursor().execute('PRAGMA user_version = 42')
          with pytest.raises(IOError):
              with bayesdb(pathname=f.name):
                  pass
  
  class DotdogBackend(bayeslite.backend.BayesDB_Backend):
      def name(self):
          return 'dotdog'
      def register(self, bdb):
          sql = '''
              INSERT OR IGNORE INTO bayesdb_backend
                  VALUES ('dotdog', 42)
          '''
          bdb.sql_execute(sql)
      def create_generator(self, bdb, generator_id, schema_tokens, **kwargs):
          pass
  
  def test_hackbackend():
      bdb = bayeslite.bayesdb_open(builtin_backends=False)
      bdb.sql_execute('CREATE TABLE t(a INTEGER, b TEXT)')
      bdb.sql_execute("INSERT INTO t (a, b) VALUES (42, 'fnord')")
      bdb.sql_execute('CREATE TABLE u AS SELECT * FROM t')
      bdb.execute('CREATE POPULATION p FOR t(b IGNORE; a NUMERICAL)')
      with pytest.raises(bayeslite.BQLError):
          bdb.execute('CREATE GENERATOR p_cc FOR p USING cgpm;')
     
  
  [... the rest is omitted ...]
files: tests/test_core.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import apsw
  import contextlib
  import itertools
  import json
  import pytest
  import tempfile
  
  import bayeslite
  import bayeslite.backend
  import bayeslite.bqlfn as bqlfn
  import bayeslite.core as core
  
  from bayeslite.backends.cgpm_backend import CGPM_Backend
  
  from bayeslite import bql_quote_name
  from bayeslite.sqlite3_util import sqlite3_connection
  from bayeslite.util import cursor_value
  
  def powerset(s):
      s = list(s)
      combinations = (itertools.combinations(s, r) for r in range(len(s) + 1))
      return itertools.chain.from_iterable(combinations)
  
  @contextlib.contextmanager
  def bayesdb(backend=None, **kwargs):
      if backend is None:
          backend = CGPM_Backend(cgpm_registry={}, multiprocess=False)
      bdb = bayeslite.bayesdb_open(builtin_backends=False, **kwargs)
      bayeslite.bayesdb_register_backend(bdb, backend)
      try:
          yield bdb
      finally:
          bdb.close()
  
  def test_bayesdb_instantiation():
      # Use bayesdb_open -- don't instantiate directly.
      with pytest.raises(TypeError):
          bayeslite.BayesDB()  # pylint: disable=no-value-for-parameter
      with pytest.raises(ValueError):
          bayeslite.BayesDB(':memory:', 0xdeadbeef)
  
  def test_prng_determinism():
      bdb = bayeslite.bayesdb_open(builtin_backends=False)
      assert bdb._prng.weakrandom_uniform(1000) == 303
      assert bdb.py_prng.uniform(0, 1) == 0.6156331606142532
      assert bdb.np_prng.uniform(0, 1) == 0.28348770982811367
  
  def test_openclose():
      with bayesdb():
          pass
  
  def test_bad_db_application_id():
      with tempfile.NamedTemporaryFile(prefix='bayeslite') as f:
          with sqlite3_connection(f.name) as db:
              db.cursor().execute('PRAGMA application_id = 42')
              db.cursor().execute('PRAGMA user_version = 3')
          with pytest.raises(IOError):
              with bayesdb(pathname=f.name):
                  pass
  
  def test_bad_db_user_version():
      # XXX Would be nice to avoid a named temporary file here.  Pass
      # the sqlite3 database connection in?
      with tempfile.NamedTemporaryFile(prefix='bayeslite') as f:
          with sqlite3_connection(f.name) as db:
              db.cursor().execute('PRAGMA application_id = 1113146434')
              db.cursor().execute('PRAGMA user_version = 42')
          with pytest.raises(IOError):
              with bayesdb(pathname=f.name):
                  pass
  
  class DotdogBackend(bayeslite.backend.BayesDB_Backend):
      def name(self):
          return 'dotdog'
      def register(self, bdb):
          sql = '''
              INSERT OR IGNORE INTO bayesdb_backend
                  VALUES ('dotdog', 42)
          '''
          bdb.sql_execute(sql)
      def create_generator(self, bdb, generator_id, schema_tokens, **kwargs):
          pass
  
  def test_hackbackend():
      bdb = bayeslite.bayesdb_open(builtin_backends=False)
      bdb.sql_execute('CREATE TABLE t(a INTEGER, b TEXT)')
      bdb.sql_execute("INSERT INTO t (a, b) VALUES (42, 'fnord')")
      bdb.sql_execute('CREATE TABLE u AS SELECT * FROM t')
      bdb.execute('CREATE POPULATION p FOR t(b IGNORE; a NUMERICAL)')
      with pytest.raises(bayeslite.BQLError):
          bdb.execute('CREATE GENERATOR p_cc FOR p USING cgpm;')
     
  
  [... the rest is omitted ...]
files: tests/test_core.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  '''Ensures that INFER EXPLICIT PREDICT col for a _rowid_ which exists in the
  base table, but not incorporated into the generator (i.e. did not exist in the
  table during INITIALIZE) correctly retrieves all other values in the row. The
  test cases creates a population of (0,1) and (1,0) -- the complement function --
  and ensures its recovery on a held-out test set.'''
  
  import pytest
  
  from bayeslite import bayesdb_open
  
  from bayeslite.core import bayesdb_get_generator
  from bayeslite.core import bayesdb_get_population
  from bayeslite.core import bayesdb_population_fresh_row_id
  from bayeslite.util import cursor_value
  
  
  @pytest.fixture(scope='module')
  def bdb():
      bdb = bayesdb_open(':memory:')
  
      # Create the population of complements.
      bdb.sql_execute('CREATE TABLE t (a TEXT, b TEXT)')
      for _ in xrange(20):
          bdb.sql_execute('INSERT INTO t (a, b) VALUES (0,1)')
      for _ in xrange(20):
          bdb.sql_execute('INSERT INTO t (a, b) VALUES (1,0)')
  
      # Create the population and generator on the existing rows.
      bdb.execute('CREATE POPULATION p FOR t (SET STATTYPES OF a, b TO NOMINAL)')
      bdb.execute('CREATE GENERATOR m FOR p;')
      bdb.execute('INITIALIZE 1 MODELS FOR m;')
      bdb.execute('ANALYZE m FOR 1000 ITERATION (OPTIMIZED);')
  
      # Add new 'hypothetical' rows into the base table to serve as out-of-
      # sample probe points; only zeros, only ones, and nothing.
      for _ in xrange(40, 50):
          bdb.sql_execute('INSERT INTO t (a) VALUES (0)')
      for _ in xrange(50, 60):
          bdb.sql_execute('INSERT INTO t (b) VALUES (1)')
      for _ in xrange(60, 80):
          bdb.sql_execute('INSERT INTO t (a,b) VALUES (NULL, NULL)')
  
      # Make sure fresh_row_id 80 from the base table, not generator.
      population_id = bayesdb_get_population(bdb, 'p')
      assert bayesdb_population_fresh_row_id(bdb, population_id) == 81
  
      # Make sure the cgpm only has 40 rowids incorporated.
      generator_id = bayesdb_get_generator(bdb, population_id, 'm')
      cursor = bdb.sql_execute('''
          SELECT MAX(table_rowid) FROM bayesdb_cgpm_individual
          WHERE generator_id = ?
      ''', (generator_id,))
      assert cursor_value(cursor) == 40
  
      # Turn off multiprocessing for sequence of queries.
      bdb.backends['cgpm'].set_multiprocess(False)
      return bdb
  
  
  def test_infer_ones(bdb):
      # All the b's should be one (check 90%).
      b_ones = bdb.execute('''
          INFER EXPLICIT
              a,
              PREDICT b CONFIDENCE conf USING 100 SAMPLES
          FROM p WHERE oid BETWEEN 41 AND 50
      ''').fetchall()
      assert all(r[0] == '0' for r in b_ones)
      assert len([r for r in b_ones if r[1] == '1' and r[2] > 0.8]) >= 9
  
  
  def test_infer_zeros(bdb):
      # All the a's should be zero (check 90%).
      a_zeros = bdb.execute('''
          INFER EXPLICIT
              PREDICT a CONFIDENCE conf USING 100 SAMPLES,
              b
          FROM p WHERE oid BETWEEN 51 AND 60
      ''').fetchall()
      assert all(r[2] == '1' for r in a_zeros)
      assert len([r for r in a_zeros if
  
  [... the rest is omitted ...]
files: tests/test_infer_hypothetical.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  '''Ensures that INFER EXPLICIT PREDICT col for a _rowid_ which exists in the
  base table, but not incorporated into the generator (i.e. did not exist in the
  table during INITIALIZE) correctly retrieves all other values in the row. The
  test cases creates a population of (0,1) and (1,0) -- the complement function --
  and ensures its recovery on a held-out test set.'''
  
  import pytest
  
  from bayeslite import bayesdb_open
  
  from bayeslite.core import bayesdb_get_generator
  from bayeslite.core import bayesdb_get_population
  from bayeslite.core import bayesdb_population_fresh_row_id
  from bayeslite.util import cursor_value
  
  
  @pytest.fixture(scope='module')
  def bdb():
      bdb = bayesdb_open(':memory:')
  
      # Create the population of complements.
      bdb.sql_execute('CREATE TABLE t (a TEXT, b TEXT)')
      for _ in xrange(20):
          bdb.sql_execute('INSERT INTO t (a, b) VALUES (0,1)')
      for _ in xrange(20):
          bdb.sql_execute('INSERT INTO t (a, b) VALUES (1,0)')
  
      # Create the population and generator on the existing rows.
      bdb.execute('CREATE POPULATION p FOR t (MODEL a, b AS NOMINAL)')
      bdb.execute('CREATE GENERATOR m FOR p;')
      bdb.execute('INITIALIZE 1 MODELS FOR m;')
      bdb.execute('ANALYZE m FOR 1000 ITERATION (OPTIMIZED);')
  
      # Add new 'hypothetical' rows into the base table to serve as out-of-
      # sample probe points; only zeros, only ones, and nothing.
      for _ in xrange(40, 50):
          bdb.sql_execute('INSERT INTO t (a) VALUES (0)')
      for _ in xrange(50, 60):
          bdb.sql_execute('INSERT INTO t (b) VALUES (1)')
      for _ in xrange(60, 80):
          bdb.sql_execute('INSERT INTO t (a,b) VALUES (NULL, NULL)')
  
      # Make sure fresh_row_id 80 from the base table, not generator.
      population_id = bayesdb_get_population(bdb, 'p')
      assert bayesdb_population_fresh_row_id(bdb, population_id) == 81
  
      # Make sure the cgpm only has 40 rowids incorporated.
      generator_id = bayesdb_get_generator(bdb, population_id, 'm')
      cursor = bdb.sql_execute('''
          SELECT MAX(table_rowid) FROM bayesdb_cgpm_individual
          WHERE generator_id = ?
      ''', (generator_id,))
      assert cursor_value(cursor) == 40
  
      # Turn off multiprocessing for sequence of queries.
      bdb.backends['cgpm'].set_multiprocess(False)
      return bdb
  
  
  def test_infer_ones(bdb):
      # All the b's should be one (check 90%).
      b_ones = bdb.execute('''
          INFER EXPLICIT
              a,
              PREDICT b CONFIDENCE conf USING 100 SAMPLES
          FROM p WHERE oid BETWEEN 41 AND 50
      ''').fetchall()
      assert all(r[0] == '0' for r in b_ones)
      assert len([r for r in b_ones if r[1] == '1' and r[2] > 0.8]) >= 9
  
  
  def test_infer_zeros(bdb):
      # All the a's should be zero (check 90%).
      a_zeros = bdb.execute('''
          INFER EXPLICIT
              PREDICT a CONFIDENCE conf USING 100 SAMPLES,
              b
          FROM p WHERE oid BETWEEN 51 AND 60
      ''').fetchall()
      assert all(r[2] == '1' for r in a_zeros)
      assert len([r for r in a_zeros if r[0
  
  [... the rest is omitted ...]
files: tests/test_infer_hypothetical.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import pytest
  
  import bayeslite.core as core
  
  from bayeslite import BQLError
  from bayeslite import bayesdb_open
  from bayeslite import bayesdb_register_backend
  from bayeslite.backends.nig_normal import NIGNormalBackend
  
  def test_nig_normal_smoke():
      with bayesdb_open(':memory:') as bdb:
          bayesdb_register_backend(bdb, NIGNormalBackend())
          bdb.sql_execute('create table t(x)')
          for x in xrange(100):
              bdb.sql_execute('insert into t(x) values(?)', (x,))
          bdb.execute('create population p for t(x numerical)')
          bdb.execute('create generator g for p using nig_normal')
          bdb.execute('initialize 1 model for g')
          bdb.execute('analyze g for 1 iteration')
          bdb.execute('estimate probability density of x = 50 from p').fetchall()
          bdb.execute('simulate x from p limit 1').fetchall()
          bdb.execute('drop models from g')
          bdb.execute('drop generator g')
          bdb.execute('drop population p')
          bdb.execute('drop table t')
  
  def test_nig_normal_latent_numbering():
      with bayesdb_open(':memory:') as bdb:
          bayesdb_register_backend(bdb, NIGNormalBackend())
          bdb.sql_execute('create table t(id integer primary key, x, y)')
          for x in xrange(100):
              bdb.sql_execute('insert into t(x, y) values(?, ?)', (x, x*x - 100))
          bdb.execute('''
              create population p for t(
                  id ignore;
                  set stattypes of x,y to numerical;
              )
          ''')
          assert core.bayesdb_has_population(bdb, 'p')
          pid = core.bayesdb_get_population(bdb, 'p')
          assert core.bayesdb_variable_numbers(bdb, pid, None) == [1, 2]
  
          bdb.execute('create generator g0 for p using nig_normal')
          bdb.execute('''
              create generator g1 for p using nig_normal(xe deviation(x))
          ''')
  
          assert core.bayesdb_has_generator(bdb, pid, 'g0')
          g0 = core.bayesdb_get_generator(bdb, pid, 'g0')
          assert core.bayesdb_has_generator(bdb, pid, 'g1')
          g1 = core.bayesdb_get_generator(bdb, pid, 'g1')
          assert core.bayesdb_variable_numbers(bdb, pid, None) == [1, 2]
          assert core.bayesdb_variable_numbers(bdb, pid, g0) == [1, 2]
          assert core.bayesdb_variable_numbers(bdb, pid, g1) == [-1, 1, 2]
  
  def test_nig_normal_latent_smoke():
      with bayesdb_open(':memory:') as bdb:
          bayesdb_register_backend(bdb, NIGNormalBackend())
          bdb.sql_execute('create table t(x)')
          for x in xrange(100):
              bdb.sql_execute('insert into t(x) values(?)', (x,))
          bdb.execute('create population p for t(x numerical)')
          bdb.execute('create generator g0 for p using nig_normal')
          bdb.execute('''
              create generator g1 for p using nig_normal(xe deviation(x))
          ''')
          bdb.execute('initialize 1 model for g0')
          bdb.execute('analyze g0 for 1 iteration')
          bdb.execute('initialize 1 model for g1')
          bdb.execute('analyze g1 for 1 iteration')
  
          # PROBABILITY DENSITY OF x = v
          bdb.execute('estimate probability density of x = 50 within p') \
              .fetchall()
          with pytest.raises(BQLError):
              bdb.execute('estimate probability density of xe = 
  
  [... the rest is omitted ...]
files: tests/test_nig_normal.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import pytest
  
  import bayeslite.core as core
  
  from bayeslite import BQLError
  from bayeslite import bayesdb_open
  from bayeslite import bayesdb_register_backend
  from bayeslite.backends.nig_normal import NIGNormalBackend
  
  def test_nig_normal_smoke():
      with bayesdb_open(':memory:') as bdb:
          bayesdb_register_backend(bdb, NIGNormalBackend())
          bdb.sql_execute('create table t(x)')
          for x in xrange(100):
              bdb.sql_execute('insert into t(x) values(?)', (x,))
          bdb.execute('create population p for t(x numerical)')
          bdb.execute('create generator g for p using nig_normal')
          bdb.execute('initialize 1 model for g')
          bdb.execute('analyze g for 1 iteration')
          bdb.execute('estimate probability density of x = 50 from p').fetchall()
          bdb.execute('simulate x from p limit 1').fetchall()
          bdb.execute('drop models from g')
          bdb.execute('drop generator g')
          bdb.execute('drop population p')
          bdb.execute('drop table t')
  
  def test_nig_normal_latent_numbering():
      with bayesdb_open(':memory:') as bdb:
          bayesdb_register_backend(bdb, NIGNormalBackend())
          bdb.sql_execute('create table t(id integer primary key, x, y)')
          for x in xrange(100):
              bdb.sql_execute('insert into t(x, y) values(?, ?)', (x, x*x - 100))
          bdb.execute('''
              create population p for t(id ignore; model x,y as numerical)
          ''')
          assert core.bayesdb_has_population(bdb, 'p')
          pid = core.bayesdb_get_population(bdb, 'p')
          assert core.bayesdb_variable_numbers(bdb, pid, None) == [1, 2]
  
          bdb.execute('create generator g0 for p using nig_normal')
          bdb.execute('''
              create generator g1 for p using nig_normal(xe deviation(x))
          ''')
  
          assert core.bayesdb_has_generator(bdb, pid, 'g0')
          g0 = core.bayesdb_get_generator(bdb, pid, 'g0')
          assert core.bayesdb_has_generator(bdb, pid, 'g1')
          g1 = core.bayesdb_get_generator(bdb, pid, 'g1')
          assert core.bayesdb_variable_numbers(bdb, pid, None) == [1, 2]
          assert core.bayesdb_variable_numbers(bdb, pid, g0) == [1, 2]
          assert core.bayesdb_variable_numbers(bdb, pid, g1) == [-1, 1, 2]
  
  def test_nig_normal_latent_smoke():
      with bayesdb_open(':memory:') as bdb:
          bayesdb_register_backend(bdb, NIGNormalBackend())
          bdb.sql_execute('create table t(x)')
          for x in xrange(100):
              bdb.sql_execute('insert into t(x) values(?)', (x,))
          bdb.execute('create population p for t(x numerical)')
          bdb.execute('create generator g0 for p using nig_normal')
          bdb.execute('''
              create generator g1 for p using nig_normal(xe deviation(x))
          ''')
          bdb.execute('initialize 1 model for g0')
          bdb.execute('analyze g0 for 1 iteration')
          bdb.execute('initialize 1 model for g1')
          bdb.execute('analyze g1 for 1 iteration')
  
          # PROBABILITY DENSITY OF x = v
          bdb.execute('estimate probability density of x = 50 within p') \
              .fetchall()
          with pytest.raises(BQLError):
              bdb.execute('estimate probability density of xe = 1 within p') \
                  .fetch
  
  [... the rest is omitted ...]
files: tests/test_nig_normal.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import contextlib
  import itertools
  import pytest
  
  import bayeslite
  import bayeslite.ast as ast
  import bayeslite.parse as parse
  
  def parse_bql_string(string):
      phrases = list(parse.parse_bql_string(string))
      phrase_pos = list(parse.parse_bql_string_pos(string))
      assert len(phrases) == len(phrase_pos)
      start = 0
      for i in range(len(phrase_pos)):
          phrase, pos = phrase_pos[i]
          assert phrases[i] == phrase
          substring = buffer(string, start, len(string) - start)
          phrase0, pos0 = parse.parse_bql_string_pos_1(substring)
          assert phrase0 == phrase
          assert pos0 == pos - start
          start = pos
      return phrases
  
  def test_empty():
      assert [] == parse_bql_string('')
      assert [] == parse_bql_string(';')
      assert [] == parse_bql_string(';;')
      assert [] == parse_bql_string(' ;')
      assert [] == parse_bql_string('; ')
      assert [] == parse_bql_string(' ; ')
      assert [] == parse_bql_string(' ; ; ')
  
  def test_multiquery():
      assert parse_bql_string('select 0; select 1;') == [
          ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitInt(0)), None)],
              None, None, None, None, None),
          ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitInt(1)), None)],
              None, None, None, None, None),
      ]
      assert parse_bql_string('select 0; select 1') == [
          ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitInt(0)), None)],
              None, None, None, None, None),
          ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitInt(1)), None)],
              None, None, None, None, None),
      ]
  
  def test_select_trivial():
      assert parse_bql_string('select null;') == \
          [ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitNull(None)), None)],
              None, None, None, None, None)]
      assert parse_bql_string("select 'x';") == \
          [ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitString('x')), None)],
              None, None, None, None, None)]
      assert parse_bql_string("select 'x''y';") == \
          [ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitString("x'y")), None)],
              None, None, None, None, None)]
      assert parse_bql_string('select "x";') == \
          [ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpCol(None, 'x'), None)],
              None, None, None, None, None)]
      assert parse_bql_string('select "x""y";') == \
          [ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpCol(None, 'x"y'), None)],
              None, None, None, None, None)]
      assert parse_bql_string('select 0;') == \
          [ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitInt(0)), None)],
              None, None, None, None, None)]
      assert parse_bql_string('select 0.;') == \
          [ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitFloat(0)), None)],
              None, None, None, None, None)]
      assert parse
  
  [... the rest is omitted ...]
files: tests/test_parse.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import contextlib
  import itertools
  import pytest
  
  import bayeslite
  import bayeslite.ast as ast
  import bayeslite.parse as parse
  
  def parse_bql_string(string):
      phrases = list(parse.parse_bql_string(string))
      phrase_pos = list(parse.parse_bql_string_pos(string))
      assert len(phrases) == len(phrase_pos)
      start = 0
      for i in range(len(phrase_pos)):
          phrase, pos = phrase_pos[i]
          assert phrases[i] == phrase
          substring = buffer(string, start, len(string) - start)
          phrase0, pos0 = parse.parse_bql_string_pos_1(substring)
          assert phrase0 == phrase
          assert pos0 == pos - start
          start = pos
      return phrases
  
  def test_empty():
      assert [] == parse_bql_string('')
      assert [] == parse_bql_string(';')
      assert [] == parse_bql_string(';;')
      assert [] == parse_bql_string(' ;')
      assert [] == parse_bql_string('; ')
      assert [] == parse_bql_string(' ; ')
      assert [] == parse_bql_string(' ; ; ')
  
  def test_multiquery():
      assert parse_bql_string('select 0; select 1;') == [
          ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitInt(0)), None)],
              None, None, None, None, None),
          ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitInt(1)), None)],
              None, None, None, None, None),
      ]
      assert parse_bql_string('select 0; select 1') == [
          ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitInt(0)), None)],
              None, None, None, None, None),
          ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitInt(1)), None)],
              None, None, None, None, None),
      ]
  
  def test_select_trivial():
      assert parse_bql_string('select null;') == \
          [ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitNull(None)), None)],
              None, None, None, None, None)]
      assert parse_bql_string("select 'x';") == \
          [ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitString('x')), None)],
              None, None, None, None, None)]
      assert parse_bql_string("select 'x''y';") == \
          [ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitString("x'y")), None)],
              None, None, None, None, None)]
      assert parse_bql_string('select "x";') == \
          [ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpCol(None, 'x'), None)],
              None, None, None, None, None)]
      assert parse_bql_string('select "x""y";') == \
          [ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpCol(None, 'x"y'), None)],
              None, None, None, None, None)]
      assert parse_bql_string('select 0;') == \
          [ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitInt(0)), None)],
              None, None, None, None, None)]
      assert parse_bql_string('select 0.;') == \
          [ast.Select(ast.SELQUANT_ALL,
              [ast.SelColExp(ast.ExpLit(ast.LitFloat(0)), None)],
              None, None, None, None, None)]
      assert parse
  
  [... the rest is omitted ...]
files: tests/test_parse.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import pytest
  
  from bayeslite import BQLError
  from bayeslite import bayesdb_register_backend
  from bayeslite.backends.cgpm_backend import CGPM_Backend
  
  from test_cgpm import cgpm_dummy_satellites_bdb
  
  
  def test_regress_bonanza__ci_integration():
      with cgpm_dummy_satellites_bdb() as bdb:
          bayesdb_register_backend(
              bdb, CGPM_Backend(dict(), multiprocess=0))
          bdb.execute('''
              CREATE POPULATION satellites FOR satellites_ucs WITH SCHEMA(
                  apogee                  NUMERICAL;
                  class_of_orbit          NOMINAL;
                  country_of_operator     NOMINAL;
                  launch_mass             NUMERICAL;
                  perigee                 NUMERICAL;
                  period                  NUMERICAL;
              )
          ''')
          bdb.execute('''
              CREATE GENERATOR m FOR satellites;
          ''')
          bdb.execute('INITIALIZE 2 MODELS FOR m;')
  
          def check_regression_variables(results, numericals, nominals):
              seen = set()
              for r in results:
                  assert len(r) == 2
                  variable = r[0]
                  assert variable not in seen
                  assert variable in numericals or \
                      any(variable.startswith('%s_dum_' % (nominal,))
                          for nominal in nominals)
                  seen.add(variable)
  
          # Regression on 1 numerical variable.
          results = bdb.execute('''
              REGRESS apogee GIVEN (perigee) USING 12 SAMPLES BY satellites;
          ''').fetchall()
          assert len(results) == 2
          check_regression_variables(results, ['intercept', 'perigee'], [])
  
          # Regression on 1 nominal variable.
          results = bdb.execute('''
              REGRESS apogee GIVEN (country_of_operator)
              USING 12 SAMPLES BY satellites;
          ''').fetchall()
          check_regression_variables(
              results, ['intercept'], ['country_of_operator'])
  
          # Regression on 1 nominal + 1 numerical variable.
          bdb.execute('''
              REGRESS apogee GIVEN (perigee, country_of_operator)
              USING 12 SAMPLES BY satellites;
          ''').fetchall()
          check_regression_variables(
              results, ['intercept', 'perigee'], ['country_of_operator'])
  
          # Regression on all variables.
          results = bdb.execute('''
              REGRESS apogee GIVEN (*) USING 12 SAMPLES BY satellites;
          ''', (3,)).fetchall()
          check_regression_variables(
              results,
              ['intercept', 'perigee', 'launch_mass', 'period',],
              ['country_of_operator', 'class_of_orbit',],
          )
  
          # Regression on column selector subexpression with a binding.
          results = bdb.execute('''
              REGRESS apogee GIVEN (
                  satellites.(
                      ESTIMATE * FROM VARIABLES OF satellites
                      ORDER BY dependence probability with apogee DESC
                      LIMIT ?
                  )
              )
              USING 12 SAMPLES BY satellites MODELED BY m USING MODEL 1;
          ''', (3,)).fetchall()
  
          cursor = bdb.execute('''
              ESTIMATE * FROM VARIABLES OF satellites
                  ORDER BY dependence probability with apogee DESC
                  LIMIT ?
          ''', (3,)).fetchall()
          top_variables = [c[0] for c in cursor]
          nominals = [
              var for var in top_variables
              if var in ['country_of_operator', 'class_of_orbit',]
          ]
          numericals = [var for var in top_variables if var not in nominals]
          check_regression_variables(
              results, numericals + ['intercept'], nominals)
  
          # Cannot mix * with other variables.
          with pytest.raises(BQLError):
              bdb.execute('''
                  REGRESS apogee GIVEN (*, class
  
  [... the rest is omitted ...]
files: tests/test_regress.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import pytest
  
  from bayeslite import BQLError
  from bayeslite import bayesdb_register_backend
  from bayeslite.backends.cgpm_backend import CGPM_Backend
  
  from test_cgpm import cgpm_dummy_satellites_bdb
  
  
  def test_regress_bonanza__ci_integration():
      with cgpm_dummy_satellites_bdb() as bdb:
          bayesdb_register_backend(
              bdb, CGPM_Backend(dict(), multiprocess=0))
          bdb.execute('''
              CREATE POPULATION satellites FOR satellites_ucs WITH SCHEMA(
                  MODEL apogee AS NUMERICAL;
                  MODEL class_of_orbit AS NOMINAL;
                  MODEL country_of_operator AS NOMINAL;
                  MODEL launch_mass AS NUMERICAL;
                  MODEL perigee AS NUMERICAL;
                  MODEL period AS NUMERICAL
              )
          ''')
          bdb.execute('''
              CREATE GENERATOR m FOR satellites;
          ''')
          bdb.execute('INITIALIZE 2 MODELS FOR m;')
  
          def check_regression_variables(results, numericals, nominals):
              seen = set()
              for r in results:
                  assert len(r) == 2
                  variable = r[0]
                  assert variable not in seen
                  assert variable in numericals or \
                      any(variable.startswith('%s_dum_' % (nominal,))
                          for nominal in nominals)
                  seen.add(variable)
  
          # Regression on 1 numerical variable.
          results = bdb.execute('''
              REGRESS apogee GIVEN (perigee) USING 12 SAMPLES BY satellites;
          ''').fetchall()
          assert len(results) == 2
          check_regression_variables(results, ['intercept', 'perigee'], [])
  
          # Regression on 1 nominal variable.
          results = bdb.execute('''
              REGRESS apogee GIVEN (country_of_operator)
              USING 12 SAMPLES BY satellites;
          ''').fetchall()
          check_regression_variables(
              results, ['intercept'], ['country_of_operator'])
  
          # Regression on 1 nominal + 1 numerical variable.
          bdb.execute('''
              REGRESS apogee GIVEN (perigee, country_of_operator)
              USING 12 SAMPLES BY satellites;
          ''').fetchall()
          check_regression_variables(
              results, ['intercept', 'perigee'], ['country_of_operator'])
  
          # Regression on all variables.
          results = bdb.execute('''
              REGRESS apogee GIVEN (*) USING 12 SAMPLES BY satellites;
          ''', (3,)).fetchall()
          check_regression_variables(
              results,
              ['intercept', 'perigee', 'launch_mass', 'period',],
              ['country_of_operator', 'class_of_orbit',],
          )
  
          # Regression on column selector subexpression with a binding.
          results = bdb.execute('''
              REGRESS apogee GIVEN (
                  satellites.(
                      ESTIMATE * FROM VARIABLES OF satellites
                      ORDER BY dependence probability with apogee DESC
                      LIMIT ?
                  )
              )
              USING 12 SAMPLES BY satellites MODELED BY m USING MODEL 1;
          ''', (3,)).fetchall()
  
          cursor = bdb.execute('''
              ESTIMATE * FROM VARIABLES OF satellites
                  ORDER BY dependence probability with apogee DESC
                  LIMIT ?
          ''', (3,)).fetchall()
          top_variables = [c[0] for c in cursor]
          nominals = [
              var for var in top_variables
              if var in ['country_of_operator', 'class_of_orbit',]
          ]
          numericals = [var for var in top_variables if var not in nominals]
          check_regression_variables(
              results, numericals + ['intercept'], nominals)
  
          # Cannot mix * with other variables.
          with pytest.raises(BQLError):
              bdb.execute('''
                  REGRESS apogee
  
  [... the rest is omitted ...]
files: tests/test_regress.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import os
  
  import pytest
  
  import bayeslite
  import bayeslite.read_csv as read_csv
  
  from bayeslite.guess import bayesdb_guess_population
  
  
  root = os.path.dirname(os.path.abspath(__file__))
  dha_csv = os.path.join(root, 'dha.csv')
  
  
  # Test that simulating a column constrained to have a specific value
  # returns an error.
  #
  # XXX This should be a backend-independent test.
  def test_simulate_drawconstraint_error__ci_slow():
      with bayeslite.bayesdb_open() as bdb:
          with open(dha_csv, 'rU') as f:
              read_csv.bayesdb_read_csv(bdb, 'dha', f, header=True, create=True)
          bdb.backends['cgpm'].set_multiprocess(False)
          bayesdb_guess_population(
              bdb, 'hospital', 'dha', overrides=[('name', 'key')])
          bdb.execute(
              'CREATE GENERATOR hospital_cc FOR hospital USING cgpm;')
          bdb.execute('INITIALIZE 1 MODEL FOR hospital_cc')
          bdb.execute('ANALYZE hospital_cc FOR 1 ITERATION (OPTIMIZED);')
          with pytest.raises(ValueError):
              bdb.execute('''
                  SIMULATE ttl_mdcr_spnd, n_death_ill FROM hospital
                      GIVEN TTL_MDCR_SPND = 40000
                      LIMIT 100
              ''').fetchall()
          samples = bdb.execute('''
              SIMULATE n_death_ill FROM hospital
                  GIVEN TTL_MDCR_SPND = 40000
                  LIMIT 100
          ''').fetchall()
          assert len(samples) == 100
          assert all(len(s) == 1 for s in samples)
  
  
  data = [
      ('foo', 56),
      ('bar', 0),
      ('baz', 1),
      ('quux', 1),
      ('zot', 0),
      ('mumble', 2),
      ('frotz', 0),
      ('gargle', 0),
      ('mumph', 1),
      ('hunf', 3),
      ('blort', 0)
  ]
  
  
  def test_simulate_given_rowid():
      # Test simulation of a variable given a rowid. Uses synthetic data with
      # one variable, in which one value of the variable is different from the
      # others by an order of magnitude. Thus, simulating the value for that row
      # should produce values that are significantly different from simulated
      # values of the variable for another row.
      with bayeslite.bayesdb_open() as bdb:
          bdb.backends['cgpm'].set_multiprocess(False)
          bdb.sql_execute('CREATE TABLE t(x TEXT, y NUMERIC)')
          for row in data:
              bdb.sql_execute('INSERT INTO t (x, y) VALUES (?, ?)', row)
          bdb.execute('''
              CREATE POPULATION t_p FOR t WITH SCHEMA {
                  IGNORE x;
                  y NUMERICAL;
              }
          ''')
          bdb.execute('''
              CREATE GENERATOR t_g FOR t_p;
          ''')
          bdb.execute('INITIALIZE 1 MODEL FOR t_g')
          bdb.execute('ANALYZE t_g FOR 3 ITERATION')
          bdb.execute('''
              CREATE TABLE row1 AS
                  SIMULATE y FROM t_p
                  GIVEN _rowid_ = 1
              LIMIT 100
          ''')
          bdb.execute('''
              CREATE TABLE row5 AS
                  SIMULATE y FROM t_p
                  GIVEN oid = 5
              LIMIT 100
          ''')
          row1_avg = bdb.execute('SELECT AVG(y) FROM row1').fetchall()[0][0]
          row5_avg = bdb.execute('SELECT AVG(y) FROM row5').fetchall()[0][0]
          # Mean of simulations for row 1 should be "significantly" larger
  
  [... the rest is omitted ...]
files: tests/test_simulate.py
files: # -*- coding: utf-8 -*-
  
  #   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
  #
  #   Licensed under the Apache License, Version 2.0 (the "License");
  #   you may not use this file except in compliance with the License.
  #   You may obtain a copy of the License at
  #
  #       http://www.apache.org/licenses/LICENSE-2.0
  #
  #   Unless required by applicable law or agreed to in writing, software
  #   distributed under the License is distributed on an "AS IS" BASIS,
  #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #   See the License for the specific language governing permissions and
  #   limitations under the License.
  
  import os
  
  import pytest
  
  import bayeslite
  import bayeslite.read_csv as read_csv
  
  from bayeslite.guess import bayesdb_guess_population
  
  
  root = os.path.dirname(os.path.abspath(__file__))
  dha_csv = os.path.join(root, 'dha.csv')
  
  
  # Test that simulating a column constrained to have a specific value
  # returns an error.
  #
  # XXX This should be a backend-independent test.
  def test_simulate_drawconstraint_error__ci_slow():
      with bayeslite.bayesdb_open() as bdb:
          with open(dha_csv, 'rU') as f:
              read_csv.bayesdb_read_csv(bdb, 'dha', f, header=True, create=True)
          bdb.backends['cgpm'].set_multiprocess(False)
          bayesdb_guess_population(
              bdb, 'hospital', 'dha', overrides=[('name', 'key')])
          bdb.execute(
              'CREATE GENERATOR hospital_cc FOR hospital USING cgpm;')
          bdb.execute('INITIALIZE 1 MODEL FOR hospital_cc')
          bdb.execute('ANALYZE hospital_cc FOR 1 ITERATION (OPTIMIZED);')
          with pytest.raises(ValueError):
              bdb.execute('''
                  SIMULATE ttl_mdcr_spnd, n_death_ill FROM hospital
                      GIVEN TTL_MDCR_SPND = 40000
                      LIMIT 100
              ''').fetchall()
          samples = bdb.execute('''
              SIMULATE n_death_ill FROM hospital
                  GIVEN TTL_MDCR_SPND = 40000
                  LIMIT 100
          ''').fetchall()
          assert len(samples) == 100
          assert all(len(s) == 1 for s in samples)
  
  
  data = [
      ('foo', 56),
      ('bar', 0),
      ('baz', 1),
      ('quux', 1),
      ('zot', 0),
      ('mumble', 2),
      ('frotz', 0),
      ('gargle', 0),
      ('mumph', 1),
      ('hunf', 3),
      ('blort', 0)
  ]
  
  
  def test_simulate_given_rowid():
      # Test simulation of a variable given a rowid. Uses synthetic data with
      # one variable, in which one value of the variable is different from the
      # others by an order of magnitude. Thus, simulating the value for that row
      # should produce values that are significantly different from simulated
      # values of the variable for another row.
      with bayeslite.bayesdb_open() as bdb:
          bdb.backends['cgpm'].set_multiprocess(False)
          bdb.sql_execute('CREATE TABLE t(x TEXT, y NUMERIC)')
          for row in data:
              bdb.sql_execute('INSERT INTO t (x, y) VALUES (?, ?)', row)
          bdb.execute('''
              CREATE POPULATION t_p FOR t WITH SCHEMA {
                  IGNORE x;
                  MODEL y AS NUMERICAL;
              }
          ''')
          bdb.execute('''
              CREATE GENERATOR t_g FOR t_p;
          ''')
          bdb.execute('INITIALIZE 1 MODEL FOR t_g')
          bdb.execute('ANALYZE t_g FOR 3 ITERATION')
          bdb.execute('''
              CREATE TABLE row1 AS
                  SIMULATE y FROM t_p
                  GIVEN _rowid_ = 1
              LIMIT 100
          ''')
          bdb.execute('''
              CREATE TABLE row5 AS
                  SIMULATE y FROM t_p
                  GIVEN oid = 5
              LIMIT 100
          ''')
          row1_avg = bdb.execute('SELECT AVG(y) FROM row1').fetchall()[0][0]
          row5_avg = bdb.execute('SELECT AVG(y) FROM row5').fetchall()[0][0]
          # Mean of simulations for row 1 should be "significantly
  
  [... the rest is omitted ...]
files: tests/test_simulate.py