<commit_before>/*ckwg +5
 * Copyright 2011 by Kitware, Inc. All Rights Reserved. Please refer to
 * KITWARE_LICENSE.TXT for licensing information, or contact General Counsel,
 * Kitware, Inc., 28 Corporate Drive, Clifton Park, NY 12065.
 */

#include "load_pipe.h"

#include "pipe_declaration_types.h"
#include "providers.h"

#include <vistk/pipeline/config.h>
#include <vistk/pipeline/process.h>

#include <boost/algorithm/string/join.hpp>
#include <boost/tuple/tuple.hpp>
#include <boost/foreach.hpp>
#include <boost/variant.hpp>

#include <algorithm>
#include <utility>
#include <vector>

/**
 * \file pipe_bakery.cxx
 *
 * \brief Implementation of baking a pipeline.
 */

namespace vistk
{

namespace
{

static config_flag_t const flag_read_only = config_flag_t("ro");

static config_provider_t const provider_config = config_provider_t("CONF");
static config_provider_t const provider_environment = config_provider_t("ENV");
static config_provider_t const provider_system = config_provider_t("SYS");

}

class VISTK_PIPELINE_UTIL_NO_EXPORT pipe_bakery
  : public boost::static_visitor<>
{
  public:
    pipe_bakery();
    ~pipe_bakery();

    void operator () (config_pipe_block const& config_block);
    void operator () (process_pipe_block const& process_block);
    void operator () (connect_pipe_block const& connect_block);
    void operator () (group_pipe_block const& group_block);

    /**
     * \note We do *not* want std::map for the groupings. With a map, we may
     * hide errors in the blocks (setting ro values, duplicate process names,
     * etc.)
     */

    typedef std::pair<config_provider_t, config::value_t> provider_request_t;
    typedef boost::variant<config::value_t, provider_request_t> config_reference_t;
    typedef boost::tuple<config_reference_t, bool> config_info_t;
    typedef std::pair<config::key_t, config_info_t> config_decl_t;
    typedef std::vector<config_decl_t> config_decls_t;

    typedef std::pair<process::name_t, process_registry::type_t> process_decl_t;
    typedef std::vector<process_decl_t> process_decls_t;

    typedef std::pair<process::port_addr_t, process::port_addr_t> connection_t;
    typedef std::vector<connection_t> connections_t;

    typedef boost::tuple<process::port_t, process::port_flags_t, process::port_addr_t> mapping_t;
    typedef std::vector<mapping_t> mappings_t;
    typedef std::pair<mappings_t, mappings_t> group_info_t;
    typedef std::pair<process::name_t, group_info_t> group_decl_t;
    typedef std::vector<group_decl_t> group_decls_t;

    static config::key_t flatten_keys(config::keys_t const& keys);
    void register_config_value(config::key_t const& root_key, config_value_t const& value);

    config_decls_t m_configs;
    process_decls_t m_processes;
    connections_t m_connections;
    group_decls_t m_groups;
};

class VISTK_PIPELINE_UTIL_NO_EXPORT provider_dereferencer
  : public boost::static_visitor<pipe_bakery::config_reference_t>
{
  public:
    provider_dereferencer();
    provider_dereferencer(config_t const conf);
    ~provider_dereferencer();

    pipe_bakery::config_reference_t operator () (config::value_t const& value) const;
    pipe_bakery::config_reference_t operator () (pipe_bakery::provider_request_t const& request) const;
  private:
    typedef std::map<config_provider_t, provider_t> provider_map_t;
    provider_map_t m_providers;
};

class VISTK_PIPELINE_UTIL_NO_EXPORT ensure_provided
  : public boost::static_visitor<config::value_t>
{
  public:
    config::value_t operator () (config::value_t const& value) const;
    config::value_t operator () (pipe_bakery::provider_request_t const& request) const;
};

pipeline_t
bake_pipe_blocks(pipe_blocks const& blocks)
{
  pipeline_t pipeline;

  pipe_bakery bakery;

  std::for_each(blocks.begin(), blocks.end(), boost::apply_visitor(bakery));

  // Build configuration.
  {
    // Dereference (non-configuration) providers.
    {
      BOOST_FOREACH (pipe_bakery::config_decl_t& decl, bakery.m_configs)
      {
        pipe_bakery::config_reference_t& ref = decl.second.get<0>();

        ref = boost::apply_visitor(provider_dereferencer(), ref);
      }
    }

    config_t conf = config::empty_config();

    BOOST_FOREACH (pipe_bakery::config_decl_t& decl, bakery.m_configs)
    {
      pipe_bakery::config_reference_t const& ref = decl.second.get<0>();

      config::key_t const key = decl.first;
      config::value_t val;

      // Only add provided configurations to the configuration.
      try
      {
        val = boost::apply_visitor(ensure_provided(), ref);
      }
      catch (...)
      {
        continue;
      }

      conf->set_value(key, val);
    }

    // Dereference configuration providers.
    {
      /// \todo Sort requests so circular requests don't occur.

      /// \todo Find config provider requests.
      config::keys_t keys;

      provider_dereferencer deref(conf);

      /// \todo This is algorithmically naive, but I'm not sure if there's a faster way.
      BOOST_FOREACH (config::key_t const& key, keys)
      {
        BOOST_FOREACH (pipe_bakery::config_decl_t& decl, bakery.m_configs)
        {
          config::key_t const& cur_key = decl.first;

          if (key != cur_key)
          {
            continue;
          }

          pipe_bakery::config_reference_t& ref = decl.second.get<0>();

          ref = boost::apply_visitor(deref, ref);

          config::value_t const val = boost::apply_visitor(ensure_provided(), ref);

          // Set the value in the intermediate configuration.
          conf->set_value(cur_key, val);
        }
      }
    }

    /// \todo Build final configuration.
  }

  /// \todo Bake pipe blocks into a pipeline.

  return pipeline;
}

pipe_bakery
::pipe_bakery()
{
}

pipe_bakery
::~pipe_bakery()
{
}

void
pipe_bakery
::operator () (config_pipe_block const& config_block)
{
  config::key_t const root_key = flatten_keys(config_block.key);

  config_values_t const& values = config_block.values;

  BOOST_FOREACH (config_value_t const& value, values)
  {
    register_config_value(root_key, value);
  }
}

void
pipe_bakery
::operator () (process_pipe_block const& process_block)
{
  config_values_t const& values = process_block.config_values;

  // Build the configuration value for the name of the process.
  config_value_t name_value;
  name_value.key.key_path.push_back(process::config_name);
  name_value.key.options.flags = config_flags_t();
  (*name_value.key.options.flags).push_back(flag_read_only);
  name_value.value = process_block.name;

  register_config_value(process_block.name, name_value);

  BOOST_FOREACH (config_value_t const& value, values)
  {
    register_config_value(process_block.name, value);
  }

  m_processes.push_back(process_decl_t(process_block.name, process_block.type));
}

void
pipe_bakery
::operator () (connect_pipe_block const& connect_block)
{
  m_connections.push_back(connection_t(connect_block.from, connect_block.to));
}

void
pipe_bakery
::operator () (group_pipe_block const& group_block)
{
  config_values_t const& values = group_block.config_values;

  BOOST_FOREACH (config_value_t const& value, values)
  {
    register_config_value(group_block.name, value);
  }

  process::port_flags_t default_flags;

  mappings_t input_mappings;

  BOOST_FOREACH (input_map_t const& map, group_block.input_mappings)
  {
    process::port_flags_t flags = default_flags;

    if (map.options.flags)
    {
      flags = *map.options.flags;
    }

    mapping_t const mapping = mapping_t(map.from, flags, map.to);

    input_mappings.push_back(mapping);
  }

  mappings_t output_mappings;

  BOOST_FOREACH (output_map_t const& map, group_block.output_mappings)
  {
    process::port_flags_t flags = default_flags;

    if (map.options.flags)
    {
      flags = *map.options.flags;
    }

    mapping_t const mapping = mapping_t(map.to, flags, map.from);

    output_mappings.push_back(mapping);
  }

  group_info_t const info = group_info_t(input_mappings, output_mappings);
  group_decl_t const decl = group_decl_t(group_block.name, info);

  m_groups.push_back(decl);
}

void
pipe_bakery
::register_config_value(config::key_t const& root_key, config_value_t const& value)
{
  config_key_t const key = value.key;

  config::key_t const subkey = flatten_keys(key.key_path);

  config_reference_t c_value;

  if (key.options.provider)
  {
    c_value = provider_request_t(*key.options.provider, value.value);
  }
  else
  {
    c_value = value.value;
  }

  bool is_readonly = false;

  if (key.options.flags)
  {
    BOOST_FOREACH (config_flag_t const& flag, *key.options.flags)
    {
      if (flag == flag_read_only)
      {
        is_readonly = true;
      }
      else
      {
        /// \todo Log warning about unrecognized flag.
      }
    }
  }

  config::key_t const full_key = root_key + config::block_sep + subkey;

  config_info_t const info = config_info_t(c_value, is_readonly);

  config_decl_t const decl = config_decl_t(full_key, info);

  m_configs.push_back(decl);
}

config::key_t
pipe_bakery
::flatten_keys(config::keys_t const& keys)
{
  return boost::join(keys, config::block_sep);
}

provider_dereferencer
::provider_dereferencer()
{
  m_providers[provider_system] = provider_t(new system_provider);
  m_providers[provider_environment] = provider_t(new environment_provider);
}

provider_dereferencer
::provider_dereferencer(config_t const conf)
{
  m_providers[provider_config] = provider_t(new config_provider(conf));
}

provider_dereferencer
::~provider_dereferencer()
{
}

pipe_bakery::config_reference_t
provider_dereferencer
::operator () (config::value_t const& value) const
{
  return value;
}

pipe_bakery::config_reference_t
provider_dereferencer
::operator () (pipe_bakery::provider_request_t const& request) const
{
  provider_map_t::const_iterator const i = m_providers.find(request.first);
  if (i == m_providers.end())
  {
    return request;
  }

  return (*i->second)(request.second);
}

config::value_t
ensure_provided
::operator () (config::value_t const& value) const
{
  return value;
}

config::value_t
ensure_provided
::operator () (pipe_bakery::provider_request_t const& /*request*/) const
{
  /// \todo Throw an exception.

  return config::value_t();
}

}
<commit_msg>Add a sorter for configuration provider requests<commit_after>/*ckwg +5
 * Copyright 2011 by Kitware, Inc. All Rights Reserved. Please refer to
 * KITWARE_LICENSE.TXT for licensing information, or contact General Counsel,
 * Kitware, Inc., 28 Corporate Drive, Clifton Park, NY 12065.
 */

#include "load_pipe.h"

#include "pipe_declaration_types.h"
#include "providers.h"

#include <vistk/pipeline/config.h>
#include <vistk/pipeline/process.h>

#include <boost/algorithm/string/join.hpp>
#include <boost/graph/directed_graph.hpp>
#include <boost/graph/topological_sort.hpp>
#include <boost/tuple/tuple.hpp>
#include <boost/foreach.hpp>
#include <boost/variant.hpp>

#include <algorithm>
#include <utility>
#include <vector>

/**
 * \file pipe_bakery.cxx
 *
 * \brief Implementation of baking a pipeline.
 */

namespace vistk
{

namespace
{

static config_flag_t const flag_read_only = config_flag_t("ro");

static config_provider_t const provider_config = config_provider_t("CONF");
static config_provider_t const provider_environment = config_provider_t("ENV");
static config_provider_t const provider_system = config_provider_t("SYS");

}

class VISTK_PIPELINE_UTIL_NO_EXPORT pipe_bakery
  : public boost::static_visitor<>
{
  public:
    pipe_bakery();
    ~pipe_bakery();

    void operator () (config_pipe_block const& config_block);
    void operator () (process_pipe_block const& process_block);
    void operator () (connect_pipe_block const& connect_block);
    void operator () (group_pipe_block const& group_block);

    /**
     * \note We do *not* want std::map for the groupings. With a map, we may
     * hide errors in the blocks (setting ro values, duplicate process names,
     * etc.)
     */

    typedef std::pair<config_provider_t, config::value_t> provider_request_t;
    typedef boost::variant<config::value_t, provider_request_t> config_reference_t;
    typedef boost::tuple<config_reference_t, bool> config_info_t;
    typedef std::pair<config::key_t, config_info_t> config_decl_t;
    typedef std::vector<config_decl_t> config_decls_t;

    typedef std::pair<process::name_t, process_registry::type_t> process_decl_t;
    typedef std::vector<process_decl_t> process_decls_t;

    typedef std::pair<process::port_addr_t, process::port_addr_t> connection_t;
    typedef std::vector<connection_t> connections_t;

    typedef boost::tuple<process::port_t, process::port_flags_t, process::port_addr_t> mapping_t;
    typedef std::vector<mapping_t> mappings_t;
    typedef std::pair<mappings_t, mappings_t> group_info_t;
    typedef std::pair<process::name_t, group_info_t> group_decl_t;
    typedef std::vector<group_decl_t> group_decls_t;

    static config::key_t flatten_keys(config::keys_t const& keys);
    void register_config_value(config::key_t const& root_key, config_value_t const& value);

    config_decls_t m_configs;
    process_decls_t m_processes;
    connections_t m_connections;
    group_decls_t m_groups;
};

class VISTK_PIPELINE_UTIL_NO_EXPORT provider_dereferencer
  : public boost::static_visitor<pipe_bakery::config_reference_t>
{
  public:
    provider_dereferencer();
    provider_dereferencer(config_t const conf);
    ~provider_dereferencer();

    pipe_bakery::config_reference_t operator () (config::value_t const& value) const;
    pipe_bakery::config_reference_t operator () (pipe_bakery::provider_request_t const& request) const;
  private:
    typedef std::map<config_provider_t, provider_t> provider_map_t;
    provider_map_t m_providers;
};

class VISTK_PIPELINE_UTIL_NO_EXPORT ensure_provided
  : public boost::static_visitor<config::value_t>
{
  public:
    config::value_t operator () (config::value_t const& value) const;
    config::value_t operator () (pipe_bakery::provider_request_t const& request) const;
};

class VISTK_PIPELINE_UTIL_NO_EXPORT config_provider_sorter
  : public boost::static_visitor<>
{
  public:
    void operator () (config::key_t const& key, config::value_t const& value) const;
    void operator () (config::key_t const& key, pipe_bakery::provider_request_t const& request);

    config::keys_t sorted() const;
  private:
    struct vertex_name_t
    {
      typedef boost::vertex_property_tag kind;
    };
    typedef boost::property<vertex_name_t, config::key_t> name_property_t;
    typedef boost::adjacency_list<boost::vecS, boost::vecS, boost::directedS, name_property_t> config_graph_t;
    typedef boost::graph_traits<config_graph_t>::vertex_descriptor vertex_t;
    typedef std::vector<vertex_t> vertices_t;
    typedef std::map<config::key_t, vertex_t> vertex_map_t;

    vertex_map_t m_vertex_map;
    config_graph_t m_graph;
};

pipeline_t
bake_pipe_blocks(pipe_blocks const& blocks)
{
  pipeline_t pipeline;

  pipe_bakery bakery;

  std::for_each(blocks.begin(), blocks.end(), boost::apply_visitor(bakery));

  // Build configuration.
  {
    // Dereference (non-configuration) providers.
    {
      BOOST_FOREACH (pipe_bakery::config_decl_t& decl, bakery.m_configs)
      {
        pipe_bakery::config_reference_t& ref = decl.second.get<0>();

        ref = boost::apply_visitor(provider_dereferencer(), ref);
      }
    }

    config_t conf = config::empty_config();

    BOOST_FOREACH (pipe_bakery::config_decl_t& decl, bakery.m_configs)
    {
      pipe_bakery::config_reference_t const& ref = decl.second.get<0>();

      config::key_t const key = decl.first;
      config::value_t val;

      // Only add provided configurations to the configuration.
      try
      {
        val = boost::apply_visitor(ensure_provided(), ref);
      }
      catch (...)
      {
        continue;
      }

      conf->set_value(key, val);
    }

    // Dereference configuration providers.
    {
      /// \todo Sort requests so circular requests don't occur.

      /// \todo Find config provider requests.
      config::keys_t keys;

      provider_dereferencer deref(conf);

      /// \todo This is algorithmically naive, but I'm not sure if there's a faster way.
      BOOST_FOREACH (config::key_t const& key, keys)
      {
        BOOST_FOREACH (pipe_bakery::config_decl_t& decl, bakery.m_configs)
        {
          config::key_t const& cur_key = decl.first;

          if (key != cur_key)
          {
            continue;
          }

          pipe_bakery::config_reference_t& ref = decl.second.get<0>();

          ref = boost::apply_visitor(deref, ref);

          config::value_t const val = boost::apply_visitor(ensure_provided(), ref);

          // Set the value in the intermediate configuration.
          conf->set_value(cur_key, val);
        }
      }
    }

    /// \todo Build final configuration.
  }

  /// \todo Bake pipe blocks into a pipeline.

  return pipeline;
}

pipe_bakery
::pipe_bakery()
{
}

pipe_bakery
::~pipe_bakery()
{
}

void
pipe_bakery
::operator () (config_pipe_block const& config_block)
{
  config::key_t const root_key = flatten_keys(config_block.key);

  config_values_t const& values = config_block.values;

  BOOST_FOREACH (config_value_t const& value, values)
  {
    register_config_value(root_key, value);
  }
}

void
pipe_bakery
::operator () (process_pipe_block const& process_block)
{
  config_values_t const& values = process_block.config_values;

  // Build the configuration value for the name of the process.
  config_value_t name_value;
  name_value.key.key_path.push_back(process::config_name);
  name_value.key.options.flags = config_flags_t();
  (*name_value.key.options.flags).push_back(flag_read_only);
  name_value.value = process_block.name;

  register_config_value(process_block.name, name_value);

  BOOST_FOREACH (config_value_t const& value, values)
  {
    register_config_value(process_block.name, value);
  }

  m_processes.push_back(process_decl_t(process_block.name, process_block.type));
}

void
pipe_bakery
::operator () (connect_pipe_block const& connect_block)
{
  m_connections.push_back(connection_t(connect_block.from, connect_block.to));
}

void
pipe_bakery
::operator () (group_pipe_block const& group_block)
{
  config_values_t const& values = group_block.config_values;

  BOOST_FOREACH (config_value_t const& value, values)
  {
    register_config_value(group_block.name, value);
  }

  process::port_flags_t default_flags;

  mappings_t input_mappings;

  BOOST_FOREACH (input_map_t const& map, group_block.input_mappings)
  {
    process::port_flags_t flags = default_flags;

    if (map.options.flags)
    {
      flags = *map.options.flags;
    }

    mapping_t const mapping = mapping_t(map.from, flags, map.to);

    input_mappings.push_back(mapping);
  }

  mappings_t output_mappings;

  BOOST_FOREACH (output_map_t const& map, group_block.output_mappings)
  {
    process::port_flags_t flags = default_flags;

    if (map.options.flags)
    {
      flags = *map.options.flags;
    }

    mapping_t const mapping = mapping_t(map.to, flags, map.from);

    output_mappings.push_back(mapping);
  }

  group_info_t const info = group_info_t(input_mappings, output_mappings);
  group_decl_t const decl = group_decl_t(group_block.name, info);

  m_groups.push_back(decl);
}

void
pipe_bakery
::register_config_value(config::key_t const& root_key, config_value_t const& value)
{
  config_key_t const key = value.key;

  config::key_t const subkey = flatten_keys(key.key_path);

  config_reference_t c_value;

  if (key.options.provider)
  {
    c_value = provider_request_t(*key.options.provider, value.value);
  }
  else
  {
    c_value = value.value;
  }

  bool is_readonly = false;

  if (key.options.flags)
  {
    BOOST_FOREACH (config_flag_t const& flag, *key.options.flags)
    {
      if (flag == flag_read_only)
      {
        is_readonly = true;
      }
      else
      {
        /// \todo Log warning about unrecognized flag.
      }
    }
  }

  config::key_t const full_key = root_key + config::block_sep + subkey;

  config_info_t const info = config_info_t(c_value, is_readonly);

  config_decl_t const decl = config_decl_t(full_key, info);

  m_configs.push_back(decl);
}

config::key_t
pipe_bakery
::flatten_keys(config::keys_t const& keys)
{
  return boost::join(keys, config::block_sep);
}

provider_dereferencer
::provider_dereferencer()
{
  m_providers[provider_system] = provider_t(new system_provider);
  m_providers[provider_environment] = provider_t(new environment_provider);
}

provider_dereferencer
::provider_dereferencer(config_t const conf)
{
  m_providers[provider_config] = provider_t(new config_provider(conf));
}

provider_dereferencer
::~provider_dereferencer()
{
}

pipe_bakery::config_reference_t
provider_dereferencer
::operator () (config::value_t const& value) const
{
  return value;
}

pipe_bakery::config_reference_t
provider_dereferencer
::operator () (pipe_bakery::provider_request_t const& request) const
{
  provider_map_t::const_iterator const i = m_providers.find(request.first);
  if (i == m_providers.end())
  {
    return request;
  }

  return (*i->second)(request.second);
}

config::value_t
ensure_provided
::operator () (config::value_t const& value) const
{
  return value;
}

config::value_t
ensure_provided
::operator () (pipe_bakery::provider_request_t const& /*request*/) const
{
  /// \todo Throw an exception.

  return config::value_t();
}

config::keys_t
config_provider_sorter
::sorted() const
{
  vertices_t vertices;

  try
  {
    boost::topological_sort(m_graph, std::back_inserter(vertices));
  }
  catch (boost::not_a_dag& e)
  {
    /// \todo Throw circular configuration provider exception.
  }

  config::keys_t keys;

  boost::property_map<config_graph_t, vertex_name_t>::const_type const key_prop = boost::get(vertex_name_t(), m_graph);

  BOOST_FOREACH (vertex_t const& vertex, vertices)
  {
    keys.push_back(boost::get(key_prop, vertex));
  }

  return keys;
}

void
config_provider_sorter
::operator () (config::key_t const& /*key*/, config::value_t const& /*value*/) const
{
}

void
config_provider_sorter
::operator () (config::key_t const& key, pipe_bakery::provider_request_t const& request)
{
  if (request.first != provider_config)
  {
    return;
  }

  boost::property_map<config_graph_t, vertex_name_t>::type key_prop = boost::get(vertex_name_t(), m_graph);

  typedef std::pair<vertex_map_t::iterator, bool> insertion_t;

  insertion_t from_iter = m_vertex_map.insert(std::make_pair(key, vertex_t()));
  insertion_t to_iter = m_vertex_map.insert(std::make_pair(request.second, vertex_t()));

  vertex_t s;
  vertex_t t;

  if (from_iter.second)
  {
    s = boost::add_vertex(m_graph);
    key_prop[s] = key;
    from_iter.first->second = s;
  }

  s = from_iter.first->second;

  if (to_iter.second)
  {
    t = boost::add_vertex(m_graph);
    key_prop[s] = request.second;
    to_iter.first->second = t;
  }

  t = to_iter.first->second;

  boost::add_edge(s, t, m_graph);
}

}
<|endoftext|>


<commit_before>/*-
 * Copyright (c) 2012, Achilleas Margaritis
 * Copyright (c) 2014, David T. Chisnall
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 *
 * * Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
#ifndef PEGMATITE_AST_HPP
#define PEGMATITE_AST_HPP

#include <cassert>
#include <list>
#include <unordered_map>
#include <memory>
#include "parser.hh"

namespace pegmatite {

class ASTNode;
template <class T, bool OPT> class ASTPtr;
template <class T> class ASTList;
template <class T> class BindAST;

/** type of AST node stack.
 */
typedef std::vector<std::unique_ptr<ASTNode>> ASTStack;

#ifdef USE_RTTI
#define PEGMATITE_RTTI(thisclass, superclass)
#else
/**
 * Define the methods required for pegmatite's lightweight RTTI replacement to
 * work.  This should be used at the end of the class definition and will
 * provide support for safe downcasting.
 */
#define PEGMATITE_RTTI(thisclass, superclass)            \
	friend ASTNode;                                      \
protected:                                               \
	static char *classKind()                             \
	{                                                    \
		static char thisclass ## id;                     \
		return &thisclass ## id;                         \
	}                                                    \
public:                                                  \
	virtual bool isa(char *x)                            \
	{                                                    \
		return (x == classKind()) ||                     \
				(superclass::isa(x));                    \
	}
#endif

/**
 * Base class for AST nodes.
 */
class ASTNode
{
public:
	/**
	 * Constructs the AST node, with a null parent.
	 */
	ASTNode() : parent_node(0) {}
	/**
	 * Copying AST nodes is not supported.
	 */
	ASTNode(const ASTNode&) = delete;

	/**
	 * Destructor does nothing, virtual for subclasses to use.
	 */
	virtual ~ASTNode() {}

	/**
	 * Returns the parent of this AST node, or `nullptr` if there isn't one
	 * (either if this is the root, or if it is still in the stack waiting to
	 * be added to the tree).
	 */
	ASTNode *parent() const { return parent_node; }

	/**
	 * Interface for constructing the AST node.  The input range `r` is the
	 * range within the source.
	 */
	virtual void construct(const InputRange &r, ASTStack &st) {}

private:
	/**
	 * The parent AST node.
	 */
	ASTNode *parent_node;

	template <class T, bool OPT> friend class ASTPtr;
	template <class T> friend class ASTList;
	template <class T> friend class BindAST;

#ifndef USE_RTTI
protected:
	/**
	 * Returns the kind of object class.  This is a unique pointer that can be
	 * tested against pointers returned by classKind() to determine whether
	 * they can be safely compared.
	 */
	virtual char *kind() { return classKind(); }
	/**
	 * Returns the unique identifier for this class.
	 */
	static char *classKind()
	{
		static char ASTNodeid;
		return &ASTNodeid;
	}
public:
	/**
	 * Root implementation of the RTTI-replacement for builds not wishing to
	 * use RTTI.  This returns true if `x` is the value returned from
	 * `classKind()`, or false otherwise.
	 */
	virtual bool isa(char *x)
	{
		return x == classKind();
	}
	/**
	 * Returns true if this object is an instance of `T`.  Note that this
	 * *only* works with single-inheritance hierarchies.  If you wish to use
	 * multiple inheritance in your AST classes, then you must define
	 * `USE_RTTI` and use the C++ RTTI mechanism.
	 */
	template <class T> bool isa()
	{
		return isa(T::classKind());
	}
	/**
	 * Returns a pointer to this object as a pointer to a child class, or
	 * `nullptr` if the cast would be unsafe.
	 *
	 * Note that AST nodes are intended to be always used as unique pointers
	 * and so the returned object is *only* valid as long as the unique pointer
	 * is valid.
	 */
	template <class T> T* get_as()
	{
		return this ? (isa<T>() ? static_cast<T*>(this) : nullptr) : nullptr;
	}
#else
public:
	template <class T> T* get_as()
	{
		return dynamic_cast<T*>(this);
	}
#endif
};

class ASTMember;

/** type of ast member vector.
 */

/**
 * The base class for non-leaf AST nodes.  Subclasses can have instances of
 * `ASTMember` subclasses as fields and will automatically construct them.
 */
class ASTContainer : public ASTNode
{
public:
	/**
	 * Constructs the container, setting a thread-local value to point to it
	 * allowing constructors in fields of the subclass to register themselves
	 * in the members vector.
	 */
	ASTContainer();

	/**
	 * Asks all members to construct themselves from the stack. The members are
	 * asked to construct themselves in reverse order from a node stack (`st`).
	 *
	 * The input range (`r`) is unused, because the leaf nodes have already
	 * constructed themselves at this point.
	 */
	virtual void construct(const InputRange &r, ASTStack &st);

private:
	/**
	 * The type used for tracking the fields of subclasses.
	 */
	typedef std::vector<ASTMember *> ASTMember_vector;
	/**
	 * References to all of the fields of the subclass that will be
	 * automatically constructed.
	 */
	ASTMember_vector members;

	friend class ASTMember;
	PEGMATITE_RTTI(ASTContainer, ASTNode)
};

/**
 * Base class for children of `ASTContainer`.
 */
class ASTMember
{
public:
	/**
	 * On construction, `ASTMember` sets its `container_node` field to the
	 * `ASTContainer` currently under construction and registers itself with
	 * the container, to be notified during the construction phase.
	 */
	ASTMember();

	/**
	 * Returns the container of which this object is a field.
	 */
	ASTContainer *container() const { return container_node; }

	/**
	 * Interface for constructing references to AST objects from the stack.
	 */
	virtual void construct(ASTStack &st) = 0;
	virtual ~ASTMember();
protected:
	/**
	 * The container that owns this object.
	 */
	ASTContainer *container_node;
};

/**
 * An `ASTPtr` is a wrapper around a pointer to an AST object.  It is intended
 * to be a member of an `ASTContainer` and will automatically pop the top item
 * from the stack and claim it when building the AST..
 */
template <class T, bool OPT = false> class ASTPtr : public ASTMember
{
public:
	/**
	 * Constructs the object in the
	 */
	ASTPtr() : ptr(nullptr) {}

	/** gets the underlying ptr value.
		@return the underlying ptr value.
	 */
	T *get() const
	{
		return ptr.get();
	}

	/** auto conversion to the underlying object ptr.
		@return the underlying ptr value.
	 */
	const std::unique_ptr<T> &operator *() const
	{
		return ptr;
	}

	/** member access.
		@return the underlying ptr value.
	 */
	const std::unique_ptr<T> &operator ->() const
	{
		assert(ptr);
		return ptr;
	}

	explicit operator bool() const noexcept
	{
		return (bool)ptr;
	}

	/**
	 * Pops the next matching object from the AST stack `st` and claims it.
	 */
	virtual void construct(ASTStack &st)
	{

		if (st.empty() && OPT)
		{
			return;
		}
		assert(!st.empty() && "Stack must not be empty");
		//get the node
		ASTNode *node = st.back().get();

		//get the object
		T *obj = node->get_as<T>();

		assert((obj || OPT) && "Required objects must exist!");
		//if the object is optional, simply return
		if (OPT && !obj)
		{
			return;
		}
		//set the new object
		ptr.reset(obj);
		//pop the node from the stack
		st.back().release();
		st.pop_back();
		ptr->parent_node = container_node;
	}

private:
	/**
	 * The node that we are pointing to.
	 */
	std::unique_ptr<T> ptr;
};

/** A list of objects.
	It pops objects of the given type from the ast stack, until no more objects can be popped.
	It assumes ownership of objects.
	@param T type of object to control.
 */
template <class T> class ASTList : public ASTMember
{
public:
	///list type.
	typedef std::list<std::unique_ptr<T>> container;

	///the default constructor.
	ASTList() {}

	/** duplicates the objects of the given list.
		@param src source object.
	 */
	ASTList(const ASTList<T> &src)
	{
		_dup(src);
	}

	/** returns the container of objects.
		@return the container of objects.
	 */
	const container &objects() const
	{
		return child_objects;
	}

	/**
	 * Pops objects of type T from the stack (`st`) until no more objects can
	 * be popped.
	 */
	virtual void construct(ASTStack &st)
	{
		for(;;)
		{
			//if the stack is empty
			if (st.empty()) break;

			//get the node
			ASTNode *node = st.back().get();

			//get the object
			T *obj = node->get_as<T>();

			//if the object was not not of the appropriate type,
			//end the list parsing
			if (!obj) return;

			//remove the node from the stack
			st.back().release();
			st.pop_back();

			//insert the object in the list, in reverse order
			child_objects.push_front(std::unique_ptr<T>(obj));

			//set the object's parent
			obj->parent_node = ASTMember::container();
		}
	}
	virtual ~ASTList() {}

private:
	//objects
	container child_objects;

	//duplicate the given list.
	void _dup(const ASTList<T> &src)
	{
		for (auto child : src.child_objects)
		{
			T *obj = new T(child.get());
			child_objects.push_back(obj);
			obj->parent_node = ASTMember::container();
		}
	}
};

/** parses the given input.
	@param i input.
	@param g root rule of grammar.
	@param ws whitespace rule.
	@param el list of errors.
	@param d user data, passed to the parse procedures.
	@return pointer to ast node created, or null if there was an error.
		The return object must be deleted by the caller.
 */
std::unique_ptr<ASTNode> parse(Input &i, const Rule &g, const Rule &ws,
                               ErrorList &el, const ParserDelegate &d);

/**
 * A parser delegate that is responsible for creating AST nodes from the input.
 *
 * This class manages a mapping from rules in some grammar to AST nodes.
 * Instances of the `BindAST` class that are fields of a subclass of this will
 * automatically register rules on creation.
 *
 * The recommended use for this class is to only register rules on construction
 * (either explicitly in the constructor or implicitly via `BindAST` members).
 * This will give a completely reentrant delegate, which can be used by
 * multiple threads to parse multiple inputs safely.
 */
class ASTParserDelegate : ParserDelegate
{
	/**
	 * BindAST is a friend so that it can call the `set_parse_proc()` function,
	 * which should never be called from anything else.
	 */
	template <class T> friend class BindAST;
	private:
	/**
	 * The map from rules to parsing handlers.
	 */
	std::unordered_map<const Rule*, parse_proc> handlers;
	/**
	 * Registers a callback in this delegate.  This should only be called from
	 * the `static` version of this function.
	 */
	void set_parse_proc(const Rule &r, parse_proc p);
	/**
	 * Registers a callback for a specific rule in the instance of this class
	 * currently under construction in this thread.  This should only ever be
	 * called by `BindAST` instances.
	 */
	static void bind_parse_proc(const Rule &r, parse_proc p);
	public:
	/**
	 * Default constructor, registers this class in thread-local storage so
	 * that it can be referenced by BindAST fields in subclasses when their
	 * constructors are run.
	 */
	ASTParserDelegate();
	virtual parse_proc get_parse_proc(const Rule &) const;
	/**
	 * Parse an input `i`, starting from rule `g` in the grammar for which
	 * this is a delegate.  The rule `ws` is used as whitespace.  Errors are
	 * returned via the `el` parameter and the root of the AST via the `ast`
	 * parameter.
	 *
	 * This function returns true on a successful parse, or false otherwise.
	 */
	template <class T> bool parse(Input &i, const Rule &g, const Rule &ws,
	                              ErrorList &el, std::unique_ptr<T> &ast) const
	{
		std::unique_ptr<ASTNode> node = pegmatite::parse(i, g, ws, el, *this);
		T *n = node->get_as<T>();
		if (n)
		{
			node.release();
			ast.reset(n);
			return true;
		}
		return false;
	}
};

/**
 * The `BindAST` class is responsible for binding an action to a rule.  The
 * template argument is the `ASTNode` subclass representing the action.  Its
 * `construct()` method will be called when the rule is matched.
 */
template <class T> class BindAST
{
public:
	/**
	 * Bind the AST class described in the grammar to the rule specified.
	 */
	BindAST(const Rule &r)
	{
		ASTParserDelegate::bind_parse_proc(r, [](const ParserPosition &b,
		                                         const ParserPosition &e, void *d)
			{
				ASTStack *st = reinterpret_cast<ASTStack *>(d);
				T *obj = new T();
				obj->construct(InputRange(b, e), *st);
				st->push_back(std::unique_ptr<ASTNode>(obj));
			});
	}
};

} //namespace pegmatite

#endif //PEGMATITE_AST_HPP
<commit_msg>Add debug logging for AST building operations.<commit_after>/*-
 * Copyright (c) 2012, Achilleas Margaritis
 * Copyright (c) 2014, David T. Chisnall
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 *
 * * Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
#ifndef PEGMATITE_AST_HPP
#define PEGMATITE_AST_HPP

#include <cassert>
#include <list>
#include <unordered_map>
#include <memory>
#include <cxxabi.h>
#include "parser.hh"

namespace pegmatite {

template <class T> void debug_log(const char *msg, int depth, T *obj)
{
#ifdef DEBUG_AST_CONSTRUCTION
	const char *mangled = typeid(T).name();
	char *buffer = (char*)malloc(strlen(mangled));
	int err;
	size_t s;
	char *demangled = abi::__cxa_demangle(mangled, buffer, &s,
		&err);
	fprintf(stderr, "[%d] %s %s (%p) off the AST stack\n",
			depth, msg,
		demangled ? demangled : mangled, obj);
	free((void*)(demangled ? demangled : buffer));
#endif // DEBUG_AST_CONSTRUCTION
}

class ASTNode;
template <class T, bool OPT> class ASTPtr;
template <class T> class ASTList;
template <class T> class BindAST;

/** type of AST node stack.
 */
typedef std::vector<std::unique_ptr<ASTNode>> ASTStack;

#ifdef USE_RTTI
#define PEGMATITE_RTTI(thisclass, superclass)
#else
/**
 * Define the methods required for pegmatite's lightweight RTTI replacement to
 * work.  This should be used at the end of the class definition and will
 * provide support for safe downcasting.
 */
#define PEGMATITE_RTTI(thisclass, superclass)            \
	friend ASTNode;                                      \
protected:                                               \
	static char *classKind()                             \
	{                                                    \
		static char thisclass ## id;                     \
		return &thisclass ## id;                         \
	}                                                    \
public:                                                  \
	virtual bool isa(char *x)                            \
	{                                                    \
		return (x == classKind()) ||                     \
				(superclass::isa(x));                    \
	}
#endif

/**
 * Base class for AST nodes.
 */
class ASTNode
{
public:
	/**
	 * Constructs the AST node, with a null parent.
	 */
	ASTNode() : parent_node(0) {}
	/**
	 * Copying AST nodes is not supported.
	 */
	ASTNode(const ASTNode&) = delete;

	/**
	 * Destructor does nothing, virtual for subclasses to use.
	 */
	virtual ~ASTNode() {}

	/**
	 * Returns the parent of this AST node, or `nullptr` if there isn't one
	 * (either if this is the root, or if it is still in the stack waiting to
	 * be added to the tree).
	 */
	ASTNode *parent() const { return parent_node; }

	/**
	 * Interface for constructing the AST node.  The input range `r` is the
	 * range within the source.
	 */
	virtual void construct(const InputRange &r, ASTStack &st) {}

private:
	/**
	 * The parent AST node.
	 */
	ASTNode *parent_node;

	template <class T, bool OPT> friend class ASTPtr;
	template <class T> friend class ASTList;
	template <class T> friend class BindAST;

#ifndef USE_RTTI
protected:
	/**
	 * Returns the kind of object class.  This is a unique pointer that can be
	 * tested against pointers returned by classKind() to determine whether
	 * they can be safely compared.
	 */
	virtual char *kind() { return classKind(); }
	/**
	 * Returns the unique identifier for this class.
	 */
	static char *classKind()
	{
		static char ASTNodeid;
		return &ASTNodeid;
	}
public:
	/**
	 * Root implementation of the RTTI-replacement for builds not wishing to
	 * use RTTI.  This returns true if `x` is the value returned from
	 * `classKind()`, or false otherwise.
	 */
	virtual bool isa(char *x)
	{
		return x == classKind();
	}
	/**
	 * Returns true if this object is an instance of `T`.  Note that this
	 * *only* works with single-inheritance hierarchies.  If you wish to use
	 * multiple inheritance in your AST classes, then you must define
	 * `USE_RTTI` and use the C++ RTTI mechanism.
	 */
	template <class T> bool isa()
	{
		return isa(T::classKind());
	}
	/**
	 * Returns a pointer to this object as a pointer to a child class, or
	 * `nullptr` if the cast would be unsafe.
	 *
	 * Note that AST nodes are intended to be always used as unique pointers
	 * and so the returned object is *only* valid as long as the unique pointer
	 * is valid.
	 */
	template <class T> T* get_as()
	{
		return this ? (isa<T>() ? static_cast<T*>(this) : nullptr) : nullptr;
	}
#else
public:
	template <class T> T* get_as()
	{
		return dynamic_cast<T*>(this);
	}
#endif
};

class ASTMember;

/** type of ast member vector.
 */

/**
 * The base class for non-leaf AST nodes.  Subclasses can have instances of
 * `ASTMember` subclasses as fields and will automatically construct them.
 */
class ASTContainer : public ASTNode
{
public:
	/**
	 * Constructs the container, setting a thread-local value to point to it
	 * allowing constructors in fields of the subclass to register themselves
	 * in the members vector.
	 */
	ASTContainer();

	/**
	 * Asks all members to construct themselves from the stack. The members are
	 * asked to construct themselves in reverse order from a node stack (`st`).
	 *
	 * The input range (`r`) is unused, because the leaf nodes have already
	 * constructed themselves at this point.
	 */
	virtual void construct(const InputRange &r, ASTStack &st);

private:
	/**
	 * The type used for tracking the fields of subclasses.
	 */
	typedef std::vector<ASTMember *> ASTMember_vector;
	/**
	 * References to all of the fields of the subclass that will be
	 * automatically constructed.
	 */
	ASTMember_vector members;

	friend class ASTMember;
	PEGMATITE_RTTI(ASTContainer, ASTNode)
};

/**
 * Base class for children of `ASTContainer`.
 */
class ASTMember
{
public:
	/**
	 * On construction, `ASTMember` sets its `container_node` field to the
	 * `ASTContainer` currently under construction and registers itself with
	 * the container, to be notified during the construction phase.
	 */
	ASTMember();

	/**
	 * Returns the container of which this object is a field.
	 */
	ASTContainer *container() const { return container_node; }

	/**
	 * Interface for constructing references to AST objects from the stack.
	 */
	virtual void construct(ASTStack &st) = 0;
	virtual ~ASTMember();
protected:
	/**
	 * The container that owns this object.
	 */
	ASTContainer *container_node;
};

/**
 * An `ASTPtr` is a wrapper around a pointer to an AST object.  It is intended
 * to be a member of an `ASTContainer` and will automatically pop the top item
 * from the stack and claim it when building the AST..
 */
template <class T, bool OPT = false> class ASTPtr : public ASTMember
{
public:
	/**
	 * Constructs the object in the
	 */
	ASTPtr() : ptr(nullptr) {}

	/** gets the underlying ptr value.
		@return the underlying ptr value.
	 */
	T *get() const
	{
		return ptr.get();
	}

	/** auto conversion to the underlying object ptr.
		@return the underlying ptr value.
	 */
	const std::unique_ptr<T> &operator *() const
	{
		return ptr;
	}

	/** member access.
		@return the underlying ptr value.
	 */
	const std::unique_ptr<T> &operator ->() const
	{
		assert(ptr);
		return ptr;
	}

	explicit operator bool() const noexcept
	{
		return (bool)ptr;
	}

	/**
	 * Pops the next matching object from the AST stack `st` and claims it.
	 */
	virtual void construct(ASTStack &st)
	{

		if (st.empty() && OPT)
		{
			return;
		}
		assert(!st.empty() && "Stack must not be empty");
		//get the node
		ASTNode *node = st.back().get();

		//get the object
		T *obj = node->get_as<T>();

		assert((obj || OPT) && "Required objects must exist!");
		//if the object is optional, simply return
		if (OPT && !obj)
		{
			return;
		}
		debug_log("Popped", st.size()-1, obj);
		//set the new object
		ptr.reset(obj);
		//pop the node from the stack
		st.back().release();
		st.pop_back();
		ptr->parent_node = container_node;
	}

private:
	/**
	 * The node that we are pointing to.
	 */
	std::unique_ptr<T> ptr;
};

/** A list of objects.
	It pops objects of the given type from the ast stack, until no more objects can be popped.
	It assumes ownership of objects.
	@param T type of object to control.
 */
template <class T> class ASTList : public ASTMember
{
public:
	///list type.
	typedef std::list<std::unique_ptr<T>> container;

	///the default constructor.
	ASTList() {}

	/** duplicates the objects of the given list.
		@param src source object.
	 */
	ASTList(const ASTList<T> &src)
	{
		_dup(src);
	}

	/** returns the container of objects.
		@return the container of objects.
	 */
	const container &objects() const
	{
		return child_objects;
	}

	/**
	 * Pops objects of type T from the stack (`st`) until no more objects can
	 * be popped.
	 */
	virtual void construct(ASTStack &st)
	{
		for(;;)
		{
			//if the stack is empty
			if (st.empty()) break;

			//get the node
			ASTNode *node = st.back().get();

			//get the object
			T *obj = node->get_as<T>();

			//if the object was not not of the appropriate type,
			//end the list parsing
			if (!obj) return;
			debug_log("Popped", st.size()-1, obj);

			//remove the node from the stack
			st.back().release();
			st.pop_back();

			//insert the object in the list, in reverse order
			child_objects.push_front(std::unique_ptr<T>(obj));

			//set the object's parent
			obj->parent_node = ASTMember::container();
		}
	}
	virtual ~ASTList() {}

private:
	//objects
	container child_objects;

	//duplicate the given list.
	void _dup(const ASTList<T> &src)
	{
		for (auto child : src.child_objects)
		{
			T *obj = new T(child.get());
			child_objects.push_back(obj);
			obj->parent_node = ASTMember::container();
		}
	}
};

/** parses the given input.
	@param i input.
	@param g root rule of grammar.
	@param ws whitespace rule.
	@param el list of errors.
	@param d user data, passed to the parse procedures.
	@return pointer to ast node created, or null if there was an error.
		The return object must be deleted by the caller.
 */
std::unique_ptr<ASTNode> parse(Input &i, const Rule &g, const Rule &ws,
                               ErrorList &el, const ParserDelegate &d);

/**
 * A parser delegate that is responsible for creating AST nodes from the input.
 *
 * This class manages a mapping from rules in some grammar to AST nodes.
 * Instances of the `BindAST` class that are fields of a subclass of this will
 * automatically register rules on creation.
 *
 * The recommended use for this class is to only register rules on construction
 * (either explicitly in the constructor or implicitly via `BindAST` members).
 * This will give a completely reentrant delegate, which can be used by
 * multiple threads to parse multiple inputs safely.
 */
class ASTParserDelegate : ParserDelegate
{
	/**
	 * BindAST is a friend so that it can call the `set_parse_proc()` function,
	 * which should never be called from anything else.
	 */
	template <class T> friend class BindAST;
	private:
	/**
	 * The map from rules to parsing handlers.
	 */
	std::unordered_map<const Rule*, parse_proc> handlers;
	/**
	 * Registers a callback in this delegate.  This should only be called from
	 * the `static` version of this function.
	 */
	void set_parse_proc(const Rule &r, parse_proc p);
	/**
	 * Registers a callback for a specific rule in the instance of this class
	 * currently under construction in this thread.  This should only ever be
	 * called by `BindAST` instances.
	 */
	static void bind_parse_proc(const Rule &r, parse_proc p);
	public:
	/**
	 * Default constructor, registers this class in thread-local storage so
	 * that it can be referenced by BindAST fields in subclasses when their
	 * constructors are run.
	 */
	ASTParserDelegate();
	virtual parse_proc get_parse_proc(const Rule &) const;
	/**
	 * Parse an input `i`, starting from rule `g` in the grammar for which
	 * this is a delegate.  The rule `ws` is used as whitespace.  Errors are
	 * returned via the `el` parameter and the root of the AST via the `ast`
	 * parameter.
	 *
	 * This function returns true on a successful parse, or false otherwise.
	 */
	template <class T> bool parse(Input &i, const Rule &g, const Rule &ws,
	                              ErrorList &el, std::unique_ptr<T> &ast) const
	{
		std::unique_ptr<ASTNode> node = pegmatite::parse(i, g, ws, el, *this);
		T *n = node->get_as<T>();
		if (n)
		{
			node.release();
			ast.reset(n);
			return true;
		}
		return false;
	}
};

/**
 * The `BindAST` class is responsible for binding an action to a rule.  The
 * template argument is the `ASTNode` subclass representing the action.  Its
 * `construct()` method will be called when the rule is matched.
 */
template <class T> class BindAST
{
public:
	/**
	 * Bind the AST class described in the grammar to the rule specified.
	 */
	BindAST(const Rule &r)
	{
		ASTParserDelegate::bind_parse_proc(r, [](const ParserPosition &b,
		                                         const ParserPosition &e, void *d)
			{
				ASTStack *st = reinterpret_cast<ASTStack *>(d);
				T *obj = new T();
				obj->construct(InputRange(b, e), *st);
				st->push_back(std::unique_ptr<ASTNode>(obj));
				debug_log("Constructed", st->size()-1, obj);
			});
	}
};

} //namespace pegmatite

#endif //PEGMATITE_AST_HPP
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2019 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "leveled_compaction_strategy.hh"
#include <algorithm>

namespace sstables {

compaction_descriptor leveled_compaction_strategy::get_sstables_for_compaction(column_family& cfs, std::vector<sstables::shared_sstable> candidates) {
    // NOTE: leveled_manifest creation may be slightly expensive, so later on,
    // we may want to store it in the strategy itself. However, the sstable
    // lists managed by the manifest may become outdated. For example, one
    // sstable in it may be marked for deletion after compacted.
    // Currently, we create a new manifest whenever it's time for compaction.
    leveled_manifest manifest = leveled_manifest::create(cfs, candidates, _max_sstable_size_in_mb, _stcs_options);
    if (!_last_compacted_keys) {
        generate_last_compacted_keys(manifest);
    }
    auto candidate = manifest.get_compaction_candidates(*_last_compacted_keys, _compaction_counter);

    if (!candidate.sstables.empty()) {
        leveled_manifest::logger.debug("leveled: Compacting {} out of {} sstables", candidate.sstables.size(), cfs.get_sstables()->size());
        return candidate;
    }

    // if there is no sstable to compact in standard way, try compacting based on droppable tombstone ratio
    // unlike stcs, lcs can look for sstable with highest droppable tombstone ratio, so as not to choose
    // a sstable which droppable data shadow data in older sstable, by starting from highest levels which
    // theoretically contain oldest non-overlapping data.
    auto gc_before = gc_clock::now() - cfs.schema()->gc_grace_seconds();
    for (auto level = int(manifest.get_level_count()); level >= 0; level--) {
        auto& sstables = manifest.get_level(level);
        // filter out sstables which droppable tombstone ratio isn't greater than the defined threshold.
        auto e = boost::range::remove_if(sstables, [this, &gc_before] (const sstables::shared_sstable& sst) -> bool {
            return !worth_dropping_tombstones(sst, gc_before);
        });
        sstables.erase(e, sstables.end());
        if (sstables.empty()) {
            continue;
        }
        auto& sst = *std::max_element(sstables.begin(), sstables.end(), [&] (auto& i, auto& j) {
            return i->estimate_droppable_tombstone_ratio(gc_before) < j->estimate_droppable_tombstone_ratio(gc_before);
        });
        return sstables::compaction_descriptor({ sst }, cfs.get_sstable_set(), service::get_local_compaction_priority(), sst->get_sstable_level());
    }
    return {};
}

compaction_descriptor leveled_compaction_strategy::get_major_compaction_job(column_family& cf, std::vector<sstables::shared_sstable> candidates) {
    if (candidates.empty()) {
        return compaction_descriptor();
    }

    auto& sst = *std::max_element(candidates.begin(), candidates.end(), [&] (sstables::shared_sstable& sst1, sstables::shared_sstable& sst2) {
        return sst1->get_sstable_level() < sst2->get_sstable_level();
    });
    return compaction_descriptor(std::move(candidates), cf.get_sstable_set(), service::get_local_compaction_priority(),
                                 sst->get_sstable_level(), _max_sstable_size_in_mb*1024*1024);
}

void leveled_compaction_strategy::notify_completion(const std::vector<shared_sstable>& removed, const std::vector<shared_sstable>& added) {
    if (removed.empty() || added.empty()) {
        return;
    }
    auto min_level = std::numeric_limits<uint32_t>::max();
    for (auto& sstable : removed) {
        min_level = std::min(min_level, sstable->get_sstable_level());
    }

    const sstables::sstable *last = nullptr;
    int target_level = 0;
    for (auto& candidate : added) {
        if (!last || last->compare_by_first_key(*candidate) < 0) {
            last = &*candidate;
        }
        target_level = std::max(target_level, int(candidate->get_sstable_level()));
    }
    _last_compacted_keys.value().at(min_level) = last->get_last_decorated_key();

    for (int i = leveled_manifest::MAX_LEVELS - 1; i > 0; i--) {
        _compaction_counter[i]++;
    }
    _compaction_counter[target_level] = 0;

    if (leveled_manifest::logger.level() == logging::log_level::debug) {
        for (auto j = 0U; j < _compaction_counter.size(); j++) {
            leveled_manifest::logger.debug("CompactionCounter: {}: {}", j, _compaction_counter[j]);
        }
    }
}

void leveled_compaction_strategy::generate_last_compacted_keys(leveled_manifest& manifest) {
    std::vector<std::optional<dht::decorated_key>> last_compacted_keys(leveled_manifest::MAX_LEVELS);
    for (auto i = 0; i < leveled_manifest::MAX_LEVELS - 1; i++) {
        if (manifest.get_level(i + 1).empty()) {
            continue;
        }

        const sstables::sstable* sstable_with_last_compacted_key = nullptr;
        std::optional<db_clock::time_point> max_creation_time;
        for (auto& sst : manifest.get_level(i + 1)) {
            auto wtime = sst->data_file_write_time();
            if (!max_creation_time || wtime >= *max_creation_time) {
                sstable_with_last_compacted_key = &*sst;
                max_creation_time = wtime;
            }
        }
        last_compacted_keys[i] = sstable_with_last_compacted_key->get_last_decorated_key();
    }
    _last_compacted_keys = std::move(last_compacted_keys);
}

int64_t leveled_compaction_strategy::estimated_pending_compactions(column_family& cf) const {
    std::vector<sstables::shared_sstable> sstables;
    sstables.reserve(cf.sstables_count());
    for (auto& entry : *cf.get_sstables()) {
        sstables.push_back(entry);
    }
    leveled_manifest manifest = leveled_manifest::create(cf, sstables, _max_sstable_size_in_mb, _stcs_options);
    return manifest.get_estimated_tasks();
}

compaction_descriptor
leveled_compaction_strategy::get_reshaping_job(std::vector<shared_sstable> input, schema_ptr schema, const ::io_priority_class& iop, reshape_mode mode) {
    std::array<std::vector<shared_sstable>, leveled_manifest::MAX_LEVELS> level_info;

    auto is_disjoint = [this, schema] (const std::vector<shared_sstable>& sstables, unsigned tolerance) {
        unsigned disjoint_sstables = 0;
        auto prev_last = dht::ring_position::min();
        for (auto& sst : sstables) {
            if (dht::ring_position(sst->get_first_decorated_key()).less_compare(*schema, prev_last)) {
                disjoint_sstables++;
            }
            prev_last = dht::ring_position(sst->get_last_decorated_key());
        }
        return disjoint_sstables > tolerance;
    };

    for (auto& sst : input) {
        auto sst_level = sst->get_sstable_level();
        if (sst_level > leveled_manifest::MAX_LEVELS) {
            leveled_manifest::logger.warn("Found SSTable with level {}, higher than the maximum {}. This is unexpected, but will fix", sst_level, leveled_manifest::MAX_LEVELS);

            // This is really unexpected, so we'll just compact it all to fix it
            compaction_descriptor desc(std::move(input), std::optional<sstables::sstable_set>(), iop, leveled_manifest::MAX_LEVELS - 1, _max_sstable_size_in_mb * 1024 * 1024);
            desc.options = compaction_options::make_reshape();
            return desc;
        }
        level_info[sst_level].push_back(sst);
    }

    for (auto& level : level_info) {
        std::sort(level.begin(), level.end(), [this, schema] (shared_sstable a, shared_sstable b) {
            return dht::ring_position(a->get_first_decorated_key()).less_compare(*schema, dht::ring_position(b->get_first_decorated_key()));
        });
    }

    unsigned max_filled_level = 0;

    size_t offstrategy_threshold = std::max(schema->min_compaction_threshold(), 4);
    size_t max_sstables = std::max(schema->max_compaction_threshold(), int(offstrategy_threshold));
    unsigned tolerance = mode == reshape_mode::strict ? 0 : leveled_manifest::leveled_fan_out * 2;

    if (level_info[0].size() > offstrategy_threshold) {
        level_info[0].resize(std::min(level_info[0].size(), max_sstables));
        compaction_descriptor desc(std::move(level_info[0]), std::optional<sstables::sstable_set>(), iop);
        desc.options = compaction_options::make_reshape();
        return desc;
    }

    for (unsigned level = leveled_manifest::MAX_LEVELS - 1; level > 0; --level) {
        if (level_info[level].empty()) {
            continue;
        }
        max_filled_level = std::max(max_filled_level, level);

        if (!is_disjoint(level_info[level], tolerance)) {
            // Unfortunately no good limit to limit input size to max_sstables for LCS major
            compaction_descriptor desc(std::move(input), std::optional<sstables::sstable_set>(), iop, max_filled_level, _max_sstable_size_in_mb * 1024 * 1024);
            desc.options = compaction_options::make_reshape();
            return desc;
        }
    }

   return compaction_descriptor();
}

}
<commit_msg>reshape: Fix reshaping procedure for LCS<commit_after>/*
 * Copyright (C) 2019 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "leveled_compaction_strategy.hh"
#include <algorithm>

namespace sstables {

compaction_descriptor leveled_compaction_strategy::get_sstables_for_compaction(column_family& cfs, std::vector<sstables::shared_sstable> candidates) {
    // NOTE: leveled_manifest creation may be slightly expensive, so later on,
    // we may want to store it in the strategy itself. However, the sstable
    // lists managed by the manifest may become outdated. For example, one
    // sstable in it may be marked for deletion after compacted.
    // Currently, we create a new manifest whenever it's time for compaction.
    leveled_manifest manifest = leveled_manifest::create(cfs, candidates, _max_sstable_size_in_mb, _stcs_options);
    if (!_last_compacted_keys) {
        generate_last_compacted_keys(manifest);
    }
    auto candidate = manifest.get_compaction_candidates(*_last_compacted_keys, _compaction_counter);

    if (!candidate.sstables.empty()) {
        leveled_manifest::logger.debug("leveled: Compacting {} out of {} sstables", candidate.sstables.size(), cfs.get_sstables()->size());
        return candidate;
    }

    // if there is no sstable to compact in standard way, try compacting based on droppable tombstone ratio
    // unlike stcs, lcs can look for sstable with highest droppable tombstone ratio, so as not to choose
    // a sstable which droppable data shadow data in older sstable, by starting from highest levels which
    // theoretically contain oldest non-overlapping data.
    auto gc_before = gc_clock::now() - cfs.schema()->gc_grace_seconds();
    for (auto level = int(manifest.get_level_count()); level >= 0; level--) {
        auto& sstables = manifest.get_level(level);
        // filter out sstables which droppable tombstone ratio isn't greater than the defined threshold.
        auto e = boost::range::remove_if(sstables, [this, &gc_before] (const sstables::shared_sstable& sst) -> bool {
            return !worth_dropping_tombstones(sst, gc_before);
        });
        sstables.erase(e, sstables.end());
        if (sstables.empty()) {
            continue;
        }
        auto& sst = *std::max_element(sstables.begin(), sstables.end(), [&] (auto& i, auto& j) {
            return i->estimate_droppable_tombstone_ratio(gc_before) < j->estimate_droppable_tombstone_ratio(gc_before);
        });
        return sstables::compaction_descriptor({ sst }, cfs.get_sstable_set(), service::get_local_compaction_priority(), sst->get_sstable_level());
    }
    return {};
}

compaction_descriptor leveled_compaction_strategy::get_major_compaction_job(column_family& cf, std::vector<sstables::shared_sstable> candidates) {
    if (candidates.empty()) {
        return compaction_descriptor();
    }

    auto& sst = *std::max_element(candidates.begin(), candidates.end(), [&] (sstables::shared_sstable& sst1, sstables::shared_sstable& sst2) {
        return sst1->get_sstable_level() < sst2->get_sstable_level();
    });
    return compaction_descriptor(std::move(candidates), cf.get_sstable_set(), service::get_local_compaction_priority(),
                                 sst->get_sstable_level(), _max_sstable_size_in_mb*1024*1024);
}

void leveled_compaction_strategy::notify_completion(const std::vector<shared_sstable>& removed, const std::vector<shared_sstable>& added) {
    if (removed.empty() || added.empty()) {
        return;
    }
    auto min_level = std::numeric_limits<uint32_t>::max();
    for (auto& sstable : removed) {
        min_level = std::min(min_level, sstable->get_sstable_level());
    }

    const sstables::sstable *last = nullptr;
    int target_level = 0;
    for (auto& candidate : added) {
        if (!last || last->compare_by_first_key(*candidate) < 0) {
            last = &*candidate;
        }
        target_level = std::max(target_level, int(candidate->get_sstable_level()));
    }
    _last_compacted_keys.value().at(min_level) = last->get_last_decorated_key();

    for (int i = leveled_manifest::MAX_LEVELS - 1; i > 0; i--) {
        _compaction_counter[i]++;
    }
    _compaction_counter[target_level] = 0;

    if (leveled_manifest::logger.level() == logging::log_level::debug) {
        for (auto j = 0U; j < _compaction_counter.size(); j++) {
            leveled_manifest::logger.debug("CompactionCounter: {}: {}", j, _compaction_counter[j]);
        }
    }
}

void leveled_compaction_strategy::generate_last_compacted_keys(leveled_manifest& manifest) {
    std::vector<std::optional<dht::decorated_key>> last_compacted_keys(leveled_manifest::MAX_LEVELS);
    for (auto i = 0; i < leveled_manifest::MAX_LEVELS - 1; i++) {
        if (manifest.get_level(i + 1).empty()) {
            continue;
        }

        const sstables::sstable* sstable_with_last_compacted_key = nullptr;
        std::optional<db_clock::time_point> max_creation_time;
        for (auto& sst : manifest.get_level(i + 1)) {
            auto wtime = sst->data_file_write_time();
            if (!max_creation_time || wtime >= *max_creation_time) {
                sstable_with_last_compacted_key = &*sst;
                max_creation_time = wtime;
            }
        }
        last_compacted_keys[i] = sstable_with_last_compacted_key->get_last_decorated_key();
    }
    _last_compacted_keys = std::move(last_compacted_keys);
}

int64_t leveled_compaction_strategy::estimated_pending_compactions(column_family& cf) const {
    std::vector<sstables::shared_sstable> sstables;
    sstables.reserve(cf.sstables_count());
    for (auto& entry : *cf.get_sstables()) {
        sstables.push_back(entry);
    }
    leveled_manifest manifest = leveled_manifest::create(cf, sstables, _max_sstable_size_in_mb, _stcs_options);
    return manifest.get_estimated_tasks();
}

compaction_descriptor
leveled_compaction_strategy::get_reshaping_job(std::vector<shared_sstable> input, schema_ptr schema, const ::io_priority_class& iop, reshape_mode mode) {
    std::array<std::vector<shared_sstable>, leveled_manifest::MAX_LEVELS> level_info;

    auto is_disjoint = [this, schema] (const std::vector<shared_sstable>& sstables, unsigned tolerance) {
        unsigned overlapping_sstables = 0;
        auto prev_last = dht::ring_position::min();
        for (auto& sst : sstables) {
            if (dht::ring_position(sst->get_first_decorated_key()).less_compare(*schema, prev_last)) {
                overlapping_sstables++;
            }
            prev_last = dht::ring_position(sst->get_last_decorated_key());
        }
        return overlapping_sstables <= tolerance;
    };

    for (auto& sst : input) {
        auto sst_level = sst->get_sstable_level();
        if (sst_level > leveled_manifest::MAX_LEVELS) {
            leveled_manifest::logger.warn("Found SSTable with level {}, higher than the maximum {}. This is unexpected, but will fix", sst_level, leveled_manifest::MAX_LEVELS);

            // This is really unexpected, so we'll just compact it all to fix it
            compaction_descriptor desc(std::move(input), std::optional<sstables::sstable_set>(), iop, leveled_manifest::MAX_LEVELS - 1, _max_sstable_size_in_mb * 1024 * 1024);
            desc.options = compaction_options::make_reshape();
            return desc;
        }
        level_info[sst_level].push_back(sst);
    }

    for (auto& level : level_info) {
        std::sort(level.begin(), level.end(), [this, schema] (shared_sstable a, shared_sstable b) {
            return dht::ring_position(a->get_first_decorated_key()).less_compare(*schema, dht::ring_position(b->get_first_decorated_key()));
        });
    }

    unsigned max_filled_level = 0;

    size_t offstrategy_threshold = std::max(schema->min_compaction_threshold(), 4);
    size_t max_sstables = std::max(schema->max_compaction_threshold(), int(offstrategy_threshold));
    unsigned tolerance = mode == reshape_mode::strict ? 0 : leveled_manifest::leveled_fan_out * 2;

    if (level_info[0].size() > offstrategy_threshold) {
        level_info[0].resize(std::min(level_info[0].size(), max_sstables));
        compaction_descriptor desc(std::move(level_info[0]), std::optional<sstables::sstable_set>(), iop);
        desc.options = compaction_options::make_reshape();
        return desc;
    }

    for (unsigned level = leveled_manifest::MAX_LEVELS - 1; level > 0; --level) {
        if (level_info[level].empty()) {
            continue;
        }
        max_filled_level = std::max(max_filled_level, level);

        if (!is_disjoint(level_info[level], tolerance)) {
            leveled_manifest::logger.warn("Turns out that level {} is not disjoint, so compacting everything on behalf of {}.{}", level, schema->ks_name(), schema->cf_name());
            // Unfortunately no good limit to limit input size to max_sstables for LCS major
            compaction_descriptor desc(std::move(input), std::optional<sstables::sstable_set>(), iop, max_filled_level, _max_sstable_size_in_mb * 1024 * 1024);
            desc.options = compaction_options::make_reshape();
            return desc;
        }
    }

   return compaction_descriptor();
}

}
<|endoftext|>


<commit_before>#ifndef STAN_MATH_OPENCL_NORMAL_ID_GLM_LPDF_HPP
#define STAN_MATH_OPENCL_NORMAL_ID_GLM_LPDF_HPP
#ifdef STAN_OPENCL

#include <stan/math/prim/meta.hpp>
#include <stan/math/prim/scal/err/check_consistent_sizes.hpp>
#include <stan/math/prim/scal/err/check_finite.hpp>
#include <stan/math/prim/scal/err/check_positive_finite.hpp>
#include <stan/math/prim/scal/fun/constants.hpp>
#include <stan/math/prim/mat/fun/value_of_rec.hpp>
#include <stan/math/prim/scal/fun/size_zero.hpp>
#include <stan/math/prim/scal/fun/sum.hpp>
#include <stan/math/prim/arr/fun/value_of_rec.hpp>
#include <stan/math/prim/mat/prob/normal_id_glm_lpdf.hpp>

#include <stan/math/opencl/kernels/normal_id_glm_lpdf.hpp>
#include <stan/math/opencl/matrix_cl.hpp>
#include <stan/math/opencl/multiply.hpp>

#include <cmath>

namespace stan {
namespace math {

/**
 * Returns the log PDF of the Generalized Linear Model (GLM)
 * with Normal distribution and id link function.
 * If containers are supplied, returns the log sum of the probabilities.
 * This is an overload of the GLM in prim/mar/prob/normal_id_glm_lpdf.hpp
 * that is implemented in OpenCL.
 * @tparam T_alpha type of the intercept(s);
 * this can be a vector (of the same length as y) of intercepts or a single
 * value (for models with constant intercept);
 * @tparam T_beta type of the weight vector;
 * this can also be a single value;
 * @tparam T_scale type of the (positive) scale(s);
 * this can be a vector (of the same length as y, for heteroskedasticity)
 * or a scalar.
 * @param y_cl vector parameter on OpenCL device
 * @param x_cl design matrix on OpenCL device
 * @param alpha intercept (in log odds)
 * @param beta weight vector
 * @param sigma (Sequence of) scale parameters for the normal
 * distribution.
 * @return log probability or log sum of probabilities
 * @throw std::domain_error if x, beta or alpha is infinite.
 * @throw std::domain_error if the scale is not positive.
 * @throw std::invalid_argument if container sizes mismatch.
 */
template <bool propto, typename T_alpha, typename T_beta, typename T_scale>
return_type_t<T_alpha, T_beta, T_scale> normal_id_glm_lpdf(
    const matrix_cl<double> &y_cl, const matrix_cl<double> &x_cl,
    const T_alpha &alpha, const T_beta &beta, const T_scale &sigma) {
  static const char *function = "normal_id_glm_lpdf(OpenCL)";
  using T_partials_return = partials_return_t<T_alpha, T_beta, T_scale>;
  using T_scale_val = typename std::conditional_t<
      is_vector<T_scale>::value,
      Eigen::Array<partials_return_t<T_scale>, -1, 1>,
      partials_return_t<T_scale>>;

  using Eigen::Array;
  using Eigen::Dynamic;
  using Eigen::Matrix;
  using Eigen::VectorXd;

  const size_t N = x_cl.rows();
  const size_t M = x_cl.cols();

  check_positive_finite(function, "Scale vector", sigma);
  check_size_match(function, "Rows of ", "x_cl", N, "rows of ", "y_cl",
                   y_cl.rows());
  check_consistent_size(function, "Weight vector", beta, M);
  if (is_vector<T_scale>::value) {
    check_size_match(function, "Rows of ", "y_cl", N, "size of ", "sigma",
                     length(sigma));
  }
  if (is_vector<T_alpha>::value) {
    check_size_match(function, "Rows of ", "y_cl", N, "size of ", "alpha",
                     length(alpha));
  }

  if (!include_summand<propto, T_alpha, T_beta, T_scale>::value) {
    return 0;
  }

  if (N == 0 || M == 0) {
    return 0;
  }

  const auto &beta_val = value_of_rec(beta);
  const auto &alpha_val = value_of_rec(alpha);
  const auto &sigma_val = value_of_rec(sigma);

  const auto &beta_val_vec = as_column_vector_or_scalar(beta_val);
  const auto &alpha_val_vec = as_column_vector_or_scalar(alpha_val);
  const auto &sigma_val_vec = as_column_vector_or_scalar(sigma_val);

  T_scale_val inv_sigma = 1 / as_array_or_scalar(sigma_val_vec);
  Matrix<T_partials_return, Dynamic, 1> y_minus_mu_over_sigma_mat(N);
  auto y_scaled = y_minus_mu_over_sigma_mat.array();

  const int local_size
      = opencl_kernels::normal_id_glm.get_option("LOCAL_SIZE_");
  const int wgs = (N + local_size - 1) / local_size;

  matrix_cl<double> beta_cl(beta_val_vec);
  matrix_cl<double> alpha_cl(alpha_val_vec);
  matrix_cl<double> sigma_cl(sigma_val_vec);

  const bool need_mu_derivative = !is_constant_all<T_beta, T_alpha>::value;
  matrix_cl<double> mu_derivative_cl(need_mu_derivative ? N : 0, 1);
  const bool need_mu_derivative_sum
      = !is_constant_all<T_alpha>::value && !is_vector<T_alpha>::value;
  matrix_cl<double> mu_derivative_sum_cl(need_mu_derivative_sum ? wgs : 0, 1);
  matrix_cl<double> y_minus_mu_over_sigma_squared_sum_cl(wgs, 1);
  const bool need_sigma_derivative
      = !is_constant_all<T_scale>::value && is_vector<T_scale>::value;
  matrix_cl<double> sigma_derivative_cl(need_sigma_derivative ? N : 0, 1);
  const bool need_log_sigma_sum
      = include_summand<propto, T_scale>::value && is_vector<T_scale>::value;
  matrix_cl<double> log_sigma_sum_cl(need_log_sigma_sum ? wgs : 0, 1);

  try {
    opencl_kernels::normal_id_glm(
        cl::NDRange(local_size * wgs), cl::NDRange(local_size),
        mu_derivative_cl, mu_derivative_sum_cl,
        y_minus_mu_over_sigma_squared_sum_cl, sigma_derivative_cl,
        log_sigma_sum_cl, y_cl, x_cl, alpha_cl, beta_cl, sigma_cl, N, M,
        length(alpha) != 1, length(sigma) != 1, need_mu_derivative,
        need_mu_derivative_sum, need_sigma_derivative, need_log_sigma_sum);
  } catch (const cl::Error &e) {
    check_opencl_error(function, e);
  }
  double y_scaled_sq_sum
      = sum(from_matrix_cl(y_minus_mu_over_sigma_squared_sum_cl));

  operands_and_partials<T_alpha, T_beta, T_scale> ops_partials(alpha, beta,
                                                               sigma);

  if (!is_constant_all<T_alpha>::value && is_vector<T_alpha>::value) {
    ops_partials.edge1_.partials_
        = from_matrix_cl<Dynamic, 1>(mu_derivative_cl);
  }
  if (need_mu_derivative_sum) {
    ops_partials.edge1_.partials_[0]
        = sum(from_matrix_cl(mu_derivative_sum_cl));
  }
  if (!is_constant_all<T_beta>::value) {
    const matrix_cl<double> mu_derivative_transpose_cl(
        mu_derivative_cl.buffer(), 1,
        mu_derivative_cl.rows());  // transposition of a vector can be done
    // without copying
    ops_partials.edge2_.partials_
        = from_matrix_cl<1, Dynamic>(mu_derivative_transpose_cl * x_cl);
  }
  if (!is_constant_all<T_scale>::value) {
    if (is_vector<T_scale>::value) {
      ops_partials.edge3_.partials_
          = from_matrix_cl<Dynamic, 1>(sigma_derivative_cl);
    } else {
      ops_partials.edge3_.partials_[0]
          = (y_scaled_sq_sum - N) * as_scalar(inv_sigma);
    }
  }

  if (!std::isfinite(y_scaled_sq_sum)) {
    check_finite(function, "Vector of dependent variables",
                 from_matrix_cl(y_cl));
    check_finite(function, "Weight vector", beta);
    check_finite(function, "Intercept", alpha);
    // if all other checks passed, next will only fail if x is not finite
    check_finite(function, "Matrix of independent variables", y_scaled_sq_sum);
  }

  // Compute log probability.
  T_partials_return logp(0.0);
  if (include_summand<propto>::value) {
    logp += NEG_LOG_SQRT_TWO_PI * N;
  }
  if (include_summand<propto, T_scale>::value) {
    if (is_vector<T_scale>::value) {
      logp -= sum(from_matrix_cl(log_sigma_sum_cl));
    } else {
      logp -= N * log(as_scalar(sigma_val));
    }
  }
  if (include_summand<propto, T_alpha, T_beta, T_scale>::value) {
    logp -= 0.5 * y_scaled_sq_sum;
  }
  return ops_partials.build(logp);
}

template <typename T_alpha, typename T_beta, typename T_scale>
inline return_type_t<T_alpha, T_beta, T_scale> normal_id_glm_lpdf(
    const matrix_cl<double> &y, const matrix_cl<double> &x,
    const T_alpha &alpha, const T_beta &beta, const T_scale &sigma) {
  return normal_id_glm_lpdf<false>(y, x, alpha, beta, sigma);
}

}  // namespace math
}  // namespace stan

#endif
#endif
<commit_msg>fix header path<commit_after>#ifndef STAN_MATH_OPENCL_NORMAL_ID_GLM_LPDF_HPP
#define STAN_MATH_OPENCL_NORMAL_ID_GLM_LPDF_HPP
#ifdef STAN_OPENCL

#include <stan/math/prim/meta.hpp>
#include <stan/math/prim/scal/err/check_consistent_sizes.hpp>
#include <stan/math/prim/scal/err/check_finite.hpp>
#include <stan/math/prim/scal/err/check_positive_finite.hpp>
#include <stan/math/prim/scal/fun/constants.hpp>
#include <stan/math/prim/mat/fun/value_of_rec.hpp>
#include <stan/math/prim/scal/fun/size_zero.hpp>
#include <stan/math/prim/scal/fun/sum.hpp>
#include <stan/math/prim/arr/fun/value_of_rec.hpp>
#include <stan/math/prim/mat/prob/normal_id_glm_lpdf.hpp>

#include <stan/math/opencl/kernels/normal_id_glm_lpdf.hpp>
#include <stan/math/opencl/matrix_cl.hpp>
#include <stan/math/opencl/prim/multiply.hpp>

#include <cmath>

namespace stan {
namespace math {

/**
 * Returns the log PDF of the Generalized Linear Model (GLM)
 * with Normal distribution and id link function.
 * If containers are supplied, returns the log sum of the probabilities.
 * This is an overload of the GLM in prim/mar/prob/normal_id_glm_lpdf.hpp
 * that is implemented in OpenCL.
 * @tparam T_alpha type of the intercept(s);
 * this can be a vector (of the same length as y) of intercepts or a single
 * value (for models with constant intercept);
 * @tparam T_beta type of the weight vector;
 * this can also be a single value;
 * @tparam T_scale type of the (positive) scale(s);
 * this can be a vector (of the same length as y, for heteroskedasticity)
 * or a scalar.
 * @param y_cl vector parameter on OpenCL device
 * @param x_cl design matrix on OpenCL device
 * @param alpha intercept (in log odds)
 * @param beta weight vector
 * @param sigma (Sequence of) scale parameters for the normal
 * distribution.
 * @return log probability or log sum of probabilities
 * @throw std::domain_error if x, beta or alpha is infinite.
 * @throw std::domain_error if the scale is not positive.
 * @throw std::invalid_argument if container sizes mismatch.
 */
template <bool propto, typename T_alpha, typename T_beta, typename T_scale>
return_type_t<T_alpha, T_beta, T_scale> normal_id_glm_lpdf(
    const matrix_cl<double> &y_cl, const matrix_cl<double> &x_cl,
    const T_alpha &alpha, const T_beta &beta, const T_scale &sigma) {
  static const char *function = "normal_id_glm_lpdf(OpenCL)";
  using T_partials_return = partials_return_t<T_alpha, T_beta, T_scale>;
  using T_scale_val = typename std::conditional_t<
      is_vector<T_scale>::value,
      Eigen::Array<partials_return_t<T_scale>, -1, 1>,
      partials_return_t<T_scale>>;

  using Eigen::Array;
  using Eigen::Dynamic;
  using Eigen::Matrix;
  using Eigen::VectorXd;

  const size_t N = x_cl.rows();
  const size_t M = x_cl.cols();

  check_positive_finite(function, "Scale vector", sigma);
  check_size_match(function, "Rows of ", "x_cl", N, "rows of ", "y_cl",
                   y_cl.rows());
  check_consistent_size(function, "Weight vector", beta, M);
  if (is_vector<T_scale>::value) {
    check_size_match(function, "Rows of ", "y_cl", N, "size of ", "sigma",
                     length(sigma));
  }
  if (is_vector<T_alpha>::value) {
    check_size_match(function, "Rows of ", "y_cl", N, "size of ", "alpha",
                     length(alpha));
  }

  if (!include_summand<propto, T_alpha, T_beta, T_scale>::value) {
    return 0;
  }

  if (N == 0 || M == 0) {
    return 0;
  }

  const auto &beta_val = value_of_rec(beta);
  const auto &alpha_val = value_of_rec(alpha);
  const auto &sigma_val = value_of_rec(sigma);

  const auto &beta_val_vec = as_column_vector_or_scalar(beta_val);
  const auto &alpha_val_vec = as_column_vector_or_scalar(alpha_val);
  const auto &sigma_val_vec = as_column_vector_or_scalar(sigma_val);

  T_scale_val inv_sigma = 1 / as_array_or_scalar(sigma_val_vec);
  Matrix<T_partials_return, Dynamic, 1> y_minus_mu_over_sigma_mat(N);
  auto y_scaled = y_minus_mu_over_sigma_mat.array();

  const int local_size
      = opencl_kernels::normal_id_glm.get_option("LOCAL_SIZE_");
  const int wgs = (N + local_size - 1) / local_size;

  matrix_cl<double> beta_cl(beta_val_vec);
  matrix_cl<double> alpha_cl(alpha_val_vec);
  matrix_cl<double> sigma_cl(sigma_val_vec);

  const bool need_mu_derivative = !is_constant_all<T_beta, T_alpha>::value;
  matrix_cl<double> mu_derivative_cl(need_mu_derivative ? N : 0, 1);
  const bool need_mu_derivative_sum
      = !is_constant_all<T_alpha>::value && !is_vector<T_alpha>::value;
  matrix_cl<double> mu_derivative_sum_cl(need_mu_derivative_sum ? wgs : 0, 1);
  matrix_cl<double> y_minus_mu_over_sigma_squared_sum_cl(wgs, 1);
  const bool need_sigma_derivative
      = !is_constant_all<T_scale>::value && is_vector<T_scale>::value;
  matrix_cl<double> sigma_derivative_cl(need_sigma_derivative ? N : 0, 1);
  const bool need_log_sigma_sum
      = include_summand<propto, T_scale>::value && is_vector<T_scale>::value;
  matrix_cl<double> log_sigma_sum_cl(need_log_sigma_sum ? wgs : 0, 1);

  try {
    opencl_kernels::normal_id_glm(
        cl::NDRange(local_size * wgs), cl::NDRange(local_size),
        mu_derivative_cl, mu_derivative_sum_cl,
        y_minus_mu_over_sigma_squared_sum_cl, sigma_derivative_cl,
        log_sigma_sum_cl, y_cl, x_cl, alpha_cl, beta_cl, sigma_cl, N, M,
        length(alpha) != 1, length(sigma) != 1, need_mu_derivative,
        need_mu_derivative_sum, need_sigma_derivative, need_log_sigma_sum);
  } catch (const cl::Error &e) {
    check_opencl_error(function, e);
  }
  double y_scaled_sq_sum
      = sum(from_matrix_cl(y_minus_mu_over_sigma_squared_sum_cl));

  operands_and_partials<T_alpha, T_beta, T_scale> ops_partials(alpha, beta,
                                                               sigma);

  if (!is_constant_all<T_alpha>::value && is_vector<T_alpha>::value) {
    ops_partials.edge1_.partials_
        = from_matrix_cl<Dynamic, 1>(mu_derivative_cl);
  }
  if (need_mu_derivative_sum) {
    ops_partials.edge1_.partials_[0]
        = sum(from_matrix_cl(mu_derivative_sum_cl));
  }
  if (!is_constant_all<T_beta>::value) {
    const matrix_cl<double> mu_derivative_transpose_cl(
        mu_derivative_cl.buffer(), 1,
        mu_derivative_cl.rows());  // transposition of a vector can be done
    // without copying
    ops_partials.edge2_.partials_
        = from_matrix_cl<1, Dynamic>(mu_derivative_transpose_cl * x_cl);
  }
  if (!is_constant_all<T_scale>::value) {
    if (is_vector<T_scale>::value) {
      ops_partials.edge3_.partials_
          = from_matrix_cl<Dynamic, 1>(sigma_derivative_cl);
    } else {
      ops_partials.edge3_.partials_[0]
          = (y_scaled_sq_sum - N) * as_scalar(inv_sigma);
    }
  }

  if (!std::isfinite(y_scaled_sq_sum)) {
    check_finite(function, "Vector of dependent variables",
                 from_matrix_cl(y_cl));
    check_finite(function, "Weight vector", beta);
    check_finite(function, "Intercept", alpha);
    // if all other checks passed, next will only fail if x is not finite
    check_finite(function, "Matrix of independent variables", y_scaled_sq_sum);
  }

  // Compute log probability.
  T_partials_return logp(0.0);
  if (include_summand<propto>::value) {
    logp += NEG_LOG_SQRT_TWO_PI * N;
  }
  if (include_summand<propto, T_scale>::value) {
    if (is_vector<T_scale>::value) {
      logp -= sum(from_matrix_cl(log_sigma_sum_cl));
    } else {
      logp -= N * log(as_scalar(sigma_val));
    }
  }
  if (include_summand<propto, T_alpha, T_beta, T_scale>::value) {
    logp -= 0.5 * y_scaled_sq_sum;
  }
  return ops_partials.build(logp);
}

template <typename T_alpha, typename T_beta, typename T_scale>
inline return_type_t<T_alpha, T_beta, T_scale> normal_id_glm_lpdf(
    const matrix_cl<double> &y, const matrix_cl<double> &x,
    const T_alpha &alpha, const T_beta &beta, const T_scale &sigma) {
  return normal_id_glm_lpdf<false>(y, x, alpha, beta, sigma);
}

}  // namespace math
}  // namespace stan

#endif
#endif
<|endoftext|>