<commit_before>#include "faze/src/new_gfx/dx12/util/dxDependencySolver.hpp"
#include "faze/src/new_gfx/definitions.hpp"
#include "core/src/global_debug.hpp"

namespace faze
{
  namespace backend
  {
    DX12DependencySolver::UsageHint DX12DependencySolver::getUsageFromAccessFlags(D3D12_RESOURCE_STATES flags)
    {
      int32_t writeMask = D3D12_RESOURCE_STATE_UNORDERED_ACCESS | D3D12_RESOURCE_STATE_DEPTH_WRITE | D3D12_RESOURCE_STATE_RENDER_TARGET
        | D3D12_RESOURCE_STATE_COPY_DEST | D3D12_RESOURCE_STATE_RESOLVE_DEST;

      if (flags == D3D12_RESOURCE_STATE_COMMON || flags & writeMask) // common is the most complex state... with value of 0
      {
        return DX12DependencySolver::UsageHint::write;
      }
      return DX12DependencySolver::UsageHint::read;
    }

    int DX12DependencySolver::addDrawCall(CommandPacket::PacketType name)
    {
      m_drawCallInfo.emplace_back(name);
      m_drawCallJobOffsets[drawCallsAdded] = static_cast<int>(m_jobs.size());
      return drawCallsAdded++;
    }

    void DX12DependencySolver::addResource(int drawCallIndex, backend::TrackedState state, D3D12_RESOURCE_STATES flags)
    {
      size_t uniqueID = state.resPtr;
      if (uniqueID == 0)
        return;

      SubresourceRange range{};

      range.sliceOffset = static_cast<int16_t>(state.slice());
      range.arraySize = static_cast<int16_t>(state.arraySize());
      range.mipOffset = static_cast<int16_t>(state.mip());
      range.mipLevels = static_cast<int16_t>(state.mipLevels());

      m_jobs.emplace_back(DependencyPacket{ drawCallIndex, uniqueID, flags, range });
      if (m_resourceStates.find(uniqueID) == m_resourceStates.end())
      {
        ResourceDependency d{};
        d.uniqueId = uniqueID;
        d.mips = static_cast<int16_t>(state.totalMipLevels());
        d.texture = reinterpret_cast<ID3D12Resource*>(state.resPtr);
        d.state = reinterpret_cast<DX12ResourceState*>(state.statePtr);
        m_resourceStates[uniqueID] = std::move(d);
      }
      m_uniqueResourcesThisChain.insert(uniqueID);
    }

    void DX12DependencySolver::addResource(int drawCallIndex, backend::TrackedState state, D3D12_RESOURCE_STATES flags, SubresourceRange range)
    {
      size_t uniqueID = state.resPtr;
      if (uniqueID == 0)
        return;
      m_jobs.emplace_back(DependencyPacket{ drawCallIndex, uniqueID, flags, range });

      m_jobs.emplace_back(DependencyPacket{ drawCallIndex, uniqueID, flags, range });
      if (m_resourceStates.find(uniqueID) == m_resourceStates.end())
      {
        ResourceDependency d{};
        d.uniqueId = uniqueID;
        d.mips = static_cast<int16_t>(state.totalMipLevels());
        d.texture = reinterpret_cast<ID3D12Resource*>(state.resPtr);
        d.state = reinterpret_cast<DX12ResourceState*>(state.statePtr);
        m_resourceStates[uniqueID] = std::move(d);
      }
      m_uniqueResourcesThisChain.insert(uniqueID);
    }

    // buffers
    /*
    void DX12DependencySolver::addBuffer(int drawCallIndex, int64_t, DX12Buffer& buffer, D3D12_RESOURCE_STATES flags)
    {
      size_t uniqueID = reinterpret_cast<size_t>(buffer.native());
      if (faze::globalconfig::graphics::GraphicsEnableReadStateCombining) // disable optimization, merges last seen usages to existing. Default on. Combines only reads.
      {
        auto currentUsage = getUsageFromAccessFlags(flags);
        if (currentUsage == UsageHint::read)
        {
          auto* obj = m_resourceUsageInLastAdd.find(uniqueID);
          if (obj)
          {
            if (obj->second.type == currentUsage)
            {
              // Try to recognize here that last time we saw the same id, we already used it in read/write state, combine the states.
              auto* jobIndex = m_drawCallJobOffsets.find(obj->second.index);
              if (jobIndex)
              {
                int index = jobIndex->second;
                while (index < static_cast<int>(m_jobs.size()) && obj->second.index == m_jobs[index].drawIndex)
                {
                  if (m_jobs[index].resource == uniqueID)
                  {
                    break;
                  }
                  index++;
                }
                m_jobs[index].access |= flags;
                return; // found and could merge... or rather forcefully merged.
              }
            }
          }
        }
        m_resourceUsageInLastAdd[uniqueID] = LastSeenUsage{ currentUsage, drawCallIndex };
      }

      m_jobs.emplace_back(DependencyPacket{ drawCallIndex, uniqueID, ResourceType::buffer,
        flags, SubresourceRange{} });

      if (m_resourceStates.find(uniqueID) == m_resourceStates.end())
      {
        TextureDependency d;
        d.uniqueId = uniqueID;
        d.mips = 1;
        d.texture = buffer.native();
        d.state = buffer.state().get();
        m_resourceStates[uniqueID] = std::move(d);
      }
      m_uniqueBuffersThisChain.insert(uniqueID);
    }
    */
    // textures
    /*
    void DX12DependencySolver::addTexture(int drawCallIndex, int64_t, DX12Texture& texture, DX12TextureView& view, int16_t mips, D3D12_RESOURCE_STATES flags)
    {
      auto uniqueID = reinterpret_cast<size_t>(texture.native());

      if (faze::globalconfig::graphics::GraphicsEnableReadStateCombining)
      {
        auto currentUsage = getUsageFromAccessFlags(flags);
        if (currentUsage == UsageHint::read)
        {
          auto* obj = m_resourceUsageInLastAdd.find(uniqueID);
          if (obj)
          {
            if (obj->second.type != currentUsage)
            {
              F_SLOG("DependencySolver", "Possible optimization here!\n");
              // Try to recognize here that last time we saw the same id, we already used it in read/write state, combine the states.
              // Texture... needs to look at all the subresources seen before and combine their state... ugh, this looks not to be easy.
            }
          }
        }
        m_resourceUsageInLastAdd[uniqueID] = LastSeenUsage{ currentUsage, drawCallIndex };
      }

      m_jobs.emplace_back(DependencyPacket{ drawCallIndex, uniqueID, ResourceType::texture,
        flags, view.range() });

      if (m_resourceStates.find(uniqueID) == m_resourceStates.end())
      {
        TextureDependency d;
        d.uniqueId = uniqueID;
        d.mips = mips;
        d.texture = texture.native();
        d.state = texture.state().get();
        m_resourceStates[uniqueID] = std::move(d);
      }

      m_uniqueResourcesThisChain.insert(uniqueID);
    }
    */

    void DX12DependencySolver::makeAllBarriers()
    {
      m_resourceCache.clear();

      for (auto&& id : m_uniqueResourcesThisChain)
      {
        auto tesState = m_resourceStates[id];
        auto flags = tesState.state->flags;
        m_resourceCache[id] = SmallResource{ tesState.texture, tesState.mips, flags };
      }
      int jobsSize = static_cast<int>(m_jobs.size());
      int jobIndex = 0;
      int barriersOffset = 0;

      int drawIndex = -1;

      while (jobIndex < jobsSize)
      {
        int draw = m_jobs[jobIndex].drawIndex;
        for (int skippedDraw = drawIndex; skippedDraw < draw; ++skippedDraw)
        {
          m_barriersOffsets.emplace_back(barriersOffset);
        }
        drawIndex = draw;

        m_uavCache.clear();

        while (jobIndex < jobsSize && m_jobs[jobIndex].drawIndex == draw)
        {
          auto& job = m_jobs[jobIndex];
          auto jobResAccess = job.access;
          /*
          if (job.type == ResourceType::buffer)
          {
            auto resource = m_bufferCache.find(job.resource);
            if (resource != m_bufferCache.end())
            {
              auto lastAccess = resource->second.flags;

              if (lastAccess & D3D12_RESOURCE_STATE_UNORDERED_ACCESS) // everytime the last access was unordered access, make uav barrier.
              {
                m_uavCache.emplace(resource->second.buffer);
              }

              if (jobResAccess != lastAccess)
              {
                barriers.emplace_back(D3D12_RESOURCE_BARRIER{
                  D3D12_RESOURCE_BARRIER_TYPE_TRANSITION,
                  D3D12_RESOURCE_BARRIER_FLAG_NONE,
                  resource->second.buffer,
                  static_cast<UINT>(-1),
                  lastAccess,
                  jobResAccess });
                resource->second.flags = jobResAccess;
                ++barriersOffset;
              }
            }
          }
          else*/
          {
            auto resource = m_resourceCache.find(job.resource);
            if (resource != m_resourceCache.end())
            {
              int16_t mipLevels = resource->second.mips;
              int16_t subresourceIndex;

              for (int16_t mip = job.range.mipOffset; mip < job.range.mipOffset + job.range.mipLevels; ++mip)
              {
                for (int16_t slice = job.range.sliceOffset; slice < job.range.sliceOffset + job.range.arraySize; ++slice)
                {
                  subresourceIndex = slice * mipLevels + mip;
                  auto& state = resource->second.states[subresourceIndex];

                  if (state & D3D12_RESOURCE_STATE_UNORDERED_ACCESS)
                  {
                    m_uavCache.emplace(resource->second.image);
                  }

                  if (state != job.access)
                  {
                    barriers.emplace_back(D3D12_RESOURCE_BARRIER{
                      D3D12_RESOURCE_BARRIER_TYPE_TRANSITION,
                      D3D12_RESOURCE_BARRIER_FLAG_NONE,
                      resource->second.image,
                      static_cast<UINT>(-1),
                      state,
                      jobResAccess });
                    state = jobResAccess;
                    ++barriersOffset;
                  }
                }
              }
            }
          }

          for (auto&& it : m_uavCache)
          {
            barriers.emplace_back(D3D12_RESOURCE_BARRIER{
              D3D12_RESOURCE_BARRIER_TYPE_UAV,
              D3D12_RESOURCE_BARRIER_FLAG_NONE,
              it });
            ++barriersOffset;
          }

          ++jobIndex;
        }
      }
      for (int skippedDraw = drawIndex; skippedDraw <= drawCallsAdded; ++skippedDraw)
      {
        m_barriersOffsets.emplace_back(static_cast<int>(barriers.size()));
      }
      m_barriersOffsets.emplace_back(static_cast<int>(barriers.size()));

      // update global state

      for (auto&& obj : m_uniqueResourcesThisChain)
      {
        auto& globalState = m_resourceStates[obj].state->flags;
        auto& localState = m_resourceCache[obj].states;
        auto globalSize = globalState.size();
        for (int i = 0; i < globalSize; ++i)
        {
          globalState[i] = localState[i];
        }
      }
      // function ends
    }

    void DX12DependencySolver::runBarrier(ID3D12GraphicsCommandList* gfx, int nextDrawCall)
    {
      if (nextDrawCall == 0)
      {
        int barrierSize = (m_barriersOffsets.size() > 1 && m_barriersOffsets[1] > 0) ? m_barriersOffsets[1] : static_cast<int>(barriers.size());
        if (barrierSize == 0)
        {
          return;
        }
        gfx->ResourceBarrier(barrierSize, barriers.data());
        return;
      }

      // after first and second, nextDrawCall == 1
      // so we need barriers from offset 0
      int barrierOffset = m_barriersOffsets[nextDrawCall];
      int barrierSize = m_barriersOffsets[nextDrawCall + 1] - barrierOffset;

      if (barrierSize == 0)
      {
        return;
      }

      gfx->ResourceBarrier(barrierSize, barriers.data() + barrierOffset);
    }

    void DX12DependencySolver::reset()
    {
      m_drawCallInfo.clear();
      m_jobs.clear();
      m_schedulingResult.clear();
      m_barriersOffsets.clear();
      barriers.clear();
      m_uniqueResourcesThisChain.clear();
    }
  }
}<commit_msg>whoops<commit_after>#include "faze/src/new_gfx/dx12/util/dxDependencySolver.hpp"
#include "faze/src/new_gfx/definitions.hpp"
#include "core/src/global_debug.hpp"

namespace faze
{
  namespace backend
  {
    DX12DependencySolver::UsageHint DX12DependencySolver::getUsageFromAccessFlags(D3D12_RESOURCE_STATES flags)
    {
      int32_t writeMask = D3D12_RESOURCE_STATE_UNORDERED_ACCESS | D3D12_RESOURCE_STATE_DEPTH_WRITE | D3D12_RESOURCE_STATE_RENDER_TARGET
        | D3D12_RESOURCE_STATE_COPY_DEST | D3D12_RESOURCE_STATE_RESOLVE_DEST;

      if (flags == D3D12_RESOURCE_STATE_COMMON || flags & writeMask) // common is the most complex state... with value of 0
      {
        return DX12DependencySolver::UsageHint::write;
      }
      return DX12DependencySolver::UsageHint::read;
    }

    int DX12DependencySolver::addDrawCall(CommandPacket::PacketType name)
    {
      m_drawCallInfo.emplace_back(name);
      m_drawCallJobOffsets[drawCallsAdded] = static_cast<int>(m_jobs.size());
      return drawCallsAdded++;
    }

    void DX12DependencySolver::addResource(int drawCallIndex, backend::TrackedState state, D3D12_RESOURCE_STATES flags)
    {
      size_t uniqueID = state.resPtr;
      if (uniqueID == 0)
        return;

      SubresourceRange range{};

      range.sliceOffset = static_cast<int16_t>(state.slice());
      range.arraySize = static_cast<int16_t>(state.arraySize());
      range.mipOffset = static_cast<int16_t>(state.mip());
      range.mipLevels = static_cast<int16_t>(state.mipLevels());

      m_jobs.emplace_back(DependencyPacket{ drawCallIndex, uniqueID, flags, range });
      if (m_resourceStates.find(uniqueID) == m_resourceStates.end())
      {
        ResourceDependency d{};
        d.uniqueId = uniqueID;
        d.mips = static_cast<int16_t>(state.totalMipLevels());
        d.texture = reinterpret_cast<ID3D12Resource*>(state.resPtr);
        d.state = reinterpret_cast<DX12ResourceState*>(state.statePtr);
        m_resourceStates[uniqueID] = std::move(d);
      }
      m_uniqueResourcesThisChain.insert(uniqueID);
    }

    void DX12DependencySolver::addResource(int drawCallIndex, backend::TrackedState state, D3D12_RESOURCE_STATES flags, SubresourceRange range)
    {
      size_t uniqueID = state.resPtr;
      if (uniqueID == 0)
        return;
      m_jobs.emplace_back(DependencyPacket{ drawCallIndex, uniqueID, flags, range });

      if (m_resourceStates.find(uniqueID) == m_resourceStates.end())
      {
        ResourceDependency d{};
        d.uniqueId = uniqueID;
        d.mips = static_cast<int16_t>(state.totalMipLevels());
        d.texture = reinterpret_cast<ID3D12Resource*>(state.resPtr);
        d.state = reinterpret_cast<DX12ResourceState*>(state.statePtr);
        m_resourceStates[uniqueID] = std::move(d);
      }
      m_uniqueResourcesThisChain.insert(uniqueID);
    }

    // buffers
    /*
    void DX12DependencySolver::addBuffer(int drawCallIndex, int64_t, DX12Buffer& buffer, D3D12_RESOURCE_STATES flags)
    {
      size_t uniqueID = reinterpret_cast<size_t>(buffer.native());
      if (faze::globalconfig::graphics::GraphicsEnableReadStateCombining) // disable optimization, merges last seen usages to existing. Default on. Combines only reads.
      {
        auto currentUsage = getUsageFromAccessFlags(flags);
        if (currentUsage == UsageHint::read)
        {
          auto* obj = m_resourceUsageInLastAdd.find(uniqueID);
          if (obj)
          {
            if (obj->second.type == currentUsage)
            {
              // Try to recognize here that last time we saw the same id, we already used it in read/write state, combine the states.
              auto* jobIndex = m_drawCallJobOffsets.find(obj->second.index);
              if (jobIndex)
              {
                int index = jobIndex->second;
                while (index < static_cast<int>(m_jobs.size()) && obj->second.index == m_jobs[index].drawIndex)
                {
                  if (m_jobs[index].resource == uniqueID)
                  {
                    break;
                  }
                  index++;
                }
                m_jobs[index].access |= flags;
                return; // found and could merge... or rather forcefully merged.
              }
            }
          }
        }
        m_resourceUsageInLastAdd[uniqueID] = LastSeenUsage{ currentUsage, drawCallIndex };
      }

      m_jobs.emplace_back(DependencyPacket{ drawCallIndex, uniqueID, ResourceType::buffer,
        flags, SubresourceRange{} });

      if (m_resourceStates.find(uniqueID) == m_resourceStates.end())
      {
        TextureDependency d;
        d.uniqueId = uniqueID;
        d.mips = 1;
        d.texture = buffer.native();
        d.state = buffer.state().get();
        m_resourceStates[uniqueID] = std::move(d);
      }
      m_uniqueBuffersThisChain.insert(uniqueID);
    }
    */
    // textures
    /*
    void DX12DependencySolver::addTexture(int drawCallIndex, int64_t, DX12Texture& texture, DX12TextureView& view, int16_t mips, D3D12_RESOURCE_STATES flags)
    {
      auto uniqueID = reinterpret_cast<size_t>(texture.native());

      if (faze::globalconfig::graphics::GraphicsEnableReadStateCombining)
      {
        auto currentUsage = getUsageFromAccessFlags(flags);
        if (currentUsage == UsageHint::read)
        {
          auto* obj = m_resourceUsageInLastAdd.find(uniqueID);
          if (obj)
          {
            if (obj->second.type != currentUsage)
            {
              F_SLOG("DependencySolver", "Possible optimization here!\n");
              // Try to recognize here that last time we saw the same id, we already used it in read/write state, combine the states.
              // Texture... needs to look at all the subresources seen before and combine their state... ugh, this looks not to be easy.
            }
          }
        }
        m_resourceUsageInLastAdd[uniqueID] = LastSeenUsage{ currentUsage, drawCallIndex };
      }

      m_jobs.emplace_back(DependencyPacket{ drawCallIndex, uniqueID, ResourceType::texture,
        flags, view.range() });

      if (m_resourceStates.find(uniqueID) == m_resourceStates.end())
      {
        TextureDependency d;
        d.uniqueId = uniqueID;
        d.mips = mips;
        d.texture = texture.native();
        d.state = texture.state().get();
        m_resourceStates[uniqueID] = std::move(d);
      }

      m_uniqueResourcesThisChain.insert(uniqueID);
    }
    */

    void DX12DependencySolver::makeAllBarriers()
    {
      m_resourceCache.clear();

      for (auto&& id : m_uniqueResourcesThisChain)
      {
        auto tesState = m_resourceStates[id];
        auto flags = tesState.state->flags;
        m_resourceCache[id] = SmallResource{ tesState.texture, tesState.mips, flags };
      }
      int jobsSize = static_cast<int>(m_jobs.size());
      int jobIndex = 0;
      int barriersOffset = 0;

      int drawIndex = -1;

      while (jobIndex < jobsSize)
      {
        int draw = m_jobs[jobIndex].drawIndex;
        for (int skippedDraw = drawIndex; skippedDraw < draw; ++skippedDraw)
        {
          m_barriersOffsets.emplace_back(barriersOffset);
        }
        drawIndex = draw;

        m_uavCache.clear();

        while (jobIndex < jobsSize && m_jobs[jobIndex].drawIndex == draw)
        {
          auto& job = m_jobs[jobIndex];
          auto jobResAccess = job.access;
          /*
          if (job.type == ResourceType::buffer)
          {
            auto resource = m_bufferCache.find(job.resource);
            if (resource != m_bufferCache.end())
            {
              auto lastAccess = resource->second.flags;

              if (lastAccess & D3D12_RESOURCE_STATE_UNORDERED_ACCESS) // everytime the last access was unordered access, make uav barrier.
              {
                m_uavCache.emplace(resource->second.buffer);
              }

              if (jobResAccess != lastAccess)
              {
                barriers.emplace_back(D3D12_RESOURCE_BARRIER{
                  D3D12_RESOURCE_BARRIER_TYPE_TRANSITION,
                  D3D12_RESOURCE_BARRIER_FLAG_NONE,
                  resource->second.buffer,
                  static_cast<UINT>(-1),
                  lastAccess,
                  jobResAccess });
                resource->second.flags = jobResAccess;
                ++barriersOffset;
              }
            }
          }
          else*/
          {
            auto resource = m_resourceCache.find(job.resource);
            if (resource != m_resourceCache.end())
            {
              int16_t mipLevels = resource->second.mips;
              int16_t subresourceIndex;

              for (int16_t mip = job.range.mipOffset; mip < job.range.mipOffset + job.range.mipLevels; ++mip)
              {
                for (int16_t slice = job.range.sliceOffset; slice < job.range.sliceOffset + job.range.arraySize; ++slice)
                {
                  subresourceIndex = slice * mipLevels + mip;
                  auto& state = resource->second.states[subresourceIndex];

                  if (state & D3D12_RESOURCE_STATE_UNORDERED_ACCESS)
                  {
                    m_uavCache.emplace(resource->second.image);
                  }

                  if (state != job.access)
                  {
                    barriers.emplace_back(D3D12_RESOURCE_BARRIER{
                      D3D12_RESOURCE_BARRIER_TYPE_TRANSITION,
                      D3D12_RESOURCE_BARRIER_FLAG_NONE,
                      resource->second.image,
                      static_cast<UINT>(-1),
                      state,
                      jobResAccess });
                    state = jobResAccess;
                    ++barriersOffset;
                  }
                }
              }
            }
          }

          for (auto&& it : m_uavCache)
          {
            barriers.emplace_back(D3D12_RESOURCE_BARRIER{
              D3D12_RESOURCE_BARRIER_TYPE_UAV,
              D3D12_RESOURCE_BARRIER_FLAG_NONE,
              it });
            ++barriersOffset;
          }

          ++jobIndex;
        }
      }
      for (int skippedDraw = drawIndex; skippedDraw <= drawCallsAdded; ++skippedDraw)
      {
        m_barriersOffsets.emplace_back(static_cast<int>(barriers.size()));
      }
      m_barriersOffsets.emplace_back(static_cast<int>(barriers.size()));

      // update global state

      for (auto&& obj : m_uniqueResourcesThisChain)
      {
        auto& globalState = m_resourceStates[obj].state->flags;
        auto& localState = m_resourceCache[obj].states;
        auto globalSize = globalState.size();
        for (int i = 0; i < globalSize; ++i)
        {
          globalState[i] = localState[i];
        }
      }
      // function ends
    }

    void DX12DependencySolver::runBarrier(ID3D12GraphicsCommandList* gfx, int nextDrawCall)
    {
      if (nextDrawCall == 0)
      {
        int barrierSize = (m_barriersOffsets.size() > 1 && m_barriersOffsets[1] > 0) ? m_barriersOffsets[1] : static_cast<int>(barriers.size());
        if (barrierSize == 0)
        {
          return;
        }
        gfx->ResourceBarrier(barrierSize, barriers.data());
        return;
      }

      // after first and second, nextDrawCall == 1
      // so we need barriers from offset 0
      int barrierOffset = m_barriersOffsets[nextDrawCall];
      int barrierSize = m_barriersOffsets[nextDrawCall + 1] - barrierOffset;

      if (barrierSize == 0)
      {
        return;
      }

      gfx->ResourceBarrier(barrierSize, barriers.data() + barrierOffset);
    }

    void DX12DependencySolver::reset()
    {
      m_drawCallInfo.clear();
      m_jobs.clear();
      m_schedulingResult.clear();
      m_barriersOffsets.clear();
      barriers.clear();
      m_uniqueResourcesThisChain.clear();
    }
  }
}<|endoftext|>


<commit_before><commit_msg>Tweak shader model error message.<commit_after><|endoftext|>


<commit_before>#include "dynet/nodes.h"
#include "dynet/dynet.h"
#include "dynet/training.h"
#include "dynet/timing.h"
#include "dynet/rnn.h"
#include "dynet/gru.h"
#include "dynet/lstm.h"
#include "dynet/fast-lstm.h"
#include "dynet/dict.h"
#include "dynet/expr.h"

#include <sys/stat.h>
#include <sys/types.h>
#include <iostream>
#include <fstream>
#include <sstream>
#include <type_traits>
#include <time.h>

#include <boost/serialization/vector.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/program_options.hpp>

#include "s2s/encdec.hpp"
#include "s2s/decode.hpp"
#include "s2s/define.hpp"
#include "s2s/comp.hpp"
#include "s2s/metrics.hpp"
#include "s2s/options.hpp"

namespace s2s {

    void train(const s2s_options &opts){
        s2s::dicts dicts;
        s2s::parallel_corpus para_corp_train;
        s2s::parallel_corpus para_corp_dev;
        dicts.set(opts);
        para_corp_train.load_src(opts.srcfile, dicts);
        para_corp_train.load_trg(opts.trgfile, dicts);
        para_corp_train.load_check();
        para_corp_dev.load_src(opts.srcvalfile, dicts);
        para_corp_dev.load_trg(opts.trgvalfile, dicts);
        para_corp_dev.load_check();
        if(opts.guided_alignment == true){
            para_corp_train.load_align(opts.alignfile);
            para_corp_train.load_check_with_align();
            para_corp_dev.load_align(opts.alignvalfile);
            para_corp_dev.load_check_with_align();
        }
        // for debug
        dicts.save(opts);
        // for debug
        dynet::Model model;
        encoder_decoder* encdec = new encoder_decoder(model, &opts);
        dynet::Trainer* trainer = nullptr;
        if(opts.optim == "sgd"){
            trainer = new dynet::SimpleSGDTrainer(model);
        }else if(opts.optim == "momentum_sgd"){
            trainer = new dynet::MomentumSGDTrainer(model);
        }else if(opts.optim == "adagrad"){
            trainer = new dynet::AdagradTrainer(model);
        }else if(opts.optim == "adadelta"){
            trainer = new dynet::AdadeltaTrainer(model);
        }else if(opts.optim == "adam"){
            trainer = new dynet::AdamTrainer(model);
        }else{
            std::cerr << "Trainer does not exist !"<< std::endl;
            assert(false);
        }
        trainer->eta0 = opts.learning_rate;
        trainer->eta_decay = opts.lr_decay;
        trainer->clip_threshold = opts.clip_threshold;
        trainer->clipping_enabled = opts.clipping_enabled;
        unsigned int epoch = 0;
        while(epoch < opts.epochs){
            // train
            para_corp_train.shuffle();
            float align_w = opts.guided_alignment_weight;
            batch one_batch;
            unsigned int bid = 0;
            while(para_corp_train.next_batch_para(one_batch, opts.max_batch_train, dicts)){
                bid++;
                auto chrono_start = std::chrono::system_clock::now();
                dynet::ComputationGraph cg;
                std::vector<dynet::expr::Expression> errs_att;
                std::vector<dynet::expr::Expression> errs_out;
                float loss_att = 0.0;
                float loss_out = 0.0;
                std::vector<dynet::expr::Expression> i_enc = encdec->encoder(one_batch, cg);
                std::vector<dynet::expr::Expression> i_feed = encdec->init_feed(one_batch, cg);
                for (unsigned int t = 0; t < one_batch.trg.size() - 1; ++t) {
                    dynet::expr::Expression i_att_t = encdec->decoder_attention(cg, one_batch.trg[t], i_feed[t], i_enc[0]);
                    if(opts.guided_alignment == true){
                        dynet::expr::Expression i_err = pickneglogsoftmax(i_att_t, one_batch.align[t]);
                        errs_att.push_back(i_err);
                    }
                    std::vector<dynet::expr::Expression> i_out_t = encdec->decoder_output(cg, i_att_t, i_enc[1]);
                    i_feed.push_back(i_out_t[1]);
                    dynet::expr::Expression i_err = pickneglogsoftmax(i_out_t[0], one_batch.trg[t+1]);
                    errs_out.push_back(i_err);
                }
                dynet::expr::Expression i_nerr_out = sum_batches(sum(errs_out));
                loss_out = as_scalar(cg.forward(i_nerr_out));
                dynet::expr::Expression i_nerr_all;
                if(opts.guided_alignment == true){
                    dynet::expr::Expression i_nerr_att = sum_batches(sum(errs_att));
                    loss_att = as_scalar(cg.incremental_forward(i_nerr_att));
                    i_nerr_all = i_nerr_out + align_w * i_nerr_att;
                }else{
                    i_nerr_all = i_nerr_out;
                }
                cg.incremental_forward(i_nerr_all);
                cg.backward(i_nerr_all);
                //cg.print_graphviz();
                trainer->update(1.0 / double(one_batch.src.at(0).at(0).size()));
                auto chrono_end = std::chrono::system_clock::now();
                auto time_used = (double)std::chrono::duration_cast<std::chrono::milliseconds>(chrono_end - chrono_start).count() / (double)1000;
                std::cerr << "batch: " << bid;
                std::cerr << ",\tsize: " << one_batch.src.at(0).at(0).size();
                std::cerr << ",\toutput loss: " << loss_out;
                std::cerr << ",\tattention loss: " << loss_att;
                std::cerr << ",\tsource length: " << one_batch.src.size();
                std::cerr << ",\ttarget length: " << one_batch.trg.size();
                std::cerr << ",\ttime: " << time_used << " [s]" << std::endl;
                std::cerr << "[epoch=" << trainer->epoch << " eta=" << trainer->eta << " clips=" << trainer->clips_since_status << " updates=" << trainer->updates_since_status << "] " << std::endl;
            }
            trainer->update_epoch();
            align_w *= opts.guided_alignment_decay;
            para_corp_train.reset_index();
            epoch++;
            // dev
            std::cerr << "dev" << std::endl;
            encdec->disable_dropout();
            ofstream dev_sents(opts.rootdir + "/dev_" + to_string(epoch) + ".txt");
            while(para_corp_dev.next_batch_para(one_batch, opts.max_batch_pred, dicts)){
                std::vector<std::vector<unsigned int> > osent;
                s2s::greedy_decode(one_batch, osent, encdec, dicts, opts);
                dev_sents << s2s::print_sents(osent, dicts);
            }
            dev_sents.close();
            para_corp_dev.reset_index();
            encdec->set_dropout(opts.dropout);
            // save Model
            ofstream model_out(opts.rootdir + "/" + opts.save_file + "_" + to_string(epoch) + ".model");
            boost::archive::text_oarchive model_oa(model_out);
            model_oa << model << *encdec;
            model_out.close();
        }
    }

    void predict(const s2s_options &opts){
        s2s::dicts dicts;
        dicts.load(opts);
        // load model
        dynet::Model model;
        encoder_decoder* encdec = new encoder_decoder(model, &opts);
        ifstream model_in(opts.modelfile);
        boost::archive::text_iarchive model_ia(model_in);
        model_ia >> model >> *encdec;
        model_in.close();
        encdec->disable_dropout();
        // predict
        s2s::monoling_corpus mono_corp;
        mono_corp.load_src(opts.srcfile, dicts);
        batch one_batch;
        ofstream predict_sents(opts.trgfile);
        while(mono_corp.next_batch_mono(one_batch, opts.max_batch_pred, dicts)){
            std::vector<std::vector<unsigned int> > osent;
            s2s::greedy_decode(one_batch, osent, encdec, dicts, opts);
            predict_sents << s2s::print_sents(osent, dicts);
        }
        predict_sents.close();
    }

};

int main(int argc, char** argv) {
    namespace po = boost::program_options;
    po::options_description bpo("h");
    s2s::s2s_options opts;
    s2s::set_s2s_options(&bpo, &opts);
    po::variables_map vm;
    po::store(po::parse_command_line(argc, argv, bpo), vm);
    po::notify(vm);
    if(vm.at("mode").as<std::string>() == "train"){
        s2s::add_s2s_options_train(&vm, &opts);
        s2s::check_s2s_options_train(&vm, opts);
        std::string file_name = opts.rootdir + "/options.txt";
        struct stat st;
        if(stat(opts.rootdir.c_str(), &st) != 0){
            mkdir(opts.rootdir.c_str(), 0775);
        }
        ofstream out(file_name);
        boost::archive::text_oarchive oa(out);
        oa << opts;
        out.close();
        dynet::initialize(argc, argv);
        s2s::train(opts);
    }else if(vm.at("mode").as<std::string>() == "predict"){
        ifstream in(opts.rootdir + "/options.txt");
        boost::archive::text_iarchive ia(in);
        ia >> opts;
        in.close();
        s2s::check_s2s_options_predict(&vm, opts);
        dynet::initialize(argc, argv);
        s2s::predict(opts);
    }else if(vm.at("mode").as<std::string>() == "test"){

    }else{
        std::cerr << "Mode does not exist !"<< std::endl;
        assert(false);
    }
}
<commit_msg>add epoch status to log<commit_after>#include "dynet/nodes.h"
#include "dynet/dynet.h"
#include "dynet/training.h"
#include "dynet/timing.h"
#include "dynet/rnn.h"
#include "dynet/gru.h"
#include "dynet/lstm.h"
#include "dynet/fast-lstm.h"
#include "dynet/dict.h"
#include "dynet/expr.h"

#include <sys/stat.h>
#include <sys/types.h>
#include <iostream>
#include <fstream>
#include <sstream>
#include <type_traits>
#include <time.h>

#include <boost/serialization/vector.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/program_options.hpp>

#include "s2s/encdec.hpp"
#include "s2s/decode.hpp"
#include "s2s/define.hpp"
#include "s2s/comp.hpp"
#include "s2s/metrics.hpp"
#include "s2s/options.hpp"

namespace s2s {

    void train(const s2s_options &opts){
        s2s::dicts dicts;
        s2s::parallel_corpus para_corp_train;
        s2s::parallel_corpus para_corp_dev;
        dicts.set(opts);
        para_corp_train.load_src(opts.srcfile, dicts);
        para_corp_train.load_trg(opts.trgfile, dicts);
        para_corp_train.load_check();
        para_corp_dev.load_src(opts.srcvalfile, dicts);
        para_corp_dev.load_trg(opts.trgvalfile, dicts);
        para_corp_dev.load_check();
        if(opts.guided_alignment == true){
            para_corp_train.load_align(opts.alignfile);
            para_corp_train.load_check_with_align();
            para_corp_dev.load_align(opts.alignvalfile);
            para_corp_dev.load_check_with_align();
        }
        // for debug
        dicts.save(opts);
        // for debug
        dynet::Model model;
        encoder_decoder* encdec = new encoder_decoder(model, &opts);
        dynet::Trainer* trainer = nullptr;
        if(opts.optim == "sgd"){
            trainer = new dynet::SimpleSGDTrainer(model);
        }else if(opts.optim == "momentum_sgd"){
            trainer = new dynet::MomentumSGDTrainer(model);
        }else if(opts.optim == "adagrad"){
            trainer = new dynet::AdagradTrainer(model);
        }else if(opts.optim == "adadelta"){
            trainer = new dynet::AdadeltaTrainer(model);
        }else if(opts.optim == "adam"){
            trainer = new dynet::AdamTrainer(model);
        }else{
            std::cerr << "Trainer does not exist !"<< std::endl;
            assert(false);
        }
        trainer->eta0 = opts.learning_rate;
        trainer->eta_decay = opts.lr_decay;
        trainer->clip_threshold = opts.clip_threshold;
        trainer->clipping_enabled = opts.clipping_enabled;
        unsigned int epoch = 0;
        while(epoch < opts.epochs){
            // train
            para_corp_train.shuffle();
            float align_w = opts.guided_alignment_weight;
            batch one_batch;
            unsigned int bid = 0;
            while(para_corp_train.next_batch_para(one_batch, opts.max_batch_train, dicts)){
                bid++;
                auto chrono_start = std::chrono::system_clock::now();
                dynet::ComputationGraph cg;
                std::vector<dynet::expr::Expression> errs_att;
                std::vector<dynet::expr::Expression> errs_out;
                float loss_att = 0.0;
                float loss_out = 0.0;
                std::vector<dynet::expr::Expression> i_enc = encdec->encoder(one_batch, cg);
                std::vector<dynet::expr::Expression> i_feed = encdec->init_feed(one_batch, cg);
                for (unsigned int t = 0; t < one_batch.trg.size() - 1; ++t) {
                    dynet::expr::Expression i_att_t = encdec->decoder_attention(cg, one_batch.trg[t], i_feed[t], i_enc[0]);
                    if(opts.guided_alignment == true){
                        dynet::expr::Expression i_err = pickneglogsoftmax(i_att_t, one_batch.align[t]);
                        errs_att.push_back(i_err);
                    }
                    std::vector<dynet::expr::Expression> i_out_t = encdec->decoder_output(cg, i_att_t, i_enc[1]);
                    i_feed.push_back(i_out_t[1]);
                    dynet::expr::Expression i_err = pickneglogsoftmax(i_out_t[0], one_batch.trg[t+1]);
                    errs_out.push_back(i_err);
                }
                dynet::expr::Expression i_nerr_out = sum_batches(sum(errs_out));
                loss_out = as_scalar(cg.forward(i_nerr_out));
                dynet::expr::Expression i_nerr_all;
                if(opts.guided_alignment == true){
                    dynet::expr::Expression i_nerr_att = sum_batches(sum(errs_att));
                    loss_att = as_scalar(cg.incremental_forward(i_nerr_att));
                    i_nerr_all = i_nerr_out + align_w * i_nerr_att;
                }else{
                    i_nerr_all = i_nerr_out;
                }
                cg.incremental_forward(i_nerr_all);
                cg.backward(i_nerr_all);
                //cg.print_graphviz();
                trainer->update(1.0 / double(one_batch.src.at(0).at(0).size()));
                auto chrono_end = std::chrono::system_clock::now();
                auto time_used = (double)std::chrono::duration_cast<std::chrono::milliseconds>(chrono_end - chrono_start).count() / (double)1000;
                std::cerr << "batch: " << bid;
                std::cerr << ",\tsize: " << one_batch.src.at(0).at(0).size();
                std::cerr << ",\toutput loss: " << loss_out;
                std::cerr << ",\tattention loss: " << loss_att;
                std::cerr << ",\tsource length: " << one_batch.src.size();
                std::cerr << ",\ttarget length: " << one_batch.trg.size();
                std::cerr << ",\ttime: " << time_used << " [s]" << std::endl;
                std::cerr << "[epoch=" << trainer->epoch << " eta=" << trainer->eta << " clips=" << trainer->clips_since_status << " updates=" << trainer->updates_since_status << "] " << std::endl;
            }
            trainer->update_epoch();
            trainer->status();
            align_w *= opts.guided_alignment_decay;
            para_corp_train.reset_index();
            epoch++;
            // dev
            std::cerr << "dev" << std::endl;
            encdec->disable_dropout();
            ofstream dev_sents(opts.rootdir + "/dev_" + to_string(epoch) + ".txt");
            while(para_corp_dev.next_batch_para(one_batch, opts.max_batch_pred, dicts)){
                std::vector<std::vector<unsigned int> > osent;
                s2s::greedy_decode(one_batch, osent, encdec, dicts, opts);
                dev_sents << s2s::print_sents(osent, dicts);
            }
            dev_sents.close();
            para_corp_dev.reset_index();
            encdec->set_dropout(opts.dropout);
            // save Model
            ofstream model_out(opts.rootdir + "/" + opts.save_file + "_" + to_string(epoch) + ".model");
            boost::archive::text_oarchive model_oa(model_out);
            model_oa << model << *encdec;
            model_out.close();
        }
    }

    void predict(const s2s_options &opts){
        s2s::dicts dicts;
        dicts.load(opts);
        // load model
        dynet::Model model;
        encoder_decoder* encdec = new encoder_decoder(model, &opts);
        ifstream model_in(opts.modelfile);
        boost::archive::text_iarchive model_ia(model_in);
        model_ia >> model >> *encdec;
        model_in.close();
        encdec->disable_dropout();
        // predict
        s2s::monoling_corpus mono_corp;
        mono_corp.load_src(opts.srcfile, dicts);
        batch one_batch;
        ofstream predict_sents(opts.trgfile);
        while(mono_corp.next_batch_mono(one_batch, opts.max_batch_pred, dicts)){
            std::vector<std::vector<unsigned int> > osent;
            s2s::greedy_decode(one_batch, osent, encdec, dicts, opts);
            predict_sents << s2s::print_sents(osent, dicts);
        }
        predict_sents.close();
    }

};

int main(int argc, char** argv) {
    namespace po = boost::program_options;
    po::options_description bpo("h");
    s2s::s2s_options opts;
    s2s::set_s2s_options(&bpo, &opts);
    po::variables_map vm;
    po::store(po::parse_command_line(argc, argv, bpo), vm);
    po::notify(vm);
    if(vm.at("mode").as<std::string>() == "train"){
        s2s::add_s2s_options_train(&vm, &opts);
        s2s::check_s2s_options_train(&vm, opts);
        std::string file_name = opts.rootdir + "/options.txt";
        struct stat st;
        if(stat(opts.rootdir.c_str(), &st) != 0){
            mkdir(opts.rootdir.c_str(), 0775);
        }
        ofstream out(file_name);
        boost::archive::text_oarchive oa(out);
        oa << opts;
        out.close();
        dynet::initialize(argc, argv);
        s2s::train(opts);
    }else if(vm.at("mode").as<std::string>() == "predict"){
        ifstream in(opts.rootdir + "/options.txt");
        boost::archive::text_iarchive ia(in);
        ia >> opts;
        in.close();
        s2s::check_s2s_options_predict(&vm, opts);
        dynet::initialize(argc, argv);
        s2s::predict(opts);
    }else if(vm.at("mode").as<std::string>() == "test"){

    }else{
        std::cerr << "Mode does not exist !"<< std::endl;
        assert(false);
    }
}
<|endoftext|>


<commit_before>// Jubatus: Online machine learning framework for distributed environment
// Copyright (C) 2011 Preferred Infrastructure and Nippon Telegraph and Telephone Corporation.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#include "util.hpp"
#include "gtest/gtest.h"
#include <cstdlib>
#include <iostream>

TEST(common,util){
  const std::string env = getenv("PATH");
  jubatus::util::append_env_path("PATH", "./path/to/hoge");
  const std::string env2 = getenv("PATH");
  ASSERT_TRUE(env.size() < env2.size());
  //  std::cout << env << std::endl;
  //  std::cout << env2 << std::endl;
  ASSERT_NE(env, env2);
}

TEST(common,util2){
  const std::string env = getenv("PATH");
  jubatus::util::append_server_path("./path/to/hoge");
  const std::string env2 = getenv("PATH");
  ASSERT_TRUE(env.size() < env2.size());
  //  std::cout << env << std::endl;
  //  std::cout << env2 << std::endl;
  ASSERT_NE(env, env2);
}

TEST(common,util_get_program_name){
  std::string path;
  EXPECT_NO_THROW({
    path = jubatus::util::get_program_name();
  });
  EXPECT_NE("", path);
}
<commit_msg>Fix test for util::get_program_name<commit_after>// Jubatus: Online machine learning framework for distributed environment
// Copyright (C) 2011 Preferred Infrastructure and Nippon Telegraph and Telephone Corporation.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#include "util.hpp"
#include "gtest/gtest.h"
#include <cstdlib>
#include <iostream>

TEST(common,util){
  const std::string env = getenv("PATH");
  jubatus::util::append_env_path("PATH", "./path/to/hoge");
  const std::string env2 = getenv("PATH");
  ASSERT_TRUE(env.size() < env2.size());
  //  std::cout << env << std::endl;
  //  std::cout << env2 << std::endl;
  ASSERT_NE(env, env2);
}

TEST(common,util2){
  const std::string env = getenv("PATH");
  jubatus::util::append_server_path("./path/to/hoge");
  const std::string env2 = getenv("PATH");
  ASSERT_TRUE(env.size() < env2.size());
  //  std::cout << env << std::endl;
  //  std::cout << env2 << std::endl;
  ASSERT_NE(env, env2);
}

TEST(common,util_get_program_name){
  std::string path;
  EXPECT_NO_THROW({
    path = jubatus::util::get_program_name();
  });
  EXPECT_EQ(std::string("util_test"), path);
}
<|endoftext|>


<commit_before>// vim:ts=2:sw=2:expandtab:autoindent:filetype=cpp:
/*
Copyright (c) 2008 Ash Berlin

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

#include "flusspferd/class.hpp"
#include "flusspferd/create.hpp"
#include "flusspferd/security.hpp"

#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif

using namespace flusspferd;

namespace {

// import hook
extern "C" void flusspferd_load(object posix)
{
  local_root_scope scope;

#ifdef HAVE_FORK
  // TODO: Need to work out how to make this play nice with the interactive repl
  create_native_function(posix, "fork", &::fork);
#endif

  create_native_function(posix, "sleep", &::sleep);

#ifdef HAVE_USLEEP
  create_native_function(posix, "usleep", usleep);
#endif
}

}

<commit_msg>posix: added errno interface<commit_after>// vim:ts=2:sw=2:expandtab:autoindent:filetype=cpp:
/*
Copyright (c) 2008 Ash Berlin

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

#include "flusspferd/class.hpp"
#include "flusspferd/create.hpp"
#include "flusspferd/security.hpp"

#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif

#include <cerrno>

using namespace flusspferd;

namespace {

int errno_getter() {
  return errno;
}

void errno_setter(int errno_) {
  errno = errno_;
}

std::string strerror_(int errno_) {
  return strerror(errno_);
}

// import hook
extern "C" void flusspferd_load(object posix)
{
  local_root_scope scope;

#ifdef HAVE_FORK
  // TODO: Need to work out how to make this play nice with the interactive repl
  create_native_function(posix, "fork", &::fork);
#endif

  create_native_function(posix, "sleep", &::sleep);

#ifdef HAVE_USLEEP
  create_native_function(posix, "usleep", usleep);
#endif

  // errno
  object errno_obj = create_object(); // TODO check if this has to be rooted
  function get_errno = create_native_function(errno_obj, "get", &errno_getter);
  function set_errno = create_native_function(errno_obj, "set", &errno_setter);
  property_attributes errno_attr(permanent_shared_property,
                                 get_errno, set_errno);
  posix.define_property("errno", errno, errno_attr);
  create_native_function(posix, "strerror", &strerror_);
  create_native_function(posix, "perror", &perror);
}

}

<|endoftext|>


<commit_before>#include "mainScreen.h"

void mainScreen::init(ds_GFX gfx){

}

void mainScreen::input(ds_GFX gfx){
	if (hidkeydown() & KEY_A){
		bg.r = rand() % 255;
		bg.g = rand() % 255;
		bg.b = rand() % 255;
	}
}

void mainScreen::update(ds_GFX gfx){

}

void mainScreen::draw(ds_GFX gfx){
	gfx.beginFrame(GFX_TOP);
	gfx.clear(bg);
}
<commit_msg>Update mainScreen.cpp<commit_after>#include "mainScreen.h"

void mainScreen::init(ds_GFX gfx){

}

void mainScreen::input(ds_GFX gfx){
	if (hidKeysDown() & KEY_A){
		bg.r = rand() % 255;
		bg.g = rand() % 255;
		bg.b = rand() % 255;
	}
}

void mainScreen::update(ds_GFX gfx){

}

void mainScreen::draw(ds_GFX gfx){
	gfx.beginFrame(GFX_TOP);
	gfx.clear(bg);
}
<|endoftext|>


<commit_before>//
//  AliEveDataSourceHLTZMQ
//
//  blame: Mikolaj Krzewicki, mikolaj.krzewicki@cern.ch
//
//

#include "AliEveDataSourceHLTZMQ.h"

#include "AliEveConfigManager.h"
#include "AliGRPPreprocessor.h"
#include <TEnv.h>
#include <TInterpreter.h>
#include <iostream>

#include "AliHLTComponent.h"
#include "AliHLTMessage.h"

#ifdef ZMQ
#include "zmq.h"
#endif

using namespace std;

AliEveDataSourceHLTZMQ::AliEveDataSourceHLTZMQ(bool storageManager) :
    AliEveDataSource("HLT"),
    fEventListenerThreadHLT(0),
    fZMQContext(NULL),
    fZMQeventQueue(NULL),
    fHLTPublisherAddress("tcp://localhost:60201")
{
  //ctor
  Init();
}

void AliEveDataSourceHLTZMQ::Init()
{
#ifdef ZMQ
  //get the address of the HLT proxy from the environment
  if (gSystem->Getenv("HLT_ZMQ_proxy"))
    fHLTPublisherAddress=gSystem->Getenv("HLT_ZMQ_proxy");
  //single ZMQ context for inter thread comm. etc.
  if (!fZMQContext) fZMQContext = zmq_ctx_new();
  //single ZMQ socket for gathering the events form various listening threads
  //must be bound before threads can connect
  int rc=0;
  if (! fZMQeventQueue)
  {
    fZMQeventQueue = zmq_socket(fZMQContext, ZMQ_PULL);
    //set default socket options
    int highWaterMarkRecv = 20;
    rc = zmq_setsockopt(fZMQeventQueue, ZMQ_RCVHWM, &highWaterMarkRecv, sizeof(highWaterMarkRecv));
    int highWaterMarkSend = 20;
    rc = zmq_setsockopt(fZMQeventQueue, ZMQ_SNDHWM, &highWaterMarkSend, sizeof(highWaterMarkSend));
    //bind the socket
    zmq_bind(fZMQeventQueue, "inproc://fCurrentEvent");
  }

  AliInfo("Starting HLT subscriber thread.");
  if(fEventListenerThreadHLT)
  {
    fEventListenerThreadHLT->Join();
    fEventListenerThreadHLT->Kill();
    delete fEventListenerThreadHLT;
    AliInfo("HLT listener thread killed and deleted");
  }
  fEventListenerThreadHLT = new TThread("fEventListenerThreadHLT",DispatchEventListenerHLT,(void*)this);
  fEventListenerThreadHLT->Run();
#endif
}

AliEveDataSourceHLTZMQ::~AliEveDataSourceHLTZMQ()
{
#ifdef ZMQ
  if (fZMQeventQueue)
  {
    int lingerValue = 0;
    int rc = zmq_setsockopt(fZMQeventQueue, ZMQ_LINGER, &lingerValue, sizeof(int));
    if (rc<0) printf("error setting linger on fZMQeventQueue\n");
    rc = zmq_close(fZMQeventQueue);
    if (rc<0) printf("error closing socket fZMQeventQueue\n");
  }

  printf("trying to destroy fZMQContext\n");
  if (fZMQContext) zmq_ctx_destroy(fZMQContext);
  printf("destroyed fZMQContext\n");
#endif

  if(fEventListenerThreadHLT)
  {
    fEventListenerThreadHLT->Join();
    fEventListenerThreadHLT->Kill();
    delete fEventListenerThreadHLT;
    cout<<"HLT listener thread killed and deleted"<<endl;
  }

}

void AliEveDataSourceHLTZMQ::PullEventFromHLT()
{
#ifdef ZMQ
  int rc = 0;

  if (!fZMQContext) return;

  //connection to the HLT data
  void* listenerSocket = zmq_socket(fZMQContext, ZMQ_SUB);
  if (!listenerSocket) {return;}
  //set default socket options
  int highWaterMarkRecv = 20;
  rc = zmq_setsockopt(listenerSocket, ZMQ_RCVHWM, &highWaterMarkRecv, sizeof(highWaterMarkRecv));
  int highWaterMarkSend = 20;
  rc = zmq_setsockopt(listenerSocket, ZMQ_SNDHWM, &highWaterMarkSend, sizeof(highWaterMarkSend));
  //connect the socket
  printf("connecting to ZMQ socket: %s\n", fHLTPublisherAddress.Data());
  rc = zmq_connect(listenerSocket, fHLTPublisherAddress.Data());
  rc = zmq_setsockopt (listenerSocket, ZMQ_SUBSCRIBE, NULL, 0);
  if (rc < 0) {return;}

  //internal publisher
  void* internalPublisher = zmq_socket(fZMQContext, ZMQ_PUSH);
  if (!internalPublisher) {return;}
  //set default socket options
  highWaterMarkRecv = 20;
  rc = zmq_setsockopt(internalPublisher, ZMQ_RCVHWM, &highWaterMarkRecv, sizeof(highWaterMarkRecv));
  highWaterMarkSend = 20;
  rc = zmq_setsockopt(internalPublisher, ZMQ_SNDHWM, &highWaterMarkSend, sizeof(highWaterMarkSend));
  //connect the socket
  rc = zmq_connect(internalPublisher, "inproc://fCurrentEvent");
  if (rc < 0) {return;}

  //create a default selection of any data:
  char requestedTopic[kAliHLTComponentDataTypeTopicSize+1]; memset(&requestedTopic, 0, sizeof(requestedTopic));
  memset(&requestedTopic, '*', kAliHLTComponentDataTypeTopicSize);
  strncpy(requestedTopic, "ALIESDV0HLT**************", kAliHLTComponentDataTypeTopicSize);
  char requestedTopic1[kAliHLTComponentDataTypeTopicSize+1]; memset(&requestedTopic1, 0, sizeof(requestedTopic1));
  memset(&requestedTopic1, '*', kAliHLTComponentDataTypeTopicSize);
  strncpy(requestedTopic1, "FLATESD**************", kAliHLTComponentDataTypeTopicSize);

  Bool_t done=false;
  while (!done)
  {
    int more = 0;
    size_t more_size = sizeof more;
    do
    {
      //here the HLT data comes in
      //receive the topic
      char receivedTopic[kAliHLTComponentDataTypeTopicSize+1]; memset(&receivedTopic, 0, sizeof(receivedTopic));
      int receivedTopicSize = zmq_recv(listenerSocket, &receivedTopic, sizeof(receivedTopic), 0);
      if (receivedTopicSize < 0 && errno==ETERM) {AliInfo("listenerSocket received ETERM, exiting main loop"); done=true; break;}

      //reive the message if there is any
      zmq_msg_t message;
      rc = zmq_msg_init(&message);
      rc = zmq_getsockopt(listenerSocket, ZMQ_RCVMORE, &more, &more_size);
      if (more)
      {
        rc = zmq_msg_recv(&message, listenerSocket, 0);
        if (rc < 0 && errno==ETERM) { zmq_msg_close(&message); done=true; break; }
      }
      rc = zmq_getsockopt(listenerSocket, ZMQ_RCVMORE, &more, &more_size);

      //after receiving a single part (topic+message) publish it internally
      //in general this will be an HLT ESD.
      //do check first if this is what we wanted
      bool isSelected = Topicncmp(requestedTopic, receivedTopic, kAliHLTComponentDataTypeTopicSize, receivedTopicSize) ||
                        Topicncmp(requestedTopic1, receivedTopic, kAliHLTComponentDataTypeTopicSize, receivedTopicSize);

      if (!isSelected)
      {
        //clean up the memory and continue
        //printf("  dropped %s\n",receivedTopic);
        zmq_msg_close(&message);
        continue;
      }

      //publish the data for further use
      rc = -1;
      //printf("pushing %s\n", receivedTopic);
      rc = zmq_send(internalPublisher, &receivedTopic, receivedTopicSize, ZMQ_SNDMORE);
      rc = zmq_msg_send(&message, internalPublisher, 0);
      if (rc < 0) zmq_msg_close(&message);

    } while (more != 0);
  }
  //cleanly exit the thread
  int lingerValue = 10;
  rc = zmq_setsockopt(listenerSocket, ZMQ_LINGER, &lingerValue, sizeof(lingerValue));
  if (rc<0) printf("error setting linger on listenerSocket, errno: %i\n", errno);
  rc = zmq_setsockopt(internalPublisher, ZMQ_LINGER, &lingerValue, sizeof(lingerValue));
  if (rc<0) printf("error setting linger on internalPublisher, errno: %i\n",errno);
  rc = zmq_close(listenerSocket);
  if (rc<0) printf("error closing listenerSocket, errno: %i\n", errno);
  rc = zmq_close(internalPublisher);
  if (rc<0) printf("error closing internalPublisher, errno: %i\n", errno);
#endif
}

void AliEveDataSourceHLTZMQ::GotoEvent(Int_t /*event*/)
{
    NextEvent();
    return;
}

void AliEveDataSourceHLTZMQ::NextEvent()
{
    static const TEveException kEH("AliEveDataSourceHLTZMQ::NextEvent ");
  //read event from queue

#ifdef ZMQ
  //init some stuff
  int rc = 0;
  AliESDEvent* esdObject = NULL;
  char topic[kAliHLTComponentDataTypeTopicSize+1]; memset(topic, 0, sizeof(topic));
  memset(topic, 0, kAliHLTComponentDataTypeTopicSize);
  zmq_msg_t message;
  rc = zmq_msg_init(&message);
  int64_t more = 0;
  size_t more_size = sizeof(more);

  //try to receive a new topic+message from the queue
  int topicSize = zmq_recv(fZMQeventQueue, &topic, kAliHLTComponentDataTypeTopicSize, ZMQ_DONTWAIT);
  int errnoTopic = errno;
  rc = zmq_getsockopt(fZMQeventQueue, ZMQ_RCVMORE, &more, &more_size);
  if (more) rc = zmq_msg_recv(&message, fZMQeventQueue, ZMQ_DONTWAIT);

  //decode the message into an AliESDEvent,
  //for HLT messages AliHLTMessage needs to be used
  if( topicSize > 0 /*&& errnoTopic != EAGAIN*/ && zmq_msg_size(&message) > 0 )
  {
    if (Topicncmp(topic, "ALIESDV0HLT", topicSize, 11))
    {
      printf("trying to decode %s with AliHLTMessage\n", topic);
      AliHLTMessage msg(zmq_msg_data(&message), zmq_msg_size(&message));
      TClass* objclass = msg.GetClass();
      esdObject = static_cast<AliESDEvent*>(msg.ReadObject(AliESDEvent::Class()));
    }
    else if (Topicncmp(topic, "ALIESDV0POR", topicSize, 11))
    {
      printf("trying to decode %s with TBufferFile\n", topic);
      TBufferFile *mess = new TBufferFile(TBuffer::kRead,
          zmq_msg_size(&message)+sizeof(UInt_t),
          zmq_msg_data(&message));
      mess->InitMap();
      mess->ReadClass();// get first the class stored in message
      mess->SetBufferOffset(sizeof(UInt_t) + sizeof(kMESS_OBJECT));
      mess->ResetMap();
      esdObject = (AliESDEvent*)(mess->ReadObjectAny(AliESDEvent::Class()));
    }
    else
    {
      printf("Don't know what to do with %s\n",topic);
    }
  }
  if (esdObject)
  {
    printf("setting new event\n");
    esdObject->GetStdContent();
    int runNumber = esdObject->GetRunNumber();

    printf("deleting current data\n");
    //replace the ESD
    fCurrentData.Clear();
    fCurrentData.fESD = esdObject;

    AliEveEventManager::GetMaster()->SetHasEvent(true);
    AliEveEventManager::GetMaster()->AfterNewEventLoaded();
    //if (AliEveEventManager::GetMaster()->GetAutoLoad()) {
    //  AliEveEventManager::GetMaster()->StartAutoLoadTimer();
    //}
  }
  else
  {
    cout<<"No new event is avaliable."<<endl;
    //AliEveEventManager::GetMaster()->NoEventLoaded();
  }
  zmq_msg_close(&message);
#endif
}

<commit_msg>AliEveDataSourceHLTZMQ: init honors fSourceURL properly<commit_after>//
//  AliEveDataSourceHLTZMQ
//
//  blame: Mikolaj Krzewicki, mikolaj.krzewicki@cern.ch
//
//

#include "AliEveDataSourceHLTZMQ.h"

#include "AliEveConfigManager.h"
#include "AliGRPPreprocessor.h"
#include <TEnv.h>
#include <TInterpreter.h>
#include <iostream>

#include "AliHLTComponent.h"
#include "AliHLTMessage.h"

#ifdef ZMQ
#include "zmq.h"
#endif

using namespace std;

AliEveDataSourceHLTZMQ::AliEveDataSourceHLTZMQ(bool storageManager) :
    AliEveDataSource("HLT"),
    fEventListenerThreadHLT(0),
    fZMQContext(NULL),
    fZMQeventQueue(NULL)
{
  //ctor
  fSourceURL="tcp://localhost:60201";
  Init();
}

void AliEveDataSourceHLTZMQ::Init()
{
#ifdef ZMQ
  //get the address of the HLT proxy from the environment if not set
  if (gSystem->Getenv("HLT_ZMQ_proxy") && fSourceURL.IsNull())
    fSourceURL=gSystem->Getenv("HLT_ZMQ_proxy");
  //single ZMQ context for inter thread comm. etc.
  if (!fZMQContext) fZMQContext = zmq_ctx_new();
  //single ZMQ socket for gathering the events form various listening threads
  //must be bound before threads can connect
  int rc=0;
  if (! fZMQeventQueue)
  {
    fZMQeventQueue = zmq_socket(fZMQContext, ZMQ_PULL);
    //set default socket options
    int highWaterMarkRecv = 20;
    rc = zmq_setsockopt(fZMQeventQueue, ZMQ_RCVHWM, &highWaterMarkRecv, sizeof(highWaterMarkRecv));
    int highWaterMarkSend = 20;
    rc = zmq_setsockopt(fZMQeventQueue, ZMQ_SNDHWM, &highWaterMarkSend, sizeof(highWaterMarkSend));
    //bind the socket
    zmq_bind(fZMQeventQueue, "inproc://fCurrentEvent");
  }

  AliInfo("Starting HLT subscriber thread.");
  if(fEventListenerThreadHLT)
  {
    fEventListenerThreadHLT->Join();
    fEventListenerThreadHLT->Kill();
    delete fEventListenerThreadHLT;
    AliInfo("HLT listener thread killed and deleted");
  }
  fEventListenerThreadHLT = new TThread("fEventListenerThreadHLT",DispatchEventListenerHLT,(void*)this);
  fEventListenerThreadHLT->Run();
#endif
}

AliEveDataSourceHLTZMQ::~AliEveDataSourceHLTZMQ()
{
#ifdef ZMQ
  if (fZMQeventQueue)
  {
    int lingerValue = 0;
    int rc = zmq_setsockopt(fZMQeventQueue, ZMQ_LINGER, &lingerValue, sizeof(int));
    if (rc<0) printf("error setting linger on fZMQeventQueue\n");
    rc = zmq_close(fZMQeventQueue);
    if (rc<0) printf("error closing socket fZMQeventQueue\n");
  }

  printf("trying to destroy fZMQContext\n");
  if (fZMQContext) zmq_ctx_destroy(fZMQContext);
  printf("destroyed fZMQContext\n");
#endif

  if(fEventListenerThreadHLT)
  {
    fEventListenerThreadHLT->Join();
    fEventListenerThreadHLT->Kill();
    delete fEventListenerThreadHLT;
    cout<<"HLT listener thread killed and deleted"<<endl;
  }

}

void AliEveDataSourceHLTZMQ::PullEventFromHLT()
{
#ifdef ZMQ
  int rc = 0;

  if (!fZMQContext) return;

  //get the URL - no lock, should be OK, since we start the thread after it is set
  const char* dataURL = fSourceURL.Data();

  //connection to the HLT data
  void* listenerSocket = zmq_socket(fZMQContext, ZMQ_SUB);
  if (!listenerSocket) {return;}
  //set default socket options
  int highWaterMarkRecv = 20;
  rc = zmq_setsockopt(listenerSocket, ZMQ_RCVHWM, &highWaterMarkRecv, sizeof(highWaterMarkRecv));
  int highWaterMarkSend = 20;
  rc = zmq_setsockopt(listenerSocket, ZMQ_SNDHWM, &highWaterMarkSend, sizeof(highWaterMarkSend));
  //connect the socket
  printf("connecting to ZMQ socket: %s\n", dataURL);
  rc = zmq_connect(listenerSocket, dataURL);
  rc = zmq_setsockopt (listenerSocket, ZMQ_SUBSCRIBE, NULL, 0);
  if (rc < 0) {printf("Cannot connect! exiting\n"); return;}

  //internal publisher
  void* internalPublisher = zmq_socket(fZMQContext, ZMQ_PUSH);
  if (!internalPublisher) {return;}
  //set default socket options
  highWaterMarkRecv = 20;
  rc = zmq_setsockopt(internalPublisher, ZMQ_RCVHWM, &highWaterMarkRecv, sizeof(highWaterMarkRecv));
  highWaterMarkSend = 20;
  rc = zmq_setsockopt(internalPublisher, ZMQ_SNDHWM, &highWaterMarkSend, sizeof(highWaterMarkSend));
  //connect the socket
  rc = zmq_connect(internalPublisher, "inproc://fCurrentEvent");
  if (rc < 0) {return;}

  //create a default selection of any data:
  char requestedTopic[kAliHLTComponentDataTypeTopicSize+1]; memset(&requestedTopic, 0, sizeof(requestedTopic));
  memset(&requestedTopic, '*', kAliHLTComponentDataTypeTopicSize);
  strncpy(requestedTopic, "ALIESDV0HLT**************", kAliHLTComponentDataTypeTopicSize);
  char requestedTopic1[kAliHLTComponentDataTypeTopicSize+1]; memset(&requestedTopic1, 0, sizeof(requestedTopic1));
  memset(&requestedTopic1, '*', kAliHLTComponentDataTypeTopicSize);
  strncpy(requestedTopic1, "FLATESD**************", kAliHLTComponentDataTypeTopicSize);

  Bool_t done=false;
  while (!done)
  {
    int more = 0;
    size_t more_size = sizeof more;
    do
    {
      //here the HLT data comes in
      //receive the topic
      char receivedTopic[kAliHLTComponentDataTypeTopicSize+1]; memset(&receivedTopic, 0, sizeof(receivedTopic));
      int receivedTopicSize = zmq_recv(listenerSocket, &receivedTopic, sizeof(receivedTopic), 0);
      if (receivedTopicSize < 0 && errno==ETERM) {AliInfo("listenerSocket received ETERM, exiting main loop"); done=true; break;}

      //reive the message if there is any
      zmq_msg_t message;
      rc = zmq_msg_init(&message);
      rc = zmq_getsockopt(listenerSocket, ZMQ_RCVMORE, &more, &more_size);
      if (more)
      {
        rc = zmq_msg_recv(&message, listenerSocket, 0);
        if (rc < 0 && errno==ETERM) { zmq_msg_close(&message); done=true; break; }
      }
      rc = zmq_getsockopt(listenerSocket, ZMQ_RCVMORE, &more, &more_size);

      //after receiving a single part (topic+message) publish it internally
      //in general this will be an HLT ESD.
      //do check first if this is what we wanted
      bool isSelected = Topicncmp(requestedTopic, receivedTopic, kAliHLTComponentDataTypeTopicSize, receivedTopicSize) ||
                        Topicncmp(requestedTopic1, receivedTopic, kAliHLTComponentDataTypeTopicSize, receivedTopicSize);

      if (!isSelected)
      {
        //clean up the memory and continue
        //printf("  dropped %s\n",receivedTopic);
        zmq_msg_close(&message);
        continue;
      }

      //publish the data for further use
      rc = -1;
      //printf("pushing %s\n", receivedTopic);
      rc = zmq_send(internalPublisher, &receivedTopic, receivedTopicSize, ZMQ_SNDMORE);
      rc = zmq_msg_send(&message, internalPublisher, 0);
      if (rc < 0) zmq_msg_close(&message);

    } while (more != 0);
  }
  //cleanly exit the thread
  int lingerValue = 10;
  rc = zmq_setsockopt(listenerSocket, ZMQ_LINGER, &lingerValue, sizeof(lingerValue));
  if (rc<0) printf("error setting linger on listenerSocket, errno: %i\n", errno);
  rc = zmq_setsockopt(internalPublisher, ZMQ_LINGER, &lingerValue, sizeof(lingerValue));
  if (rc<0) printf("error setting linger on internalPublisher, errno: %i\n",errno);
  rc = zmq_close(listenerSocket);
  if (rc<0) printf("error closing listenerSocket, errno: %i\n", errno);
  rc = zmq_close(internalPublisher);
  if (rc<0) printf("error closing internalPublisher, errno: %i\n", errno);
#endif
}

void AliEveDataSourceHLTZMQ::GotoEvent(Int_t /*event*/)
{
    NextEvent();
    return;
}

void AliEveDataSourceHLTZMQ::NextEvent()
{
    static const TEveException kEH("AliEveDataSourceHLTZMQ::NextEvent ");
  //read event from queue

#ifdef ZMQ
  //init some stuff
  int rc = 0;
  AliESDEvent* esdObject = NULL;
  char topic[kAliHLTComponentDataTypeTopicSize+1]; memset(topic, 0, sizeof(topic));
  memset(topic, 0, kAliHLTComponentDataTypeTopicSize);
  zmq_msg_t message;
  rc = zmq_msg_init(&message);
  int64_t more = 0;
  size_t more_size = sizeof(more);

  //try to receive a new topic+message from the queue
  int topicSize = zmq_recv(fZMQeventQueue, &topic, kAliHLTComponentDataTypeTopicSize, ZMQ_DONTWAIT);
  int errnoTopic = errno;
  rc = zmq_getsockopt(fZMQeventQueue, ZMQ_RCVMORE, &more, &more_size);
  if (more) rc = zmq_msg_recv(&message, fZMQeventQueue, ZMQ_DONTWAIT);

  //decode the message into an AliESDEvent,
  //for HLT messages AliHLTMessage needs to be used
  if( topicSize > 0 /*&& errnoTopic != EAGAIN*/ && zmq_msg_size(&message) > 0 )
  {
    if (Topicncmp(topic, "ALIESDV0HLT", topicSize, 11))
    {
      printf("trying to decode %s with AliHLTMessage\n", topic);
      AliHLTMessage msg(zmq_msg_data(&message), zmq_msg_size(&message));
      TClass* objclass = msg.GetClass();
      esdObject = static_cast<AliESDEvent*>(msg.ReadObject(AliESDEvent::Class()));
    }
    else if (Topicncmp(topic, "ALIESDV0POR", topicSize, 11))
    {
      printf("trying to decode %s with TBufferFile\n", topic);
      TBufferFile *mess = new TBufferFile(TBuffer::kRead,
          zmq_msg_size(&message)+sizeof(UInt_t),
          zmq_msg_data(&message));
      mess->InitMap();
      mess->ReadClass();// get first the class stored in message
      mess->SetBufferOffset(sizeof(UInt_t) + sizeof(kMESS_OBJECT));
      mess->ResetMap();
      esdObject = (AliESDEvent*)(mess->ReadObjectAny(AliESDEvent::Class()));
    }
    else
    {
      printf("Don't know what to do with %s\n",topic);
    }
  }
  if (esdObject)
  {
    printf("setting new event\n");
    esdObject->GetStdContent();
    int runNumber = esdObject->GetRunNumber();

    printf("deleting current data\n");
    //replace the ESD
    fCurrentData.Clear();
    fCurrentData.fESD = esdObject;

    AliEveEventManager::GetMaster()->SetHasEvent(true);
    AliEveEventManager::GetMaster()->AfterNewEventLoaded();
    //if (AliEveEventManager::GetMaster()->GetAutoLoad()) {
    //  AliEveEventManager::GetMaster()->StartAutoLoadTimer();
    //}
  }
  else
  {
    cout<<"No new event is avaliable."<<endl;
    //AliEveEventManager::GetMaster()->NoEventLoaded();
  }
  zmq_msg_close(&message);
#endif
}

<|endoftext|>


<commit_before>#include "mainwindow.h"

#include <QtOpenGL>
#include <QDesktopWidget>

#include <osg/ref_ptr>
#include <osgDB/ReadFile>
#include <osgQt/GraphicsWindowQt>

#include "viewer.h"

MainWindow::MainWindow(QWidget * parent, Qt::WindowFlags flags)
:   QMainWindow(parent, flags)
{
    // configure window
    this->setWindowTitle("Troen");

	// set window size
	QDesktopWidget desktopWidget;
	QRect displaySize = desktopWidget.screenGeometry();
	this->resize(displaySize.width() / 3 * 2, displaySize.height() / 3 * 2);

	this->m_viewer = new Viewer();
	this->setCentralWidget(this->m_viewer);

	// FMOD TEST
	/*
	m_fmodManager = cFMODManager::GetInstance();
	m_fmodManager->InitFMOD(MAX_CHANNELS, FMOD_INIT_NORMAL);
	uint soundIndex0 = m_fmodManager->LoadSound("sound.mp3", false, false, 1.0f);
	uint soundIndex1 = m_fmodManager->LoadSound("sound.mp3", false, false, 1.0f);
	uint soundIndex2 = m_fmodManager->LoadSound("sound.mp3", false, false, 1.0f);

	std::cout << soundIndex0 << soundIndex1 << soundIndex2 << std::endl;
	m_fmodManager->PlayFMODSound(soundIndex2);
	*/

}

MainWindow::~MainWindow()
{
	delete m_viewerWidget;
}<commit_msg>line ending commit test<commit_after>#include "mainwindow.h"

#include <QtOpenGL>
#include <QDesktopWidget>

#include <osg/ref_ptr>
#include <osgDB/ReadFile>
#include <osgQt/GraphicsWindowQt>

#include "viewer.h"

MainWindow::MainWindow(QWidget * parent, Qt::WindowFlags flags)
:   QMainWindow(parent, flags)
{
    // configure window
    this->setWindowTitle("Troen");

	// set window size
	QDesktopWidget desktopWidget;
	QRect displaySize = desktopWidget.screenGeometry();
	this->resize(displaySize.width() / 3 * 2, displaySize.height() / 3 * 2);

	this->m_viewer = new Viewer();
	this->setCentralWidget(this->m_viewer);

	// FMOD TEST
	/*
	m_fmodManager = cFMODManager::GetInstance();
	m_fmodManager->InitFMOD(MAX_CHANNELS, FMOD_INIT_NORMAL);
	uint soundIndex0 = m_fmodManager->LoadSound("sound.mp3", false, false, 1.0f);
	uint soundIndex1 = m_fmodManager->LoadSound("sound.mp3", false, false, 1.0f);
	uint soundIndex2 = m_fmodManager->LoadSound("sound.mp3", false, false, 1.0f);

	std::cout << soundIndex0 << soundIndex1 << soundIndex2 << std::endl;
	m_fmodManager->PlayFMODSound(soundIndex2);
	*/

}

MainWindow::~MainWindow()
{
	delete m_viewer;
}<|endoftext|>


<commit_before>///
/// @file   PreSieve.cpp
/// @brief  Pre-sieve multiples of small primes to speed up the
///         sieve of Eratosthenes.
///
/// Copyright (C) 2016 Kim Walisch, <kim.walisch@gmail.com>
///
/// This file is distributed under the BSD License. See the COPYING
/// file in the top level directory.
///

#include <primesieve/config.hpp>
#include <primesieve/PreSieve.hpp>
#include <primesieve/EratSmall.hpp>

#include <stdint.h>
#include <cstring>

using namespace std;
using namespace primesieve;

namespace {

const uint_t primes[] = { 2, 3, 5, 7, 11, 13, 17, 19, 23 };

const uint_t primeProduct[] = { 2, 6, 30, 210, 2310, 30030, 510510, 9699690, 223092870 };

}

namespace primesieve {

PreSieve::PreSieve(uint64_t start, uint64_t stop) :
  limit_(7),
  preSieve_(NULL)
{
  uint64_t interval = stop - start;

  // only use a large preSieve_ buffer if the sieve
  // interval is sufficiently large
  for (int i = 4; i < 8; i++)
    if (interval / 10 < primeProduct[i])
      limit_ = primes_[i];

  init();
}

PreSieve::~PreSieve()
{
  delete[] preSieve_;
}

/// Cross-off the multiples of small primes <= limit_
/// in the preSieve_ array.
///
void PreSieve::init()
{
  for (int i = 0; primes[i] <= limit_; i++)
    primeProduct_ = primeProduct[i];

  size_ = primeProduct_ / NUMBERS_PER_BYTE;
  preSieve_ = new byte_t[size_];
  memset(preSieve_, 0xff, size_);

  uint_t stop = primeProduct_ * 2;
  EratSmall eratSmall(stop, size_, limit_);

  for (int i = 3; primes[i] <= limit_; i++)
    eratSmall.addSievingPrime(primes[i], primeProduct_);

  // sieve [primeProduct_, primeProduct_ * 2]
  eratSmall.crossOff(preSieve_, &preSieve_[size_]);
}

/// Pre-sieve the multiples of small primes <= limit_
/// by copying preSieve_ to sieve.
///
void PreSieve::doIt(byte_t* sieve,
                    uint_t sieveSize,
                    uint64_t segmentLow) const
{
  // map segmentLow to the preSieve_ array
  uint_t remainder = static_cast<uint_t>(segmentLow % primeProduct_);
  uint_t index = remainder / NUMBERS_PER_BYTE;
  uint_t sizeLeft = size_ - index;

  if (sieveSize <= sizeLeft)
    memcpy(sieve, &preSieve_[index], sieveSize);
  else
  {
    // copy the last remaining bytes at the end of preSieve_
    // to the beginning of the sieve array
    memcpy(sieve, &preSieve_[index], sizeLeft);

    // restart copying at the beginning of preSieve_
    for (index = sizeLeft; index + size_ < sieveSize; index += size_)
      memcpy(&sieve[index], preSieve_, size_);

    memcpy(&sieve[index], preSieve_, sieveSize - index);
  }
}

} // namespace
<commit_msg>Fix typo<commit_after>///
/// @file   PreSieve.cpp
/// @brief  Pre-sieve multiples of small primes to speed up the
///         sieve of Eratosthenes.
///
/// Copyright (C) 2016 Kim Walisch, <kim.walisch@gmail.com>
///
/// This file is distributed under the BSD License. See the COPYING
/// file in the top level directory.
///

#include <primesieve/config.hpp>
#include <primesieve/PreSieve.hpp>
#include <primesieve/EratSmall.hpp>

#include <stdint.h>
#include <cstring>

using namespace std;
using namespace primesieve;

namespace {

const uint_t primes[] = { 2, 3, 5, 7, 11, 13, 17, 19, 23 };

const uint_t primeProduct[] = { 2, 6, 30, 210, 2310, 30030, 510510, 9699690, 223092870 };

}

namespace primesieve {

PreSieve::PreSieve(uint64_t start, uint64_t stop) :
  limit_(7),
  preSieve_(NULL)
{
  uint64_t interval = stop - start;

  // only use a large preSieve_ buffer if the sieve
  // interval is sufficiently large
  for (int i = 4; i < 8; i++)
    if (interval / 10 < primeProduct[i])
      limit_ = primes[i];

  init();
}

PreSieve::~PreSieve()
{
  delete[] preSieve_;
}

/// Cross-off the multiples of small primes <= limit_
/// in the preSieve_ array.
///
void PreSieve::init()
{
  for (int i = 0; primes[i] <= limit_; i++)
    primeProduct_ = primeProduct[i];

  size_ = primeProduct_ / NUMBERS_PER_BYTE;
  preSieve_ = new byte_t[size_];
  memset(preSieve_, 0xff, size_);

  uint_t stop = primeProduct_ * 2;
  EratSmall eratSmall(stop, size_, limit_);

  for (int i = 3; primes[i] <= limit_; i++)
    eratSmall.addSievingPrime(primes[i], primeProduct_);

  // sieve [primeProduct_, primeProduct_ * 2]
  eratSmall.crossOff(preSieve_, &preSieve_[size_]);
}

/// Pre-sieve the multiples of small primes <= limit_
/// by copying preSieve_ to sieve.
///
void PreSieve::doIt(byte_t* sieve,
                    uint_t sieveSize,
                    uint64_t segmentLow) const
{
  // map segmentLow to the preSieve_ array
  uint_t remainder = static_cast<uint_t>(segmentLow % primeProduct_);
  uint_t index = remainder / NUMBERS_PER_BYTE;
  uint_t sizeLeft = size_ - index;

  if (sieveSize <= sizeLeft)
    memcpy(sieve, &preSieve_[index], sieveSize);
  else
  {
    // copy the last remaining bytes at the end of preSieve_
    // to the beginning of the sieve array
    memcpy(sieve, &preSieve_[index], sizeLeft);

    // restart copying at the beginning of preSieve_
    for (index = sizeLeft; index + size_ < sieveSize; index += size_)
      memcpy(&sieve[index], preSieve_, size_);

    memcpy(&sieve[index], preSieve_, sieveSize - index);
  }
}

} // namespace
<|endoftext|>


<commit_before>
#include "atFile.h++"

#ifdef _MSC_VER

   #include <Windows.h>
   #include <Strsafe.h>

   int chdir(const char * path)
   {
      // Change to the directory
      _chdir(path);
   }

   bool createDirectory(char * path)
   {
      // Attempt to create the directory (passing NULL as the second parameter
      // means that we will use the security and permission profile of the
      // calling user)
      if (CreateDirectory(path, NULL))
      {
         // The operation succeeded
         return true;
      }

      // Indicate failure
      return false;
   }

   int listFiles(char * path, char ** results, int count)
   {
      char              searchTarget[MAX_PATH];
      HANDLE            searchHandle;
      WIN32_FIND_DATA   fileData;
      DWORD             errorCode;
      int               fileCount;
      int               lengthBytes;

      // Prepare the path string for use with FindFile functions (first, copy
      // the string to a buffer, then append '\*' to the directory name)
      StringCchCopy(searchTarget, MAX_PATH, path);
      StringCchCat(searchTarget, MAX_PATH, TEXT("\\*"));

      // Find the first file in the directory
      searchHandle = FindFirstFile(searchTarget, &fileData);
      if (searchHandle == INVALID_HANDLE_VALUE)
      {
         // errorCode = GetLastError();
         return -1;
      }

      // Indicate that we haven't yet found any files
      fileCount = 0;

      // Proceed until our condition is met
      do
      {
         // Check whether the name is part of the list
         if (((fileData.dwFileAttributes) & (FILE_ATTRIBUTE_DIRECTORY)) == 0)
         {
            // Allocate a new string of this length
            lengthBytes = strlen(fileData.cFileName) + 1;
            results[fileCount] = (char *)malloc(lengthBytes * sizeof(char));

            // Copy the file name into the string we just allocated
            StringCchCopy(results[fileCount], lengthBytes, fileData.cFileName);

            // Account for the file we just copied
            fileCount++;
         }
      }
      // Confirm that we haven't run out of file names or space to store them
      while ((FindNextFile(searchHandle, &fileData) != 0) &&
             (fileCount < count));

      // FindNextFile will always set an error code if it returns non-zero
      // (check whether this is actually an error or whether we've just reached
      // the end of the list)
      errorCode = GetLastError();
      if (errorCode == ERROR_NO_MORE_FILES)
      {
         // Close the search handle now that we're done with it
         FindClose(searchHandle);

         // Return the number of filenames we successfully placed in the array
         return fileCount;
      }
      else
      {
         // Close the search handle now that we're done with it (this may fail
         // if the handle is invalid, but there's little we can do about that)
         FindClose(searchHandle);

         // This is a real error (so free the memory we've been allocating)
         while (fileCount >= 0)
         {
            // Free the memory
            free(results[fileCount]);

            // Move down to the previous entry
            fileCount--;
         }

         // Indicate failure
         return -1;
      }
   }

#else

   #include <glob.h>
   #include <stdio.h>
   #include <stdlib.h>
   #include <string.h>
   #include <sys/stat.h>
   #include <sys/types.h>

   bool createDirectory(char * path)
   {
      // Attempt to create the directory (0755 is the octal value for the
      // desired permissions: -rwxr-xr-x)
      if (mkdir(path, 0755) == 0)
      {
         // The operation succeeded
         return true;
      }

      // Indicate failure
      return false;
   }

   int listFiles(char * path, char ** results, int count)
   {
      char     targetPath[512];
      glob_t   fileList;
      int      fileCount;
      char *   currentToken;
      char *   nextToken;
      char     pathDelimiter[16];

      // Convert the path into the search pattern
      // TODO: Make sure the path is properly formatted
      sprintf(targetPath, "%s/*", path);

      // Get a list of all the files in the target location
      glob(targetPath, 0, NULL, &fileList);

      // Initialize our result counter
      fileCount = 0;

      // Print the directory separator to the path delimiter string
      sprintf(pathDelimiter, "%c", DIRECTORY_SEPARATOR);

      // Loop until we run out of files or out of space to store them
      while ((fileCount < count) &&
             (fileCount < (int ) fileList.gl_pathc))
      {
         // We want to take only the files themselves, but the vector contains
         // paths as well (tokenizing on the delimiter string will allow us to
         // extract only the filename. Begin by pulling off the first token)
         currentToken = strtok(fileList.gl_pathv[fileCount], pathDelimiter);

         // Keep attempting to fetch the next token so long as there is one
         nextToken = strtok(NULL, pathDelimiter);
         while (nextToken)
         {
            // Move the current token forward
            currentToken = nextToken;

            // Query the next token
            nextToken = strtok(NULL, pathDelimiter);
         }

         // The next token is NULL, which means there are no more delimiters,
         // which means that our current token contains only the file name so
         // allocate a new string of this length.
         results[fileCount] = (char *)
            malloc((strlen(currentToken) + 1) * sizeof(char));

         // Copy the file name into the string we just allocated
         strcpy(results[fileCount], currentToken);

         // Account for the file we just copied
         fileCount++;
      }

      // Free the original path list
      globfree(&fileList);

      // Return the number of files we found
      return fileCount;
   }

#endif

<commit_msg>Put in return value for chdir.<commit_after>
#include "atFile.h++"

#ifdef _MSC_VER

   #include <Windows.h>
   #include <Strsafe.h>

   int chdir(const char * path)
   {
      // Change to the directory
      return _chdir(path);
   }

   bool createDirectory(char * path)
   {
      // Attempt to create the directory (passing NULL as the second parameter
      // means that we will use the security and permission profile of the
      // calling user)
      if (CreateDirectory(path, NULL))
      {
         // The operation succeeded
         return true;
      }

      // Indicate failure
      return false;
   }

   int listFiles(char * path, char ** results, int count)
   {
      char              searchTarget[MAX_PATH];
      HANDLE            searchHandle;
      WIN32_FIND_DATA   fileData;
      DWORD             errorCode;
      int               fileCount;
      int               lengthBytes;

      // Prepare the path string for use with FindFile functions (first, copy
      // the string to a buffer, then append '\*' to the directory name)
      StringCchCopy(searchTarget, MAX_PATH, path);
      StringCchCat(searchTarget, MAX_PATH, TEXT("\\*"));

      // Find the first file in the directory
      searchHandle = FindFirstFile(searchTarget, &fileData);
      if (searchHandle == INVALID_HANDLE_VALUE)
      {
         // errorCode = GetLastError();
         return -1;
      }

      // Indicate that we haven't yet found any files
      fileCount = 0;

      // Proceed until our condition is met
      do
      {
         // Check whether the name is part of the list
         if (((fileData.dwFileAttributes) & (FILE_ATTRIBUTE_DIRECTORY)) == 0)
         {
            // Allocate a new string of this length
            lengthBytes = strlen(fileData.cFileName) + 1;
            results[fileCount] = (char *)malloc(lengthBytes * sizeof(char));

            // Copy the file name into the string we just allocated
            StringCchCopy(results[fileCount], lengthBytes, fileData.cFileName);

            // Account for the file we just copied
            fileCount++;
         }
      }
      // Confirm that we haven't run out of file names or space to store them
      while ((FindNextFile(searchHandle, &fileData) != 0) &&
             (fileCount < count));

      // FindNextFile will always set an error code if it returns non-zero
      // (check whether this is actually an error or whether we've just reached
      // the end of the list)
      errorCode = GetLastError();
      if (errorCode == ERROR_NO_MORE_FILES)
      {
         // Close the search handle now that we're done with it
         FindClose(searchHandle);

         // Return the number of filenames we successfully placed in the array
         return fileCount;
      }
      else
      {
         // Close the search handle now that we're done with it (this may fail
         // if the handle is invalid, but there's little we can do about that)
         FindClose(searchHandle);

         // This is a real error (so free the memory we've been allocating)
         while (fileCount >= 0)
         {
            // Free the memory
            free(results[fileCount]);

            // Move down to the previous entry
            fileCount--;
         }

         // Indicate failure
         return -1;
      }
   }

#else

   #include <glob.h>
   #include <stdio.h>
   #include <stdlib.h>
   #include <string.h>
   #include <sys/stat.h>
   #include <sys/types.h>

   bool createDirectory(char * path)
   {
      // Attempt to create the directory (0755 is the octal value for the
      // desired permissions: -rwxr-xr-x)
      if (mkdir(path, 0755) == 0)
      {
         // The operation succeeded
         return true;
      }

      // Indicate failure
      return false;
   }

   int listFiles(char * path, char ** results, int count)
   {
      char     targetPath[512];
      glob_t   fileList;
      int      fileCount;
      char *   currentToken;
      char *   nextToken;
      char     pathDelimiter[16];

      // Convert the path into the search pattern
      // TODO: Make sure the path is properly formatted
      sprintf(targetPath, "%s/*", path);

      // Get a list of all the files in the target location
      glob(targetPath, 0, NULL, &fileList);

      // Initialize our result counter
      fileCount = 0;

      // Print the directory separator to the path delimiter string
      sprintf(pathDelimiter, "%c", DIRECTORY_SEPARATOR);

      // Loop until we run out of files or out of space to store them
      while ((fileCount < count) &&
             (fileCount < (int ) fileList.gl_pathc))
      {
         // We want to take only the files themselves, but the vector contains
         // paths as well (tokenizing on the delimiter string will allow us to
         // extract only the filename. Begin by pulling off the first token)
         currentToken = strtok(fileList.gl_pathv[fileCount], pathDelimiter);

         // Keep attempting to fetch the next token so long as there is one
         nextToken = strtok(NULL, pathDelimiter);
         while (nextToken)
         {
            // Move the current token forward
            currentToken = nextToken;

            // Query the next token
            nextToken = strtok(NULL, pathDelimiter);
         }

         // The next token is NULL, which means there are no more delimiters,
         // which means that our current token contains only the file name so
         // allocate a new string of this length.
         results[fileCount] = (char *)
            malloc((strlen(currentToken) + 1) * sizeof(char));

         // Copy the file name into the string we just allocated
         strcpy(results[fileCount], currentToken);

         // Account for the file we just copied
         fileCount++;
      }

      // Free the original path list
      globfree(&fileList);

      // Return the number of files we found
      return fileCount;
   }

#endif

<|endoftext|>