<commit_before>#include "pch.h"
#include "AudioRenderer.h"

namespace SaneAudioRenderer
{
    AudioRenderer::AudioRenderer(ISettings* pSettings, IMyClock* pClock, CAMEvent& bufferFilled, HRESULT& result)
        : m_deviceManager(result)
        , m_myClock(pClock)
        , m_flush(TRUE/*manual reset*/)
        , m_dspVolume(*this)
        , m_dspBalance(*this)
        , m_bufferFilled(bufferFilled)
        , m_settings(pSettings)
    {
        if (FAILED(result))
            return;

        try
        {
            if (!m_settings || !m_myClock)
                throw E_UNEXPECTED;

            ThrowIfFailed(m_myClock->QueryInterface(IID_PPV_ARGS(&m_myGraphClock)));

            if (static_cast<HANDLE>(m_flush) == NULL ||
                static_cast<HANDLE>(m_bufferFilled) == NULL)
            {
                throw E_OUTOFMEMORY;
            }
        }
        catch (HRESULT ex)
        {
            result = ex;
        }
    }

    AudioRenderer::~AudioRenderer()
    {
        // Just in case.
        if (m_state != State_Stopped)
            Stop();
    }

    void AudioRenderer::SetClock(IReferenceClock* pClock)
    {
        CAutoLock objectLock(this);

        m_graphClock = pClock;

        if (m_graphClock && m_graphClock != m_myGraphClock)
        {
            if (!m_externalClock)
                ClearDevice();

            m_externalClock = true;
        }
        else
        {
            if (m_externalClock)
                ClearDevice();

            m_externalClock = false;
        }
    }

    bool AudioRenderer::OnExternalClock()
    {
        CAutoLock objectLock(this);

        return m_externalClock;
    }

    bool AudioRenderer::Enqueue(IMediaSample* pSample, AM_SAMPLE2_PROPERTIES& sampleProps)
    {
        DspChunk chunk;

        {
            CAutoLock objectLock(this);
            assert(m_inputFormat);
            assert(m_state != State_Stopped);

            try
            {
                // Clear the device if related settings were changed.
                CheckDeviceSettings();

                // Create the device if needed.
                if (!m_device)
                    CreateDevice();

                // Apply sample corrections (pad, crop, guess timings).
                chunk = m_sampleCorrection.ProcessSample(pSample, sampleProps);

                // Apply clock corrections (graph clock and rate dsp).
                if (m_device && m_state == State_Running)
                    ApplyClockCorrection();

                // Apply dsp chain.
                if (m_device && !m_device->bitstream)
                {
                    auto f = [&](DspBase* pDsp)
                    {
                        pDsp->Process(chunk);
                    };

                    EnumerateProcessors(f);

                    DspChunk::ToFormat(m_device->dspFormat, chunk);
                }
            }
            catch (std::bad_alloc&)
            {
                ClearDevice();
                chunk = DspChunk();
            }
        }

        // Send processed sample to the device.
        return Push(chunk);
    }

    bool AudioRenderer::Finish(bool blockUntilEnd)
    {
        DspChunk chunk;

        {
            CAutoLock objectLock(this);
            assert(m_state != State_Stopped);

            // No device - nothing to block on.
            if (!m_device)
                blockUntilEnd = false;

            try
            {
                // Apply dsp chain.
                if (m_device && !m_device->bitstream)
                {
                    auto f = [&](DspBase* pDsp)
                    {
                        pDsp->Finish(chunk);
                    };

                    EnumerateProcessors(f);

                    DspChunk::ToFormat(m_device->dspFormat, chunk);
                }
            }
            catch (std::bad_alloc&)
            {
                chunk = DspChunk();
                assert(chunk.IsEmpty());
            }
        }

        auto doBlock = [this]
        {
            // Increase system timer resolution.
            TimePeriodHelper timePeriodHelper(1);

            // Unslave the clock because no more samples are going to be pushed.
            m_myClock->UnslaveClockFromAudio();

            for (;;)
            {
                int64_t actual = INT64_MAX;
                int64_t target;

                {
                    CAutoLock objectLock(this);

                    if (!m_device)
                        return true;

                    UINT64 deviceClockFrequency, deviceClockPosition;

                    try
                    {
                        ThrowIfFailed(m_device->audioClock->GetFrequency(&deviceClockFrequency));
                        ThrowIfFailed(m_device->audioClock->GetPosition(&deviceClockPosition, nullptr));
                    }
                    catch (HRESULT)
                    {
                        ClearDevice();
                        return true;
                    }

                    const auto previous = actual;
                    actual = llMulDiv(deviceClockPosition, OneSecond, deviceClockFrequency, 0);
                    target = llMulDiv(m_pushedFrames, OneSecond, m_device->waveFormat->nSamplesPerSec, 0);

                    // Return if the end of stream is reached.
                    if (actual == target)
                        return true;

                    // Stalling protection.
                    if (actual == previous && m_state == State_Running)
                        return true;
                }

                // Sleep until predicted end of stream.
                if (m_flush.Wait(std::max(1, (int32_t)((target - actual) * 1000 / OneSecond))))
                    return false;
            }
        };

        // Send processed sample to the device, and block until the buffer is drained (if requested).
        return Push(chunk) && (!blockUntilEnd || doBlock());
    }

    void AudioRenderer::BeginFlush()
    {
        m_flush.Set();
    }

    void AudioRenderer::EndFlush()
    {
        CAutoLock objectLock(this);
        assert(m_state != State_Running);

        if (m_device)
        {
            m_device->audioClient->Reset();
            m_bufferFilled.Reset();
            m_sampleCorrection.NewBuffer();
            m_pushedFrames = 0;
        }

        m_flush.Reset();
    }

    bool AudioRenderer::CheckFormat(SharedWaveFormat inputFormat)
    {
        assert(inputFormat);

        if (DspFormatFromWaveFormat(*inputFormat) != DspFormat::Unknown)
            return true;

        BOOL exclusive;
        m_settings->GetOuputDevice(nullptr, &exclusive, nullptr);
        BOOL bitstreamingAllowed;
        m_settings->GetAllowBitstreaming(&bitstreamingAllowed);

        if (!exclusive || !bitstreamingAllowed)
            return false;

        CAutoLock objectLock(this);

        return m_deviceManager.BitstreamFormatSupported(inputFormat, m_settings);
    }

    void AudioRenderer::SetFormat(SharedWaveFormat inputFormat)
    {
        CAutoLock objectLock(this);

        m_inputFormat = inputFormat;

        m_sampleCorrection.NewFormat(inputFormat);

        ClearDevice();
    }

    void AudioRenderer::NewSegment(double rate)
    {
        CAutoLock objectLock(this);

        m_startClockOffset = 0;
        m_rate = rate;

        m_sampleCorrection.NewSegment(m_rate);

        assert(m_inputFormat);
        if (m_device)
            InitializeProcessors();
    }

    void AudioRenderer::Play(REFERENCE_TIME startTime)
    {
        CAutoLock objectLock(this);
        assert(m_state != State_Running);
        m_state = State_Running;

        m_startTime = startTime;
        StartDevice();
    }

    void AudioRenderer::Pause()
    {
        CAutoLock objectLock(this);
        m_state = State_Paused;

        if (m_device)
        {
            m_myClock->UnslaveClockFromAudio();
            m_device->audioClient->Stop();
        }
    }

    void AudioRenderer::Stop()
    {
        CAutoLock objectLock(this);
        m_state = State_Stopped;

        ClearDevice();
    }

    SharedWaveFormat AudioRenderer::GetInputFormat()
    {
        CAutoLock objectLock(this);

        return m_inputFormat;
    }

    SharedAudioDevice AudioRenderer::GetAudioDevice()
    {
        CAutoLock objectLock(this);

        return m_device;
    }

    std::vector<std::wstring> AudioRenderer::GetActiveProcessors()
    {
        CAutoLock objectLock(this);

        std::vector<std::wstring> ret;

        if (m_inputFormat && m_device && !m_device->bitstream)
        {
            auto f = [&](DspBase* pDsp)
            {
                if (pDsp->Active())
                    ret.emplace_back(pDsp->Name());
            };

            EnumerateProcessors(f);
        }

        return ret;
    }

    void AudioRenderer::CheckDeviceSettings()
    {
        CAutoLock objectLock(this);

        UINT32 serial = m_settings->GetSerial();

        if (m_device && m_deviceSettingsSerial != serial)
        {
            LPWSTR pDeviceName = nullptr;
            BOOL exclusive;
            UINT32 buffer;
            if (SUCCEEDED(m_settings->GetOuputDevice(&pDeviceName, &exclusive, &buffer)))
            {
                if (m_device->exclusive != !!exclusive ||
                    m_device->bufferDuration != buffer ||
                    (pDeviceName && *pDeviceName && wcscmp(pDeviceName, m_device->friendlyName->c_str())) ||
                    ((!pDeviceName || !*pDeviceName) && !m_device->default))
                {
                    ClearDevice();
                    assert(!m_device);
                }
                else
                {
                    m_deviceSettingsSerial = serial;
                }
                CoTaskMemFree(pDeviceName);
            }
        }
    }

    void AudioRenderer::StartDevice()
    {
        CAutoLock objectLock(this);
        assert(m_state == State_Running);

        if (m_device)
        {
            m_myClock->SlaveClockToAudio(m_device->audioClock, m_startTime + m_startClockOffset);
            m_startClockOffset = 0;
            m_device->audioClient->Start();
            //assert(m_bufferFilled.Check());
        }
    }

    void AudioRenderer::CreateDevice()
    {
        CAutoLock objectLock(this);

        assert(!m_device);
        assert(m_inputFormat);

        m_deviceSettingsSerial = m_settings->GetSerial();
        m_device = m_deviceManager.CreateDevice(m_inputFormat, m_settings);

        if (m_device)
        {
            m_sampleCorrection.NewBuffer();
            m_pushedFrames = 0;

            InitializeProcessors();

            m_startClockOffset = m_sampleCorrection.GetLastSampleEnd();

            if (m_state == State_Running)
                StartDevice();
        }
    }

    void AudioRenderer::ClearDevice()
    {
        CAutoLock objectLock(this);

        if (m_device)
        {
            m_myClock->UnslaveClockFromAudio();
            m_device->audioClient->Stop();
            m_bufferFilled.Reset();
            m_sampleCorrection.NewBuffer();
            m_pushedFrames = 0;
            m_device = nullptr;
            m_deviceManager.ReleaseDevice();
        }
    }

    void AudioRenderer::ApplyClockCorrection()
    {
        CAutoLock objectLock(this);
        assert(m_inputFormat);
        assert(m_device);
        assert(m_state == State_Running);

        // Apply corrections to internal clock.
        {
            REFERENCE_TIME offset = m_sampleCorrection.GetTimingsError() - m_myClock->GetSlavedClockOffset();
            if (std::abs(offset) > 1000)
            {
                m_myClock->OffsetSlavedClock(offset);
                DebugOut("AudioRenderer offset internal clock by", offset, "units");
            }
        }

        // Try to match internal clock with graph clock if they're different.
        // We do it in the roundabout way by dynamically changing audio sampling rate.
        if (m_externalClock && !m_device->bitstream)
        {
            assert(m_dspRate.Active());
            REFERENCE_TIME graphTime, myTime, myStartTime;
            if (SUCCEEDED(m_myClock->GetAudioClockStartTime(&myStartTime)) &&
                SUCCEEDED(m_myClock->GetAudioClockTime(&myTime, nullptr)) &&
                SUCCEEDED(GetGraphTime(graphTime)) &&
                myTime > myStartTime)
            {
                REFERENCE_TIME offset = graphTime - myTime - m_correctedWithRateDsp;
                if (std::abs(offset) > MILLISECONDS_TO_100NS_UNITS(2))
                {
                    m_dspRate.Adjust(offset);
                    m_correctedWithRateDsp += offset;
                    DebugOut("AudioRenderer offset internal clock indirectly by", offset, "units");
                }
            }
        }
    }

    HRESULT AudioRenderer::GetGraphTime(REFERENCE_TIME& time)
    {
        CAutoLock objectLock(this);

        return m_graphClock ?
                   m_graphClock->GetTime(&time) :
                   m_myGraphClock->GetTime(&time);
    }

    void AudioRenderer::InitializeProcessors()
    {
        CAutoLock objectLock(this);
        assert(m_inputFormat);
        assert(m_device);

        m_correctedWithRateDsp = 0;

        if (m_device->bitstream)
            return;

        const auto inRate = m_inputFormat->nSamplesPerSec;
        const auto inChannels = m_inputFormat->nChannels;
        const auto inMask = DspMatrix::GetChannelMask(*m_inputFormat);
        const auto outRate = m_device->waveFormat->nSamplesPerSec;
        const auto outChannels = m_device->waveFormat->nChannels;
        const auto outMask = DspMatrix::GetChannelMask(*m_device->waveFormat);

        m_dspMatrix.Initialize(inChannels, inMask, outChannels, outMask);
        m_dspRate.Initialize(m_externalClock, inRate, outRate, outChannels);
        m_dspTempo.Initialize(m_rate, outRate, outChannels);
        m_dspCrossfeed.Initialize(m_settings, outRate, outChannels, outMask);
        m_dspVolume.Initialize(m_device->exclusive);
        m_dspLimiter.Initialize(m_settings, outRate, m_device->exclusive);
        m_dspDither.Initialize(m_device->dspFormat);
    }

    bool AudioRenderer::Push(DspChunk& chunk)
    {
        if (chunk.IsEmpty())
            return true;

        const uint32_t frameSize = chunk.GetFrameSize();
        const size_t chunkFrames = chunk.GetFrameCount();

        uint32_t sleepDuration = 0;
        bool firstIteration = true;
        for (size_t doneFrames = 0; doneFrames < chunkFrames;)
        {
            // The device buffer is full or almost full at the beginning of the second and subsequent iterations.
            // Sleep until the buffer may have significant amount of free space. Unless interrupted.
            if (!firstIteration && m_flush.Wait(sleepDuration))
                return false;

            firstIteration = false;

            CAutoLock objectLock(this);

            assert(m_state != State_Stopped);

            if (m_device)
            {
                try
                {
                    // Get up-to-date information on the device buffer.
                    UINT32 bufferFrames, bufferPadding;
                    ThrowIfFailed(m_device->audioClient->GetBufferSize(&bufferFrames));
                    ThrowIfFailed(m_device->audioClient->GetCurrentPadding(&bufferPadding));

                    // Find out how many frames we can write this time.
                    const UINT32 doFrames = std::min(bufferFrames - bufferPadding, (UINT32)(chunkFrames - doneFrames));

                    sleepDuration = m_device->bufferDuration / 4;

                    if (doFrames == 0)
                        continue;

                    // Write frames to the device buffer.
                    BYTE* deviceBuffer;
                    ThrowIfFailed(m_device->audioRenderClient->GetBuffer(doFrames, &deviceBuffer));
                    assert(frameSize == (m_device->waveFormat->wBitsPerSample / 8 * m_device->waveFormat->nChannels));
                    memcpy(deviceBuffer, chunk.GetConstData() + doneFrames * frameSize, doFrames * frameSize);
                    ThrowIfFailed(m_device->audioRenderClient->ReleaseBuffer(doFrames, 0));

                    // If the buffer is fully filled, set the corresponding event
                    // and stop delaying Paused->Running transition.
                    if (bufferPadding + doFrames == bufferFrames &&
                        !m_bufferFilled.Check())
                    {
                        m_bufferFilled.Set();
                    }

                    doneFrames += doFrames;
                    m_pushedFrames += doFrames;

                    continue;
                }
                catch (HRESULT)
                {
                    ClearDevice();
                }
            }

            // The code below emulates null audio device.
            assert(!m_device);

            // Don't delay Paused->Running transition, because we don't have a buffer to fill anyway.
            m_bufferFilled.Set();

            sleepDuration = 1;

            // Loop until the graph time passes the current sample end.
            REFERENCE_TIME graphTime;
            if (m_state == State_Running &&
                SUCCEEDED(GetGraphTime(graphTime)) &&
                graphTime > m_startTime + m_sampleCorrection.GetLastSampleEnd())
            {
                break;
            }
        }

        return true;
    }
}
<commit_msg>Alter debug output a bit<commit_after>#include "pch.h"
#include "AudioRenderer.h"

namespace SaneAudioRenderer
{
    AudioRenderer::AudioRenderer(ISettings* pSettings, IMyClock* pClock, CAMEvent& bufferFilled, HRESULT& result)
        : m_deviceManager(result)
        , m_myClock(pClock)
        , m_flush(TRUE/*manual reset*/)
        , m_dspVolume(*this)
        , m_dspBalance(*this)
        , m_bufferFilled(bufferFilled)
        , m_settings(pSettings)
    {
        if (FAILED(result))
            return;

        try
        {
            if (!m_settings || !m_myClock)
                throw E_UNEXPECTED;

            ThrowIfFailed(m_myClock->QueryInterface(IID_PPV_ARGS(&m_myGraphClock)));

            if (static_cast<HANDLE>(m_flush) == NULL ||
                static_cast<HANDLE>(m_bufferFilled) == NULL)
            {
                throw E_OUTOFMEMORY;
            }
        }
        catch (HRESULT ex)
        {
            result = ex;
        }
    }

    AudioRenderer::~AudioRenderer()
    {
        // Just in case.
        if (m_state != State_Stopped)
            Stop();
    }

    void AudioRenderer::SetClock(IReferenceClock* pClock)
    {
        CAutoLock objectLock(this);

        m_graphClock = pClock;

        if (m_graphClock && m_graphClock != m_myGraphClock)
        {
            if (!m_externalClock)
                ClearDevice();

            m_externalClock = true;
        }
        else
        {
            if (m_externalClock)
                ClearDevice();

            m_externalClock = false;
        }
    }

    bool AudioRenderer::OnExternalClock()
    {
        CAutoLock objectLock(this);

        return m_externalClock;
    }

    bool AudioRenderer::Enqueue(IMediaSample* pSample, AM_SAMPLE2_PROPERTIES& sampleProps)
    {
        DspChunk chunk;

        {
            CAutoLock objectLock(this);
            assert(m_inputFormat);
            assert(m_state != State_Stopped);

            try
            {
                // Clear the device if related settings were changed.
                CheckDeviceSettings();

                // Create the device if needed.
                if (!m_device)
                    CreateDevice();

                // Apply sample corrections (pad, crop, guess timings).
                chunk = m_sampleCorrection.ProcessSample(pSample, sampleProps);

                // Apply clock corrections (graph clock and rate dsp).
                if (m_device && m_state == State_Running)
                    ApplyClockCorrection();

                // Apply dsp chain.
                if (m_device && !m_device->bitstream)
                {
                    auto f = [&](DspBase* pDsp)
                    {
                        pDsp->Process(chunk);
                    };

                    EnumerateProcessors(f);

                    DspChunk::ToFormat(m_device->dspFormat, chunk);
                }
            }
            catch (std::bad_alloc&)
            {
                ClearDevice();
                chunk = DspChunk();
            }
        }

        // Send processed sample to the device.
        return Push(chunk);
    }

    bool AudioRenderer::Finish(bool blockUntilEnd)
    {
        DspChunk chunk;

        {
            CAutoLock objectLock(this);
            assert(m_state != State_Stopped);

            // No device - nothing to block on.
            if (!m_device)
                blockUntilEnd = false;

            try
            {
                // Apply dsp chain.
                if (m_device && !m_device->bitstream)
                {
                    auto f = [&](DspBase* pDsp)
                    {
                        pDsp->Finish(chunk);
                    };

                    EnumerateProcessors(f);

                    DspChunk::ToFormat(m_device->dspFormat, chunk);
                }
            }
            catch (std::bad_alloc&)
            {
                chunk = DspChunk();
                assert(chunk.IsEmpty());
            }
        }

        auto doBlock = [this]
        {
            // Increase system timer resolution.
            TimePeriodHelper timePeriodHelper(1);

            // Unslave the clock because no more samples are going to be pushed.
            m_myClock->UnslaveClockFromAudio();

            for (;;)
            {
                int64_t actual = INT64_MAX;
                int64_t target;

                {
                    CAutoLock objectLock(this);

                    if (!m_device)
                        return true;

                    UINT64 deviceClockFrequency, deviceClockPosition;

                    try
                    {
                        ThrowIfFailed(m_device->audioClock->GetFrequency(&deviceClockFrequency));
                        ThrowIfFailed(m_device->audioClock->GetPosition(&deviceClockPosition, nullptr));
                    }
                    catch (HRESULT)
                    {
                        ClearDevice();
                        return true;
                    }

                    const auto previous = actual;
                    actual = llMulDiv(deviceClockPosition, OneSecond, deviceClockFrequency, 0);
                    target = llMulDiv(m_pushedFrames, OneSecond, m_device->waveFormat->nSamplesPerSec, 0);

                    // Return if the end of stream is reached.
                    if (actual == target)
                        return true;

                    // Stalling protection.
                    if (actual == previous && m_state == State_Running)
                        return true;
                }

                // Sleep until predicted end of stream.
                if (m_flush.Wait(std::max(1, (int32_t)((target - actual) * 1000 / OneSecond))))
                    return false;
            }
        };

        // Send processed sample to the device, and block until the buffer is drained (if requested).
        return Push(chunk) && (!blockUntilEnd || doBlock());
    }

    void AudioRenderer::BeginFlush()
    {
        m_flush.Set();
    }

    void AudioRenderer::EndFlush()
    {
        CAutoLock objectLock(this);
        assert(m_state != State_Running);

        if (m_device)
        {
            m_device->audioClient->Reset();
            m_bufferFilled.Reset();
            m_sampleCorrection.NewBuffer();
            m_pushedFrames = 0;
        }

        m_flush.Reset();
    }

    bool AudioRenderer::CheckFormat(SharedWaveFormat inputFormat)
    {
        assert(inputFormat);

        if (DspFormatFromWaveFormat(*inputFormat) != DspFormat::Unknown)
            return true;

        BOOL exclusive;
        m_settings->GetOuputDevice(nullptr, &exclusive, nullptr);
        BOOL bitstreamingAllowed;
        m_settings->GetAllowBitstreaming(&bitstreamingAllowed);

        if (!exclusive || !bitstreamingAllowed)
            return false;

        CAutoLock objectLock(this);

        return m_deviceManager.BitstreamFormatSupported(inputFormat, m_settings);
    }

    void AudioRenderer::SetFormat(SharedWaveFormat inputFormat)
    {
        CAutoLock objectLock(this);

        m_inputFormat = inputFormat;

        m_sampleCorrection.NewFormat(inputFormat);

        ClearDevice();
    }

    void AudioRenderer::NewSegment(double rate)
    {
        CAutoLock objectLock(this);

        m_startClockOffset = 0;
        m_rate = rate;

        m_sampleCorrection.NewSegment(m_rate);

        assert(m_inputFormat);
        if (m_device)
            InitializeProcessors();
    }

    void AudioRenderer::Play(REFERENCE_TIME startTime)
    {
        CAutoLock objectLock(this);
        assert(m_state != State_Running);
        m_state = State_Running;

        m_startTime = startTime;
        StartDevice();
    }

    void AudioRenderer::Pause()
    {
        CAutoLock objectLock(this);
        m_state = State_Paused;

        if (m_device)
        {
            m_myClock->UnslaveClockFromAudio();
            m_device->audioClient->Stop();
        }
    }

    void AudioRenderer::Stop()
    {
        CAutoLock objectLock(this);
        m_state = State_Stopped;

        ClearDevice();
    }

    SharedWaveFormat AudioRenderer::GetInputFormat()
    {
        CAutoLock objectLock(this);

        return m_inputFormat;
    }

    SharedAudioDevice AudioRenderer::GetAudioDevice()
    {
        CAutoLock objectLock(this);

        return m_device;
    }

    std::vector<std::wstring> AudioRenderer::GetActiveProcessors()
    {
        CAutoLock objectLock(this);

        std::vector<std::wstring> ret;

        if (m_inputFormat && m_device && !m_device->bitstream)
        {
            auto f = [&](DspBase* pDsp)
            {
                if (pDsp->Active())
                    ret.emplace_back(pDsp->Name());
            };

            EnumerateProcessors(f);
        }

        return ret;
    }

    void AudioRenderer::CheckDeviceSettings()
    {
        CAutoLock objectLock(this);

        UINT32 serial = m_settings->GetSerial();

        if (m_device && m_deviceSettingsSerial != serial)
        {
            LPWSTR pDeviceName = nullptr;
            BOOL exclusive;
            UINT32 buffer;
            if (SUCCEEDED(m_settings->GetOuputDevice(&pDeviceName, &exclusive, &buffer)))
            {
                if (m_device->exclusive != !!exclusive ||
                    m_device->bufferDuration != buffer ||
                    (pDeviceName && *pDeviceName && wcscmp(pDeviceName, m_device->friendlyName->c_str())) ||
                    ((!pDeviceName || !*pDeviceName) && !m_device->default))
                {
                    ClearDevice();
                    assert(!m_device);
                }
                else
                {
                    m_deviceSettingsSerial = serial;
                }
                CoTaskMemFree(pDeviceName);
            }
        }
    }

    void AudioRenderer::StartDevice()
    {
        CAutoLock objectLock(this);
        assert(m_state == State_Running);

        if (m_device)
        {
            m_myClock->SlaveClockToAudio(m_device->audioClock, m_startTime + m_startClockOffset);
            m_startClockOffset = 0;
            m_device->audioClient->Start();
            //assert(m_bufferFilled.Check());
        }
    }

    void AudioRenderer::CreateDevice()
    {
        CAutoLock objectLock(this);

        assert(!m_device);
        assert(m_inputFormat);

        m_deviceSettingsSerial = m_settings->GetSerial();
        m_device = m_deviceManager.CreateDevice(m_inputFormat, m_settings);

        if (m_device)
        {
            m_sampleCorrection.NewBuffer();
            m_pushedFrames = 0;

            InitializeProcessors();

            m_startClockOffset = m_sampleCorrection.GetLastSampleEnd();

            if (m_state == State_Running)
                StartDevice();
        }
    }

    void AudioRenderer::ClearDevice()
    {
        CAutoLock objectLock(this);

        if (m_device)
        {
            m_myClock->UnslaveClockFromAudio();
            m_device->audioClient->Stop();
            m_bufferFilled.Reset();
            m_sampleCorrection.NewBuffer();
            m_pushedFrames = 0;
            m_device = nullptr;
            m_deviceManager.ReleaseDevice();
        }
    }

    void AudioRenderer::ApplyClockCorrection()
    {
        CAutoLock objectLock(this);
        assert(m_inputFormat);
        assert(m_device);
        assert(m_state == State_Running);

        // Apply corrections to internal clock.
        {
            REFERENCE_TIME offset = m_sampleCorrection.GetTimingsError() - m_myClock->GetSlavedClockOffset();
            if (std::abs(offset) > 1000)
            {
                m_myClock->OffsetSlavedClock(offset);
                DebugOut("AudioRenderer offset internal clock by", offset / 10000., "ms");
            }
        }

        // Try to match internal clock with graph clock if they're different.
        // We do it in the roundabout way by dynamically changing audio sampling rate.
        if (m_externalClock && !m_device->bitstream)
        {
            assert(m_dspRate.Active());
            REFERENCE_TIME graphTime, myTime, myStartTime;
            if (SUCCEEDED(m_myClock->GetAudioClockStartTime(&myStartTime)) &&
                SUCCEEDED(m_myClock->GetAudioClockTime(&myTime, nullptr)) &&
                SUCCEEDED(GetGraphTime(graphTime)) &&
                myTime > myStartTime)
            {
                REFERENCE_TIME offset = graphTime - myTime - m_correctedWithRateDsp;
                if (std::abs(offset) > MILLISECONDS_TO_100NS_UNITS(2))
                {
                    m_dspRate.Adjust(offset);
                    m_correctedWithRateDsp += offset;
                    DebugOut("AudioRenderer offset internal clock indirectly by", offset / 10000., "ms");
                }
            }
        }
    }

    HRESULT AudioRenderer::GetGraphTime(REFERENCE_TIME& time)
    {
        CAutoLock objectLock(this);

        return m_graphClock ?
                   m_graphClock->GetTime(&time) :
                   m_myGraphClock->GetTime(&time);
    }

    void AudioRenderer::InitializeProcessors()
    {
        CAutoLock objectLock(this);
        assert(m_inputFormat);
        assert(m_device);

        m_correctedWithRateDsp = 0;

        if (m_device->bitstream)
            return;

        const auto inRate = m_inputFormat->nSamplesPerSec;
        const auto inChannels = m_inputFormat->nChannels;
        const auto inMask = DspMatrix::GetChannelMask(*m_inputFormat);
        const auto outRate = m_device->waveFormat->nSamplesPerSec;
        const auto outChannels = m_device->waveFormat->nChannels;
        const auto outMask = DspMatrix::GetChannelMask(*m_device->waveFormat);

        m_dspMatrix.Initialize(inChannels, inMask, outChannels, outMask);
        m_dspRate.Initialize(m_externalClock, inRate, outRate, outChannels);
        m_dspTempo.Initialize(m_rate, outRate, outChannels);
        m_dspCrossfeed.Initialize(m_settings, outRate, outChannels, outMask);
        m_dspVolume.Initialize(m_device->exclusive);
        m_dspLimiter.Initialize(m_settings, outRate, m_device->exclusive);
        m_dspDither.Initialize(m_device->dspFormat);
    }

    bool AudioRenderer::Push(DspChunk& chunk)
    {
        if (chunk.IsEmpty())
            return true;

        const uint32_t frameSize = chunk.GetFrameSize();
        const size_t chunkFrames = chunk.GetFrameCount();

        uint32_t sleepDuration = 0;
        bool firstIteration = true;
        for (size_t doneFrames = 0; doneFrames < chunkFrames;)
        {
            // The device buffer is full or almost full at the beginning of the second and subsequent iterations.
            // Sleep until the buffer may have significant amount of free space. Unless interrupted.
            if (!firstIteration && m_flush.Wait(sleepDuration))
                return false;

            firstIteration = false;

            CAutoLock objectLock(this);

            assert(m_state != State_Stopped);

            if (m_device)
            {
                try
                {
                    // Get up-to-date information on the device buffer.
                    UINT32 bufferFrames, bufferPadding;
                    ThrowIfFailed(m_device->audioClient->GetBufferSize(&bufferFrames));
                    ThrowIfFailed(m_device->audioClient->GetCurrentPadding(&bufferPadding));

                    // Find out how many frames we can write this time.
                    const UINT32 doFrames = std::min(bufferFrames - bufferPadding, (UINT32)(chunkFrames - doneFrames));

                    sleepDuration = m_device->bufferDuration / 4;

                    if (doFrames == 0)
                        continue;

                    // Write frames to the device buffer.
                    BYTE* deviceBuffer;
                    ThrowIfFailed(m_device->audioRenderClient->GetBuffer(doFrames, &deviceBuffer));
                    assert(frameSize == (m_device->waveFormat->wBitsPerSample / 8 * m_device->waveFormat->nChannels));
                    memcpy(deviceBuffer, chunk.GetConstData() + doneFrames * frameSize, doFrames * frameSize);
                    ThrowIfFailed(m_device->audioRenderClient->ReleaseBuffer(doFrames, 0));

                    // If the buffer is fully filled, set the corresponding event
                    // and stop delaying Paused->Running transition.
                    if (bufferPadding + doFrames == bufferFrames &&
                        !m_bufferFilled.Check())
                    {
                        m_bufferFilled.Set();
                    }

                    doneFrames += doFrames;
                    m_pushedFrames += doFrames;

                    continue;
                }
                catch (HRESULT)
                {
                    ClearDevice();
                }
            }

            // The code below emulates null audio device.
            assert(!m_device);

            // Don't delay Paused->Running transition, because we don't have a buffer to fill anyway.
            m_bufferFilled.Set();

            sleepDuration = 1;

            // Loop until the graph time passes the current sample end.
            REFERENCE_TIME graphTime;
            if (m_state == State_Running &&
                SUCCEEDED(GetGraphTime(graphTime)) &&
                graphTime > m_startTime + m_sampleCorrection.GetLastSampleEnd())
            {
                break;
            }
        }

        return true;
    }
}
<|endoftext|>


<commit_before>#include <ir/index_manager/index/Indexer.h>
#include <ir/index_manager/index/CollectionIndexer.h>
#include <ir/index_manager/index/TermReader.h>
#include <ir/index_manager/index/IndexBarrelWriter.h>
#include <ir/index_manager/index/IndexerPropertyConfig.h>

#include <ir/index_manager/utility/StringUtils.h>

#include <ir/index_manager/store/IndexOutput.h>

#include <boost/variant.hpp>
#include <boost/shared_ptr.hpp>

#include <algorithm>
#include <deque>
#include <cstring>

using namespace std;
using namespace izenelib::util;

using namespace izenelib::ir::indexmanager;

CollectionIndexer::CollectionIndexer(collectionid_t id, MemCache* pCache, Indexer* pIndexer)
        :colID_(id)
        ,pMemCache_(pCache)
        ,pIndexer_(pIndexer)
        ,pDocLengthWriter_(NULL)
        ,docLengthWidth_(0)
{
    pFieldsInfo_ = new FieldsInfo();
}

CollectionIndexer::~CollectionIndexer()
{
    delete pFieldsInfo_;
    pMemCache_ = NULL;
    pIndexer_ = NULL;
    if (pDocLengthWriter_)
    {
        delete pDocLengthWriter_;
        pDocLengthWriter_ = NULL;
    }
}

void CollectionIndexer::setSchema(const IndexerCollectionMeta& schema)
{
    pFieldsInfo_->setSchema(schema);
    if (pIndexer_->getIndexManagerConfig()->indexStrategy_.indexDocLength_)
    {
        pDocLengthWriter_ = new DocLengthWriter(schema.getDocumentSchema(), pIndexer_->getDirectory());
        docLengthWidth_ = pDocLengthWriter_->get_num_properties();
    }
}

void CollectionIndexer::setFieldIndexers()
{
    size_t memCacheSize = (size_t)pIndexer_->getIndexManagerConfig()->indexStrategy_.memory_;

    pFieldsInfo_->startIterator();
    FieldInfo* pFieldInfo = NULL;
    size_t indexedProperties = 0;
    while (pFieldsInfo_->hasNext())
    {
        pFieldInfo = pFieldsInfo_->next();
        if (pFieldInfo->isIndexed()&&pFieldInfo->isAnalyzed())
            indexedProperties++;
    }

    memCacheSize = (memCacheSize/indexedProperties) < 10*1024*1024 ?
                              10*1024*1024:(memCacheSize/indexedProperties) ;

    pFieldsInfo_->startIterator();

    while (pFieldsInfo_->hasNext())
    {
        pFieldInfo = pFieldsInfo_->next();
        if (pFieldInfo->isIndexed()&&pFieldInfo->isAnalyzed())
        {
            FieldIndexer* pFieldIndexer = new FieldIndexer(pFieldInfo->getName(), pMemCache_,pIndexer_);
            fieldIndexerMap_.insert(make_pair(pFieldInfo->getName(),pFieldIndexer));
            pFieldIndexer->setHitBuffer(memCacheSize);
        }
    }
}

void CollectionIndexer::addDocument(IndexerDocument& doc)
{
    DocId uniqueID;
    doc.getDocId(uniqueID);

    map<IndexerPropertyConfig, IndexerDocumentPropertyType> propertyValueList;
    doc.getPropertyList(propertyValueList);

    doclen_t docLength[docLengthWidth_];
    memset(docLength, 0, docLengthWidth_*sizeof(doclen_t));

    for (map<IndexerPropertyConfig, IndexerDocumentPropertyType>::iterator iter
                            = propertyValueList.begin(); iter != propertyValueList.end(); ++iter)
    {
        if (!iter->first.isIndex())
            continue;

        if (iter->first.isFilter())
        {
            if(iter->first.isMultiValue())
            {
                MultiValuePropertyType prop;
                if (iter->first.isAnalyzed())
                    prop = boost::get<MultiValueIndexPropertyType >(iter->second).second;
                else
                    prop = boost::get<MultiValuePropertyType>(iter->second);

                for(MultiValuePropertyType::iterator it = prop.begin(); it != prop.end(); ++it)
                    pIndexer_->getBTreeIndexer()->add(uniqueID.colId, iter->first.getPropertyId(), *it, uniqueID.docId);
            }
            else
            {
                PropertyType prop;
                if (iter->first.isAnalyzed())
                    prop = boost::get<IndexPropertyType >(iter->second).second;
                else
                    prop = boost::get<PropertyType>(iter->second);

                pIndexer_->getBTreeIndexer()->add(uniqueID.colId, iter->first.getPropertyId(), prop, uniqueID.docId);
            }
        }

        if (iter->first.isAnalyzed())
        {
            map<string, boost::shared_ptr<FieldIndexer> >::iterator it = fieldIndexerMap_.find(iter->first.getName());
            if (it == fieldIndexerMap_.end())
                // This field is not indexed.
                continue;

            boost::shared_ptr<LAInput> laInput;
            if (iter->first.isFilter())
                if(iter->first.isMultiValue())
                    laInput = boost::get<MultiValueIndexPropertyType >(iter->second).first;
                else
                    laInput = boost::get<IndexPropertyType >(iter->second).first;
            else
                laInput = boost::get<boost::shared_ptr<LAInput> >(iter->second);

            it->second->addField(uniqueID.docId, laInput);

            if (pDocLengthWriter_ && iter->first.isStoreDocLen())
                pDocLengthWriter_->fill(iter->first.getPropertyId(), laInput->size(), docLength);
        }
    }

    if (pDocLengthWriter_)
        pDocLengthWriter_->add(uniqueID.docId, docLength);
}

void CollectionIndexer::updateDocument(IndexerDocument& oldDoc, IndexerDocument& doc)
{
    DocId uniqueID;
    doc.getDocId(uniqueID);

    map<IndexerPropertyConfig, IndexerDocumentPropertyType> propertyValueList;
    doc.getPropertyList(propertyValueList);
    map<IndexerPropertyConfig, IndexerDocumentPropertyType> oldPropertyValueList;
    oldDoc.getPropertyList(oldPropertyValueList);
    for (map<IndexerPropertyConfig, IndexerDocumentPropertyType>::iterator iter
                = propertyValueList.begin(); iter != propertyValueList.end(); ++iter)
    {
        if (iter->first.isFilter())
        {
            map<IndexerPropertyConfig, IndexerDocumentPropertyType>::iterator it;
            if( (it = oldPropertyValueList.find(iter->first)) != oldPropertyValueList.end() )
            {
                PropertyType oldProp;
                oldProp = boost::get<PropertyType>(it->second);
                pIndexer_->getBTreeIndexer()->remove(uniqueID.colId, iter->first.getPropertyId(), oldProp, uniqueID.docId);
            }
            PropertyType prop;
            prop = boost::get<PropertyType>(iter->second);
            pIndexer_->getBTreeIndexer()->add(uniqueID.colId, iter->first.getPropertyId(), prop, uniqueID.docId);
        }
    }
}

void CollectionIndexer::write(OutputDescriptor* desc)
{
    IndexOutput* pVocOutput = desc->getVocOutput();
    IndexOutput* pDOutput = desc->getDPostingOutput();
    IndexOutput* pPOutput = desc->getPPostingOutput();

    fileoffset_t vocOff1,vocOff2,dfiOff1,dfiOff2,ptiOff1,ptiOff2;
    fileoffset_t vocOffset;

    FieldIndexer* pFieldIndexer;
    for (map<string, boost::shared_ptr<FieldIndexer> >::iterator iter =
                    fieldIndexerMap_.begin(); iter != fieldIndexerMap_.end(); ++iter)
    {
        pFieldIndexer = iter->second.get();
        if (pFieldIndexer == NULL)
            continue;
        vocOff1 = pVocOutput->getFilePointer();
        dfiOff1 = pDOutput->getFilePointer();
        ptiOff1 = pPOutput->getFilePointer();
        vocOffset = pFieldIndexer->write(desc);///write field index data

        pFieldsInfo_->setDistinctNumTerms(iter->first,pFieldIndexer->distinctNumTerms());
        pFieldsInfo_->setFieldOffset(iter->first,vocOffset);

        vocOff2 = pVocOutput->getFilePointer();
        dfiOff2 = pDOutput->getFilePointer();
        ptiOff2 = pPOutput->getFilePointer();

        pFieldsInfo_->getField(iter->first)->
                setLength(vocOff2-vocOff1,dfiOff2-dfiOff1,ptiOff2-ptiOff1);
    }
    if (pDocLengthWriter_)
        pDocLengthWriter_->flush();
}

void CollectionIndexer::reset()
{
    for (map<string, boost::shared_ptr<FieldIndexer> >::iterator iter =
                fieldIndexerMap_.begin(); iter != fieldIndexerMap_.end(); ++iter)
    {
        iter->second->reset();
    }
}

FieldIndexer* CollectionIndexer::getFieldIndexer(const char* field)
{
    string fieldstr(field);
    FieldIndexer* fieldIndexer = fieldIndexerMap_[fieldstr].get();
    return fieldIndexer;
}

<commit_msg>add multi value case for r-type indexing<commit_after>#include <ir/index_manager/index/Indexer.h>
#include <ir/index_manager/index/CollectionIndexer.h>
#include <ir/index_manager/index/TermReader.h>
#include <ir/index_manager/index/IndexBarrelWriter.h>
#include <ir/index_manager/index/IndexerPropertyConfig.h>

#include <ir/index_manager/utility/StringUtils.h>

#include <ir/index_manager/store/IndexOutput.h>

#include <boost/variant.hpp>
#include <boost/shared_ptr.hpp>

#include <algorithm>
#include <deque>
#include <cstring>

using namespace std;
using namespace izenelib::util;

using namespace izenelib::ir::indexmanager;

CollectionIndexer::CollectionIndexer(collectionid_t id, MemCache* pCache, Indexer* pIndexer)
        :colID_(id)
        ,pMemCache_(pCache)
        ,pIndexer_(pIndexer)
        ,pDocLengthWriter_(NULL)
        ,docLengthWidth_(0)
{
    pFieldsInfo_ = new FieldsInfo();
}

CollectionIndexer::~CollectionIndexer()
{
    delete pFieldsInfo_;
    pMemCache_ = NULL;
    pIndexer_ = NULL;
    if (pDocLengthWriter_)
    {
        delete pDocLengthWriter_;
        pDocLengthWriter_ = NULL;
    }
}

void CollectionIndexer::setSchema(const IndexerCollectionMeta& schema)
{
    pFieldsInfo_->setSchema(schema);
    if (pIndexer_->getIndexManagerConfig()->indexStrategy_.indexDocLength_)
    {
        pDocLengthWriter_ = new DocLengthWriter(schema.getDocumentSchema(), pIndexer_->getDirectory());
        docLengthWidth_ = pDocLengthWriter_->get_num_properties();
    }
}

void CollectionIndexer::setFieldIndexers()
{
    size_t memCacheSize = (size_t)pIndexer_->getIndexManagerConfig()->indexStrategy_.memory_;

    pFieldsInfo_->startIterator();
    FieldInfo* pFieldInfo = NULL;
    size_t indexedProperties = 0;
    while (pFieldsInfo_->hasNext())
    {
        pFieldInfo = pFieldsInfo_->next();
        if (pFieldInfo->isIndexed()&&pFieldInfo->isAnalyzed())
            indexedProperties++;
    }

    memCacheSize = (memCacheSize/indexedProperties) < 10*1024*1024 ?
                              10*1024*1024:(memCacheSize/indexedProperties) ;

    pFieldsInfo_->startIterator();

    while (pFieldsInfo_->hasNext())
    {
        pFieldInfo = pFieldsInfo_->next();
        if (pFieldInfo->isIndexed()&&pFieldInfo->isAnalyzed())
        {
            FieldIndexer* pFieldIndexer = new FieldIndexer(pFieldInfo->getName(), pMemCache_,pIndexer_);
            fieldIndexerMap_.insert(make_pair(pFieldInfo->getName(),pFieldIndexer));
            pFieldIndexer->setHitBuffer(memCacheSize);
        }
    }
}

void CollectionIndexer::addDocument(IndexerDocument& doc)
{
    DocId uniqueID;
    doc.getDocId(uniqueID);

    map<IndexerPropertyConfig, IndexerDocumentPropertyType> propertyValueList;
    doc.getPropertyList(propertyValueList);

    doclen_t docLength[docLengthWidth_];
    memset(docLength, 0, docLengthWidth_*sizeof(doclen_t));

    for (map<IndexerPropertyConfig, IndexerDocumentPropertyType>::iterator iter
                            = propertyValueList.begin(); iter != propertyValueList.end(); ++iter)
    {
        if (!iter->first.isIndex())
            continue;

        if (iter->first.isFilter())
        {
            if(iter->first.isMultiValue())
            {
                MultiValuePropertyType prop;
                if (iter->first.isAnalyzed())
                    prop = boost::get<MultiValueIndexPropertyType >(iter->second).second;
                else
                    prop = boost::get<MultiValuePropertyType>(iter->second);

                for(MultiValuePropertyType::iterator it = prop.begin(); it != prop.end(); ++it)
                    pIndexer_->getBTreeIndexer()->add(uniqueID.colId, iter->first.getPropertyId(), *it, uniqueID.docId);
            }
            else
            {
                PropertyType prop;
                if (iter->first.isAnalyzed())
                    prop = boost::get<IndexPropertyType >(iter->second).second;
                else
                    prop = boost::get<PropertyType>(iter->second);

                pIndexer_->getBTreeIndexer()->add(uniqueID.colId, iter->first.getPropertyId(), prop, uniqueID.docId);
            }
        }

        if (iter->first.isAnalyzed())
        {
            map<string, boost::shared_ptr<FieldIndexer> >::iterator it = fieldIndexerMap_.find(iter->first.getName());
            if (it == fieldIndexerMap_.end())
                // This field is not indexed.
                continue;

            boost::shared_ptr<LAInput> laInput;
            if (iter->first.isFilter())
                if(iter->first.isMultiValue())
                    laInput = boost::get<MultiValueIndexPropertyType >(iter->second).first;
                else
                    laInput = boost::get<IndexPropertyType >(iter->second).first;
            else
                laInput = boost::get<boost::shared_ptr<LAInput> >(iter->second);

            it->second->addField(uniqueID.docId, laInput);

            if (pDocLengthWriter_ && iter->first.isStoreDocLen())
                pDocLengthWriter_->fill(iter->first.getPropertyId(), laInput->size(), docLength);
        }
    }

    if (pDocLengthWriter_)
        pDocLengthWriter_->add(uniqueID.docId, docLength);
}

void CollectionIndexer::updateDocument(IndexerDocument& oldDoc, IndexerDocument& doc)
{
    DocId uniqueID;
    doc.getDocId(uniqueID);

    map<IndexerPropertyConfig, IndexerDocumentPropertyType> propertyValueList;
    doc.getPropertyList(propertyValueList);
    map<IndexerPropertyConfig, IndexerDocumentPropertyType> oldPropertyValueList;
    oldDoc.getPropertyList(oldPropertyValueList);
    for (map<IndexerPropertyConfig, IndexerDocumentPropertyType>::iterator iter
                = propertyValueList.begin(); iter != propertyValueList.end(); ++iter)
    {
        if (iter->first.isFilter())
        {
            map<IndexerPropertyConfig, IndexerDocumentPropertyType>::iterator it;
            if( (it = oldPropertyValueList.find(iter->first)) != oldPropertyValueList.end() )
            {
                if(it->first.isMultiValue())
                {
                    MultiValuePropertyType oldProp;
                    oldProp = boost::get<MultiValuePropertyType>(it->second);
                    for(MultiValuePropertyType::iterator multiIt = oldProp.begin(); multiIt != oldProp.end(); ++multiIt)
                       pIndexer_->getBTreeIndexer()->remove(uniqueID.colId, iter->first.getPropertyId(), *multiIt, uniqueID.docId);
                }
                else
                {
                    PropertyType oldProp;
                    oldProp = boost::get<PropertyType>(it->second);
                    pIndexer_->getBTreeIndexer()->remove(uniqueID.colId, iter->first.getPropertyId(), oldProp, uniqueID.docId);
                }
            }
            if(iter->first.isMultiValue())
            {
                MultiValuePropertyType prop;
                prop = boost::get<MultiValuePropertyType>(iter->second);
                for(MultiValuePropertyType::iterator multiIt = prop.begin(); multiIt != prop.end(); ++multiIt)
                    pIndexer_->getBTreeIndexer()->add(uniqueID.colId, iter->first.getPropertyId(), *multiIt, uniqueID.docId);
            }
            else
            {
                PropertyType prop;
                prop = boost::get<PropertyType>(iter->second);
                pIndexer_->getBTreeIndexer()->add(uniqueID.colId, iter->first.getPropertyId(), prop, uniqueID.docId);
            }
        }
    }
}

void CollectionIndexer::write(OutputDescriptor* desc)
{
    IndexOutput* pVocOutput = desc->getVocOutput();
    IndexOutput* pDOutput = desc->getDPostingOutput();
    IndexOutput* pPOutput = desc->getPPostingOutput();

    fileoffset_t vocOff1,vocOff2,dfiOff1,dfiOff2,ptiOff1,ptiOff2;
    fileoffset_t vocOffset;

    FieldIndexer* pFieldIndexer;
    for (map<string, boost::shared_ptr<FieldIndexer> >::iterator iter =
                    fieldIndexerMap_.begin(); iter != fieldIndexerMap_.end(); ++iter)
    {
        pFieldIndexer = iter->second.get();
        if (pFieldIndexer == NULL)
            continue;
        vocOff1 = pVocOutput->getFilePointer();
        dfiOff1 = pDOutput->getFilePointer();
        ptiOff1 = pPOutput->getFilePointer();
        vocOffset = pFieldIndexer->write(desc);///write field index data

        pFieldsInfo_->setDistinctNumTerms(iter->first,pFieldIndexer->distinctNumTerms());
        pFieldsInfo_->setFieldOffset(iter->first,vocOffset);

        vocOff2 = pVocOutput->getFilePointer();
        dfiOff2 = pDOutput->getFilePointer();
        ptiOff2 = pPOutput->getFilePointer();

        pFieldsInfo_->getField(iter->first)->
                setLength(vocOff2-vocOff1,dfiOff2-dfiOff1,ptiOff2-ptiOff1);
    }
    if (pDocLengthWriter_)
        pDocLengthWriter_->flush();
}

void CollectionIndexer::reset()
{
    for (map<string, boost::shared_ptr<FieldIndexer> >::iterator iter =
                fieldIndexerMap_.begin(); iter != fieldIndexerMap_.end(); ++iter)
    {
        iter->second->reset();
    }
}

FieldIndexer* CollectionIndexer::getFieldIndexer(const char* field)
{
    string fieldstr(field);
    FieldIndexer* fieldIndexer = fieldIndexerMap_[fieldstr].get();
    return fieldIndexer;
}

<|endoftext|>


<commit_before>//
//  request.hpp
//  ql_fetcher
//
//  Created by Yarrik Shkurat on 21/09/2017.
//  Copyright © 2017 Yarrik Sh. All rights reserved.
//
//  Permission is hereby granted, free of charge, to any person obtaining a copy
//  of this software and associated documentation files (the "Software"), to deal
//  in the Software without restriction, including without limitation the rights
//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
//  copies of the Software, and to permit persons to whom the Software is furnished
//  to do so, subject to the following conditions:
//
//  The above copyright notice and this permission notice shall be included in all
//  copies or substantial portions of the Software.
//
//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
//  WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
//  CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//

#ifndef ql_request_hpp
#define ql_request_hpp

#include "request_params.hpp"
#include "response.hpp"
#include "networking.hpp"
#include <string>
#include <future>
#include <functional>

namespace ql_fetcher
{
    std::string& get_api_key();

    class request
    {
    public:
        request(const std::string& data_code, const std::string& source_code)
        {
            dataset(data_code);
            database(source_code);
            params._api_key = get_api_key();
        }

        request& database(const std::string& source_code)
        {
            params.update(params._database_code != source_code);

            params._database_code = source_code;
            return *this;
        }

        request& dataset(const std::string& data_code)
        {
            params.update(params._dataset_code != data_code);

            params._dataset_code = data_code;
            return *this;
        }

        request& limit(size_t n)
        {
            params.update(params._limit != n);

            params._limit = n;
            return *this;
        }

        request& column(size_t n)
        {
            params.update(params._column_index != n);

            params._column_index = n;
            return *this;
        }

        request& begin(const std::string& yyyy_mm_dd)
        {
            params.update(params._start_date != yyyy_mm_dd);

            params._start_date = yyyy_mm_dd;
            return *this;
        }

        request& end(const std::string& yyyy_mm_dd)
        {
            params.update(params._end_date != yyyy_mm_dd);

            params._end_date = yyyy_mm_dd;
            return *this;
        }

        request& order(sort_order _order)
        {
            params.update(params._order != _order);

            params._order = _order;
            return *this;
        }

        request& timeframe(sampling_freq freq)
        {
            params.update(params._timeframe != freq);

            params._timeframe = freq;
            return *this;
        }

        request& transform(calc_kind calc)
        {
            params.update(params._transform != calc);

            params._transform = calc;
            return *this;
        }

        request& api_key(const std::string& key)
        {
            params.update(params._api_key != key);

            params._api_key = key;
            return *this;
        }

        request& as(result_kind data_kind)
        {
            params.update(params._fetch_as != data_kind);

            params._fetch_as = data_kind;
            return *this;
        }

        const std::string& database() const
        {
            return params._database_code;
        }

        const std::string& dataset() const
        {
            return params._dataset_code;
        }

        size_t limit() const
        {
            return params._limit;
        }

        size_t column() const
        {
            return params._column_index;
        }

        const std::string& begin() const
        {
            return params._start_date;
        }

        const std::string& end() const
        {
            return params._end_date;
        }

        sort_order order() const
        {
            return params._order;
        }

        sampling_freq timeframe() const
        {
            return params._timeframe;
        }

        calc_kind transform() const
        {
            return params._transform;
        }

        const std::string& api_key() const
        {
            return params._api_key;
        }

        std::string url() const
        {
            return form_request(params);
        }

        template <typename CallBack>
        auto with(CallBack cb)
        {
            make();
            return std::async(std::launch::deferred,
            [=] {
                return cb(make().get());
            });
        }

        template <typename CallBack, typename FallBack>
        auto with_or(CallBack cb, FallBack fb)
        {
            return with([=](response& resp)
            {
                if (resp.empty() || resp.fail())
                    return fb(resp);
                else
                    return cb(resp);
            });
        }

        std::future<response&> make()
        {
            if (params.updated())
                cached_resp = response{ fetch_raw(url()) };

            return std::async(std::launch::async,
            [&] () -> response& {
                return cached_resp;
            });
        }

    private:
        request_params params;
        response cached_resp { "" };
    };
}

#endif /* ql_request_hpp */
<commit_msg>Duplicate network request with callback fix<commit_after>//
//  request.hpp
//  ql_fetcher
//
//  Created by Yarrik Shkurat on 21/09/2017.
//  Copyright © 2017 Yarrik Sh. All rights reserved.
//
//  Permission is hereby granted, free of charge, to any person obtaining a copy
//  of this software and associated documentation files (the "Software"), to deal
//  in the Software without restriction, including without limitation the rights
//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
//  copies of the Software, and to permit persons to whom the Software is furnished
//  to do so, subject to the following conditions:
//
//  The above copyright notice and this permission notice shall be included in all
//  copies or substantial portions of the Software.
//
//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
//  WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
//  CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//

#ifndef ql_request_hpp
#define ql_request_hpp

#include "request_params.hpp"
#include "response.hpp"
#include "networking.hpp"
#include <string>
#include <future>
#include <functional>

namespace ql_fetcher
{
    std::string& get_api_key();

    class request
    {
    public:
        request(const std::string& data_code, const std::string& source_code)
        {
            dataset(data_code);
            database(source_code);
            params._api_key = get_api_key();
        }

        request& database(const std::string& source_code)
        {
            params.update(params._database_code != source_code);

            params._database_code = source_code;
            return *this;
        }

        request& dataset(const std::string& data_code)
        {
            params.update(params._dataset_code != data_code);

            params._dataset_code = data_code;
            return *this;
        }

        request& limit(size_t n)
        {
            params.update(params._limit != n);

            params._limit = n;
            return *this;
        }

        request& column(size_t n)
        {
            params.update(params._column_index != n);

            params._column_index = n;
            return *this;
        }

        request& begin(const std::string& yyyy_mm_dd)
        {
            params.update(params._start_date != yyyy_mm_dd);

            params._start_date = yyyy_mm_dd;
            return *this;
        }

        request& end(const std::string& yyyy_mm_dd)
        {
            params.update(params._end_date != yyyy_mm_dd);

            params._end_date = yyyy_mm_dd;
            return *this;
        }

        request& order(sort_order _order)
        {
            params.update(params._order != _order);

            params._order = _order;
            return *this;
        }

        request& timeframe(sampling_freq freq)
        {
            params.update(params._timeframe != freq);

            params._timeframe = freq;
            return *this;
        }

        request& transform(calc_kind calc)
        {
            params.update(params._transform != calc);

            params._transform = calc;
            return *this;
        }

        request& api_key(const std::string& key)
        {
            params.update(params._api_key != key);

            params._api_key = key;
            return *this;
        }

        request& as(result_kind data_kind)
        {
            params.update(params._fetch_as != data_kind);

            params._fetch_as = data_kind;
            return *this;
        }

        const std::string& database() const
        {
            return params._database_code;
        }

        const std::string& dataset() const
        {
            return params._dataset_code;
        }

        size_t limit() const
        {
            return params._limit;
        }

        size_t column() const
        {
            return params._column_index;
        }

        const std::string& begin() const
        {
            return params._start_date;
        }

        const std::string& end() const
        {
            return params._end_date;
        }

        sort_order order() const
        {
            return params._order;
        }

        sampling_freq timeframe() const
        {
            return params._timeframe;
        }

        calc_kind transform() const
        {
            return params._transform;
        }

        const std::string& api_key() const
        {
            return params._api_key;
        }

        std::string url() const
        {
            return form_request(params);
        }

        template <typename CallBack>
        auto with(CallBack cb)
        {
            return std::async(std::launch::deferred,
            [=] {
                return cb(make().get());
            });
        }

        template <typename CallBack, typename FallBack>
        auto with_or(CallBack cb, FallBack fb)
        {
            return with([=](response& resp)
            {
                if (resp.empty() || resp.fail())
                    return fb(resp);
                else
                    return cb(resp);
            });
        }

        std::future<response&> make()
        {
            if (params.updated())
                cached_resp = response{ fetch_raw(url()) };

            return std::async(std::launch::async,
            [&] () -> response& {
                return cached_resp;
            });
        }

    private:
        request_params params;
        response cached_resp { "" };
    };
}

#endif /* ql_request_hpp */
<|endoftext|>


<commit_before>/// @brief provides CORDIC for arccos function
/// @ref see H. Dawid, H. Meyr, "CORDIC Algorithms and Architectures"

#include "./../../fixed_point_lib/src/CORDIC/lut/lut.hpp"

namespace std {
    template<typename T, size_t n, size_t f, class op, class up>
    utils::number<T, n, f, op, up> acos(utils::number<T, n, f, op, up> val)
    {
        typedef utils::number<T, n, f, op, up> fixed_point;
        using utils::cordic::lut;

        // one need integer part bits to represent pi number
        #define pi fixed_point(3.141592653589793)
        assert(("fixed-point format is illegal", n - f >= 2));

        bool const is_negative = (val < fixed_point(0));
        val = std::fabs(val);   // make the value positive
        assert(("argument has to be from interval [0.0, 1.0]", val <= fixed_point(1.0)));

        typedef lut<f, fixed_point> lut_type;
        static lut_type const angles = lut_type::build_arctan_lut();
        static lut_type const scales = lut_type::build_circular_scales_lut();

        // rotation mode: see page 6
        // shift sequence is just 0, 1, ... (circular coordinate system)
        fixed_point x(1.0), y(0.0), z(0.0);
        fixed_point x1, y1;
        BOOST_FOREACH(size_t i, boost::irange<size_t>(0, f, 1))
        {
            int s, sign(0);
            if (y < 0.0) {
                s = -1;
            }
            else {
                s = +1;
            }

            if (val <= x) {
                sign = +s;
            }
            else {
                sign = -s;
            }

            fixed_point const x_scaled = fixed_point::wrap(sign * (x.value() >> i));
            fixed_point const y_scaled = fixed_point::wrap(sign * (y.value() >> i));

            x1 = fixed_point(x - y_scaled);
            y1 = fixed_point(y + x_scaled);

            x = x1; y = y1;
            z = z + ((sign > 0) ? angles[i] : -angles[i]);

            // multiply by square of scale parameter K(n)
            val = fixed_point(val * scales[i]);
        }

        return (is_negative) ? fixed_point(pi - std::fabs(z)) : std::fabs(z);

        #undef pi
    }
}
<commit_msg>arccos.inl: refactoring<commit_after>/// @brief provides CORDIC for arccos function
/// @ref see H. Dawid, H. Meyr, "CORDIC Algorithms and Architectures"

#include "./../../fixed_point_lib/src/CORDIC/lut/lut.hpp"

namespace std {
    template<typename T, size_t n, size_t f, class op, class up>
    utils::number<T, n, f, op, up> acos(utils::number<T, n, f, op, up> val)
    {
        typedef utils::number<T, n, f, op, up> fp;
        typedef utils::cordic::lut<f, fp> lut;

        BOOST_STATIC_ASSERT(n - f >= 2); // to enable pi representative
        assert(("argument has to be from interval [1.0, 1.0]", val <= fp(1.0)));
        if (val > fp(1.0)) {
            throw std::exception("acos: argument must be from [-1.0, 1.0]");
        }
        if (val == fp(1.0)) {
            return fp::wrap(0);
        }
        else if (val == fp(-1.0)) {
            return fp::CONST_PI;
        }
        else if (val == fp::wrap(0)) {
            return fp::CONST_PI_2;
        }

        bool const is_negative = val.value() < 0;
        val = std::fabs(val);

        static lut const angles = lut::build_arctan_lut();
        static lut const scales = lut::build_circular_scales_lut();

        // rotation mode: see page 6
        // shift sequence is just 0, 1, ... (circular coordinate system)
        fp x(1.0), y(0.0), z(0.0);
        BOOST_FOREACH(size_t i, boost::irange<size_t>(0, f, 1))
        {
            int sign(0);
            if (val <= x) {
                sign = (y < 0.0) ? -1 : +1;
            }
            else {
                sign = (y < 0.0) ? +1 : -1;
            }

            fp::value_type const storage(x.value());
            x = x - fp::wrap(sign * (y.value() >> i));
            y = y + fp::wrap(sign * (storage >> i));
            z = (sign > 0)? z + angles[i] : z - angles[i];
            val = val * scales[i]; // multiply by square of K(n)
        }

        return (is_negative) ? fp(fp::CONST_PI - std::fabs(z)) : std::fabs(z);
    }
}
<|endoftext|>


<commit_before>void LoadLibraries();
void AddAnalysisTasks();
class AliAnalysisAlien;
AliAnalysisAlien* CreateAlienHandler(const char *plugin_mode);

Bool_t doQAsym        = 1;   // output ok
Bool_t doVZERO        = 1;   // output ok but there is a 2nd file
Bool_t doVertex       = 1;   // output ok
Bool_t doSPD          = 1;   // output ok, needs RP
Bool_t doFMD          = 1;   // output ok
Bool_t doTPC          = 1;   // output ok
Bool_t doEventStat    = 1;   // output ok
Bool_t doSDD          = 1;   // outout ok needs RP

void PilotAnalysis(const char *plugin_mode = "full")
{
  TGrid::Connect("alien://");
  if (!gGrid || !gGrid->IsConnected()) {
    ::Error("PilotAnalysis", "No grid connection");
    return;
  }
  // Load libraries
  gSystem->SetIncludePath("-I. -I$ROOTSYS/include -I$ALICE_ROOT/include -I$ALICE_ROOT/ITS");
  LoadLibraries();
  // Create manager
  AliAnalysisManager *mgr  = new AliAnalysisManager("PilotAnalysis", "Production train");
  mgr->SetNSysInfo(1);
  // Input handler
  AliESDInputHandlerRP *esdHandler = new AliESDInputHandlerRP();
  esdHandler->SetActiveBranches("ESDfriend");
  mgr->SetInputEventHandler(esdHandler);

  mgr->SetDebugLevel(3);

  // AnalysisTasks
  AddAnalysisTasks();
  // Grid handler
  AliAnalysisGrid *alienHandler = CreateAlienHandler(plugin_mode);
  mgr->SetGridHandler(alienHandler);
  if (mgr->InitAnalysis()) {
    mgr->PrintStatus();
    mgr->StartAnalysis("grid");
  }
}

void LoadLibraries()
{
  gSystem->Load("libANALYSIS");
  gSystem->Load("libANALYSISalice");
  gSystem->Load("libTENDER");
  gSystem->Load("libPWG0base.so");
  gSystem->Load("libPWG0dep.so");
  gSystem->Load("libPWG0selectors.so");
  gSystem->Load("libPWG1.so");
  gSystem->Load("libPWG2.so");
  gSystem->Load("libPWG2forward.so");

  if (doSPD) {
    TFile::Cp(gSystem->ExpandPathName("$ALICE_ROOT/PWG1/PilotTrain/AliAnalysisTaskSPD.cxx"), "AliAnalysisTaskSPD.cxx");
    TFile::Cp(gSystem->ExpandPathName("$ALICE_ROOT/PWG1/PilotTrain/AliAnalysisTaskSPD.h"), "AliAnalysisTaskSPD.h");
    gROOT->LoadMacro("AliAnalysisTaskSPD.cxx++g");
  }
  if (doSDD) {
    TFile::Cp(gSystem->ExpandPathName("$ALICE_ROOT/PWG1/PilotTrain/AliAnalysisTaskSDDRP.cxx"), "AliAnalysisTaskSDDRP.cxx");
    TFile::Cp(gSystem->ExpandPathName("$ALICE_ROOT/PWG1/PilotTrain/AliAnalysisTaskSDDRP.h"), "AliAnalysisTaskSDDRP.h");
    gROOT->LoadMacro("AliAnalysisTaskSDDRP.cxx++g");
  }
}

void AddAnalysisTasks()
{
  // Vertexing (A. Dainese)
  //
  if (doVertex) {
    gROOT->LoadMacro("$ALICE_ROOT/PWG1/macros/AddTaskVertexESD.C");
    AliAnalysisTaskVertexESD* taskvertexesd =  AddTaskVertexESD();
    taskvertexesd->SelectCollisionCandidates();
  }

  // TPC QA (E. Sicking)
  //
  if (doQAsym) {
    gROOT->LoadMacro("$ALICE_ROOT/PWG1/PilotTrain/AddTaskQAsym.C");
    AliAnalysisTaskSE * taskqasim = AddTaskQAsym();
    taskqasim->SelectCollisionCandidates();
  }
  //
  // VZERO QA  (C. Cheshkov)
  //
  if (doVZERO) {
    gROOT->LoadMacro("$ALICE_ROOT/PWG1/PilotTrain/AddTaskVZEROQA.C");
    AliAnalysisTaskSE * taskv0qa = AddTaskVZEROQA(0);
//  taskv0qa->SelectCollisionCandidates();
  }
  // FMD (Hans Hjersing Dalsgaard)
  //
  if (doFMD) {
    gROOT->LoadMacro("$ALICE_ROOT/PWG1/PilotTrain/AddTaskFMD.C");
    AliAnalysisTaskSE* taskfmd = AddTaskFMD();
    taskfmd->SelectCollisionCandidates();
  }
  //
  // TPC (Jacek Otwinowski)
  //
  if (doTPC) {
    gROOT->LoadMacro("$(ALICE_ROOT)/PWG1/TPC/macros/AddTaskPerformanceTPCQA.C");
    AliPerformanceTask *tpcQA = AddTaskPerformanceTPCQA(kFALSE, kTRUE);
  }
  //
  // SPD (A. Mastroserio)
  //
  if (doSPD) {
    gROOT->LoadMacro("$ALICE_ROOT/PWG1/PilotTrain/AddTaskSPDQA.C");
    AliAnalysisTaskSE* taskspdqa = AddTaskSPDQA();
    taskspdqa->SelectCollisionCandidates();
  }
  //
  // SDD (F. Prino)
  //
  if (doSDD) {
    gROOT->LoadMacro("$ALICE_ROOT/PWG1/PilotTrain/AddSDDPoints.C");
    AliAnalysisTaskSE* tasksdd = AddSDDPoints();
    tasksdd->SelectCollisionCandidates();
  }
  //
  // Event Statistics (Jan Fiete)
  //

  if (doEventStat) {
      gROOT->LoadMacro("$ALICE_ROOT/PWG1/PilotTrain/AddTaskPhysicsSelection.C");
      AliPhysicsSelectionTask* physSelTask = AddTaskPhysicsSelection();
      AliPhysicsSelection* physSel = physSelTask->GetPhysicsSelection();
      physSel->AddBackgroundIdentification(new AliBackgroundSelection());
  }
}

//______________________________________________________________________________
AliAnalysisAlien* CreateAlienHandler(const char *plugin_mode)
{
// Check if user has a valid token, otherwise make one. This has limitations.
// One can always follow the standard procedure of calling alien-token-init then
//   source /tmp/gclient_env_$UID in the current shell.
   if (!AliAnalysisGrid::CreateToken()) return NULL;
   AliAnalysisAlien *plugin = new AliAnalysisAlien();
// Set the run mode (can be "full", "test", "offline", "submit" or "terminate")
   plugin->SetRunMode(plugin_mode);
//   if (useProductionMode) plugin->SetProductionMode();
   plugin->SetJobTag("Pilot analysis train");
   plugin->SetNtestFiles(1);
// Set versions of used packages
//   plugin->SetAPIVersion("V2.4");
   plugin->SetAPIVersion("V1.1x");
   plugin->SetROOTVersion("v5-25-04-3");
   plugin->SetAliROOTVersion("v4-18-14-AN-1");
// Declare input data to be processed.
// Method 1: Create automatically XML collections using alien 'find' command.
// Define production directory LFN
   plugin->SetGridDataDir("/alice/data/2009/LHC09d");
// Set data search pattern
   plugin->SetDataPattern("*ESD.tag.root");
// ...then add run numbers to be considered
//   if (!iAODanalysis) plugin->SetRunRange(run_range[0], run_range[1]);
   plugin->SetRunPrefix("000");
   plugin->SetOutputSingleFolder("output");
   plugin->SetOutputToRunNo();
   Int_t run_numbers[30] = {104065, 104155, 104157, 104159, 104160, 104315, 104316, 104320, 104321, 104439,
                            104792, 104793, 104799, 104800, 104801, 104802, 104803, 104821, 104824, 104825,
                            104841, 104845, 104849, 104852, 104865, 104867, 104876, 104892, 105143, 105160};
   for (Int_t i=0; i<30; i++) {
      plugin->AddRunNumber(run_numbers[i]);
   }
// Define alien work directory where all files will be copied. Relative to alien $HOME.
   plugin->SetGridWorkingDir("analysisDATA");
// Declare alien output directory. Relative to working directory.
   plugin->SetGridOutputDir("pilotAnalysis2");

//   plugin->EnablePackage("");

// Declare the analysis source files names separated by blancs. To be compiled runtime
// using ACLiC on the worker nodes.
// Declare all libraries (other than the default ones for the framework. These will be
// loaded by the generated analysis macro. Add all extra files (task .cxx/.h) here.
   plugin->AddIncludePath("-I$ROOTSYS/include -I$ALICE_ROOT/include  -I$ALICE_ROOT/ITS");
   plugin->SetAnalysisSource("AliAnalysisTaskSPD.cxx AliAnalysisTaskSDDRP.cxx");
   plugin->SetAdditionalLibs("libTENDER.so libPWG0base.so libPWG0dep.so libPWG0selectors.so libPWG1.so libPWG2.so libPWG2forward.so AliAnalysisTaskSPD.h AliAnalysisTaskSPD.cxx AliAnalysisTaskSDDRP.h AliAnalysisTaskSDDRP.cxx");

// Declare the output file names separated by blancs.
// (can be like: file.root or file.root@ALICE::Niham::File)
   plugin->SetDefaultOutputs();
//   plugin->SetMergeExcludes(mergeExclude);
   plugin->SetMaxMergeFiles(100);
   plugin->SetNrunsPerMaster(1);
// Optionally define the files to be archived.
//   plugin->SetOutputArchive("log_archive.zip:stdout,stderr@ALICE::NIHAM::File root_archive.zip:AliAOD.root,AOD.tag.root@ALICE::NIHAM::File");

   // Put default output files to archive
   TString listhists = "";
   TString listaods  = "";
   AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
   TIter next(mgr->GetOutputs());
   AliAnalysisDataContainer *output;
   while ((output=(AliAnalysisDataContainer*)next())) {
      const char *filename = output->GetFileName();
      if (!(strcmp(filename, "default"))) {
         if (!mgr->GetOutputEventHandler()) continue;
         filename = mgr->GetOutputEventHandler()->GetOutputFileName();
         if (listaods.Length()) listaods += ",";
         listaods += filename;
      } else {
         if (listhists.Contains(filename)) continue;
         if (listhists.Length()) listhists += ",";
         listhists += filename;
      }
   }
   if (mgr->GetExtraFiles().Length()) {
      if (listaods.Length()) listaods += ",";
      listaods += mgr->GetExtraFiles();
      listaods.ReplaceAll(" ", ",");
   }
   if (listhists.Length()) listhists = Form("hist_archive.zip:%s", listhists.Data());
   if (listaods.Length())  listaods  = Form("aod_archive.zip:%s", listaods.Data());
   if (!listhists.Length()) {
      ::Fatal("AnalysisTrainNew", "No task output !");
   }
   TString outputArchive = "log_archive.zip:stdout,stderr";
   if (listaods.Length()) {
      outputArchive += " ";
      outputArchive += listaods;
   }
   if (listhists.Length()) {
      outputArchive += " ";
      outputArchive += listhists;
   }
// Set friends
//   if (iAODanalysis && iPWG3d2h)
//      plugin->SetFriendChainName("AliAOD.VertexingHF.root");
   plugin->SetOutputArchive(outputArchive);
// Optionally set a name for the generated analysis macro (default MyAnalysis.C)
   plugin->SetAnalysisMacro("pilotAnalysis002.C");
// Optionally set maximum number of input files/subjob (default 100, put 0 to ignore)
   plugin->SetSplitMaxInputFileNumber(1000);
// Optionally set number of failed jobs that will trigger killing waiting sub-jobs.
//   plugin->SetMaxInitFailed(5);
// Optionally resubmit threshold.
//   plugin->SetMasterResubmitThreshold(90);
// Optionally set time to live (default 30000 sec)
   plugin->SetTTL(70000);
// Optionally set input format (default xml-single)
   plugin->SetInputFormat("xml-single");
// Optionally modify the name of the generated JDL (default analysis.jdl)
   plugin->SetJDLName("pilotAnalysis002.jdl");
// Optionally modify the executable name (default analysis.sh)
   plugin->SetExecutable("pilotAnalysis002.sh");
// Optionally modify job price (default 1)
   plugin->SetPrice(1);
// Optionally modify split mode (default 'se')
   plugin->SetSplitMode("se");
   plugin->SetExecutableCommand("aliroot -b -q");
   return plugin;
}
<commit_msg>Updates. (M. Gheata)<commit_after>#include "Riostream.h"
void LoadLibraries();
void AddAnalysisTasks();
class AliAnalysisAlien;
AliAnalysisAlien* CreateAlienHandler(const char *plugin_mode);

Bool_t doQAsym        = 1;   // output ok
Bool_t doVZERO        = 1;   // output ok but there is a 2nd file
Bool_t doVertex       = 1;   // output ok
Bool_t doSPD          = 1;   // output ok, needs RP
Bool_t doFMD          = 1;   // output ok
Bool_t doTPC          = 1;   // output ok
Bool_t doEventStat    = 1;   // output ok
Bool_t doSDD          = 1;   // outout ok needs RP

TString     train_name         = "QA001_PASS4";
TString     job_tag            = "QA001: PWG1 QA train";
TString     root_version       = "v5-26-00b";
TString     aliroot_version    = "v4-19-04-AN";
TString     grid_datadir       = "/alice/data/2009/LHC09d";
TString     data_pattern       = "*ESDs/pass4/*ESDs.root";
//TString     alien_outdir       = "";
TString     alien_outdir       = "/alice/data/2009/LHC09d/analysis/QA001_PASS4";

Bool_t useProductionMode       = kTRUE;

void PilotAnalysis(const char *plugin_mode = "full")
{
  TGrid::Connect("alien://");
  if (!gGrid || !gGrid->IsConnected()) {
    ::Error("PilotAnalysis", "No grid connection");
    return;
  }
  // Write configuration
  TString cdir = gSystem->WorkingDirectory();
  gSystem->MakeDirectory(train_name);
  gSystem->ChangeDirectory(train_name);
  ofstream out;
  out.open(Form("%sConfig.C",train_name.Data()), ios::out);
  out << "{" << endl;
  out << "   train_name      = " << "\"" << train_name.Data() << "\";" << endl;
  out << "   root_version    = " << "\"" << root_version.Data() << "\";" << endl;
  out << "   aliroot_version = " << "\"" << aliroot_version.Data() << "\";" << endl;
  out << "   grid_datadir   = " << "\"" << grid_datadir.Data() << "\";" << endl;
  if (!alien_outdir.Length()) alien_outdir = Form("output_%s",train_name.Data());
  out << "   alien_outdir    = " << "\"" << alien_outdir.Data() << "\";" << endl;
  out << "   doQAsim         = " << doQAsym << ";" << endl;
  out << "   doVZERO         = " << doVZERO << ";" << endl;
  out << "   doVertex        = " << doVertex << ";" << endl;
  out << "   doSPD           = " << doSPD << ";" << endl;
  out << "   doSDD           = " << doSDD << ";" << endl;
  out << "   doFMD           = " << doFMD << ";" << endl;
  out << "   doTPC           = " << doTPC << ";" << endl;
  out << "   doEventStat     = " << doEventStat << ";" << endl;
  out << "}" << endl;
  out.close();

  // Load libraries
  gSystem->SetIncludePath("-I. -I$ROOTSYS/include -I$ALICE_ROOT/include -I$ALICE_ROOT/ITS");
  LoadLibraries();
  // Create manager
  AliAnalysisManager *mgr  = new AliAnalysisManager("PilotAnalysis", "Production train");
  mgr->SetNSysInfo(1);
  // Input handler
  AliESDInputHandlerRP *esdHandler = new AliESDInputHandlerRP();
  esdHandler->SetActiveBranches("ESDfriend");
  mgr->SetInputEventHandler(esdHandler);

  mgr->SetDebugLevel(3);
  mgr->SetSaveCanvases(kTRUE);

  // AnalysisTasks
  AddAnalysisTasks();
  // Grid handler
  AliAnalysisAlien *alienHandler = CreateAlienHandler(plugin_mode);
  mgr->SetGridHandler(alienHandler);
  if (mgr->InitAnalysis()) {
    mgr->PrintStatus();
    mgr->StartAnalysis("grid");
    TString alien_workdir = gGrid->GetHomeDirectory();
    alien_workdir += "analysisDATA";
    TString configName = Form("%s/%sConfig.C", alien_workdir.Data(), train_name.Data());
    if (strcmp(plugin_mode, "test")) {
      printf("=== Registering configuration file <%s>===\n", configName.Data());
      if (AliAnalysisAlien::FileExists(configName.Data())) gGrid->Rm(configName.Data());
      TFile::Cp(Form("file:%sConfig.C",train_name.Data()), Form("alien://%s", configName.Data()));
    }
  }
}

void LoadLibraries()
{
  gSystem->Load("libANALYSIS");
  gSystem->Load("libANALYSISalice");
  gSystem->Load("libTENDER");
  gSystem->Load("libPWG0base.so");
  gSystem->Load("libPWG0dep.so");
  gSystem->Load("libPWG0selectors.so");
  gSystem->Load("libPWG1.so");
  gSystem->Load("libPWG2.so");
  gSystem->Load("libPWG2forward.so");

  if (doSPD) {
    TFile::Cp(gSystem->ExpandPathName("$ALICE_ROOT/PWG1/ITS/AliAnalysisTaskSPD.cxx"), "AliAnalysisTaskSPD.cxx");
    TFile::Cp(gSystem->ExpandPathName("$ALICE_ROOT/PWG1/ITS/AliAnalysisTaskSPD.h"), "AliAnalysisTaskSPD.h");
    gROOT->LoadMacro("AliAnalysisTaskSPD.cxx++g");
  }
  if (doSDD) {
    TFile::Cp(gSystem->ExpandPathName("$ALICE_ROOT/PWG1/ITS/AliAnalysisTaskSDDRP.cxx"), "AliAnalysisTaskSDDRP.cxx");
    TFile::Cp(gSystem->ExpandPathName("$ALICE_ROOT/PWG1/ITS/AliAnalysisTaskSDDRP.h"), "AliAnalysisTaskSDDRP.h");
//    gROOT->LoadMacro("AliAnalysisTaskSDDRP.cxx++g");
  }
}

void AddAnalysisTasks()
{
  // Vertexing (A. Dainese)
  //
  if (doVertex) {
    gROOT->LoadMacro("$ALICE_ROOT/PWG1/macros/AddTaskVertexESD.C");
    AliAnalysisTaskVertexESD* taskvertexesd =  AddTaskVertexESD();
    taskvertexesd->SelectCollisionCandidates();
  }

  // TPC QA (E. Sicking)
  //
  if (doQAsym) {
    gROOT->LoadMacro("$ALICE_ROOT/PWG1/PilotTrain/AddTaskQAsym.C");
    AliAnalysisTaskSE * taskqasim = AddTaskQAsym();
    taskqasim->SelectCollisionCandidates();
  }
  //
  // VZERO QA  (C. Cheshkov)
  //
  if (doVZERO) {
    gROOT->LoadMacro("$ALICE_ROOT/PWG1/PilotTrain/AddTaskVZEROQA.C");
    AliAnalysisTaskSE * taskv0qa = AddTaskVZEROQA(0);
//  taskv0qa->SelectCollisionCandidates();
  }
  // FMD (Hans Hjersing Dalsgaard)
  //
  if (doFMD) {
    gROOT->LoadMacro("$ALICE_ROOT/PWG1/PilotTrain/AddTaskFMD.C");
    AliAnalysisTaskSE* taskfmd = AddTaskFMD();
    taskfmd->SelectCollisionCandidates();
  }
  //
  // TPC (Jacek Otwinowski)
  //
  if (doTPC) {
    gROOT->LoadMacro("$(ALICE_ROOT)/PWG1/TPC/macros/AddTaskPerformanceTPCQA.C");
    AliPerformanceTask *tpcQA = AddTaskPerformanceTPCQA(kFALSE, kTRUE);
  }
  //
  // SPD (A. Mastroserio)
  //
  if (doSPD) {
    gROOT->LoadMacro("$ALICE_ROOT/PWG1/PilotTrain/AddTaskSPDQA.C");
    AliAnalysisTaskSE* taskspdqa = AddTaskSPDQA();
    taskspdqa->SelectCollisionCandidates();
  }
  //
  // SDD (F. Prino)
  //
  if (doSDD) {
    gROOT->LoadMacro("$ALICE_ROOT/PWG1/PilotTrain/AddSDDPoints.C");
    AliAnalysisTaskSE* tasksdd = AddSDDPoints();
    tasksdd->SelectCollisionCandidates();
  }
  //
  // Event Statistics (Jan Fiete)
  //

  if (doEventStat) {
      gROOT->LoadMacro("$ALICE_ROOT/PWG1/PilotTrain/AddTaskPhysicsSelection.C");
      AliPhysicsSelectionTask* physSelTask = AddTaskPhysicsSelection();
      AliPhysicsSelection* physSel = physSelTask->GetPhysicsSelection();
      physSel->AddBackgroundIdentification(new AliBackgroundSelection());
      AliAnalysisManager::GetAnalysisManager()->RegisterExtraFile("event_stat.root");
  }
}

//______________________________________________________________________________
AliAnalysisAlien* CreateAlienHandler(const char *plugin_mode)
{
// Check if user has a valid token, otherwise make one. This has limitations.
// One can always follow the standard procedure of calling alien-token-init then
//   source /tmp/gclient_env_$UID in the current shell.
   if (!AliAnalysisGrid::CreateToken()) return NULL;
   AliAnalysisAlien *plugin = new AliAnalysisAlien();
// Set the run mode (can be "full", "test", "offline", "submit" or "terminate")
   plugin->SetRunMode(plugin_mode);
   if (useProductionMode) plugin->SetProductionMode();
   plugin->SetJobTag(job_tag);
   plugin->SetNtestFiles(1);
// Set versions of used packages
   plugin->SetAPIVersion("V1.1x");
   plugin->SetROOTVersion(root_version);
   plugin->SetAliROOTVersion(aliroot_version);
// Declare input data to be processed.
// Method 1: Create automatically XML collections using alien 'find' command.
// Define production directory LFN
   plugin->SetGridDataDir(grid_datadir);
// Set data search pattern
   plugin->SetDataPattern(data_pattern);
// ...then add run numbers to be considered
//   if (!iAODanalysis) plugin->SetRunRange(run_range[0], run_range[1]);
   plugin->SetRunPrefix("000");
   plugin->SetOutputSingleFolder("output");
   plugin->SetOutputToRunNo();
   Int_t run_numbers[30] = {104065, 104155, 104157, 104159, 104160, 104315, 104316, 104320, 104321, 104439,
                            104792, 104793, 104799, 104800, 104801, 104802, 104803, 104821, 104824, 104825,
                            104841, 104845, 104849, 104852, 104865, 104867, 104876, 104892, 105143, 105160};
   for (Int_t i=0; i<30; i++) {
      plugin->AddRunNumber(run_numbers[i]);
   }
// Define alien work directory where all files will be copied. Relative to alien $HOME.
   plugin->SetGridWorkingDir("analysisDATA");
// Declare alien output directory. Relative to working directory.
   if (alien_outdir.IsNull()) alien_outdir = Form("output_%s",train_name.Data());
   plugin->SetGridOutputDir(alien_outdir);

//   plugin->EnablePackage("");

// Declare the analysis source files names separated by blancs. To be compiled runtime
// using ACLiC on the worker nodes.
// Declare all libraries (other than the default ones for the framework. These will be
// loaded by the generated analysis macro. Add all extra files (task .cxx/.h) here.
   plugin->AddIncludePath("-I$ROOTSYS/include -I$ALICE_ROOT/include  -I$ALICE_ROOT/ITS");
   plugin->SetAnalysisSource("AliAnalysisTaskSPD.cxx AliAnalysisTaskSDDRP.cxx");
   plugin->SetAdditionalLibs("libTENDER.so libPWG0base.so libPWG0dep.so libPWG0selectors.so libPWG1.so libPWG2.so libPWG2forward.so AliAnalysisTaskSPD.h AliAnalysisTaskSPD.cxx AliAnalysisTaskSDDRP.h AliAnalysisTaskSDDRP.cxx");

// Declare the output file names separated by blancs.
// (can be like: file.root or file.root@ALICE::Niham::File)
   plugin->SetDefaultOutputs();
//   plugin->SetMergeExcludes(mergeExclude);
   plugin->SetMaxMergeFiles(100);
   plugin->SetNrunsPerMaster(1);
// Optionally define the files to be archived.
//   plugin->SetOutputArchive("log_archive.zip:stdout,stderr@ALICE::NIHAM::File root_archive.zip:AliAOD.root,AOD.tag.root@ALICE::NIHAM::File");

   // Put default output files to archive
   TString listhists = "";
   TString listaods  = "";
   AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
   TIter next(mgr->GetOutputs());
   AliAnalysisDataContainer *output;
   while ((output=(AliAnalysisDataContainer*)next())) {
      const char *filename = output->GetFileName();
      if (!(strcmp(filename, "default"))) {
         if (!mgr->GetOutputEventHandler()) continue;
         filename = mgr->GetOutputEventHandler()->GetOutputFileName();
         if (listaods.Length()) listaods += ",";
         listaods += filename;
      } else {
         if (listhists.Contains(filename)) continue;
         if (listhists.Length()) listhists += ",";
         listhists += filename;
      }
   }
   if (mgr->GetExtraFiles().Length()) {
      if (listhists.Length()) listhists += ",";
      listhists += mgr->GetExtraFiles();
      listhists.ReplaceAll(" ", ",");
   }
   if (listhists.Length()) listhists = Form("hist_archive.zip:%s", listhists.Data());
   if (listaods.Length())  listaods  = Form("aod_archive.zip:%s", listaods.Data());
   if (!listhists.Length()) {
      ::Fatal("AnalysisTrainNew", "No task output !");
   }
   TString outputArchive = "log_archive.zip:stdout,stderr@disk=4";
   if (listaods.Length()) {
      outputArchive += " ";
      outputArchive += listaods;
      outputArchive += "@disk=4";
   }
   if (listhists.Length()) {
      outputArchive += " ";
      outputArchive += listhists;
      outputArchive += "@disk=4";
   }
// Set friends
//   if (iAODanalysis && iPWG3d2h)
//      plugin->SetFriendChainName("AliAOD.VertexingHF.root");
   plugin->SetOutputArchive(outputArchive);
// Optionally set a name for the generated analysis macro (default MyAnalysis.C)
   plugin->SetAnalysisMacro(Form("%s.C", train_name.Data()));
// Optionally set maximum number of input files/subjob (default 100, put 0 to ignore)
   plugin->SetSplitMaxInputFileNumber(1000);
// Optionally set number of failed jobs that will trigger killing waiting sub-jobs.
//   plugin->SetMaxInitFailed(5);
// Optionally resubmit threshold.
//   plugin->SetMasterResubmitThreshold(90);
// Optionally set time to live (default 30000 sec)
   plugin->SetTTL(70000);
// Optionally set input format (default xml-single)
   plugin->SetInputFormat("xml-single");
// Optionally modify the name of the generated JDL (default analysis.jdl)
   plugin->SetJDLName(Form("%s.jdl", train_name.Data()));
// Optionally modify the executable name (default analysis.sh)
   plugin->SetExecutable(Form("%s.sh", train_name.Data()));
// Optionally modify job price (default 1)
   plugin->SetPrice(1);
// Optionally modify split mode (default 'se')
   plugin->SetSplitMode("se");
   plugin->SetExecutableCommand("aliroot -b -q");
   return plugin;
}
<|endoftext|>