<commit_before>// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/command_line.h"
#include "base/time.h"
#include "chrome/browser/chromeos/cros/cros_in_process_browser_test.h"
#include "chrome/browser/chromeos/cros/mock_cryptohome_library.h"
#include "chrome/browser/chromeos/cros/mock_library_loader.h"
#include "chrome/browser/chromeos/cros/mock_network_library.h"
#include "chrome/browser/chromeos/cros/mock_power_library.h"
#include "chrome/browser/chromeos/cros/mock_screen_lock_library.h"
#include "chrome/browser/profiles/profile_manager.h"
#include "chrome/browser/ui/browser.h"
#include "chrome/common/chrome_switches.h"
#include "chrome/test/base/in_process_browser_test.h"
#include "chrome/test/base/ui_test_utils.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"

namespace chromeos {
using ::testing::_;
using ::testing::AtLeast;
using ::testing::Return;

class LoginTestBase : public CrosInProcessBrowserTest {
 public:
  LoginTestBase() : mock_cryptohome_library_(NULL),
                    mock_screen_lock_library_(NULL) {
  }

 protected:
  virtual void SetUpInProcessBrowserTestFixture() {
    cros_mock_->InitStatusAreaMocks();
    cros_mock_->SetStatusAreaMocksExpectations();
    cros_mock_->InitMockCryptohomeLibrary();
    cros_mock_->InitMockScreenLockLibrary();
    mock_cryptohome_library_ = cros_mock_->mock_cryptohome_library();
    mock_screen_lock_library_ = cros_mock_->mock_screen_lock_library();
    EXPECT_CALL(*mock_cryptohome_library_, IsMounted())
        .WillRepeatedly(Return(true));
  }

  MockCryptohomeLibrary* mock_cryptohome_library_;
  MockScreenLockLibrary* mock_screen_lock_library_;

 private:
  DISALLOW_COPY_AND_ASSIGN(LoginTestBase);
};

class LoginUserTest : public LoginTestBase {
 protected:
  virtual void SetUpInProcessBrowserTestFixture() {
    LoginTestBase::SetUpInProcessBrowserTestFixture();
    // TODO(nkostylev): Remove this once Aura build includes ScreenLocker.
#if !defined(USE_AURA)
    EXPECT_CALL(*mock_screen_lock_library_, AddObserver(_))
       .Times(AtLeast(1))
       .WillRepeatedly(Return());
    EXPECT_CALL(*mock_screen_lock_library_, RemoveObserver(_))
       .Times(AtLeast(1))
       .WillRepeatedly(Return());
#endif
  }

  virtual void SetUpCommandLine(CommandLine* command_line) {
    command_line->AppendSwitchASCII(switches::kLoginUser, "TestUser@gmail.com");
    command_line->AppendSwitchASCII(switches::kLoginProfile, "user");
    command_line->AppendSwitch(switches::kNoFirstRun);
  }
};

class LoginProfileTest : public LoginUserTest {
 protected:
  virtual void SetUpCommandLine(CommandLine* command_line) {
    command_line->AppendSwitchASCII(switches::kLoginProfile, "user");
    command_line->AppendSwitch(switches::kNoFirstRun);
  }
};

// After a chrome crash, the session manager will restart chrome with
// the -login-user flag indicating that the user is already logged in.
// This profile should NOT be an OTR profile.
IN_PROC_BROWSER_TEST_F(LoginUserTest, UserPassed) {
  Profile* profile = browser()->profile();
  EXPECT_EQ("user", profile->GetPath().BaseName().value());
  EXPECT_FALSE(profile->IsOffTheRecord());
}

// On initial launch, we should get the OTR default profile.
IN_PROC_BROWSER_TEST_F(LoginProfileTest, UserNotPassed) {
  Profile* profile = browser()->profile();
  EXPECT_EQ("Default", profile->GetPath().BaseName().value());
  EXPECT_TRUE(profile->IsOffTheRecord());
  // Ensure there's extension service for this profile.
  EXPECT_TRUE(profile->GetExtensionService());
}

} // namespace chromeos
<commit_msg>Remove obsolete ifdef in login browsertests.<commit_after>// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/command_line.h"
#include "base/time.h"
#include "chrome/browser/chromeos/cros/cros_in_process_browser_test.h"
#include "chrome/browser/chromeos/cros/mock_cryptohome_library.h"
#include "chrome/browser/chromeos/cros/mock_library_loader.h"
#include "chrome/browser/chromeos/cros/mock_network_library.h"
#include "chrome/browser/chromeos/cros/mock_power_library.h"
#include "chrome/browser/chromeos/cros/mock_screen_lock_library.h"
#include "chrome/browser/profiles/profile_manager.h"
#include "chrome/browser/ui/browser.h"
#include "chrome/common/chrome_switches.h"
#include "chrome/test/base/in_process_browser_test.h"
#include "chrome/test/base/ui_test_utils.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"

namespace chromeos {
using ::testing::_;
using ::testing::AtLeast;
using ::testing::Return;

class LoginTestBase : public CrosInProcessBrowserTest {
 public:
  LoginTestBase() : mock_cryptohome_library_(NULL),
                    mock_screen_lock_library_(NULL) {
  }

 protected:
  virtual void SetUpInProcessBrowserTestFixture() {
    cros_mock_->InitStatusAreaMocks();
    cros_mock_->SetStatusAreaMocksExpectations();
    cros_mock_->InitMockCryptohomeLibrary();
    cros_mock_->InitMockScreenLockLibrary();
    mock_cryptohome_library_ = cros_mock_->mock_cryptohome_library();
    mock_screen_lock_library_ = cros_mock_->mock_screen_lock_library();
    EXPECT_CALL(*mock_cryptohome_library_, IsMounted())
        .WillRepeatedly(Return(true));
  }

  MockCryptohomeLibrary* mock_cryptohome_library_;
  MockScreenLockLibrary* mock_screen_lock_library_;

 private:
  DISALLOW_COPY_AND_ASSIGN(LoginTestBase);
};

class LoginUserTest : public LoginTestBase {
 protected:
  virtual void SetUpInProcessBrowserTestFixture() {
    LoginTestBase::SetUpInProcessBrowserTestFixture();
    EXPECT_CALL(*mock_screen_lock_library_, AddObserver(_))
       .Times(AtLeast(1))
       .WillRepeatedly(Return());
    EXPECT_CALL(*mock_screen_lock_library_, RemoveObserver(_))
       .Times(AtLeast(1))
       .WillRepeatedly(Return());
  }

  virtual void SetUpCommandLine(CommandLine* command_line) {
    command_line->AppendSwitchASCII(switches::kLoginUser, "TestUser@gmail.com");
    command_line->AppendSwitchASCII(switches::kLoginProfile, "user");
    command_line->AppendSwitch(switches::kNoFirstRun);
  }
};

class LoginProfileTest : public LoginUserTest {
 protected:
  virtual void SetUpCommandLine(CommandLine* command_line) {
    command_line->AppendSwitchASCII(switches::kLoginProfile, "user");
    command_line->AppendSwitch(switches::kNoFirstRun);
  }
};

// After a chrome crash, the session manager will restart chrome with
// the -login-user flag indicating that the user is already logged in.
// This profile should NOT be an OTR profile.
IN_PROC_BROWSER_TEST_F(LoginUserTest, UserPassed) {
  Profile* profile = browser()->profile();
  EXPECT_EQ("user", profile->GetPath().BaseName().value());
  EXPECT_FALSE(profile->IsOffTheRecord());
}

// On initial launch, we should get the OTR default profile.
IN_PROC_BROWSER_TEST_F(LoginProfileTest, UserNotPassed) {
  Profile* profile = browser()->profile();
  EXPECT_EQ("Default", profile->GetPath().BaseName().value());
  EXPECT_TRUE(profile->IsOffTheRecord());
  // Ensure there's extension service for this profile.
  EXPECT_TRUE(profile->GetExtensionService());
}

} // namespace chromeos
<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: macro_expander.cxx,v $
 *
 *  $Revision: 1.8 $
 *
 *  last change: $Author: rt $ $Date: 2005-09-08 09:27:49 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#include <rtl/bootstrap.hxx>

#include <cppuhelper/factory.hxx>
#include <cppuhelper/implbase3.hxx>
#include <cppuhelper/compbase3.hxx>
#include <cppuhelper/component_context.hxx>

#include <com/sun/star/lang/XServiceInfo.hpp>
#include <com/sun/star/lang/XInitialization.hpp>
#include <com/sun/star/util/XMacroExpander.hpp>
#include "com/sun/star/uno/RuntimeException.hpp"

#define OUSTR(x) ::rtl::OUString( RTL_CONSTASCII_USTRINGPARAM(x) )
#define SERVICE_NAME_A "com.sun.star.lang.MacroExpander"
#define SERVICE_NAME_B "com.sun.star.lang.BootstrapMacroExpander"
#define IMPL_NAME "com.sun.star.lang.comp.cppuhelper.BootstrapMacroExpander"

using namespace ::rtl;
using namespace ::osl;
using namespace ::com::sun::star;
using namespace ::com::sun::star::uno;

namespace cppu
{
//---- private forward -----------------------------------------------------------------------------
Bootstrap const & get_unorc() SAL_THROW( () );
}

namespace
{
inline OUString s_impl_name() { return OUSTR(IMPL_NAME); }
static Sequence< OUString > const & s_get_service_names()
{
    static Sequence< OUString > const * s_pnames = 0;
    if (! s_pnames)
    {
        MutexGuard guard( Mutex::getGlobalMutex() );
        if (! s_pnames)
        {
            static Sequence< OUString > s_names( 2 );
            s_names[ 0 ] = OUSTR(SERVICE_NAME_A);
            s_names[ 1 ] = OUSTR(SERVICE_NAME_B);
            s_pnames = &s_names;
        }
    }
    return *s_pnames;
}

typedef ::cppu::WeakComponentImplHelper3<
    util::XMacroExpander, lang::XServiceInfo, lang::XInitialization > t_uno_impl;

struct mutex_holder
{
    Mutex m_mutex;
};
class Bootstrap_MacroExpander : public mutex_holder, public t_uno_impl
{
    rtlBootstrapHandle m_bstrap;
    OUString m_rc_path;

protected:
    virtual void SAL_CALL disposing();

public:
    inline Bootstrap_MacroExpander( Reference< XComponentContext > const & ) SAL_THROW( () )
        : t_uno_impl( m_mutex ),
          m_bstrap( 0 )
        {}
    virtual ~Bootstrap_MacroExpander()
        SAL_THROW( () );

    // XMacroExpander impl
    virtual OUString SAL_CALL expandMacros( OUString const & exp )
        throw (lang::IllegalArgumentException);
    //  XInitialization impl
    virtual void SAL_CALL initialize(
        Sequence< Any > const & arguments )
        throw (Exception);
    // XServiceInfo impl
    virtual OUString SAL_CALL getImplementationName()
        throw (RuntimeException);
    virtual sal_Bool SAL_CALL supportsService( OUString const & serviceName )
        throw (RuntimeException);
    virtual Sequence< OUString > SAL_CALL getSupportedServiceNames()
        throw (RuntimeException);
};

//__________________________________________________________________________________________________
void Bootstrap_MacroExpander::disposing()
{
    if (m_bstrap)
    {
        rtl_bootstrap_args_close( m_bstrap );
        m_bstrap = 0;
    }
}
//__________________________________________________________________________________________________
Bootstrap_MacroExpander::~Bootstrap_MacroExpander() SAL_THROW( () )
{
    if (m_bstrap)
    {
        rtl_bootstrap_args_close( m_bstrap );
        m_bstrap = 0;
    }
}

// XServiceInfo impl
//__________________________________________________________________________________________________
OUString Bootstrap_MacroExpander::getImplementationName()
    throw (RuntimeException)
{
    return s_impl_name();
}
//__________________________________________________________________________________________________
sal_Bool Bootstrap_MacroExpander::supportsService( OUString const & serviceName )
    throw (RuntimeException)
{
    Sequence< OUString > const & service_names = s_get_service_names();
    OUString const * p = service_names.getConstArray();
    for ( sal_Int32 nPos = service_names.getLength(); nPos--; )
    {
        if (p[ nPos ].equals( serviceName ))
            return sal_True;
    }
    return sal_False;
}
//__________________________________________________________________________________________________
Sequence< OUString > Bootstrap_MacroExpander::getSupportedServiceNames()
    throw (RuntimeException)
{
    return s_get_service_names();
}
//  XInitialization impl
//__________________________________________________________________________________________________
void SAL_CALL Bootstrap_MacroExpander::initialize(
    Sequence< Any > const & arguments )
    throw (Exception)
{
    if (m_bstrap)
    {
        throw RuntimeException(
            OUSTR("already initialized!"),
            Reference< XInterface >() );
    }
    if (1 != arguments.getLength())
    {
        throw lang::IllegalArgumentException(
            OUSTR("invalid number of args given!  give single file url!"),
            Reference< XInterface >(),
            0 );
    }
    if (! (arguments[ 0 ] >>= m_rc_path))
    {
        throw lang::IllegalArgumentException(
            OUSTR("give file url!"),
            Reference< XInterface >(),
            0 );
    }
}

// XMacroExpander impl
//__________________________________________________________________________________________________
OUString Bootstrap_MacroExpander::expandMacros( OUString const & exp )
    throw (lang::IllegalArgumentException)
{
    // determine bootstrap handle
    rtlBootstrapHandle bstrap;
    if (m_rc_path.getLength())
    {
        // late init
        if (! m_bstrap)
        {
            rtlBootstrapHandle bstrap = rtl_bootstrap_args_open( m_rc_path.pData );
            ClearableMutexGuard guard( Mutex::getGlobalMutex() );
            if (m_bstrap)
            {
                guard.clear();
                rtl_bootstrap_args_close( bstrap );
            }
            else
            {
                m_bstrap = bstrap;
            }
        }
        bstrap = m_bstrap;
    }
    else
    {
        bstrap = ::cppu::get_unorc().getHandle();
    }

    // expand
    OUString ret( exp );
    rtl_bootstrap_expandMacros_from_handle( bstrap, &ret.pData );
    return ret;
}

//==================================================================================================
Reference< XInterface > SAL_CALL service_create(
    Reference< XComponentContext > const & xComponentContext )
    SAL_THROW( (RuntimeException) )
{
    return static_cast< ::cppu::OWeakObject * >( new Bootstrap_MacroExpander( xComponentContext ) );
}

}

namespace cppu
{

//##################################################################################################
Reference< lang::XSingleComponentFactory > create_boostrap_macro_expander_factory() SAL_THROW( () )
{
    return ::cppu::createSingleComponentFactory(
        service_create,
        s_impl_name(),
        s_get_service_names() );
}

}
<commit_msg>INTEGRATION: CWS warnings01 (1.7.42); FILE MERGED 2005/09/22 15:38:58 sb 1.7.42.2: RESYNC: (1.7-1.8); FILE MERGED 2005/09/07 11:05:30 sb 1.7.42.1: #i53898# Made code warning-free.<commit_after>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: macro_expander.cxx,v $
 *
 *  $Revision: 1.9 $
 *
 *  last change: $Author: hr $ $Date: 2006-06-19 10:34:22 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#include <rtl/bootstrap.hxx>

#include <cppuhelper/factory.hxx>
#include <cppuhelper/implbase3.hxx>
#include <cppuhelper/compbase3.hxx>
#include <cppuhelper/component_context.hxx>

#include <com/sun/star/lang/XServiceInfo.hpp>
#include <com/sun/star/lang/XInitialization.hpp>
#include <com/sun/star/util/XMacroExpander.hpp>
#include "com/sun/star/uno/RuntimeException.hpp"

#define OUSTR(x) ::rtl::OUString( RTL_CONSTASCII_USTRINGPARAM(x) )
#define SERVICE_NAME_A "com.sun.star.lang.MacroExpander"
#define SERVICE_NAME_B "com.sun.star.lang.BootstrapMacroExpander"
#define IMPL_NAME "com.sun.star.lang.comp.cppuhelper.BootstrapMacroExpander"

using namespace ::rtl;
using namespace ::osl;
using namespace ::com::sun::star;
using namespace ::com::sun::star::uno;

namespace cppu
{
//---- private forward -----------------------------------------------------------------------------
Bootstrap const & get_unorc() SAL_THROW( () );
}

namespace
{
inline OUString s_impl_name() { return OUSTR(IMPL_NAME); }
static Sequence< OUString > const & s_get_service_names()
{
    static Sequence< OUString > const * s_pnames = 0;
    if (! s_pnames)
    {
        MutexGuard guard( Mutex::getGlobalMutex() );
        if (! s_pnames)
        {
            static Sequence< OUString > s_names( 2 );
            s_names[ 0 ] = OUSTR(SERVICE_NAME_A);
            s_names[ 1 ] = OUSTR(SERVICE_NAME_B);
            s_pnames = &s_names;
        }
    }
    return *s_pnames;
}

typedef ::cppu::WeakComponentImplHelper3<
    util::XMacroExpander, lang::XServiceInfo, lang::XInitialization > t_uno_impl;

struct mutex_holder
{
    Mutex m_mutex;
};
class Bootstrap_MacroExpander : public mutex_holder, public t_uno_impl
{
    rtlBootstrapHandle m_bstrap;
    OUString m_rc_path;

protected:
    virtual void SAL_CALL disposing();

public:
    inline Bootstrap_MacroExpander( Reference< XComponentContext > const & ) SAL_THROW( () )
        : t_uno_impl( m_mutex ),
          m_bstrap( 0 )
        {}
    virtual ~Bootstrap_MacroExpander()
        SAL_THROW( () );

    // XMacroExpander impl
    virtual OUString SAL_CALL expandMacros( OUString const & exp )
        throw (lang::IllegalArgumentException);
    //  XInitialization impl
    virtual void SAL_CALL initialize(
        Sequence< Any > const & arguments )
        throw (Exception);
    // XServiceInfo impl
    virtual OUString SAL_CALL getImplementationName()
        throw (RuntimeException);
    virtual sal_Bool SAL_CALL supportsService( OUString const & serviceName )
        throw (RuntimeException);
    virtual Sequence< OUString > SAL_CALL getSupportedServiceNames()
        throw (RuntimeException);
};

//__________________________________________________________________________________________________
void Bootstrap_MacroExpander::disposing()
{
    rtlBootstrapHandle h;
    {
        osl::MutexGuard g(m_mutex);
        h = m_bstrap;
        m_bstrap = 0;
    }
    if (h) {
        rtl_bootstrap_args_close(h);
    }
}
//__________________________________________________________________________________________________
Bootstrap_MacroExpander::~Bootstrap_MacroExpander() SAL_THROW( () )
{
    disposing();
}

// XServiceInfo impl
//__________________________________________________________________________________________________
OUString Bootstrap_MacroExpander::getImplementationName()
    throw (RuntimeException)
{
    return s_impl_name();
}
//__________________________________________________________________________________________________
sal_Bool Bootstrap_MacroExpander::supportsService( OUString const & serviceName )
    throw (RuntimeException)
{
    Sequence< OUString > const & service_names = s_get_service_names();
    OUString const * p = service_names.getConstArray();
    for ( sal_Int32 nPos = service_names.getLength(); nPos--; )
    {
        if (p[ nPos ].equals( serviceName ))
            return sal_True;
    }
    return sal_False;
}
//__________________________________________________________________________________________________
Sequence< OUString > Bootstrap_MacroExpander::getSupportedServiceNames()
    throw (RuntimeException)
{
    return s_get_service_names();
}
//  XInitialization impl
//__________________________________________________________________________________________________
void SAL_CALL Bootstrap_MacroExpander::initialize(
    Sequence< Any > const & arguments )
    throw (Exception)
{
    if (m_bstrap)
    {
        throw RuntimeException(
            OUSTR("already initialized!"),
            Reference< XInterface >() );
    }
    if (1 != arguments.getLength())
    {
        throw lang::IllegalArgumentException(
            OUSTR("invalid number of args given!  give single file url!"),
            Reference< XInterface >(),
            0 );
    }
    if (! (arguments[ 0 ] >>= m_rc_path))
    {
        throw lang::IllegalArgumentException(
            OUSTR("give file url!"),
            Reference< XInterface >(),
            0 );
    }
}

// XMacroExpander impl
//__________________________________________________________________________________________________
OUString Bootstrap_MacroExpander::expandMacros( OUString const & exp )
    throw (lang::IllegalArgumentException)
{
    rtlBootstrapHandle h;
    if (m_rc_path.getLength() != 0) {
        osl::MutexGuard g(m_mutex);
        if (!m_bstrap) {
            m_bstrap = rtl_bootstrap_args_open(m_rc_path.pData);
        }
        h = m_bstrap;
    } else {
        h = cppu::get_unorc().getHandle();
    }
    OUString ret( exp );
    rtl_bootstrap_expandMacros_from_handle( h, &ret.pData );
    return ret;
}

//==================================================================================================
Reference< XInterface > SAL_CALL service_create(
    Reference< XComponentContext > const & xComponentContext )
    SAL_THROW( (RuntimeException) )
{
    return static_cast< ::cppu::OWeakObject * >( new Bootstrap_MacroExpander( xComponentContext ) );
}

}

namespace cppu
{

//##################################################################################################
Reference< lang::XSingleComponentFactory > create_boostrap_macro_expander_factory() SAL_THROW( () )
{
    return ::cppu::createSingleComponentFactory(
        service_create,
        s_impl_name(),
        s_get_service_names() );
}

}
<|endoftext|>


<commit_before>#include "chrome/browser/extensions/extension_browsertest.h"

#include "base/command_line.h"
#include "base/file_path.h"
#include "base/path_service.h"
#include "chrome/browser/browser.h"
#include "chrome/browser/extensions/extension_host.h"
#include "chrome/browser/extensions/extensions_service.h"
#include "chrome/browser/profile.h"
#include "chrome/common/chrome_switches.h"
#include "chrome/common/chrome_paths.h"
#include "chrome/common/notification_registrar.h"
#include "chrome/common/notification_service.h"
#include "chrome/common/notification_type.h"
#include "chrome/test/ui_test_utils.h"

namespace {
// Amount of time to wait to load an extension. This is purposely obscenely
// long because it will only get used in the case of failure and we want to
// minimize false positives.
static const int kTimeoutMs = 60 * 1000;  // 1 minute
};

// Base class for extension browser tests. Provides utilities for loading,
// unloading, and installing extensions.
void ExtensionBrowserTest::SetUpCommandLine(CommandLine* command_line) {
  // This enables DOM automation for tab contentses.
  EnableDOMAutomation();

  // This enables it for extension hosts.
  ExtensionHost::EnableDOMAutomation();

  command_line->AppendSwitch(switches::kEnableExtensions);

  PathService::Get(chrome::DIR_TEST_DATA, &test_data_dir_);
  test_data_dir_ = test_data_dir_.AppendASCII("extensions");
}

bool ExtensionBrowserTest::LoadExtension(const FilePath& path) {
  ExtensionsService* service = browser()->profile()->GetExtensionsService();
  size_t num_before = service->extensions()->size();
  registrar_.Add(this, NotificationType::EXTENSIONS_LOADED,
                 NotificationService::AllSources());
  service->LoadExtension(path);
  MessageLoop::current()->PostDelayedTask(FROM_HERE, new MessageLoop::QuitTask,
      kTimeoutMs);
  ui_test_utils::RunMessageLoop();
  registrar_.Remove(this, NotificationType::EXTENSIONS_LOADED,
                    NotificationService::AllSources());
  size_t num_after = service->extensions()->size();
  if (num_after != (num_before + 1))
    return false;

  return WaitForExtensionHostsToLoad();
}

bool ExtensionBrowserTest::InstallExtension(const FilePath& path) {
  ExtensionsService* service = browser()->profile()->GetExtensionsService();
  service->set_show_extensions_prompts(false);
  size_t num_before = service->extensions()->size();

  registrar_.Add(this, NotificationType::EXTENSION_INSTALLED,
                 NotificationService::AllSources());
  service->InstallExtension(path);
  MessageLoop::current()->PostDelayedTask(FROM_HERE, new MessageLoop::QuitTask,
      kTimeoutMs);
  ui_test_utils::RunMessageLoop();
  registrar_.Remove(this, NotificationType::EXTENSION_INSTALLED,
                    NotificationService::AllSources());
  size_t num_after = service->extensions()->size();
  if (num_after != (num_before + 1)) {
    std::cout << "Num extensions before: " << IntToString(num_before)
              << "num after: " << IntToString(num_after)
              << "Installed extensions are:\n";
    for (size_t i = 0; i < service->extensions()->size(); ++i)
      std::cout << "  " << service->extensions()->at(i)->id() << "\n";
    return false;
  }

  return WaitForExtensionHostsToLoad();
}

void ExtensionBrowserTest::UninstallExtension(const std::string& extension_id) {
  ExtensionsService* service = browser()->profile()->GetExtensionsService();
  service->UninstallExtension(extension_id, false);
}

bool ExtensionBrowserTest::WaitForExtensionHostsToLoad() {
  // Wait for all the extension hosts that exist to finish loading.
  // NOTE: This assumes that the extension host list is not changing while
  // this method is running.
  ExtensionProcessManager* manager =
        browser()->profile()->GetExtensionProcessManager();
  base::Time start_time = base::Time::Now();
  for (ExtensionProcessManager::const_iterator iter = manager->begin();
     iter != manager->end(); ++iter) {
    while (!(*iter)->did_stop_loading()) {
      if ((base::Time::Now() - start_time).InMilliseconds() > kTimeoutMs) {
        std::cout << "Extension host did not load for URL: "
                  << (*iter)->GetURL().spec();
        return false;
      }

      MessageLoop::current()->PostDelayedTask(FROM_HERE,
          new MessageLoop::QuitTask, 200);
      ui_test_utils::RunMessageLoop();
    }
  }

  return true;
}

void ExtensionBrowserTest::Observe(NotificationType type,
                     const NotificationSource& source,
                     const NotificationDetails& details) {
  switch (type.value) {
    case NotificationType::EXTENSION_INSTALLED:
    case NotificationType::EXTENSIONS_LOADED:
      MessageLoopForUI::current()->Quit();
      break;
    default:
      NOTREACHED();
      break;
  }
}
<commit_msg>Add some more logging. Still trying to figure out flakey test.<commit_after>#include "chrome/browser/extensions/extension_browsertest.h"

#include "base/command_line.h"
#include "base/file_path.h"
#include "base/path_service.h"
#include "chrome/browser/browser.h"
#include "chrome/browser/extensions/extension_host.h"
#include "chrome/browser/extensions/extensions_service.h"
#include "chrome/browser/profile.h"
#include "chrome/common/chrome_switches.h"
#include "chrome/common/chrome_paths.h"
#include "chrome/common/extensions/extension_error_reporter.h"
#include "chrome/common/notification_registrar.h"
#include "chrome/common/notification_service.h"
#include "chrome/common/notification_type.h"
#include "chrome/test/ui_test_utils.h"

namespace {
// Amount of time to wait to load an extension. This is purposely obscenely
// long because it will only get used in the case of failure and we want to
// minimize false positives.
static const int kTimeoutMs = 60 * 1000;  // 1 minute
};

// Base class for extension browser tests. Provides utilities for loading,
// unloading, and installing extensions.
void ExtensionBrowserTest::SetUpCommandLine(CommandLine* command_line) {
  // This enables DOM automation for tab contentses.
  EnableDOMAutomation();

  // This enables it for extension hosts.
  ExtensionHost::EnableDOMAutomation();

  command_line->AppendSwitch(switches::kEnableExtensions);

  PathService::Get(chrome::DIR_TEST_DATA, &test_data_dir_);
  test_data_dir_ = test_data_dir_.AppendASCII("extensions");
}

bool ExtensionBrowserTest::LoadExtension(const FilePath& path) {
  ExtensionsService* service = browser()->profile()->GetExtensionsService();
  size_t num_before = service->extensions()->size();
  registrar_.Add(this, NotificationType::EXTENSIONS_LOADED,
                 NotificationService::AllSources());
  service->LoadExtension(path);
  MessageLoop::current()->PostDelayedTask(FROM_HERE, new MessageLoop::QuitTask,
      kTimeoutMs);
  ui_test_utils::RunMessageLoop();
  registrar_.Remove(this, NotificationType::EXTENSIONS_LOADED,
                    NotificationService::AllSources());
  size_t num_after = service->extensions()->size();
  if (num_after != (num_before + 1))
    return false;

  return WaitForExtensionHostsToLoad();
}

bool ExtensionBrowserTest::InstallExtension(const FilePath& path) {
  ExtensionsService* service = browser()->profile()->GetExtensionsService();
  service->set_show_extensions_prompts(false);
  size_t num_before = service->extensions()->size();

  registrar_.Add(this, NotificationType::EXTENSION_INSTALLED,
                 NotificationService::AllSources());
  service->InstallExtension(path);
  MessageLoop::current()->PostDelayedTask(FROM_HERE, new MessageLoop::QuitTask,
      kTimeoutMs);
  ui_test_utils::RunMessageLoop();
  registrar_.Remove(this, NotificationType::EXTENSION_INSTALLED,
                    NotificationService::AllSources());
  size_t num_after = service->extensions()->size();
  if (num_after != (num_before + 1)) {
    std::cout << "Num extensions before: " << IntToString(num_before) << " "
              << "num after: " << IntToString(num_after) << " "
              << "Installed extensions follow:\n";

    for (size_t i = 0; i < service->extensions()->size(); ++i)
      std::cout << "  " << service->extensions()->at(i)->id() << "\n";

    std::cout << "Errors follow:\n";
    const std::vector<std::string>* errors =
        ExtensionErrorReporter::GetInstance()->GetErrors();
    for (std::vector<std::string>::const_iterator iter = errors->begin();
         iter != errors->end(); ++iter) {
      std::cout << *iter << "\n";
    }

    return false;
  }

  return WaitForExtensionHostsToLoad();
}

void ExtensionBrowserTest::UninstallExtension(const std::string& extension_id) {
  ExtensionsService* service = browser()->profile()->GetExtensionsService();
  service->UninstallExtension(extension_id, false);
}

bool ExtensionBrowserTest::WaitForExtensionHostsToLoad() {
  // Wait for all the extension hosts that exist to finish loading.
  // NOTE: This assumes that the extension host list is not changing while
  // this method is running.
  ExtensionProcessManager* manager =
        browser()->profile()->GetExtensionProcessManager();
  base::Time start_time = base::Time::Now();
  for (ExtensionProcessManager::const_iterator iter = manager->begin();
     iter != manager->end(); ++iter) {
    while (!(*iter)->did_stop_loading()) {
      if ((base::Time::Now() - start_time).InMilliseconds() > kTimeoutMs) {
        std::cout << "Extension host did not load for URL: "
                  << (*iter)->GetURL().spec();
        return false;
      }

      MessageLoop::current()->PostDelayedTask(FROM_HERE,
          new MessageLoop::QuitTask, 200);
      ui_test_utils::RunMessageLoop();
    }
  }

  return true;
}

void ExtensionBrowserTest::Observe(NotificationType type,
                     const NotificationSource& source,
                     const NotificationDetails& details) {
  switch (type.value) {
    case NotificationType::EXTENSION_INSTALLED:
      std::cout << "Got EXTENSION_INSTALLED notification.\n";
      MessageLoopForUI::current()->Quit();
      break;

    case NotificationType::EXTENSIONS_LOADED:
      std::cout << "Got EXTENSION_LOADED notification.\n";
      MessageLoopForUI::current()->Quit();
      break;

    default:
      NOTREACHED();
      break;
  }
}
<|endoftext|>


<commit_before>#ifndef GCN_SDL_HPP
#define GCN_SDL_HPP

#include <guichan/sdl/sdlgraphics.hpp>
#include <guichan/sdl/sdlimage.hpp>
#include <guichan/sdl/sdlinput.hpp>

#endif // end GCN_SDL_HPP
<commit_msg>Removed sdlimage.hpp added sdlimageloader.hpp<commit_after>#ifndef GCN_SDL_HPP
#define GCN_SDL_HPP

#include <guichan/sdl/sdlgraphics.hpp>
#include <guichan/sdl/sdlimageloader.hpp>
#include <guichan/sdl/sdlinput.hpp>

#endif // end GCN_SDL_HPP
<|endoftext|>


<commit_before>#ifndef __ENVIRE_CORE_ITEM_BASE__
#define __ENVIRE_CORE_ITEM_BASE__

#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_io.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/serialization/access.hpp>
#include <boost/serialization/string.hpp>
#include <boost/serialization/export.hpp>
#include <boost/serialization/nvp.hpp>
#include <base/Time.hpp>
#include <string>
#include <type_traits>
#include <typeindex>
#include <envire_core/serialization/BoostTypes.hpp>

namespace envire { namespace core
{
    using FrameId = std::string;

    /**@class ItemBase
    *
    * ItemBase class
    */
    class ItemBase
    {
    public:
        template<class T>
        using PtrType = boost::shared_ptr<T>;

        typedef PtrType<ItemBase> Ptr;

    protected:

        base::Time time; /** Timestamp */

        boost::uuids::uuid uuid; /** Unique Identifier */

        FrameId frame_name; /** Frame name in which the Item is located */

        // TBD: do we want/need this pointer?
        void* user_data_ptr; /** Pointer to the user data */

    public:

        ItemBase();
        ItemBase(const ItemBase& item);
        ItemBase(ItemBase&& item);
        virtual ~ItemBase() {}

        ItemBase& operator=(const ItemBase& item);
        ItemBase& operator=(ItemBase&& item);

        /**@brief setTime
        *
        * Sets the timestamp of the item
        *
        */
        void setTime(const base::Time& time) { this->time = time; }

        /**@brief getTime
        *
        * Returns the timestamp of the item
        *
        */
        const base::Time& getTime() const { return this->time; }

        /**@brief setID
        *
        * Sets the unique identifier of the item
        *
        */
        void setID(const boost::uuids::uuid& id) { this->uuid = id; }

        /**@brief getID
        *TARGET
        * Returns the unique identifier of the item
        *
        */
        const boost::uuids::uuid getID() const { return this->uuid; }
        const std::string getIDString() const { return boost::uuids::to_string(this->uuid); }

        /**@brief setFrame
        *
        * Sets the frame name of the item
        *
        */
        void setFrame(const std::string& frame_name) { this->frame_name = frame_name; }

        /**@brief getFrame
        *
        * Returns the frame name of the item
        *
        */
        const std::string& getFrame() const { return this->frame_name; }

        /**@brief getClassName
        *
        * Returns the class name of the item
        *
        */
        virtual std::string getClassName() const { return "UnknownItem"; }

        virtual std::type_index getTypeIndex() const = 0;

        void* getRawData() const { return user_data_ptr; }

    private:
        /**Grands access to boost serialization */
        friend class boost::serialization::access;

        /**Serializes the members of this class*/
        template <typename Archive>
        void serialize(Archive &ar, const unsigned int version)
        {
            ar & boost::serialization::make_nvp("time", time.microseconds);
            ar & BOOST_SERIALIZATION_NVP(uuid);
            ar & BOOST_SERIALIZATION_NVP(frame_name);
        }
    };

    /**Mark this class as abstract class */
    BOOST_SERIALIZATION_ASSUME_ABSTRACT(envire::core::ItemBase);

    template <class TARGET>
    struct ItemBaseCaster
    {
        ItemBase::PtrType<TARGET> operator()(const ItemBase::Ptr p) const
        {
            //FIXME static_assert that TARGET is ItemBase::PtrType<X>
            return boost::dynamic_pointer_cast<TARGET>(p);
        }
    };
}}

#endif

<commit_msg>fixed missing reference on return type<commit_after>#ifndef __ENVIRE_CORE_ITEM_BASE__
#define __ENVIRE_CORE_ITEM_BASE__

#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_io.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/serialization/access.hpp>
#include <boost/serialization/string.hpp>
#include <boost/serialization/export.hpp>
#include <boost/serialization/nvp.hpp>
#include <base/Time.hpp>
#include <string>
#include <type_traits>
#include <typeindex>
#include <envire_core/serialization/BoostTypes.hpp>

namespace envire { namespace core
{
    using FrameId = std::string;

    /**@class ItemBase
    *
    * ItemBase class
    */
    class ItemBase
    {
    public:
        template<class T>
        using PtrType = boost::shared_ptr<T>;

        typedef PtrType<ItemBase> Ptr;

    protected:

        base::Time time; /** Timestamp */

        boost::uuids::uuid uuid; /** Unique Identifier */

        FrameId frame_name; /** Frame name in which the Item is located */

        // TBD: do we want/need this pointer?
        void* user_data_ptr; /** Pointer to the user data */

    public:

        ItemBase();
        ItemBase(const ItemBase& item);
        ItemBase(ItemBase&& item);
        virtual ~ItemBase() {}

        ItemBase& operator=(const ItemBase& item);
        ItemBase& operator=(ItemBase&& item);

        /**@brief setTime
        *
        * Sets the timestamp of the item
        *
        */
        void setTime(const base::Time& time) { this->time = time; }

        /**@brief getTime
        *
        * Returns the timestamp of the item
        *
        */
        const base::Time& getTime() const { return this->time; }

        /**@brief setID
        *
        * Sets the unique identifier of the item
        *
        */
        void setID(const boost::uuids::uuid& id) { this->uuid = id; }

        /**@brief getID
        *TARGET
        * Returns the unique identifier of the item
        *
        */
        const boost::uuids::uuid& getID() const { return this->uuid; }
        const std::string getIDString() const { return boost::uuids::to_string(this->uuid); }

        /**@brief setFrame
        *
        * Sets the frame name of the item
        *
        */
        void setFrame(const std::string& frame_name) { this->frame_name = frame_name; }

        /**@brief getFrame
        *
        * Returns the frame name of the item
        *
        */
        const std::string& getFrame() const { return this->frame_name; }

        /**@brief getClassName
        *
        * Returns the class name of the item
        *
        */
        virtual std::string getClassName() const { return "UnknownItem"; }

        virtual std::type_index getTypeIndex() const = 0;

        void* getRawData() const { return user_data_ptr; }

    private:
        /**Grands access to boost serialization */
        friend class boost::serialization::access;

        /**Serializes the members of this class*/
        template <typename Archive>
        void serialize(Archive &ar, const unsigned int version)
        {
            ar & boost::serialization::make_nvp("time", time.microseconds);
            ar & BOOST_SERIALIZATION_NVP(uuid);
            ar & BOOST_SERIALIZATION_NVP(frame_name);
        }
    };

    /**Mark this class as abstract class */
    BOOST_SERIALIZATION_ASSUME_ABSTRACT(envire::core::ItemBase);

    template <class TARGET>
    struct ItemBaseCaster
    {
        ItemBase::PtrType<TARGET> operator()(const ItemBase::Ptr p) const
        {
            //FIXME static_assert that TARGET is ItemBase::PtrType<X>
            return boost::dynamic_pointer_cast<TARGET>(p);
        }
    };
}}

#endif

<|endoftext|>


<commit_before><commit_msg>Fix some issues with bookmark bar folder menu.<commit_after><|endoftext|>


<commit_before>//
//  Enterprise.cpp
//  Clock Signal
//
//  Created by Thomas Harte on 10/06/2021.
//  Copyright © 2021 Thomas Harte. All rights reserved.
//

#include "Enterprise.hpp"

#include "../MachineTypes.hpp"

#include "../../Processors/Z80/Z80.hpp"

#include "../../Analyser/Static/Enterprise/Target.hpp"

namespace Enterprise {

class ConcreteMachine:
	public CPU::Z80::BusHandler,
	public Machine,
	public MachineTypes::ScanProducer,
	public MachineTypes::TimedMachine {
	public:
		ConcreteMachine([[maybe_unused]] const Analyser::Static::Enterprise::Target &target, const ROMMachine::ROMFetcher &rom_fetcher) :
			z80_(*this) {
			// Request a clock of 4Mhz; this'll be mapped upwards for Nick and Dave elsewhere.
			set_clock_rate(4'000'000);

			const auto request = ROM::Request(ROM::Name::EnterpriseEXOS);
			auto roms = rom_fetcher(request);
			if(!request.validate(roms)) {
				throw ROMMachine::Error::MissingROMs;
			}

			// Take a reasonable guess at the initial memory configuration.
			page<0>(0x00);
			page<1>(0x01);
			page<2>(0xfe);
			page<3>(0xff);
		}

		// MARK: - Z80::BusHandler.
		forceinline HalfCycles perform_machine_cycle(const CPU::Z80::PartialMachineCycle &cycle) {
			using PartialMachineCycle = CPU::Z80::PartialMachineCycle;
			const uint16_t address = cycle.address ? *cycle.address : 0x0000;

			// TODO: possibly apply an access penalty.

			switch(cycle.operation) {
				default: break;

				case CPU::Z80::PartialMachineCycle::Input:
				case CPU::Z80::PartialMachineCycle::Output:
					assert(false);
				break;

				case CPU::Z80::PartialMachineCycle::Read:
				case CPU::Z80::PartialMachineCycle::ReadOpcode:
					if(read_pointers_[address >> 14]) {
						*cycle.value = read_pointers_[address >> 14][address];
					} else {
						*cycle.value = 0xff;
					}
				break;

				case CPU::Z80::PartialMachineCycle::Write:
					if(write_pointers_[address >> 14]) {
						write_pointers_[address >> 14][address] = *cycle.value;
					}
				break;
			}

			return HalfCycles(0);
		}

	private:
		CPU::Z80::Processor<ConcreteMachine, false, false> z80_;

		std::array<uint8_t, 32 * 1024> exos_;
		std::array<uint8_t, 256 * 1024> ram_;
		const uint8_t min_ram_slot_ = 0xff - 3;

		const uint8_t *read_pointers_[4];
		uint8_t *write_pointers_[4];
		uint8_t pages_[4];

		template <size_t slot> void page(uint8_t offset) {
			pages_[slot] = offset;

			if(offset < 2) {
				page<slot>(&exos_[offset * 0x4000], nullptr);
				return;
			}

			if(offset >= min_ram_slot_) {
				const size_t address = (offset - min_ram_slot_) * 0x4000;
				page<slot>(&ram_[address], &ram_[address]);
				return;
			}

			page<slot>(nullptr, nullptr);
		}

		template <size_t slot> void page(const uint8_t *read, uint8_t *write) {
			read_pointers_[slot] = read ? read - (slot * 0x4000) : nullptr;
			write_pointers_[slot] = write ? write - (slot * 0x4000) : nullptr;
		}

		// MARK: - ScanProducer
		void set_scan_target(Outputs::Display::ScanTarget *scan_target) override {
			(void)scan_target;
		}

		Outputs::Display::ScanStatus get_scaled_scan_status() const override {
			return Outputs::Display::ScanStatus();
		}

		// MARK: - TimedMachine
		void run_for(const Cycles cycles) override {
			z80_.run_for(cycles);
		}
};

}

using namespace Enterprise;

Machine *Machine::Enterprise(const Analyser::Static::Target *target, const ROMMachine::ROMFetcher &rom_fetcher) {
	using Target = Analyser::Static::Enterprise::Target;
	const Target *const enterprise_target = dynamic_cast<const Target *>(target);

	return new Enterprise::ConcreteMachine(*enterprise_target, rom_fetcher);
}

Machine::~Machine() {}
<commit_msg>Revised guess; there's a jump to C02E almost immediately.<commit_after>//
//  Enterprise.cpp
//  Clock Signal
//
//  Created by Thomas Harte on 10/06/2021.
//  Copyright © 2021 Thomas Harte. All rights reserved.
//

#include "Enterprise.hpp"

#include "../MachineTypes.hpp"

#include "../../Processors/Z80/Z80.hpp"

#include "../../Analyser/Static/Enterprise/Target.hpp"

namespace Enterprise {

class ConcreteMachine:
	public CPU::Z80::BusHandler,
	public Machine,
	public MachineTypes::ScanProducer,
	public MachineTypes::TimedMachine {
	public:
		ConcreteMachine([[maybe_unused]] const Analyser::Static::Enterprise::Target &target, const ROMMachine::ROMFetcher &rom_fetcher) :
			z80_(*this) {
			// Request a clock of 4Mhz; this'll be mapped upwards for Nick and Dave elsewhere.
			set_clock_rate(4'000'000);

			constexpr ROM::Name exos_name = ROM::Name::EnterpriseEXOS;
			const auto request = ROM::Request(exos_name);
			auto roms = rom_fetcher(request);
			if(!request.validate(roms)) {
				throw ROMMachine::Error::MissingROMs;
			}

			const auto &exos = roms.find(exos_name)->second;
			memcpy(exos_.data(), exos.data(), std::min(exos_.size(), exos.size()));

			// Take a reasonable guess at the initial memory configuration.
			page<0>(0x00);
			page<1>(0x00);
			page<2>(0x00);
			page<3>(0x00);
		}

		// MARK: - Z80::BusHandler.
		forceinline HalfCycles perform_machine_cycle(const CPU::Z80::PartialMachineCycle &cycle) {
			using PartialMachineCycle = CPU::Z80::PartialMachineCycle;
			const uint16_t address = cycle.address ? *cycle.address : 0x0000;

			// TODO: possibly apply an access penalty.

			switch(cycle.operation) {
				default: break;

				case CPU::Z80::PartialMachineCycle::Input:
					printf("Unhandled input: %04x\n", address);
					assert(false);
				break;

				case CPU::Z80::PartialMachineCycle::Output:
					printf("Unhandled output: %04x\n", address);
					assert(false);
				break;

				case CPU::Z80::PartialMachineCycle::Read:
				case CPU::Z80::PartialMachineCycle::ReadOpcode:
					if(read_pointers_[address >> 14]) {
						*cycle.value = read_pointers_[address >> 14][address];
					} else {
						*cycle.value = 0xff;
					}
				break;

				case CPU::Z80::PartialMachineCycle::Write:
					if(write_pointers_[address >> 14]) {
						write_pointers_[address >> 14][address] = *cycle.value;
					}
				break;
			}

			return HalfCycles(0);
		}

	private:
		CPU::Z80::Processor<ConcreteMachine, false, false> z80_;

		std::array<uint8_t, 32 * 1024> exos_;
		std::array<uint8_t, 256 * 1024> ram_;
		const uint8_t min_ram_slot_ = 0xff - 3;

		const uint8_t *read_pointers_[4];
		uint8_t *write_pointers_[4];
		uint8_t pages_[4];

		template <size_t slot> void page(uint8_t offset) {
			pages_[slot] = offset;

			if(offset < 2) {
				page<slot>(&exos_[offset * 0x4000], nullptr);
				return;
			}

			if(offset >= min_ram_slot_) {
				const size_t address = (offset - min_ram_slot_) * 0x4000;
				page<slot>(&ram_[address], &ram_[address]);
				return;
			}

			page<slot>(nullptr, nullptr);
		}

		template <size_t slot> void page(const uint8_t *read, uint8_t *write) {
			read_pointers_[slot] = read ? read - (slot * 0x4000) : nullptr;
			write_pointers_[slot] = write ? write - (slot * 0x4000) : nullptr;
		}

		// MARK: - ScanProducer
		void set_scan_target(Outputs::Display::ScanTarget *scan_target) override {
			(void)scan_target;
		}

		Outputs::Display::ScanStatus get_scaled_scan_status() const override {
			return Outputs::Display::ScanStatus();
		}

		// MARK: - TimedMachine
		void run_for(const Cycles cycles) override {
			z80_.run_for(cycles);
		}
};

}

using namespace Enterprise;

Machine *Machine::Enterprise(const Analyser::Static::Target *target, const ROMMachine::ROMFetcher &rom_fetcher) {
	using Target = Analyser::Static::Enterprise::Target;
	const Target *const enterprise_target = dynamic_cast<const Target *>(target);

	return new Enterprise::ConcreteMachine(*enterprise_target, rom_fetcher);
}

Machine::~Machine() {}
<|endoftext|>


<commit_before>#pragma once

#include <glm/glm.hpp>

namespace mos {
namespace gfx {

/** Visual bounding box. */
class Box final {
public:
  Box() = default;
  Box(const glm::mat4 &transform,
      const glm::vec3 &extent);
  glm::mat4 transform;
  glm::vec3 extent;
  glm::vec3 position() const;
  glm::vec3 size() const;
  glm::vec3 min() const;
  glm::vec3 max() const;
  bool inside(const glm::vec3 &point) const;
};
}
}
<commit_msg>Constructor fix.<commit_after>#pragma once

#include <glm/glm.hpp>

namespace mos {
namespace gfx {

/** Visual bounding box. */
class Box final {
public:
  Box();
  Box(const glm::mat4 &transform,
      const glm::vec3 &extent);
  glm::mat4 transform;
  glm::vec3 extent;
  glm::vec3 position() const;
  glm::vec3 size() const;
  glm::vec3 min() const;
  glm::vec3 max() const;
  bool inside(const glm::vec3 &point) const;
};
}
}
<|endoftext|>


<commit_before>/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "process_thread_impl.h"
#include "module.h"
#include "trace.h"

namespace webrtc {
ProcessThread::~ProcessThread()
{
}

ProcessThread* ProcessThread::CreateProcessThread()
{
    return new ProcessThreadImpl();
}

void ProcessThread::DestroyProcessThread(ProcessThread* module)
{
    delete module;
}

ProcessThreadImpl::ProcessThreadImpl()
    : _timeEvent(*EventWrapper::Create()),
      _critSectModules(CriticalSectionWrapper::CreateCriticalSection()),
      _thread(NULL)
{
    WEBRTC_TRACE(kTraceMemory, kTraceUtility, -1, "%s created", __FUNCTION__);
}

ProcessThreadImpl::~ProcessThreadImpl()
{
    delete _critSectModules;
    delete &_timeEvent;
    WEBRTC_TRACE(kTraceMemory, kTraceUtility, -1, "%s deleted", __FUNCTION__);
}

WebRtc_Word32 ProcessThreadImpl::Start()
{
    CriticalSectionScoped lock(_critSectModules);
    if(_thread)
    {
        return -1;
    }
    _thread = ThreadWrapper::CreateThread(Run, this, kNormalPriority,
                                          "ProcessThread");
    unsigned int id;
    WebRtc_Word32 retVal = _thread->Start(id);
    if(retVal >= 0)
    {
        return 0;
    }
    delete _thread;
    _thread = NULL;
    return -1;
}

WebRtc_Word32 ProcessThreadImpl::Stop()
{
    _critSectModules->Enter();
    if(_thread)
    {
        _thread->SetNotAlive();

        ThreadWrapper* thread = _thread;
        _thread = NULL;

        _timeEvent.Set();
        _critSectModules->Leave();

        if(thread->Stop())
        {
            delete thread;
        } else {
            return -1;
        }
    } else {
        _critSectModules->Leave();
    }
    return 0;
}

WebRtc_Word32 ProcessThreadImpl::RegisterModule(const Module* module)
{
    CriticalSectionScoped lock(_critSectModules);

    // Only allow module to be registered once.
    ListItem* item = _modules.First();
    for(WebRtc_UWord32 i = 0; i < _modules.GetSize() && item; i++)
    {
        if(module == item->GetItem())
        {
            return -1;
        }
        item = _modules.Next(item);
    }

    _modules.PushFront(module);
    WEBRTC_TRACE(kTraceInfo, kTraceUtility, -1,
                 "number of registered modules has increased to %d",
                 _modules.GetSize());
    // Wake the thread calling ProcessThreadImpl::Process() to update the
    // waiting time. The waiting time for the just registered module may be
    // shorter than all other registered modules.
    _timeEvent.Set();
    return 0;
}

WebRtc_Word32 ProcessThreadImpl::DeRegisterModule(const Module* module)
{
    CriticalSectionScoped lock(_critSectModules);

    ListItem* item = _modules.First();
    for(WebRtc_UWord32 i = 0; i < _modules.GetSize() && item; i++)
    {
        if(module == item->GetItem())
        {
            int res = _modules.Erase(item);
            WEBRTC_TRACE(kTraceInfo, kTraceUtility, -1,
                         "number of registered modules has decreased to %d",
                         _modules.GetSize());
            return res;
        }
        item = _modules.Next(item);
    }
    return -1;
}

bool ProcessThreadImpl::Run(void* obj)
{
    return static_cast<ProcessThreadImpl*>(obj)->Process();
}

bool ProcessThreadImpl::Process()
{
    // Wait for the module that should be called next, but don't block thread
    // longer than 100 ms.
    WebRtc_Word32 minTimeToNext = 100;
    {
        CriticalSectionScoped lock(_critSectModules);
        ListItem* item = _modules.First();
        for(WebRtc_UWord32 i = 0; i < _modules.GetSize() && item; i++)
        {
            WebRtc_Word32 timeToNext =
                static_cast<Module*>(item->GetItem())->TimeUntilNextProcess();
            if(minTimeToNext > timeToNext)
            {
                minTimeToNext = timeToNext;
            }
            item = _modules.Next(item);
        }
    }

    if(minTimeToNext > 0)
    {
        if(kEventError == _timeEvent.Wait(minTimeToNext))
        {
            return true;
        }
        if(!_thread)
        {
            return false;
        }
    }
    {
        CriticalSectionScoped lock(_critSectModules);
        ListItem* item = _modules.First();
        for(WebRtc_UWord32 i = 0; i < _modules.GetSize() && item; i++)
        {
            WebRtc_Word32 timeToNext =
                static_cast<Module*>(item->GetItem())->TimeUntilNextProcess();
            if(timeToNext < 1)
            {
                static_cast<Module*>(item->GetItem())->Process();
            }
            item = _modules.Next(item);
        }
    }
    return true;
}
} // namespace webrtc
<commit_msg>Fixes data race in WebRTCAudioDeviceTest.Construct reported by ThreadSanitizer<commit_after>/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "process_thread_impl.h"
#include "module.h"
#include "trace.h"

namespace webrtc {
ProcessThread::~ProcessThread()
{
}

ProcessThread* ProcessThread::CreateProcessThread()
{
    return new ProcessThreadImpl();
}

void ProcessThread::DestroyProcessThread(ProcessThread* module)
{
    delete module;
}

ProcessThreadImpl::ProcessThreadImpl()
    : _timeEvent(*EventWrapper::Create()),
      _critSectModules(CriticalSectionWrapper::CreateCriticalSection()),
      _thread(NULL)
{
    WEBRTC_TRACE(kTraceMemory, kTraceUtility, -1, "%s created", __FUNCTION__);
}

ProcessThreadImpl::~ProcessThreadImpl()
{
    delete _critSectModules;
    delete &_timeEvent;
    WEBRTC_TRACE(kTraceMemory, kTraceUtility, -1, "%s deleted", __FUNCTION__);
}

WebRtc_Word32 ProcessThreadImpl::Start()
{
    CriticalSectionScoped lock(_critSectModules);
    if(_thread)
    {
        return -1;
    }
    _thread = ThreadWrapper::CreateThread(Run, this, kNormalPriority,
                                          "ProcessThread");
    unsigned int id;
    WebRtc_Word32 retVal = _thread->Start(id);
    if(retVal >= 0)
    {
        return 0;
    }
    delete _thread;
    _thread = NULL;
    return -1;
}

WebRtc_Word32 ProcessThreadImpl::Stop()
{
    _critSectModules->Enter();
    if(_thread)
    {
        _thread->SetNotAlive();

        ThreadWrapper* thread = _thread;
        _thread = NULL;

        _timeEvent.Set();
        _critSectModules->Leave();

        if(thread->Stop())
        {
            delete thread;
        } else {
            return -1;
        }
    } else {
        _critSectModules->Leave();
    }
    return 0;
}

WebRtc_Word32 ProcessThreadImpl::RegisterModule(const Module* module)
{
    CriticalSectionScoped lock(_critSectModules);

    // Only allow module to be registered once.
    ListItem* item = _modules.First();
    for(WebRtc_UWord32 i = 0; i < _modules.GetSize() && item; i++)
    {
        if(module == item->GetItem())
        {
            return -1;
        }
        item = _modules.Next(item);
    }

    _modules.PushFront(module);
    WEBRTC_TRACE(kTraceInfo, kTraceUtility, -1,
                 "number of registered modules has increased to %d",
                 _modules.GetSize());
    // Wake the thread calling ProcessThreadImpl::Process() to update the
    // waiting time. The waiting time for the just registered module may be
    // shorter than all other registered modules.
    _timeEvent.Set();
    return 0;
}

WebRtc_Word32 ProcessThreadImpl::DeRegisterModule(const Module* module)
{
    CriticalSectionScoped lock(_critSectModules);

    ListItem* item = _modules.First();
    for(WebRtc_UWord32 i = 0; i < _modules.GetSize() && item; i++)
    {
        if(module == item->GetItem())
        {
            int res = _modules.Erase(item);
            WEBRTC_TRACE(kTraceInfo, kTraceUtility, -1,
                         "number of registered modules has decreased to %d",
                         _modules.GetSize());
            return res;
        }
        item = _modules.Next(item);
    }
    return -1;
}

bool ProcessThreadImpl::Run(void* obj)
{
    return static_cast<ProcessThreadImpl*>(obj)->Process();
}

bool ProcessThreadImpl::Process()
{
    // Wait for the module that should be called next, but don't block thread
    // longer than 100 ms.
    WebRtc_Word32 minTimeToNext = 100;
    {
        CriticalSectionScoped lock(_critSectModules);
        ListItem* item = _modules.First();
        for(WebRtc_UWord32 i = 0; i < _modules.GetSize() && item; i++)
        {
            WebRtc_Word32 timeToNext =
                static_cast<Module*>(item->GetItem())->TimeUntilNextProcess();
            if(minTimeToNext > timeToNext)
            {
                minTimeToNext = timeToNext;
            }
            item = _modules.Next(item);
        }
    }

    if(minTimeToNext > 0)
    {
        if(kEventError == _timeEvent.Wait(minTimeToNext))
        {
            return true;
        }
        CriticalSectionScoped lock(_critSectModules);
        if(!_thread)
        {
            return false;
        }
    }
    {
        CriticalSectionScoped lock(_critSectModules);
        ListItem* item = _modules.First();
        for(WebRtc_UWord32 i = 0; i < _modules.GetSize() && item; i++)
        {
            WebRtc_Word32 timeToNext =
                static_cast<Module*>(item->GetItem())->TimeUntilNextProcess();
            if(timeToNext < 1)
            {
                static_cast<Module*>(item->GetItem())->Process();
            }
            item = _modules.Next(item);
        }
    }
    return true;
}
} // namespace webrtc
<|endoftext|>


<commit_before>// This file is part of MLDB. Copyright 2015 Datacratic. All rights reserved.

/** for_each_line.cc
    Jeremy Barnes, 29 November 2013
    Copyright (c) 2013 Datacratic Inc.  All rights reserved.

*/

#include <atomic>
#include <exception>
#include <mutex>
#include "for_each_line.h"
#include "mldb/arch/threads.h"
#include <chrono>
#include <thread>
#include <cstring>
#include "mldb/jml/utils/ring_buffer.h"
#include "mldb/vfs/filter_streams.h"
#include "mldb/base/thread_pool.h"
#include "mldb/base/exc_assert.h"
#include "mldb/types/date.h"

using namespace std;
using namespace ML;

namespace {

struct Processing {

    Processing()
        : shutdown(false), hasException_(false),
          excPtr(nullptr), decompressedLines(16000)
    {
    }

    void takeLastException()
    {
        std::lock_guard<std::mutex> guard(excPtrLock);
        excPtr = std::current_exception();
        hasException_ = true;
    }

    bool hasException()
    {
        return hasException_;
    }

    atomic<bool> shutdown;
    atomic<bool> hasException_;
    std::mutex excPtrLock;
    std::exception_ptr excPtr;

    RingBufferSWMR<pair<int64_t, vector<string> > > decompressedLines;
};

}

namespace Datacratic {

/*****************************************************************************/
/* PARALLEL LINE PROCESSOR                                                   */
/*****************************************************************************/

static size_t
readStream(std::istream & stream,
           Processing & processing,
           bool ignoreStreamExceptions,
           int64_t maxLines)
{
    Date start = Date::now();

    pair<int64_t, vector<string> > current;
    current.second.reserve(1000);

    Date lastCheck = start;

    int64_t done = current.first = 0;  // 32 bit not enough

    try {
        while (stream && !stream.eof() && (maxLines == -1 || done < maxLines)) {

            if (processing.hasException()) {
                break;
            }
            string line;
            getline(stream, line);
            current.second.emplace_back(std::move(line));

            ++done;

            if (current.second.size() == 1000) {
                processing.decompressedLines.push(std::move(current));
                current.first = done;
                current.second.clear();
                //current.clear();
                ExcAssertEqual(current.second.size(), 0);
                current.second.reserve(1000);
            }

            if (done % 1000000 == 0) {

                //cerr << "done " << done << " lines" << endl;
                Date now = Date::now();

                double elapsed = now.secondsSince(start);
                double instElapsed = now.secondsSince(lastCheck);
                cerr << ML::format("doing %.3fMlines/second total, %.3f instantaneous",
                                   done / elapsed / 1000000.0,
                                   1000000 / instElapsed / 1000000.0)
                     << endl;
                lastCheck = now;
            }
        }
    } catch (const std::exception & exc) {
        if (!ignoreStreamExceptions) {
            processing.takeLastException();
        }
        else {
            cerr << "stream threw ignored exception: " << exc.what() << endl;
        }
    }

    if (!current.second.empty() && !processing.hasException()) {
        processing.decompressedLines.push(std::move(current));
    }

    return done;
};

static void
parseLinesThreadStr(Processing & processing,
                    const std::function<void (const std::string &,
                                              int64_t lineNum)> & processLine)
{
    while (!processing.hasException()) {
        std::pair<int64_t, vector<string> > lines;
        if (!processing.decompressedLines.tryPop(lines)) {
            if (processing.shutdown) {
                break;
            }
            std::this_thread::sleep_for(std::chrono::milliseconds(1));
            continue;
        }

        for (unsigned i = 0;  i < lines.second.size();  ++i) {
            if (processing.hasException()) {
                break;
            }
            const string & line = lines.second[i];
            try {
                processLine(line, lines.first + i);
            } catch (const std::exception & exc) {
                processing.takeLastException();
                cerr << "error dealing with line " << line
                     << ": " << exc.what() << endl;
            } catch (...) {
                processing.takeLastException();
            }
        }
    }
};

size_t
forEachLine(std::istream & stream,
            const std::function<void (const char *, size_t,
                                      int64_t)> & processLine,
            int numThreads,
            bool ignoreStreamExceptions,
            int64_t maxLines)
{
    auto onLineStr = [&] (const string & line, int64_t lineNum) {
        processLine(line.c_str(), line.size(), lineNum);
    };

    return forEachLineStr(stream, onLineStr, numThreads,
                          ignoreStreamExceptions, maxLines);
}

size_t
forEachLineStr(std::istream & stream,
               const std::function<void (const std::string &,
                                         int64_t)> & processLine,
               int numThreads,
               bool ignoreStreamExceptions,
               int64_t maxLines)
{
    Processing processing;

    std::vector<std::thread> threads;
    for (unsigned i = 0;  i < numThreads;  ++i)
        threads.emplace_back(std::bind(parseLinesThreadStr,
                                       std::ref(processing),
                                       std::ref(processLine)));

    size_t result = readStream(stream, processing,
                               ignoreStreamExceptions, maxLines);

    processing.shutdown = true;

    for (auto & t: threads)
        t.join();

    if (processing.hasException()) {
        std::rethrow_exception(processing.excPtr);
    }

    return result;
}

size_t
forEachLine(const std::string & filename,
            const std::function<void (const char *, size_t, int64_t)> & processLine,
            int numThreads,
            bool ignoreStreamExceptions,
            int64_t maxLines)
{
    filter_istream stream(filename);
    return forEachLine(stream, processLine, numThreads,
                       ignoreStreamExceptions, maxLines);
}

size_t
forEachLineStr(const std::string & filename,
               const std::function<void (const std::string &, int64_t)> & processLine,
               int numThreads,
               bool ignoreStreamExceptions,
               int64_t maxLines)
{
    filter_istream stream(filename);
    return forEachLineStr(stream, processLine, numThreads,
                          ignoreStreamExceptions, maxLines);
}

/*****************************************************************************/
/* FOR EACH LINE BLOCK                                                       */
/*****************************************************************************/

void forEachLineBlock(std::istream & stream,
                      std::function<bool (const char * line,
                                          size_t lineLength,
                                          int64_t blockNumber,
                                          int64_t lineNumber)> onLine,
                      int64_t maxLines,
                      int maxParallelism,
                      std::function<bool (int64_t blockNumber, int64_t lineNumber)> startBlock,
                      std::function<bool (int64_t blockNumber, int64_t lineNumber)> endBlock)
{
    //static constexpr int64_t BLOCK_SIZE = 100000000;  // 100MB blocks
    static constexpr int64_t BLOCK_SIZE = 10000000;  // 10MB blocks
    static constexpr int64_t READ_SIZE = 200000;  // read&scan 200kb to fit in cache

    std::atomic<int64_t> doneLines(0); //number of lines processed but not yet returned
    std::atomic<int64_t> returnedLines(0); //number of lines returned
    std::atomic<int64_t> byteOffset(0);
    std::atomic<int> chunkNumber(0);

    ThreadPool tp(maxParallelism);

    // Memory map if possible
    const char * mapped = nullptr;
    size_t mappedSize = 0;

    filter_istream * fistream = dynamic_cast<filter_istream *>(&stream);

    if (fistream) {
        // Can we get a memory mapped version of our stream?  It
        // saves us having to copy data.  mapped will be set to
        // nullptr if it's not possible to memory map this stream.
        std::tie(mapped, mappedSize) = fistream->mapped();
    }

    std::atomic<int> hasExc(false);
    std::exception_ptr exc;

    std::function<void ()> doBlock = [&] ()
        {
            //cerr << "block starting at line " << doneLines << endl;

            std::shared_ptr<const char> blockOut;

            int64_t startOffset = byteOffset;
            int64_t startLine = doneLines;
            vector<size_t> lineOffsets = {0};
            bool lastBlock = false;
            size_t myChunkNumber = 0;

            try {
                //MLDB-1426
                if (mapped && false) {
                    const char * start = mapped + stream.tellg();
                    const char * current = start;
                    const char * end = mapped + mappedSize;

                    while (current && current < end && (current - start) < BLOCK_SIZE
                           && (maxLines == -1 || doneLines < maxLines)) { //stop processing new line when we have enough)
                        current = (const char *)memchr(current, '\n', end - current);
                        if (current && current < end) {
                            ExcAssertEqual(*current, '\n');
                            lineOffsets.push_back(current - start);
                            ++doneLines;
                            ++current;
                        }
                    }

                    if (current)
                        stream.seekg(current - start, ios::cur);
                    else {
                        // Last line has no newline
                        lineOffsets.push_back(end - start);
                        ++doneLines;
                    }

                    myChunkNumber = chunkNumber++;

                    if (current && current < end &&
                        (maxLines == -1 || doneLines < maxLines)) // don't schedule a new block if we have enough lines
                        {
                            // Ready for another chunk
                            tp.add(doBlock);
                        } else if (current == end) {
                        lastBlock = true;
                    }

                    blockOut = std::shared_ptr<const char>(start,
                                                           [] (const char *) {});
                }
                else {
                    // How far through our block are we?
                    size_t offset = 0;

                    // How much extra space to allocate for the last line?
                    static constexpr size_t EXTRA_SIZE = 10000;

                    std::shared_ptr<char> block(new char[BLOCK_SIZE + EXTRA_SIZE],
                                                [] (char * c) { delete[] c; });
                    blockOut = block;

                    // First line starts at offset 0

                    while (stream && !stream.eof()
                           && (maxLines == -1 || doneLines < maxLines)  //stop processing new line when we have enough
                           && (byteOffset - startOffset < BLOCK_SIZE)) {

                        stream.read((char *)block.get() + offset,
                                    std::min<size_t>(READ_SIZE, BLOCK_SIZE - offset));

                        // Check how many bytes we actually read
                        size_t bytesRead = stream.gcount();

                        offset += bytesRead;

                        // Scan for end of line characters
                        const char * current = block.get() + lineOffsets.back();
                        const char * end = block.get() + offset;

                        while (current && current < end) {
                            current = (const char *)memchr(current, '\n', end - current);
                            if (current && current < end) {
                                ExcAssertEqual(*current, '\n');
                                if (lineOffsets.back() != current - block.get()) {
                                    lineOffsets.push_back(current - block.get());
                                    //cerr << "got line at offset " << lineOffsets.back() << endl;
                                    ++doneLines;
                                }
                                ++current;
                            }
                        }

                        byteOffset += bytesRead;
                    }

                    if (stream.eof()) {
                        // If we are at the end of the stream
                        // make sure we include the last line
                        // if there was no newline
                        if (lineOffsets.back() != offset - 1) {
                            lineOffsets.push_back(offset);
                            ++doneLines;
                        }
                    }
                    else {
                        // If we are not at the end of the stream
                        // get the last line, as we probably got just a partial
                        // line in the last one
                        std::string lastLine;
                        getline(stream, lastLine);

                        size_t cnt = stream.gcount();

                        if (cnt != 0) {
                            // Check for overflow on the buffer size
                            if (offset + lastLine.size() + 1 > BLOCK_SIZE + EXTRA_SIZE) {
                                // reallocate and copy
                                std::shared_ptr<char> newBlock(new char[offset + lastLine.size() + 1],
                                                               [] (char * c) { delete[] c; });
                                std::copy(block.get(), block.get() + offset,
                                          newBlock.get());
                                block = newBlock;
                                blockOut = block;
                            }

                            std::copy(lastLine.data(), lastLine.data() + lastLine.length(),
                                      block.get() + offset);

                            lineOffsets.emplace_back(offset + lastLine.length());
                            ++doneLines;
                            offset += cnt;
                        }
                    }

                    myChunkNumber = chunkNumber++;

                    if (stream && !stream.eof() &&
                        (maxLines == -1 || doneLines < maxLines)) // don't schedule a new block if we have enough lines
                        {
                            // Ready for another chunk
                            tp.add(doBlock);
                        } else if (stream.eof()) {
                        lastBlock = true;
                    }
                }

                //cerr << "processing block of " << lineOffsets.size() - 1
                //     << " lines starting at " << startLine << endl;

                int64_t chunkLineNumber = startLine;
                size_t lastLineOffset = lineOffsets[0];

                if (startBlock)
                    if (!startBlock(myChunkNumber, chunkLineNumber))
                        return;

                for (unsigned i = 1;  i < lineOffsets.size() && (maxLines == -1 || returnedLines++ < maxLines);  ++i) {
                    if (hasExc.load(std::memory_order_relaxed))
                        return;
                    const char * line = blockOut.get() + lastLineOffset;
                    size_t len = lineOffsets[i] - lastLineOffset;

                    // Skip \r for DOS line endings
                    if (len > 0 && line[len - 1] == '\r')
                        --len;

                    // if we are not at the last line
                    if (!lastBlock || len != 0 || i != lineOffsets.size() - 1)
                        if (!onLine(line, len, chunkNumber, chunkLineNumber++))
                            return;

                    lastLineOffset = lineOffsets[i] + 1;

                }

                if (endBlock)
                    if (!endBlock(myChunkNumber, chunkLineNumber))
                        return;

            } JML_CATCH_ALL {
                if (hasExc.fetch_add(1) == 0) {
                    exc = std::current_exception();
                }
            }
        };

    tp.add(doBlock);
    tp.waitForAll();

    // If there was an exception, rethrow it rather than returning
    // cleanly
    if (hasExc) {
        std::rethrow_exception(exc);
    }
}

} // namespace Datacratic
<commit_msg>Use 20MB blocks (MLDB-1507)<commit_after>// This file is part of MLDB. Copyright 2015 Datacratic. All rights reserved.

/** for_each_line.cc
    Jeremy Barnes, 29 November 2013
    Copyright (c) 2013 Datacratic Inc.  All rights reserved.

*/

#include <atomic>
#include <exception>
#include <mutex>
#include "for_each_line.h"
#include "mldb/arch/threads.h"
#include <chrono>
#include <thread>
#include <cstring>
#include "mldb/jml/utils/ring_buffer.h"
#include "mldb/vfs/filter_streams.h"
#include "mldb/base/thread_pool.h"
#include "mldb/base/exc_assert.h"
#include "mldb/types/date.h"

using namespace std;
using namespace ML;

namespace {

struct Processing {

    Processing()
        : shutdown(false), hasException_(false),
          excPtr(nullptr), decompressedLines(16000)
    {
    }

    void takeLastException()
    {
        std::lock_guard<std::mutex> guard(excPtrLock);
        excPtr = std::current_exception();
        hasException_ = true;
    }

    bool hasException()
    {
        return hasException_;
    }

    atomic<bool> shutdown;
    atomic<bool> hasException_;
    std::mutex excPtrLock;
    std::exception_ptr excPtr;

    RingBufferSWMR<pair<int64_t, vector<string> > > decompressedLines;
};

}

namespace Datacratic {

/*****************************************************************************/
/* PARALLEL LINE PROCESSOR                                                   */
/*****************************************************************************/

static size_t
readStream(std::istream & stream,
           Processing & processing,
           bool ignoreStreamExceptions,
           int64_t maxLines)
{
    Date start = Date::now();

    pair<int64_t, vector<string> > current;
    current.second.reserve(1000);

    Date lastCheck = start;

    int64_t done = current.first = 0;  // 32 bit not enough

    try {
        while (stream && !stream.eof() && (maxLines == -1 || done < maxLines)) {

            if (processing.hasException()) {
                break;
            }
            string line;
            getline(stream, line);
            current.second.emplace_back(std::move(line));

            ++done;

            if (current.second.size() == 1000) {
                processing.decompressedLines.push(std::move(current));
                current.first = done;
                current.second.clear();
                //current.clear();
                ExcAssertEqual(current.second.size(), 0);
                current.second.reserve(1000);
            }

            if (done % 1000000 == 0) {

                //cerr << "done " << done << " lines" << endl;
                Date now = Date::now();

                double elapsed = now.secondsSince(start);
                double instElapsed = now.secondsSince(lastCheck);
                cerr << ML::format("doing %.3fMlines/second total, %.3f instantaneous",
                                   done / elapsed / 1000000.0,
                                   1000000 / instElapsed / 1000000.0)
                     << endl;
                lastCheck = now;
            }
        }
    } catch (const std::exception & exc) {
        if (!ignoreStreamExceptions) {
            processing.takeLastException();
        }
        else {
            cerr << "stream threw ignored exception: " << exc.what() << endl;
        }
    }

    if (!current.second.empty() && !processing.hasException()) {
        processing.decompressedLines.push(std::move(current));
    }

    return done;
};

static void
parseLinesThreadStr(Processing & processing,
                    const std::function<void (const std::string &,
                                              int64_t lineNum)> & processLine)
{
    while (!processing.hasException()) {
        std::pair<int64_t, vector<string> > lines;
        if (!processing.decompressedLines.tryPop(lines)) {
            if (processing.shutdown) {
                break;
            }
            std::this_thread::sleep_for(std::chrono::milliseconds(1));
            continue;
        }

        for (unsigned i = 0;  i < lines.second.size();  ++i) {
            if (processing.hasException()) {
                break;
            }
            const string & line = lines.second[i];
            try {
                processLine(line, lines.first + i);
            } catch (const std::exception & exc) {
                processing.takeLastException();
                cerr << "error dealing with line " << line
                     << ": " << exc.what() << endl;
            } catch (...) {
                processing.takeLastException();
            }
        }
    }
};

size_t
forEachLine(std::istream & stream,
            const std::function<void (const char *, size_t,
                                      int64_t)> & processLine,
            int numThreads,
            bool ignoreStreamExceptions,
            int64_t maxLines)
{
    auto onLineStr = [&] (const string & line, int64_t lineNum) {
        processLine(line.c_str(), line.size(), lineNum);
    };

    return forEachLineStr(stream, onLineStr, numThreads,
                          ignoreStreamExceptions, maxLines);
}

size_t
forEachLineStr(std::istream & stream,
               const std::function<void (const std::string &,
                                         int64_t)> & processLine,
               int numThreads,
               bool ignoreStreamExceptions,
               int64_t maxLines)
{
    Processing processing;

    std::vector<std::thread> threads;
    for (unsigned i = 0;  i < numThreads;  ++i)
        threads.emplace_back(std::bind(parseLinesThreadStr,
                                       std::ref(processing),
                                       std::ref(processLine)));

    size_t result = readStream(stream, processing,
                               ignoreStreamExceptions, maxLines);

    processing.shutdown = true;

    for (auto & t: threads)
        t.join();

    if (processing.hasException()) {
        std::rethrow_exception(processing.excPtr);
    }

    return result;
}

size_t
forEachLine(const std::string & filename,
            const std::function<void (const char *, size_t, int64_t)> & processLine,
            int numThreads,
            bool ignoreStreamExceptions,
            int64_t maxLines)
{
    filter_istream stream(filename);
    return forEachLine(stream, processLine, numThreads,
                       ignoreStreamExceptions, maxLines);
}

size_t
forEachLineStr(const std::string & filename,
               const std::function<void (const std::string &, int64_t)> & processLine,
               int numThreads,
               bool ignoreStreamExceptions,
               int64_t maxLines)
{
    filter_istream stream(filename);
    return forEachLineStr(stream, processLine, numThreads,
                          ignoreStreamExceptions, maxLines);
}

/*****************************************************************************/
/* FOR EACH LINE BLOCK                                                       */
/*****************************************************************************/

void forEachLineBlock(std::istream & stream,
                      std::function<bool (const char * line,
                                          size_t lineLength,
                                          int64_t blockNumber,
                                          int64_t lineNumber)> onLine,
                      int64_t maxLines,
                      int maxParallelism,
                      std::function<bool (int64_t blockNumber, int64_t lineNumber)> startBlock,
                      std::function<bool (int64_t blockNumber, int64_t lineNumber)> endBlock)
{
    //static constexpr int64_t BLOCK_SIZE = 100000000;  // 100MB blocks
    static constexpr int64_t BLOCK_SIZE = 20000000;  // 20MB blocks
    static constexpr int64_t READ_SIZE = 200000;  // read&scan 200kb to fit in cache

    std::atomic<int64_t> doneLines(0); //number of lines processed but not yet returned
    std::atomic<int64_t> returnedLines(0); //number of lines returned
    std::atomic<int64_t> byteOffset(0);
    std::atomic<int> chunkNumber(0);

    ThreadPool tp(maxParallelism);

    // Memory map if possible
    const char * mapped = nullptr;
    size_t mappedSize = 0;

    filter_istream * fistream = dynamic_cast<filter_istream *>(&stream);

    if (fistream) {
        // Can we get a memory mapped version of our stream?  It
        // saves us having to copy data.  mapped will be set to
        // nullptr if it's not possible to memory map this stream.
        std::tie(mapped, mappedSize) = fistream->mapped();
    }

    std::atomic<int> hasExc(false);
    std::exception_ptr exc;

    std::function<void ()> doBlock = [&] ()
        {
            //cerr << "block starting at line " << doneLines << endl;

            std::shared_ptr<const char> blockOut;

            int64_t startOffset = byteOffset;
            int64_t startLine = doneLines;
            vector<size_t> lineOffsets = {0};
            bool lastBlock = false;
            size_t myChunkNumber = 0;

            try {
                //MLDB-1426
                if (mapped && false) {
                    const char * start = mapped + stream.tellg();
                    const char * current = start;
                    const char * end = mapped + mappedSize;

                    while (current && current < end && (current - start) < BLOCK_SIZE
                           && (maxLines == -1 || doneLines < maxLines)) { //stop processing new line when we have enough)
                        current = (const char *)memchr(current, '\n', end - current);
                        if (current && current < end) {
                            ExcAssertEqual(*current, '\n');
                            lineOffsets.push_back(current - start);
                            ++doneLines;
                            ++current;
                        }
                    }

                    if (current)
                        stream.seekg(current - start, ios::cur);
                    else {
                        // Last line has no newline
                        lineOffsets.push_back(end - start);
                        ++doneLines;
                    }

                    myChunkNumber = chunkNumber++;

                    if (current && current < end &&
                        (maxLines == -1 || doneLines < maxLines)) // don't schedule a new block if we have enough lines
                        {
                            // Ready for another chunk
                            tp.add(doBlock);
                        } else if (current == end) {
                        lastBlock = true;
                    }

                    blockOut = std::shared_ptr<const char>(start,
                                                           [] (const char *) {});
                }
                else {
                    // How far through our block are we?
                    size_t offset = 0;

                    // How much extra space to allocate for the last line?
                    static constexpr size_t EXTRA_SIZE = 10000;

                    std::shared_ptr<char> block(new char[BLOCK_SIZE + EXTRA_SIZE],
                                                [] (char * c) { delete[] c; });
                    blockOut = block;

                    // First line starts at offset 0

                    while (stream && !stream.eof()
                           && (maxLines == -1 || doneLines < maxLines)  //stop processing new line when we have enough
                           && (byteOffset - startOffset < BLOCK_SIZE)) {

                        stream.read((char *)block.get() + offset,
                                    std::min<size_t>(READ_SIZE, BLOCK_SIZE - offset));

                        // Check how many bytes we actually read
                        size_t bytesRead = stream.gcount();

                        offset += bytesRead;

                        // Scan for end of line characters
                        const char * current = block.get() + lineOffsets.back();
                        const char * end = block.get() + offset;

                        while (current && current < end) {
                            current = (const char *)memchr(current, '\n', end - current);
                            if (current && current < end) {
                                ExcAssertEqual(*current, '\n');
                                if (lineOffsets.back() != current - block.get()) {
                                    lineOffsets.push_back(current - block.get());
                                    //cerr << "got line at offset " << lineOffsets.back() << endl;
                                    ++doneLines;
                                }
                                ++current;
                            }
                        }

                        byteOffset += bytesRead;
                    }

                    if (stream.eof()) {
                        // If we are at the end of the stream
                        // make sure we include the last line
                        // if there was no newline
                        if (lineOffsets.back() != offset - 1) {
                            lineOffsets.push_back(offset);
                            ++doneLines;
                        }
                    }
                    else {
                        // If we are not at the end of the stream
                        // get the last line, as we probably got just a partial
                        // line in the last one
                        std::string lastLine;
                        getline(stream, lastLine);

                        size_t cnt = stream.gcount();

                        if (cnt != 0) {
                            // Check for overflow on the buffer size
                            if (offset + lastLine.size() + 1 > BLOCK_SIZE + EXTRA_SIZE) {
                                // reallocate and copy
                                std::shared_ptr<char> newBlock(new char[offset + lastLine.size() + 1],
                                                               [] (char * c) { delete[] c; });
                                std::copy(block.get(), block.get() + offset,
                                          newBlock.get());
                                block = newBlock;
                                blockOut = block;
                            }

                            std::copy(lastLine.data(), lastLine.data() + lastLine.length(),
                                      block.get() + offset);

                            lineOffsets.emplace_back(offset + lastLine.length());
                            ++doneLines;
                            offset += cnt;
                        }
                    }

                    myChunkNumber = chunkNumber++;

                    if (stream && !stream.eof() &&
                        (maxLines == -1 || doneLines < maxLines)) // don't schedule a new block if we have enough lines
                        {
                            // Ready for another chunk
                            tp.add(doBlock);
                        } else if (stream.eof()) {
                        lastBlock = true;
                    }
                }

                //cerr << "processing block of " << lineOffsets.size() - 1
                //     << " lines starting at " << startLine << endl;

                int64_t chunkLineNumber = startLine;
                size_t lastLineOffset = lineOffsets[0];

                if (startBlock)
                    if (!startBlock(myChunkNumber, chunkLineNumber))
                        return;

                for (unsigned i = 1;  i < lineOffsets.size() && (maxLines == -1 || returnedLines++ < maxLines);  ++i) {
                    if (hasExc.load(std::memory_order_relaxed))
                        return;
                    const char * line = blockOut.get() + lastLineOffset;
                    size_t len = lineOffsets[i] - lastLineOffset;

                    // Skip \r for DOS line endings
                    if (len > 0 && line[len - 1] == '\r')
                        --len;

                    // if we are not at the last line
                    if (!lastBlock || len != 0 || i != lineOffsets.size() - 1)
                        if (!onLine(line, len, chunkNumber, chunkLineNumber++))
                            return;

                    lastLineOffset = lineOffsets[i] + 1;

                }

                if (endBlock)
                    if (!endBlock(myChunkNumber, chunkLineNumber))
                        return;

            } JML_CATCH_ALL {
                if (hasExc.fetch_add(1) == 0) {
                    exc = std::current_exception();
                }
            }
        };

    tp.add(doBlock);
    tp.waitForAll();

    // If there was an exception, rethrow it rather than returning
    // cleanly
    if (hasExc) {
        std::rethrow_exception(exc);
    }
}

} // namespace Datacratic
<|endoftext|>