<commit_before>//===- unittests/DirectoryWatcher/DirectoryWatcherTest.cpp ----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "clang/DirectoryWatcher/DirectoryWatcher.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Mutex.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
#include "gtest/gtest.h"
#include <condition_variable>
#include <future>
#include <mutex>
#include <thread>

using namespace llvm;
using namespace llvm::sys;
using namespace llvm::sys::fs;
using namespace clang;

namespace clang {
static bool operator==(const DirectoryWatcher::Event &lhs,
                       const DirectoryWatcher::Event &rhs) {
  return lhs.Filename == rhs.Filename &&
         static_cast<int>(lhs.Kind) == static_cast<int>(rhs.Kind);
}
} // namespace clang

namespace {

typedef DirectoryWatcher::Event::EventKind EventKind;

struct DirectoryWatcherTestFixture {
  std::string TestRootDir;
  std::string TestWatchedDir;

  DirectoryWatcherTestFixture() {
    SmallString<128> pathBuf;
#ifndef NDEBUG
    std::error_code UniqDirRes =
#endif
    createUniqueDirectory("dirwatcher", pathBuf);
    assert(!UniqDirRes);
    TestRootDir = pathBuf.str();
    path::append(pathBuf, "watch");
    TestWatchedDir = pathBuf.str();
#ifndef NDEBUG
    std::error_code CreateDirRes =
#endif
    create_directory(TestWatchedDir, false);
    assert(!CreateDirRes);
  }

  ~DirectoryWatcherTestFixture() { remove_directories(TestRootDir); }

  SmallString<128> getPathInWatched(const std::string &testFile) {
    SmallString<128> pathBuf;
    pathBuf = TestWatchedDir;
    path::append(pathBuf, testFile);
    return pathBuf;
  }

  void addFile(const std::string &testFile) {
    Expected<file_t> ft = openNativeFileForWrite(getPathInWatched(testFile),
                                                 CD_CreateNew, OF_None);
    if (ft) {
      closeFile(*ft);
    } else {
      llvm::errs() << llvm::toString(ft.takeError()) << "\n";
      llvm::errs() << getPathInWatched(testFile) << "\n";
      llvm_unreachable("Couldn't create test file.");
    }
  }

  void deleteFile(const std::string &testFile) {
    std::error_code EC =
        remove(getPathInWatched(testFile), /*IgnoreNonExisting=*/false);
    ASSERT_FALSE(EC);
  }
};

std::string eventKindToString(const EventKind K) {
  switch (K) {
  case EventKind::Removed:
    return "Removed";
  case EventKind::Modified:
    return "Modified";
  case EventKind::WatchedDirRemoved:
    return "WatchedDirRemoved";
  case EventKind::WatcherGotInvalidated:
    return "WatcherGotInvalidated";
  }
  llvm_unreachable("unknown event kind");
}

struct VerifyingConsumer {
  std::vector<DirectoryWatcher::Event> ExpectedInitial;
  const std::vector<DirectoryWatcher::Event> ExpectedInitialCopy;
  std::vector<DirectoryWatcher::Event> ExpectedNonInitial;
  const std::vector<DirectoryWatcher::Event> ExpectedNonInitialCopy;
  std::vector<DirectoryWatcher::Event> OptionalNonInitial;
  std::vector<DirectoryWatcher::Event> UnexpectedInitial;
  std::vector<DirectoryWatcher::Event> UnexpectedNonInitial;
  std::mutex Mtx;
  std::condition_variable ResultIsReady;

  VerifyingConsumer(
      const std::vector<DirectoryWatcher::Event> &ExpectedInitial,
      const std::vector<DirectoryWatcher::Event> &ExpectedNonInitial,
      const std::vector<DirectoryWatcher::Event> &OptionalNonInitial = {})
      : ExpectedInitial(ExpectedInitial), ExpectedInitialCopy(ExpectedInitial),
        ExpectedNonInitial(ExpectedNonInitial), ExpectedNonInitialCopy(ExpectedNonInitial),
        OptionalNonInitial(OptionalNonInitial) {}

  // This method is used by DirectoryWatcher.
  void consume(DirectoryWatcher::Event E, bool IsInitial) {
    if (IsInitial)
      consumeInitial(E);
    else
      consumeNonInitial(E);
  }

  void consumeInitial(DirectoryWatcher::Event E) {
    std::unique_lock<std::mutex> L(Mtx);
    auto It = std::find(ExpectedInitial.begin(), ExpectedInitial.end(), E);
    if (It == ExpectedInitial.end()) {
      UnexpectedInitial.push_back(E);
    } else {
      ExpectedInitial.erase(It);
    }
    if (result())
      ResultIsReady.notify_one();
  }

  void consumeNonInitial(DirectoryWatcher::Event E) {
    std::unique_lock<std::mutex> L(Mtx);
    auto It =
        std::find(ExpectedNonInitial.begin(), ExpectedNonInitial.end(), E);
    if (It == ExpectedNonInitial.end()) {
      auto OptIt =
          std::find(OptionalNonInitial.begin(), OptionalNonInitial.end(), E);
      if (OptIt != OptionalNonInitial.end()) {
        OptionalNonInitial.erase(OptIt);
      } else {
        UnexpectedNonInitial.push_back(E);
      }
    } else {
      ExpectedNonInitial.erase(It);
    }
    if (result())
      ResultIsReady.notify_one();
  }

  // This method is used by DirectoryWatcher.
  void consume(llvm::ArrayRef<DirectoryWatcher::Event> Es, bool IsInitial) {
    for (const auto &E : Es)
      consume(E, IsInitial);
  }

  // Not locking - caller has to lock Mtx.
  llvm::Optional<bool> result() const {
    if (ExpectedInitial.empty() && ExpectedNonInitial.empty() &&
        UnexpectedInitial.empty() && UnexpectedNonInitial.empty())
      return true;
    if (!UnexpectedInitial.empty() || !UnexpectedNonInitial.empty())
      return false;
    return llvm::None;
  }

  // This method is used by tests.
  // \returns true on success
  bool blockUntilResult() {
    std::unique_lock<std::mutex> L(Mtx);
    while (true) {
      if (result())
        return *result();

      ResultIsReady.wait(L, [this]() { return result().hasValue(); });
    }
    return false; // Just to make compiler happy.
  }

  void printUnmetExpectations(llvm::raw_ostream &OS) {
    // If there was any issue, print the expected state
    if (
      !ExpectedInitial.empty()
      ||
      !ExpectedNonInitial.empty()
      ||
      !UnexpectedInitial.empty()
      ||
      !UnexpectedNonInitial.empty()
    ) {
      OS << "Expected initial events: \n";
      for (const auto &E : ExpectedInitialCopy) {
        OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
      }
      OS << "Expected non-initial events: \n";
      for (const auto &E : ExpectedNonInitialCopy) {
        OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
      }
    }

    if (!ExpectedInitial.empty()) {
      OS << "Expected but not seen initial events: \n";
      for (const auto &E : ExpectedInitial) {
        OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
      }
    }
    if (!ExpectedNonInitial.empty()) {
      OS << "Expected but not seen non-initial events: \n";
      for (const auto &E : ExpectedNonInitial) {
        OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
      }
    }
    if (!UnexpectedInitial.empty()) {
      OS << "Unexpected initial events seen: \n";
      for (const auto &E : UnexpectedInitial) {
        OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
      }
    }
    if (!UnexpectedNonInitial.empty()) {
      OS << "Unexpected non-initial events seen: \n";
      for (const auto &E : UnexpectedNonInitial) {
        OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
      }
    }
  }
};

void checkEventualResultWithTimeout(VerifyingConsumer &TestConsumer) {
  std::packaged_task<int(void)> task(
      [&TestConsumer]() { return TestConsumer.blockUntilResult(); });
  std::future<int> WaitForExpectedStateResult = task.get_future();
  std::thread worker(std::move(task));
  worker.detach();

  EXPECT_TRUE(WaitForExpectedStateResult.wait_for(std::chrono::seconds(3)) ==
              std::future_status::ready)
      << "The expected result state wasn't reached before the time-out.";
  std::unique_lock<std::mutex> L(TestConsumer.Mtx);
  EXPECT_TRUE(TestConsumer.result().hasValue());
  if (TestConsumer.result().hasValue()) {
    EXPECT_TRUE(*TestConsumer.result());
  }
  if ((TestConsumer.result().hasValue() && !TestConsumer.result().getValue()) ||
      !TestConsumer.result().hasValue())
    TestConsumer.printUnmetExpectations(llvm::outs());
}
} // namespace

TEST(DirectoryWatcherTest, InitialScanSync) {
  DirectoryWatcherTestFixture fixture;

  fixture.addFile("a");
  fixture.addFile("b");
  fixture.addFile("c");

  VerifyingConsumer TestConsumer{
      {{EventKind::Modified, "a"},
       {EventKind::Modified, "b"},
       {EventKind::Modified, "c"}},
      {},
      // We have to ignore these as it's a race between the test process
      // which is scanning the directory and kernel which is sending
      // notification.
      {{EventKind::Modified, "a"},
       {EventKind::Modified, "b"},
       {EventKind::Modified, "c"}}
      };

  auto DW = DirectoryWatcher::create(
      fixture.TestWatchedDir,
      [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
                      bool IsInitial) {
        TestConsumer.consume(Events, IsInitial);
      },
      /*waitForInitialSync=*/true);

  checkEventualResultWithTimeout(TestConsumer);
}

TEST(DirectoryWatcherTest, InitialScanAsync) {
  DirectoryWatcherTestFixture fixture;

  fixture.addFile("a");
  fixture.addFile("b");
  fixture.addFile("c");

  VerifyingConsumer TestConsumer{
      {{EventKind::Modified, "a"},
       {EventKind::Modified, "b"},
       {EventKind::Modified, "c"}},
      {},
      // We have to ignore these as it's a race between the test process
      // which is scanning the directory and kernel which is sending
      // notification.
      {{EventKind::Modified, "a"},
       {EventKind::Modified, "b"},
       {EventKind::Modified, "c"}}
       };

  auto DW = DirectoryWatcher::create(
      fixture.TestWatchedDir,
      [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
                      bool IsInitial) {
        TestConsumer.consume(Events, IsInitial);
      },
      /*waitForInitialSync=*/false);

  checkEventualResultWithTimeout(TestConsumer);
}

TEST(DirectoryWatcherTest, AddFiles) {
  DirectoryWatcherTestFixture fixture;

  VerifyingConsumer TestConsumer{
      {},
      {{EventKind::Modified, "a"},
       {EventKind::Modified, "b"},
       {EventKind::Modified, "c"}}};

  auto DW = DirectoryWatcher::create(
      fixture.TestWatchedDir,
      [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
                      bool IsInitial) {
        TestConsumer.consume(Events, IsInitial);
      },
      /*waitForInitialSync=*/true);

  fixture.addFile("a");
  fixture.addFile("b");
  fixture.addFile("c");

  checkEventualResultWithTimeout(TestConsumer);
}

TEST(DirectoryWatcherTest, ModifyFile) {
  DirectoryWatcherTestFixture fixture;

  fixture.addFile("a");

  VerifyingConsumer TestConsumer{
      {{EventKind::Modified, "a"}},
      {{EventKind::Modified, "a"}},
      {{EventKind::Modified, "a"}}};

  auto DW = DirectoryWatcher::create(
      fixture.TestWatchedDir,
      [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
                      bool IsInitial) {
        TestConsumer.consume(Events, IsInitial);
      },
      /*waitForInitialSync=*/true);

  // modify the file
  {
    std::error_code error;
    llvm::raw_fd_ostream bStream(fixture.getPathInWatched("a"), error,
                                 CD_OpenExisting);
    assert(!error);
    bStream << "foo";
  }

  checkEventualResultWithTimeout(TestConsumer);
}

TEST(DirectoryWatcherTest, DeleteFile) {
  DirectoryWatcherTestFixture fixture;

  fixture.addFile("a");

  VerifyingConsumer TestConsumer{
      {{EventKind::Modified, "a"}},
      {{EventKind::Removed, "a"}},
      {{EventKind::Modified, "a"}, {EventKind::Removed, "a"}}};

  auto DW = DirectoryWatcher::create(
      fixture.TestWatchedDir,
      [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
                      bool IsInitial) {
        TestConsumer.consume(Events, IsInitial);
      },
      /*waitForInitialSync=*/true);

  fixture.deleteFile("a");

  checkEventualResultWithTimeout(TestConsumer);
}

TEST(DirectoryWatcherTest, DeleteWatchedDir) {
  DirectoryWatcherTestFixture fixture;

  VerifyingConsumer TestConsumer{
      {},
      {{EventKind::WatchedDirRemoved, ""},
       {EventKind::WatcherGotInvalidated, ""}}};

  auto DW = DirectoryWatcher::create(
      fixture.TestWatchedDir,
      [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
                      bool IsInitial) {
        TestConsumer.consume(Events, IsInitial);
      },
      /*waitForInitialSync=*/true);

  remove_directories(fixture.TestWatchedDir);

  checkEventualResultWithTimeout(TestConsumer);
}

TEST(DirectoryWatcherTest, InvalidatedWatcher) {
  DirectoryWatcherTestFixture fixture;

  VerifyingConsumer TestConsumer{
      {}, {{EventKind::WatcherGotInvalidated, ""}}};

  {
    auto DW = DirectoryWatcher::create(
        fixture.TestWatchedDir,
        [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
                        bool IsInitial) {
          TestConsumer.consume(Events, IsInitial);
        },
        /*waitForInitialSync=*/true);
  } // DW is destructed here.

  checkEventualResultWithTimeout(TestConsumer);
}<commit_msg>[NFC][DirectoryWatchedTests] Unlocks mutexes before signaling condition variable<commit_after>//===- unittests/DirectoryWatcher/DirectoryWatcherTest.cpp ----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "clang/DirectoryWatcher/DirectoryWatcher.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Mutex.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
#include "gtest/gtest.h"
#include <condition_variable>
#include <future>
#include <mutex>
#include <thread>

using namespace llvm;
using namespace llvm::sys;
using namespace llvm::sys::fs;
using namespace clang;

namespace clang {
static bool operator==(const DirectoryWatcher::Event &lhs,
                       const DirectoryWatcher::Event &rhs) {
  return lhs.Filename == rhs.Filename &&
         static_cast<int>(lhs.Kind) == static_cast<int>(rhs.Kind);
}
} // namespace clang

namespace {

typedef DirectoryWatcher::Event::EventKind EventKind;

struct DirectoryWatcherTestFixture {
  std::string TestRootDir;
  std::string TestWatchedDir;

  DirectoryWatcherTestFixture() {
    SmallString<128> pathBuf;
#ifndef NDEBUG
    std::error_code UniqDirRes =
#endif
    createUniqueDirectory("dirwatcher", pathBuf);
    assert(!UniqDirRes);
    TestRootDir = pathBuf.str();
    path::append(pathBuf, "watch");
    TestWatchedDir = pathBuf.str();
#ifndef NDEBUG
    std::error_code CreateDirRes =
#endif
    create_directory(TestWatchedDir, false);
    assert(!CreateDirRes);
  }

  ~DirectoryWatcherTestFixture() { remove_directories(TestRootDir); }

  SmallString<128> getPathInWatched(const std::string &testFile) {
    SmallString<128> pathBuf;
    pathBuf = TestWatchedDir;
    path::append(pathBuf, testFile);
    return pathBuf;
  }

  void addFile(const std::string &testFile) {
    Expected<file_t> ft = openNativeFileForWrite(getPathInWatched(testFile),
                                                 CD_CreateNew, OF_None);
    if (ft) {
      closeFile(*ft);
    } else {
      llvm::errs() << llvm::toString(ft.takeError()) << "\n";
      llvm::errs() << getPathInWatched(testFile) << "\n";
      llvm_unreachable("Couldn't create test file.");
    }
  }

  void deleteFile(const std::string &testFile) {
    std::error_code EC =
        remove(getPathInWatched(testFile), /*IgnoreNonExisting=*/false);
    ASSERT_FALSE(EC);
  }
};

std::string eventKindToString(const EventKind K) {
  switch (K) {
  case EventKind::Removed:
    return "Removed";
  case EventKind::Modified:
    return "Modified";
  case EventKind::WatchedDirRemoved:
    return "WatchedDirRemoved";
  case EventKind::WatcherGotInvalidated:
    return "WatcherGotInvalidated";
  }
  llvm_unreachable("unknown event kind");
}

struct VerifyingConsumer {
  std::vector<DirectoryWatcher::Event> ExpectedInitial;
  const std::vector<DirectoryWatcher::Event> ExpectedInitialCopy;
  std::vector<DirectoryWatcher::Event> ExpectedNonInitial;
  const std::vector<DirectoryWatcher::Event> ExpectedNonInitialCopy;
  std::vector<DirectoryWatcher::Event> OptionalNonInitial;
  std::vector<DirectoryWatcher::Event> UnexpectedInitial;
  std::vector<DirectoryWatcher::Event> UnexpectedNonInitial;
  std::mutex Mtx;
  std::condition_variable ResultIsReady;

  VerifyingConsumer(
      const std::vector<DirectoryWatcher::Event> &ExpectedInitial,
      const std::vector<DirectoryWatcher::Event> &ExpectedNonInitial,
      const std::vector<DirectoryWatcher::Event> &OptionalNonInitial = {})
      : ExpectedInitial(ExpectedInitial), ExpectedInitialCopy(ExpectedInitial),
        ExpectedNonInitial(ExpectedNonInitial), ExpectedNonInitialCopy(ExpectedNonInitial),
        OptionalNonInitial(OptionalNonInitial) {}

  // This method is used by DirectoryWatcher.
  void consume(DirectoryWatcher::Event E, bool IsInitial) {
    if (IsInitial)
      consumeInitial(E);
    else
      consumeNonInitial(E);
  }

  void consumeInitial(DirectoryWatcher::Event E) {
    std::unique_lock<std::mutex> L(Mtx);
    auto It = std::find(ExpectedInitial.begin(), ExpectedInitial.end(), E);
    if (It == ExpectedInitial.end()) {
      UnexpectedInitial.push_back(E);
    } else {
      ExpectedInitial.erase(It);
    }
    if (result()) {
      L.unlock();
      ResultIsReady.notify_one();
    }
  }

  void consumeNonInitial(DirectoryWatcher::Event E) {
    std::unique_lock<std::mutex> L(Mtx);
    auto It =
        std::find(ExpectedNonInitial.begin(), ExpectedNonInitial.end(), E);
    if (It == ExpectedNonInitial.end()) {
      auto OptIt =
          std::find(OptionalNonInitial.begin(), OptionalNonInitial.end(), E);
      if (OptIt != OptionalNonInitial.end()) {
        OptionalNonInitial.erase(OptIt);
      } else {
        UnexpectedNonInitial.push_back(E);
      }
    } else {
      ExpectedNonInitial.erase(It);
    }
    if (result()) {
      L.unlock();
      ResultIsReady.notify_one();
    }
  }

  // This method is used by DirectoryWatcher.
  void consume(llvm::ArrayRef<DirectoryWatcher::Event> Es, bool IsInitial) {
    for (const auto &E : Es)
      consume(E, IsInitial);
  }

  // Not locking - caller has to lock Mtx.
  llvm::Optional<bool> result() const {
    if (ExpectedInitial.empty() && ExpectedNonInitial.empty() &&
        UnexpectedInitial.empty() && UnexpectedNonInitial.empty())
      return true;
    if (!UnexpectedInitial.empty() || !UnexpectedNonInitial.empty())
      return false;
    return llvm::None;
  }

  // This method is used by tests.
  // \returns true on success
  bool blockUntilResult() {
    std::unique_lock<std::mutex> L(Mtx);
    while (true) {
      if (result())
        return *result();

      ResultIsReady.wait(L, [this]() { return result().hasValue(); });
    }
    return false; // Just to make compiler happy.
  }

  void printUnmetExpectations(llvm::raw_ostream &OS) {
    // If there was any issue, print the expected state
    if (
      !ExpectedInitial.empty()
      ||
      !ExpectedNonInitial.empty()
      ||
      !UnexpectedInitial.empty()
      ||
      !UnexpectedNonInitial.empty()
    ) {
      OS << "Expected initial events: \n";
      for (const auto &E : ExpectedInitialCopy) {
        OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
      }
      OS << "Expected non-initial events: \n";
      for (const auto &E : ExpectedNonInitialCopy) {
        OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
      }
    }

    if (!ExpectedInitial.empty()) {
      OS << "Expected but not seen initial events: \n";
      for (const auto &E : ExpectedInitial) {
        OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
      }
    }
    if (!ExpectedNonInitial.empty()) {
      OS << "Expected but not seen non-initial events: \n";
      for (const auto &E : ExpectedNonInitial) {
        OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
      }
    }
    if (!UnexpectedInitial.empty()) {
      OS << "Unexpected initial events seen: \n";
      for (const auto &E : UnexpectedInitial) {
        OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
      }
    }
    if (!UnexpectedNonInitial.empty()) {
      OS << "Unexpected non-initial events seen: \n";
      for (const auto &E : UnexpectedNonInitial) {
        OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
      }
    }
  }
};

void checkEventualResultWithTimeout(VerifyingConsumer &TestConsumer) {
  std::packaged_task<int(void)> task(
      [&TestConsumer]() { return TestConsumer.blockUntilResult(); });
  std::future<int> WaitForExpectedStateResult = task.get_future();
  std::thread worker(std::move(task));
  worker.detach();

  EXPECT_TRUE(WaitForExpectedStateResult.wait_for(std::chrono::seconds(3)) ==
              std::future_status::ready)
      << "The expected result state wasn't reached before the time-out.";
  std::unique_lock<std::mutex> L(TestConsumer.Mtx);
  EXPECT_TRUE(TestConsumer.result().hasValue());
  if (TestConsumer.result().hasValue()) {
    EXPECT_TRUE(*TestConsumer.result());
  }
  if ((TestConsumer.result().hasValue() && !TestConsumer.result().getValue()) ||
      !TestConsumer.result().hasValue())
    TestConsumer.printUnmetExpectations(llvm::outs());
}
} // namespace

TEST(DirectoryWatcherTest, InitialScanSync) {
  DirectoryWatcherTestFixture fixture;

  fixture.addFile("a");
  fixture.addFile("b");
  fixture.addFile("c");

  VerifyingConsumer TestConsumer{
      {{EventKind::Modified, "a"},
       {EventKind::Modified, "b"},
       {EventKind::Modified, "c"}},
      {},
      // We have to ignore these as it's a race between the test process
      // which is scanning the directory and kernel which is sending
      // notification.
      {{EventKind::Modified, "a"},
       {EventKind::Modified, "b"},
       {EventKind::Modified, "c"}}
      };

  auto DW = DirectoryWatcher::create(
      fixture.TestWatchedDir,
      [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
                      bool IsInitial) {
        TestConsumer.consume(Events, IsInitial);
      },
      /*waitForInitialSync=*/true);

  checkEventualResultWithTimeout(TestConsumer);
}

TEST(DirectoryWatcherTest, InitialScanAsync) {
  DirectoryWatcherTestFixture fixture;

  fixture.addFile("a");
  fixture.addFile("b");
  fixture.addFile("c");

  VerifyingConsumer TestConsumer{
      {{EventKind::Modified, "a"},
       {EventKind::Modified, "b"},
       {EventKind::Modified, "c"}},
      {},
      // We have to ignore these as it's a race between the test process
      // which is scanning the directory and kernel which is sending
      // notification.
      {{EventKind::Modified, "a"},
       {EventKind::Modified, "b"},
       {EventKind::Modified, "c"}}
       };

  auto DW = DirectoryWatcher::create(
      fixture.TestWatchedDir,
      [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
                      bool IsInitial) {
        TestConsumer.consume(Events, IsInitial);
      },
      /*waitForInitialSync=*/false);

  checkEventualResultWithTimeout(TestConsumer);
}

TEST(DirectoryWatcherTest, AddFiles) {
  DirectoryWatcherTestFixture fixture;

  VerifyingConsumer TestConsumer{
      {},
      {{EventKind::Modified, "a"},
       {EventKind::Modified, "b"},
       {EventKind::Modified, "c"}}};

  auto DW = DirectoryWatcher::create(
      fixture.TestWatchedDir,
      [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
                      bool IsInitial) {
        TestConsumer.consume(Events, IsInitial);
      },
      /*waitForInitialSync=*/true);

  fixture.addFile("a");
  fixture.addFile("b");
  fixture.addFile("c");

  checkEventualResultWithTimeout(TestConsumer);
}

TEST(DirectoryWatcherTest, ModifyFile) {
  DirectoryWatcherTestFixture fixture;

  fixture.addFile("a");

  VerifyingConsumer TestConsumer{
      {{EventKind::Modified, "a"}},
      {{EventKind::Modified, "a"}},
      {{EventKind::Modified, "a"}}};

  auto DW = DirectoryWatcher::create(
      fixture.TestWatchedDir,
      [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
                      bool IsInitial) {
        TestConsumer.consume(Events, IsInitial);
      },
      /*waitForInitialSync=*/true);

  // modify the file
  {
    std::error_code error;
    llvm::raw_fd_ostream bStream(fixture.getPathInWatched("a"), error,
                                 CD_OpenExisting);
    assert(!error);
    bStream << "foo";
  }

  checkEventualResultWithTimeout(TestConsumer);
}

TEST(DirectoryWatcherTest, DeleteFile) {
  DirectoryWatcherTestFixture fixture;

  fixture.addFile("a");

  VerifyingConsumer TestConsumer{
      {{EventKind::Modified, "a"}},
      {{EventKind::Removed, "a"}},
      {{EventKind::Modified, "a"}, {EventKind::Removed, "a"}}};

  auto DW = DirectoryWatcher::create(
      fixture.TestWatchedDir,
      [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
                      bool IsInitial) {
        TestConsumer.consume(Events, IsInitial);
      },
      /*waitForInitialSync=*/true);

  fixture.deleteFile("a");

  checkEventualResultWithTimeout(TestConsumer);
}

TEST(DirectoryWatcherTest, DeleteWatchedDir) {
  DirectoryWatcherTestFixture fixture;

  VerifyingConsumer TestConsumer{
      {},
      {{EventKind::WatchedDirRemoved, ""},
       {EventKind::WatcherGotInvalidated, ""}}};

  auto DW = DirectoryWatcher::create(
      fixture.TestWatchedDir,
      [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
                      bool IsInitial) {
        TestConsumer.consume(Events, IsInitial);
      },
      /*waitForInitialSync=*/true);

  remove_directories(fixture.TestWatchedDir);

  checkEventualResultWithTimeout(TestConsumer);
}

TEST(DirectoryWatcherTest, InvalidatedWatcher) {
  DirectoryWatcherTestFixture fixture;

  VerifyingConsumer TestConsumer{
      {}, {{EventKind::WatcherGotInvalidated, ""}}};

  {
    auto DW = DirectoryWatcher::create(
        fixture.TestWatchedDir,
        [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
                        bool IsInitial) {
          TestConsumer.consume(Events, IsInitial);
        },
        /*waitForInitialSync=*/true);
  } // DW is destructed here.

  checkEventualResultWithTimeout(TestConsumer);
}<|endoftext|>


<commit_before>#ifdef USE_ALLEGRO
#include <allegro.h>
#endif
#ifdef USE_SDL
#include <SDL.h>
#endif
#ifdef USE_ALLEGRO5
#include <allegro5/allegro.h>
#endif
#include "funcs.h"
#include "globals.h"
#include "debug.h"
#include <vector>
#include <string>
#include <string.h>
#include <algorithm>
#include "file-system.h"
#include "bitmap.h"
#include "trans-bitmap.h"
#include "font.h"
#include "init.h"
#include <sstream>
#include "file-system.h"

#ifndef USE_ALLEGRO
/* FIXME: move this to the filesystem module */
#include "sfl/sfl.h"
#include "sfl/sflfile.h"
#endif

#ifndef WINDOWS
#include <unistd.h>
#endif

#ifdef max
#undef max
#endif

#ifdef min
#undef min
#endif

using namespace std;

/* remove this once cmake and scons properly set DATA_PATH */
#ifndef DATA_PATH
#define DATA_PATH "data"
#endif

/* the build system should define DATA_PATH */
static string dataPath = DATA_PATH;

const double Util::pi = 3.14159265358979323846;

double Util::radians(double degree){
    return degree * pi / 180.0;
}

/*
inline int rnd( int q ){
	if ( q <= 0 ) return 0;
	return (int)( rand() % q );
}
*/

int Util::max(int a, int b){
    if (a>b){
        return a;
    }
    return b;
}

double Util::min(double a, double b){
    if (a < b){
        return a;
    }
    return b;
}

double Util::max(double a, double b){
    if (a > b){
        return a;
    }
    return b;
}

/*
int Util::min(int a, int b){
    if (a<b){
        return a;
    }
    return b;
}
*/

int Util::clamp(int value, int min, int max){
    return Util::min(Util::max(value, min), max);
}

double Util::clamp(double value, double min, double max){
    return Util::min(Util::max(value, min), max);
}

int Util::rnd( int q, int min, int range ){
	return q - min + rnd( range );
}

int Util::rnd( int min, int max ){
	return rnd( max - min ) + min;
}

/* sleep for `x' milliseconds */
void Util::rest( int x ){
#ifdef USE_ALLEGRO
    ::rest(x);
#endif
#ifdef USE_SDL
    SDL_Delay(x);
#endif
#ifdef USE_ALLEGRO5
    al_rest((double) x / 1000.0);
#endif
}

void Util::restSeconds(double x){
    Util::rest((int)(x * 1000));
}

bool Util::checkVersion(int version){
    if (version == Global::getVersion()){
        return true;
    }

    /* when an incompatible version is made, add a check here, like
     *  version < getVersion(3, 5)
     * would mean any client below version 3.5 is incompatible.
     *
     * assume versions of client's greater than ourself is compatible, but
     * this may not be true. There is no way to check this.
     */
    if (version < 0){
        return false;
    }

    return true;
}

void Util::setDataPath( const string & str ){
    dataPath = str;
}

Filesystem::AbsolutePath Util::getDataPath2(){
    return Filesystem::AbsolutePath(dataPath + "/");
}

/* FIXME: remove this method */
bool Util::exists( const string & file ){
    return Storage::instance().exists(Filesystem::AbsolutePath(file));
/*
#ifdef USE_ALLEGRO
    return ::exists(file.c_str()) != 0;
#else
    return file_exists(file.c_str());
#endif
*/
}

/*
vector<Filesystem::AbsolutePath> Util::getFiles(const Filesystem::AbsolutePath & dataPath, const string & find){
    return Filesystem::getFiles(dataPath, find);
}
*/

string Util::trim(const std::string & str){
    string s;
    size_t startpos = str.find_first_not_of(" \t");
    size_t endpos = str.find_last_not_of(" \t");
    // if all spaces or empty return an empty string
    if ((string::npos == startpos) ||
        (string::npos == endpos)){
        return "";
    } else {
        return str.substr(startpos, endpos-startpos+1);
    }
    return str;
}

/* makes the first letter of a string upper case */
string Util::upcase(std::string str){
    if ( str.length() > 0 && (str[0] >= 'a' && str[0] <= 'z') ){
        str[0] = str[0] - 'a' + 'A';
    }
    return str;
}

static int lowerCase(int c){
    return tolower(c);
}

static int upperCase(int c){
    return toupper(c);
}

string Util::upperCaseAll(std::string str){
    std::transform(str.begin(), str.end(), str.begin(), upperCase);
    return str;
}

string Util::lowerCaseAll(std::string str){
    std::transform(str.begin(), str.end(), str.begin(), lowerCase);
    /*
    for (unsigned int i = 0; i < str.length(); i++){
        if (str[0] >= 'A' && str[0] <= 'Z'){
            str[0] = str[0] - 'A' + 'a';
        }
    }
    */
    return str;
}

/*Gets the minimum of three values*/
static int minimum(int a,int b,int c){
  int min=a;
  if(b<min)
    min=b;
  if(c<min)
    min=c;
  return min;
}

/* Compute levenshtein distance between s and t
 * from: http://www.merriampark.com/ldc.htm
 */
static int levenshtein_distance(const char *s, const char *t){
  //Step 1
  int k,i,j,n,m,cost,distance;
  int * d;
  n=strlen(s);
  m=strlen(t);
  if(n!=0&&m!=0){
    d = (int*) malloc((sizeof(int))*(m+1)*(n+1));
    m++;
    n++;
    //Step 2
    for(k=0;k<n;k++)
	d[k]=k;
    for(k=0;k<m;k++)
      d[k*n]=k;
    //Step 3 and 4
    for(i=1;i<n;i++)
      for(j=1;j<m;j++)
	{
        //Step 5
        if(s[i-1]==t[j-1])
          cost=0;
        else
          cost=1;
        //Step 6
        d[j*n+i]=minimum(d[(j-1)*n+i]+1,d[j*n+i-1]+1,d[(j-1)*n+i-1]+cost);
      }
    distance=d[n*m-1];
    free(d);
    return distance;
  } else {
    return -1; //a negative return value means that one or both strings are empty.
  }
}

int Util::levenshtein(const std::string & str1, const std::string & str2){
    return levenshtein_distance(str1.c_str(), str2.c_str());
}

void Util::limitPrintf(char * buffer, int size, const char * format, va_list args){
#ifdef USE_ALLEGRO
    uvszprintf(buffer, size, format, args);
#else
    vsnprintf(buffer, size, format, args);
#endif
}

#ifndef WINDOWS
int Util::getPipe(int files[2]){
#ifdef PS3
    return 0;
#else
    return pipe(files);
#endif
}
#endif

void Util::showError(const Graphics::Bitmap & screen, const Exception::Base & exception, const string & info){
    screen.BlitFromScreen(0, 0);
    Graphics::Bitmap error(screen.getWidth() - 100, screen.getHeight() - 100);
    error.fill(Graphics::darken(Graphics::makeColor(160, 0, 0), 3));
    error.border(1, 2, Graphics::makeColor(240, 0, 0));
    const Font & font = Font::getFont(Global::DEFAULT_FONT, 17, 17);
    int y = 10;
    std::ostringstream out;
    out << info;
    out << " " << exception.getTrace();
    Global::debug(0) << out.str() << std::endl;
    font.printfWrap(10, 10, Graphics::makeColor(240, 240, 240), error, error.getWidth() - 20, out.str(), 0);

    Graphics::Bitmap::transBlender(0, 0, 0, 220);
    error.translucent().draw(50, 50, screen);
    screen.BlitToScreen();
}

void Util::showError(const Exception::Base & exception, const std::string & info){
    Graphics::Bitmap screen(GFX_X, GFX_Y);
    showError(screen, exception, info);
}

static double twoDigits(double x){
    return (int) (x * 100) / 100.0;
}

string Util::niceSize(unsigned long size){
    const char sizes[] = {'b', 'k', 'm', 'g', 't'};
    double real = size;
    for (unsigned int i = 0; i < sizeof(sizes) / sizeof(const char); i++){
        if (real > 1024){
            real /= 1024.0;
        } else {
            ostringstream in;
            in << twoDigits(real) << sizes[i];
            return in.str();
        }
    }
    ostringstream in;
    in << twoDigits(real) << "t";
    return in.str();
}

std::string Util::join(const std::vector<std::string> & list, const std::string & middle){
    ostringstream out;

    bool first = true;
    for (std::vector<std::string>::const_iterator it = list.begin(); it != list.end(); it++){
        if (!first){
            out << middle;
        }

        out << *it;
        first = false;
    }

    return out.str();
}

double Util::distance(double x1, double y1, double x2, double y2){
    double xs = x1 - x2;
    double ys = y1 - y2;
    return sqrt(xs * xs + ys * ys);
}
<commit_msg>need math.h<commit_after>#ifdef USE_ALLEGRO
#include <allegro.h>
#endif
#ifdef USE_SDL
#include <SDL.h>
#endif
#ifdef USE_ALLEGRO5
#include <allegro5/allegro.h>
#endif
#include "funcs.h"
#include "globals.h"
#include "debug.h"
#include <vector>
#include <string>
#include <string.h>
#include <algorithm>
#include "file-system.h"
#include "bitmap.h"
#include "trans-bitmap.h"
#include "font.h"
#include "init.h"
#include <sstream>
#include "file-system.h"
#include <math.h>

#ifndef USE_ALLEGRO
/* FIXME: move this to the filesystem module */
#include "sfl/sfl.h"
#include "sfl/sflfile.h"
#endif

#ifndef WINDOWS
#include <unistd.h>
#endif

#ifdef max
#undef max
#endif

#ifdef min
#undef min
#endif

using namespace std;

/* remove this once cmake and scons properly set DATA_PATH */
#ifndef DATA_PATH
#define DATA_PATH "data"
#endif

/* the build system should define DATA_PATH */
static string dataPath = DATA_PATH;

const double Util::pi = 3.14159265358979323846;

double Util::radians(double degree){
    return degree * pi / 180.0;
}

/*
inline int rnd( int q ){
	if ( q <= 0 ) return 0;
	return (int)( rand() % q );
}
*/

int Util::max(int a, int b){
    if (a>b){
        return a;
    }
    return b;
}

double Util::min(double a, double b){
    if (a < b){
        return a;
    }
    return b;
}

double Util::max(double a, double b){
    if (a > b){
        return a;
    }
    return b;
}

/*
int Util::min(int a, int b){
    if (a<b){
        return a;
    }
    return b;
}
*/

int Util::clamp(int value, int min, int max){
    return Util::min(Util::max(value, min), max);
}

double Util::clamp(double value, double min, double max){
    return Util::min(Util::max(value, min), max);
}

int Util::rnd( int q, int min, int range ){
	return q - min + rnd( range );
}

int Util::rnd( int min, int max ){
	return rnd( max - min ) + min;
}

/* sleep for `x' milliseconds */
void Util::rest( int x ){
#ifdef USE_ALLEGRO
    ::rest(x);
#endif
#ifdef USE_SDL
    SDL_Delay(x);
#endif
#ifdef USE_ALLEGRO5
    al_rest((double) x / 1000.0);
#endif
}

void Util::restSeconds(double x){
    Util::rest((int)(x * 1000));
}

bool Util::checkVersion(int version){
    if (version == Global::getVersion()){
        return true;
    }

    /* when an incompatible version is made, add a check here, like
     *  version < getVersion(3, 5)
     * would mean any client below version 3.5 is incompatible.
     *
     * assume versions of client's greater than ourself is compatible, but
     * this may not be true. There is no way to check this.
     */
    if (version < 0){
        return false;
    }

    return true;
}

void Util::setDataPath( const string & str ){
    dataPath = str;
}

Filesystem::AbsolutePath Util::getDataPath2(){
    return Filesystem::AbsolutePath(dataPath + "/");
}

/* FIXME: remove this method */
bool Util::exists( const string & file ){
    return Storage::instance().exists(Filesystem::AbsolutePath(file));
/*
#ifdef USE_ALLEGRO
    return ::exists(file.c_str()) != 0;
#else
    return file_exists(file.c_str());
#endif
*/
}

/*
vector<Filesystem::AbsolutePath> Util::getFiles(const Filesystem::AbsolutePath & dataPath, const string & find){
    return Filesystem::getFiles(dataPath, find);
}
*/

string Util::trim(const std::string & str){
    string s;
    size_t startpos = str.find_first_not_of(" \t");
    size_t endpos = str.find_last_not_of(" \t");
    // if all spaces or empty return an empty string
    if ((string::npos == startpos) ||
        (string::npos == endpos)){
        return "";
    } else {
        return str.substr(startpos, endpos-startpos+1);
    }
    return str;
}

/* makes the first letter of a string upper case */
string Util::upcase(std::string str){
    if ( str.length() > 0 && (str[0] >= 'a' && str[0] <= 'z') ){
        str[0] = str[0] - 'a' + 'A';
    }
    return str;
}

static int lowerCase(int c){
    return tolower(c);
}

static int upperCase(int c){
    return toupper(c);
}

string Util::upperCaseAll(std::string str){
    std::transform(str.begin(), str.end(), str.begin(), upperCase);
    return str;
}

string Util::lowerCaseAll(std::string str){
    std::transform(str.begin(), str.end(), str.begin(), lowerCase);
    /*
    for (unsigned int i = 0; i < str.length(); i++){
        if (str[0] >= 'A' && str[0] <= 'Z'){
            str[0] = str[0] - 'A' + 'a';
        }
    }
    */
    return str;
}

/*Gets the minimum of three values*/
static int minimum(int a,int b,int c){
  int min=a;
  if(b<min)
    min=b;
  if(c<min)
    min=c;
  return min;
}

/* Compute levenshtein distance between s and t
 * from: http://www.merriampark.com/ldc.htm
 */
static int levenshtein_distance(const char *s, const char *t){
  //Step 1
  int k,i,j,n,m,cost,distance;
  int * d;
  n=strlen(s);
  m=strlen(t);
  if(n!=0&&m!=0){
    d = (int*) malloc((sizeof(int))*(m+1)*(n+1));
    m++;
    n++;
    //Step 2
    for(k=0;k<n;k++)
	d[k]=k;
    for(k=0;k<m;k++)
      d[k*n]=k;
    //Step 3 and 4
    for(i=1;i<n;i++)
      for(j=1;j<m;j++)
	{
        //Step 5
        if(s[i-1]==t[j-1])
          cost=0;
        else
          cost=1;
        //Step 6
        d[j*n+i]=minimum(d[(j-1)*n+i]+1,d[j*n+i-1]+1,d[(j-1)*n+i-1]+cost);
      }
    distance=d[n*m-1];
    free(d);
    return distance;
  } else {
    return -1; //a negative return value means that one or both strings are empty.
  }
}

int Util::levenshtein(const std::string & str1, const std::string & str2){
    return levenshtein_distance(str1.c_str(), str2.c_str());
}

void Util::limitPrintf(char * buffer, int size, const char * format, va_list args){
#ifdef USE_ALLEGRO
    uvszprintf(buffer, size, format, args);
#else
    vsnprintf(buffer, size, format, args);
#endif
}

#ifndef WINDOWS
int Util::getPipe(int files[2]){
#ifdef PS3
    return 0;
#else
    return pipe(files);
#endif
}
#endif

void Util::showError(const Graphics::Bitmap & screen, const Exception::Base & exception, const string & info){
    screen.BlitFromScreen(0, 0);
    Graphics::Bitmap error(screen.getWidth() - 100, screen.getHeight() - 100);
    error.fill(Graphics::darken(Graphics::makeColor(160, 0, 0), 3));
    error.border(1, 2, Graphics::makeColor(240, 0, 0));
    const Font & font = Font::getFont(Global::DEFAULT_FONT, 17, 17);
    int y = 10;
    std::ostringstream out;
    out << info;
    out << " " << exception.getTrace();
    Global::debug(0) << out.str() << std::endl;
    font.printfWrap(10, 10, Graphics::makeColor(240, 240, 240), error, error.getWidth() - 20, out.str(), 0);

    Graphics::Bitmap::transBlender(0, 0, 0, 220);
    error.translucent().draw(50, 50, screen);
    screen.BlitToScreen();
}

void Util::showError(const Exception::Base & exception, const std::string & info){
    Graphics::Bitmap screen(GFX_X, GFX_Y);
    showError(screen, exception, info);
}

static double twoDigits(double x){
    return (int) (x * 100) / 100.0;
}

string Util::niceSize(unsigned long size){
    const char sizes[] = {'b', 'k', 'm', 'g', 't'};
    double real = size;
    for (unsigned int i = 0; i < sizeof(sizes) / sizeof(const char); i++){
        if (real > 1024){
            real /= 1024.0;
        } else {
            ostringstream in;
            in << twoDigits(real) << sizes[i];
            return in.str();
        }
    }
    ostringstream in;
    in << twoDigits(real) << "t";
    return in.str();
}

std::string Util::join(const std::vector<std::string> & list, const std::string & middle){
    ostringstream out;

    bool first = true;
    for (std::vector<std::string>::const_iterator it = list.begin(); it != list.end(); it++){
        if (!first){
            out << middle;
        }

        out << *it;
        first = false;
    }

    return out.str();
}

double Util::distance(double x1, double y1, double x2, double y2){
    double xs = x1 - x2;
    double ys = y1 - y2;
    return sqrt(xs * xs + ys * ys);
}
<|endoftext|>


<commit_before>/*
 * t_qs.cpp
 *
 *  Created on: 2010-4-26
 *      Author: jinglei
 */

#include <idmlib/query-suggestion/Reminder.h>
#include "qs_types.h"

#include <sdb/SequentialDB.h>

#include <iostream>
#include <string>
#include <vector>
#include <fstream>
#include <algorithm>

#include <wiselib/ustring/UString.h>
#include <boost/algorithm/string.hpp>
#include <boost/filesystem.hpp>
#include <boost/unordered_map.hpp>

using namespace idmlib;

const uint32_t MAX_READ_BUF_SIZE=1024;
const uint32_t MAX_TIME_SERIRES=2;

void parseFile(const std::string& fileName, std::list<std::pair<wiselib::UString,uint32_t> >& logItems)
{
    fstream fileIn(fileName.c_str(), ios::in | ios::binary);
    if (!fileIn.good())
    {
        return;
    }
    boost::unordered_map<std::string, uint32_t> queryFreqMap;
	while(!fileIn.eof())
	{
		string strLine;
		getline(fileIn,strLine);
		int posBegin=strLine.find("[");
		int posEnd=strLine.find("]");
		std::string strQuery=strLine.substr(posBegin+1, posEnd-posBegin-1);
		queryFreqMap[strQuery]+=1;
	}
	boost::unordered_map<std::string, uint32_t>::iterator iter=queryFreqMap.begin();
	for(;iter!=queryFreqMap.end();iter++)
	{
		wiselib::UString ustrQuery(iter->first, wiselib::UString::GB2312);
		std::pair<wiselib::UString, uint32_t> item(ustrQuery, iter->second);
		logItems.push_back(item);
	}
}

//void parseDirectory(const std::string& path, std::list<std::pair<wiselib::UString,int> >& logItems)
//{
//    boost::filesystem::directory_iterator item_begin(path);
//    boost::filesystem::directory_iterator item_end;
//    boost::unordered_map<std::string, int> queryFreqMap;
//    int count=0;
//    for (; item_begin != item_end&&count<MAX_TIME_SERIRES; item_begin++, count++)
//    {
//      	std::cout<<"parsing file"<<item_begin -> path().file_string()<<std::endl;
//        if (boost::filesystem::is_directory(*item_begin))
//            continue;
//        std::list<std::pair<wiselib::UString,int> > tempItems;
//        parseFile(item_begin -> path().file_string(), tempItems);
//        std::list<std::pair<wiselib::UString,int> >::iterator it=tempItems.begin();
//        for(;it!=tempItems.end();it++)
//        {
//        	std::string strItem;
//        	it->first.convertString(strItem,wiselib::UString::GB2312);
//        	queryFreqMap[strItem]+=it->second;
//        }
//    }
//
//	boost::unordered_map<std::string, int>::iterator iter=queryFreqMap.begin();
//	for(;iter!=queryFreqMap.end();iter++)
//	{
//		wiselib::UString ustrQuery(iter->first, wiselib::UString::GB2312);
//		std::pair<wiselib::UString, int> item(ustrQuery, iter->second);
//		logItems.push_back(item);
//	}
//}

bool test()
{
	std::cout<<"Test for query suggestion!"<<std::endl;
    std::string path("/home/jinglei/sohuq");
    std::string basicPath(".");
	TestIDManager* idManager= new TestIDManager(".");
    Reminder<TestIDManager> reminder(idManager, basicPath);

	//Control the experiment log slice number.
	uint32_t count=0;
    try
    {
        boost::filesystem::directory_iterator item_begin(path);
        boost::filesystem::directory_iterator item_end;
        uint32_t timeId=1;
        std::vector<std::string> fileNames;
        for (;item_begin != item_end&& count<MAX_TIME_SERIRES;item_begin++, count++)
        {
            if (boost::filesystem::is_directory(*item_begin))
                continue;
            else
            {
            	std::string tempFileName((item_begin -> path().file_string()));
            	fileNames.push_back(tempFileName);
            }
        }
        std::sort(fileNames.begin(), fileNames.end());
        for (size_t i=0;i<fileNames.size();i++, count++)
        {
        	std::cout<<fileNames[i]<<std::endl;
            std::list<std::pair<wiselib::UString,uint32_t> > logItems;
            parseFile(fileNames[i], logItems);
            if(i==fileNames.size()-1)
            	reminder.indexQueryLog(timeId, logItems, true);
            else
            	reminder.indexQueryLog(timeId, logItems);
            timeId++;
        }
    } catch (...)
    {
        std::cout << "Error : log built fail.\n";
    }

    reminder.indexQuery();
    std::vector<wiselib::UString> realItems;
    reminder.getRealTimeQuery(realItems);
    std::cout<<"realItems.size()"<<realItems.size()<<std::endl;
    for(uint32_t i=0;i<realItems.size();i++)
    {
    	realItems[i].displayStringValue(wiselib::UString::UTF_8);
    	std::cout<<std::endl;
    }

    std::vector<wiselib::UString> popularItems;
    reminder.getPopularQuery(popularItems);
    std::cout<<"popularItems.size()"<<popularItems.size()<<std::endl;
    for(uint32_t i=0;i<popularItems.size();i++)
    {
    	popularItems[i].displayStringValue(wiselib::UString::UTF_8);
    	std::cout<<std::endl;
    }

    if(idManager)
    	delete idManager;

	return 0;

}

bool testPopularQuery()
{
//	std::cout<<"Test for query suggestion!"<<std::endl;
//    std::string path("/home/jinglei/sohuq");
//    std::string basicPath(".");
//    Recommender recommender(basicPath);
//
//	//Control the experiment log slice number.
//	int count=0;
//    try
//    {
//
//    	std::cout<<"entering inside"<<std::endl;
//        boost::filesystem::directory_iterator item_begin(path);
//        boost::filesystem::directory_iterator item_end;
//        uint32_t timeId=1;
//        for (;item_begin != item_end&& count<MAX_TIME_SERIRES;item_begin++, count++)
//        {
//        	std::cout<<"parsing file"<<item_begin -> path().file_string()<<std::endl;
//            if (boost::filesystem::is_directory(*item_begin))
//                continue;
//            std::list<std::pair<wiselib::UString,int> > logItems;
//            parseFile(item_begin -> path().file_string(), logItems);
//            recommender.indexLog(timeId, logItems);
//            timeId++;
//        }
//        std::list<std::pair<wiselib::UString,int> > allLogItems;
//        parseDirectory(path, allLogItems);
//        recommender.indexPopularQuery(allLogItems);
//    } catch (...)
//    {
//        std::cout << "Error : log built fail.\n";
//    } // end - try-catch
//
//    std::vector<wiselib::UString> popularItems;
//    recommender.getPopularQuery(popularItems);
//    for(uint32_t i=0;i<popularItems.size();i++)
//    {
//    	popularItems[i].displayStringValue(wiselib::UString::UTF_8);
//    	std::cout<<std::endl;
//    }

	return 0;

}

int main()
{

//	testRealTimeQuery();
//	testPopularQuery();
	test();
	return 0;

}
<commit_msg>add log convert<commit_after>/*
 * t_qs.cpp
 *
 *  Created on: 2010-4-26
 *      Author: jinglei
 */

#include <idmlib/query-suggestion/Reminder.h>
#include "qs_types.h"

#include <sdb/SequentialDB.h>

#include <iostream>
#include <string>
#include <vector>
#include <fstream>
#include <algorithm>

#include <wiselib/ustring/UString.h>
#include <boost/algorithm/string.hpp>
#include <boost/filesystem.hpp>
#include <boost/unordered_map.hpp>

using namespace idmlib;

const uint32_t MAX_READ_BUF_SIZE=1024;
const uint32_t MAX_TIME_SERIRES=2;

void parseFile(const std::string& fileName, std::list<std::pair<wiselib::UString,uint32_t> >& logItems)
{
    fstream fileIn(fileName.c_str(), ios::in | ios::binary);
    if (!fileIn.good())
    {
        return;
    }
    boost::unordered_map<std::string, uint32_t> queryFreqMap;
	while(!fileIn.eof())
	{
		string strLine;
		getline(fileIn,strLine);
		int posBegin=strLine.find("[");
		int posEnd=strLine.find("]");
		std::string strQuery=strLine.substr(posBegin+1, posEnd-posBegin-1);
		queryFreqMap[strQuery]+=1;
	}
	boost::unordered_map<std::string, uint32_t>::iterator iter=queryFreqMap.begin();
	for(;iter!=queryFreqMap.end();iter++)
	{
		wiselib::UString ustrQuery(iter->first, wiselib::UString::GB2312);
		std::pair<wiselib::UString, uint32_t> item(ustrQuery, iter->second);
		logItems.push_back(item);
	}
}

void parseTosf1v5LogFile(const std::string& fileName, const std::string& logFileName)
{
    std::ifstream fileIn(fileName.c_str());
    if (!fileIn.good())
    {
        return;
    }
    std::ofstream fileOut(logFileName.c_str());
    string strLine;
    while( getline(fileIn,strLine) )
    {
//         std::cout<<"Have "<<strLine<<std::endl;

        std::vector<std::string> items;
        boost::algorithm::split( items, strLine, boost::algorithm::is_any_of("\t") );
//         std::cout<<"size "<<items.size()<<" "<<items[2]<<std::endl;
        if(items.size()!=4) continue;
        std::vector<std::string> subItems;
        boost::algorithm::split( subItems, items[2], boost::algorithm::is_any_of(" ") );
        if( subItems.size()!=2 ) continue;
        if( subItems[1] != "1" ) continue;
        std::string outputLine = "Query\t";
        outputLine += items[0] + "\t20100517T151422\t00:00:01.027035\t";
        std::string query = items[1];
        query = query.substr(1, query.length()-2);
        wiselib::UString uquery(query, wiselib::UString::GB2312);
        std::string tmp;
        uquery.convertString(tmp, wiselib::UString::UTF_8);
//         std::cout<<"append "<<tmp<<std::endl;

        outputLine += tmp + "\t100\t0\t10";
        fileOut<<outputLine<<std::endl;
    }
    fileIn.close();
    fileOut.close();

}

void convertLog(const std::string& sogouPath, const std::string& objPath)
{
    boost::filesystem::create_directories(objPath);
    boost::posix_time::ptime time = boost::posix_time::second_clock::local_time();
    boost::gregorian::days oneday(1);
    for(uint32_t i=20060831;i>=20060801;i--)
    {
        std::string fileName = sogouPath+"/"+"access_log."+boost::lexical_cast<std::string>(i)+".decode.filter";
        if( !boost::filesystem::exists(fileName) )
        {
            continue;
        }
        std::string dateTimeStr = boost::posix_time::to_iso_string(time).substr(0,8);
        std::string outputFileName = objPath+"/"+dateTimeStr+".log";
        std::cout<<"Converting "<<fileName<<" to "<<outputFileName<<std::endl;
        parseTosf1v5LogFile( fileName, outputFileName);
        time -= oneday;
    }
}

//void parseDirectory(const std::string& path, std::list<std::pair<wiselib::UString,int> >& logItems)
//{
//    boost::filesystem::directory_iterator item_begin(path);
//    boost::filesystem::directory_iterator item_end;
//    boost::unordered_map<std::string, int> queryFreqMap;
//    int count=0;
//    for (; item_begin != item_end&&count<MAX_TIME_SERIRES; item_begin++, count++)
//    {
//      	std::cout<<"parsing file"<<item_begin -> path().file_string()<<std::endl;
//        if (boost::filesystem::is_directory(*item_begin))
//            continue;
//        std::list<std::pair<wiselib::UString,int> > tempItems;
//        parseFile(item_begin -> path().file_string(), tempItems);
//        std::list<std::pair<wiselib::UString,int> >::iterator it=tempItems.begin();
//        for(;it!=tempItems.end();it++)
//        {
//        	std::string strItem;
//        	it->first.convertString(strItem,wiselib::UString::GB2312);
//        	queryFreqMap[strItem]+=it->second;
//        }
//    }
//
//	boost::unordered_map<std::string, int>::iterator iter=queryFreqMap.begin();
//	for(;iter!=queryFreqMap.end();iter++)
//	{
//		wiselib::UString ustrQuery(iter->first, wiselib::UString::GB2312);
//		std::pair<wiselib::UString, int> item(ustrQuery, iter->second);
//		logItems.push_back(item);
//	}
//}

bool test()
{
	std::cout<<"Test for query suggestion!"<<std::endl;
    std::string path("/home/jinglei/sohuq");
    std::string basicPath(".");
	TestIDManager* idManager= new TestIDManager(".");
    Reminder<TestIDManager> reminder(idManager, basicPath);

	//Control the experiment log slice number.
	uint32_t count=0;
    try
    {
        boost::filesystem::directory_iterator item_begin(path);
        boost::filesystem::directory_iterator item_end;
        uint32_t timeId=1;
        std::vector<std::string> fileNames;
        for (;item_begin != item_end&& count<MAX_TIME_SERIRES;item_begin++, count++)
        {
            if (boost::filesystem::is_directory(*item_begin))
                continue;
            else
            {
            	std::string tempFileName((item_begin -> path().file_string()));
            	fileNames.push_back(tempFileName);
            }
        }
        std::sort(fileNames.begin(), fileNames.end());
        for (size_t i=0;i<fileNames.size();i++, count++)
        {
        	std::cout<<fileNames[i]<<std::endl;
            std::list<std::pair<wiselib::UString,uint32_t> > logItems;
            parseFile(fileNames[i], logItems);
            if(i==fileNames.size()-1)
            	reminder.indexQueryLog(timeId, logItems, true);
            else
            	reminder.indexQueryLog(timeId, logItems);
            timeId++;
        }
    } catch (...)
    {
        std::cout << "Error : log built fail.\n";
    }

    reminder.indexQuery();
    std::vector<wiselib::UString> realItems;
    reminder.getRealTimeQuery(realItems);
    std::cout<<"realItems.size()"<<realItems.size()<<std::endl;
    for(uint32_t i=0;i<realItems.size();i++)
    {
    	realItems[i].displayStringValue(wiselib::UString::UTF_8);
    	std::cout<<std::endl;
    }

    std::vector<wiselib::UString> popularItems;
    reminder.getPopularQuery(popularItems);
    std::cout<<"popularItems.size()"<<popularItems.size()<<std::endl;
    for(uint32_t i=0;i<popularItems.size();i++)
    {
    	popularItems[i].displayStringValue(wiselib::UString::UTF_8);
    	std::cout<<std::endl;
    }

    if(idManager)
    	delete idManager;

	return 0;

}

bool testPopularQuery()
{
//	std::cout<<"Test for query suggestion!"<<std::endl;
//    std::string path("/home/jinglei/sohuq");
//    std::string basicPath(".");
//    Recommender recommender(basicPath);
//
//	//Control the experiment log slice number.
//	int count=0;
//    try
//    {
//
//    	std::cout<<"entering inside"<<std::endl;
//        boost::filesystem::directory_iterator item_begin(path);
//        boost::filesystem::directory_iterator item_end;
//        uint32_t timeId=1;
//        for (;item_begin != item_end&& count<MAX_TIME_SERIRES;item_begin++, count++)
//        {
//        	std::cout<<"parsing file"<<item_begin -> path().file_string()<<std::endl;
//            if (boost::filesystem::is_directory(*item_begin))
//                continue;
//            std::list<std::pair<wiselib::UString,int> > logItems;
//            parseFile(item_begin -> path().file_string(), logItems);
//            recommender.indexLog(timeId, logItems);
//            timeId++;
//        }
//        std::list<std::pair<wiselib::UString,int> > allLogItems;
//        parseDirectory(path, allLogItems);
//        recommender.indexPopularQuery(allLogItems);
//    } catch (...)
//    {
//        std::cout << "Error : log built fail.\n";
//    } // end - try-catch
//
//    std::vector<wiselib::UString> popularItems;
//    recommender.getPopularQuery(popularItems);
//    for(uint32_t i=0;i<popularItems.size();i++)
//    {
//    	popularItems[i].displayStringValue(wiselib::UString::UTF_8);
//    	std::cout<<std::endl;
//    }

	return 0;

}

int main()
{
    convertLog("/home/jarvis/data/SogouQ", "/home/jarvis/data/SogouQ/sf1v5");
//	testRealTimeQuery();
//	testPopularQuery();
	test();
	return 0;

}
<|endoftext|>


<commit_before>#include "Halide.h"

using namespace Halide;
using namespace Halide::Internal;

class ValidateInterleavedPipeline: public IRMutator {
protected:
//
// This is roughly the structure that we are trying to validate in this custom pass:
//
// parallel<Renderscript> (result$2.s0.y$2.__block_id_y, 0, result$2.extent.1) {
//   parallel<Renderscript> (result$2.s0.x$2.__block_id_x, 0, result$2.extent.0) {
//     ...
//     parallel<Renderscript> (.__thread_id_x, 0, 1) {
//       for<Renderscript> (result$2.s0.c$2, 0, 4) {
//         image_store("result$2",
//                     result$2.buffer,
//                     (result$2.s0.x$2.__block_id_x + result$2.min.0),
//                     (result$2.s0.y$2.__block_id_y + result$2.min.1),
//                     result$2.s0.c$2,
//                     image_load("input",
//                                input.buffer,
//                                ((result$2.s0.x$2.__block_id_x + result$2.min.0) - input.min.0),
//                                input.extent.0,
//                                ((result$2.s0.y$2.__block_id_y + result$2.min.1) - input.min.1),
//                                input.extent.1,
//                                result$2.s0.c$2,
//                                4))
//       }
//     }
//     ...
//   }
// }
//
    using IRMutator::visit;

    virtual void visit(const Call *call) {
        if (in_pipeline && call->is_intrinsic(Call::image_store)) {
            assert(for_nest_level == 4);

            std::map<std::string, Expr> matches;

            assert(expr_match(
                StringImm::make("result"),
                call->args[0],
                matches));
            assert(expr_match(
                Variable::make(Handle(1), "result.buffer"),
                call->args[1],
                matches));
            assert(expr_match(
                Variable::make(Int(32), "result.s0.x.__block_id_x") + Variable::make(Int(32), "result.min.0"),
                call->args[2],
                matches));
            assert(expr_match(
                Variable::make(Int(32), "result.s0.y.__block_id_y") + Variable::make(Int(32), "result.min.1"),
                call->args[3],
                matches));
            assert(expr_match(
                Variable::make(Int(32), "result.s0.c"),
                call->args[4],
                matches));
            assert(expr_match(
                Call::make(
                    UInt(8),
                    Call::image_load,
                    {
                        StringImm::make("input"),
                        Variable::make(Handle(1), "input.buffer"),
                        (Variable::make(Int(32), "result.s0.x.__block_id_x") +
                         Variable::make(Int(32), "result.min.0")) -
                         Variable::make(Int(32), "input.min.0"),
                        Variable::make(Int(32), "input.extent.0"),
                        (Variable::make(Int(32), "result.s0.y.__block_id_y") +
                         Variable::make(Int(32), "result.min.1")) -
                         Variable::make(Int(32), "input.min.1"),
                        Variable::make(Int(32), "input.extent.1"),
                        Variable::make(Int(32), "result.s0.c"),
                        channels
                    },
                    Call::CallType::PureIntrinsic),
                call->args[5],
                matches));
        }
        IRMutator::visit(call);
    }

    void visit(const For *op) {
        if (in_pipeline) {
            assert(for_nest_level >= 0); // For-loop should show up in Pipeline.
            for_nest_level++;
            if (for_nest_level <= 3) {
                assert(op->for_type == ForType::Parallel);
                assert(op->device_api == DeviceAPI::Renderscript);
            }
        }
        IRMutator::visit(op);
    }

    void visit(const ProducerConsumer *op) {
        assert(!in_pipeline); // There should be only one pipeline in the test.
        for_nest_level = 0;
        in_pipeline = true;

        assert(op->produce.defined());
        assert(!op->update.defined());
        assert(op->consume.defined());

        IRMutator::visit(op);
        stmt = Stmt();
    }

    int for_nest_level = -1;
    bool in_pipeline = false;
    int channels;

public:
    ValidateInterleavedPipeline(int _channels) : channels(_channels) {}
};

class ValidateInterleavedVectorizedPipeline: public ValidateInterleavedPipeline {
    using IRMutator::visit;
    using ValidateInterleavedPipeline::visit;

    virtual void visit(const Call *call) {
        if (in_pipeline && call->is_intrinsic(Call::image_store)) {
            assert(for_nest_level == 3); // Should be three nested for-loops before we get to the first call.

            std::map<std::string, Expr> matches;

            assert(expr_match(
                Broadcast::make(StringImm::make("result"), channels),
                call->args[0],
                matches));
            assert(expr_match(
                Broadcast::make(Variable::make(Handle(1), "result.buffer"), channels),
                call->args[1],
                matches));
            assert(expr_match(
                Broadcast::make(Variable::make(Int(32), "result.s0.x.__block_id_x") + Variable::make(Int(32), "result.min.0"), channels),
                call->args[2],
                matches));
            assert(expr_match(
                Broadcast::make(Variable::make(Int(32), "result.s0.y.__block_id_y") + Variable::make(Int(32), "result.min.1"), channels),
                call->args[3],
                matches));
            assert(expr_match(
                Ramp::make(0, 1, channels), call->args[4], matches));
            assert(expr_match(
                Call::make(
                    UInt(8, channels),
                    Call::image_load,
                    {
                        Broadcast::make(StringImm::make("input"), channels),
                        Broadcast::make(Variable::make(Handle(1), "input.buffer"), channels),
                        Broadcast::make(
                            Add::make(
                                Variable::make(Int(32), "result.s0.x.__block_id_x"),
                                Variable::make(Int(32), "result.min.0")) -
                            Variable::make(Int(32), "input.min.0"),
                            channels),
                        Broadcast::make(Variable::make(Int(32), "input.extent.0"), channels),
                        Broadcast::make(
                            Add::make(
                                Variable::make(Int(32), "result.s0.y.__block_id_y"),
                                Variable::make(Int(32), "result.min.1")) -
                            Variable::make(Int(32), "input.min.1"),
                            channels),
                        Broadcast::make(Variable::make(Int(32), "input.extent.1"), channels),
                        Ramp::make(0, 1, channels),
                        Broadcast::make(channels, channels)
                    },
                    Call::CallType::PureIntrinsic),
                call->args[5],
                matches));
        }
        IRMutator::visit(call);
    }
public:
    ValidateInterleavedVectorizedPipeline(int _channels) : ValidateInterleavedPipeline(_channels) {}
};

Image<uint8_t> make_interleaved_image(uint8_t *host, int W, int H, int channels) {
    buffer_t buf = { 0 };
    buf.host = host;
    buf.extent[0] = W;
    buf.stride[0] = channels;
    buf.extent[1] = H;
    buf.stride[1] = buf.stride[0] * buf.extent[0];
    buf.extent[2] = channels;
    buf.stride[2] = 1;
    buf.elem_size = 1;
    return Image<uint8_t>(&buf);
}

void copy_interleaved(bool vectorized = false, int channels = 4) {
    ImageParam input8(UInt(8), 3, "input");
    input8.set_stride(0, channels)
        .set_stride(1, Halide::Expr())
        .set_stride(2, 1)
        .set_bounds(2, 0, channels);  // expecting interleaved image
    uint8_t *in_buf = new uint8_t[128 * 128 * channels];
    Image<uint8_t> in = make_interleaved_image(in_buf, 128, 128, channels);
    input8.set(in);

    Var x, y, c;
    Func result("result");
    result(x, y, c) = input8(x, y, c);
    result.output_buffer()
        .set_stride(0, channels)
        .set_stride(1, Halide::Expr())
        .set_stride(2, 1)
        .set_bounds(2, 0, channels);  // expecting interleaved image

    result.bound(c, 0, channels);
    result.shader(x, y, c, DeviceAPI::Renderscript);
    if (vectorized) result.vectorize(c);

    result.add_custom_lowering_pass(
        vectorized?
            new ValidateInterleavedVectorizedPipeline(channels):
            new ValidateInterleavedPipeline(channels));

    result.compile_jit();

    delete[] in_buf;
}

int main(int argc, char **argv) {
    copy_interleaved(true, 4);
    copy_interleaved(false, 4);
    copy_interleaved(true, 3);
    copy_interleaved(false, 3);

    std::cout << "Done!" << std::endl;
    return 0;
}
<commit_msg>Fix renderscript test<commit_after>#include "Halide.h"

using namespace Halide;
using namespace Halide::Internal;

class ValidateInterleavedPipeline: public IRMutator {
protected:
//
// This is roughly the structure that we are trying to validate in this custom pass:
//
// parallel<Renderscript> (result$2.s0.y$2.__block_id_y, 0, result$2.extent.1) {
//   parallel<Renderscript> (result$2.s0.x$2.__block_id_x, 0, result$2.extent.0) {
//     ...
//     parallel<Renderscript> (.__thread_id_x, 0, 1) {
//       for<Renderscript> (result$2.s0.c$2, 0, 4) {
//         image_store("result$2",
//                     result$2.buffer,
//                     (result$2.s0.x$2.__block_id_x + result$2.min.0),
//                     (result$2.s0.y$2.__block_id_y + result$2.min.1),
//                     result$2.s0.c$2,
//                     image_load("input",
//                                input.buffer,
//                                ((result$2.s0.x$2.__block_id_x + result$2.min.0) - input.min.0),
//                                input.extent.0,
//                                ((result$2.s0.y$2.__block_id_y + result$2.min.1) - input.min.1),
//                                input.extent.1,
//                                result$2.s0.c$2,
//                                4))
//       }
//     }
//     ...
//   }
// }
//
    using IRMutator::visit;

    virtual void visit(const Call *call) {
        if (in_pipeline && call->is_intrinsic(Call::image_store)) {
            assert(for_nest_level == 4);

            std::map<std::string, Expr> matches;

            assert(expr_match(
                StringImm::make("result"),
                call->args[0],
                matches));
            assert(expr_match(
                Variable::make(Handle(1), "result.buffer"),
                call->args[1],
                matches));
            assert(expr_match(
                Variable::make(Int(32), "result.s0.x.__block_id_x") + Variable::make(Int(32), "result.min.0"),
                call->args[2],
                matches));
            assert(expr_match(
                Variable::make(Int(32), "result.s0.y.__block_id_y") + Variable::make(Int(32), "result.min.1"),
                call->args[3],
                matches));
            assert(expr_match(
                Variable::make(Int(32), "result.s0.c"),
                call->args[4],
                matches));
            assert(expr_match(
                Call::make(
                    UInt(8),
                    Call::image_load,
                    {
                        StringImm::make("input"),
                        Variable::make(Handle(1), "input.buffer"),
                        (Variable::make(Int(32), "result.s0.x.__block_id_x") +
                         Variable::make(Int(32), "result.min.0")) -
                         Variable::make(Int(32), "input.min.0"),
                        Variable::make(Int(32), "input.extent.0"),
                        (Variable::make(Int(32), "result.s0.y.__block_id_y") +
                         Variable::make(Int(32), "result.min.1")) -
                         Variable::make(Int(32), "input.min.1"),
                        Variable::make(Int(32), "input.extent.1"),
                        Variable::make(Int(32), "result.s0.c"),
                        channels
                    },
                    Call::CallType::PureIntrinsic),
                call->args[5],
                matches));
        }
        IRMutator::visit(call);
    }

    void visit(const For *op) {
        if (in_pipeline) {
            assert(for_nest_level >= 0); // For-loop should show up in Pipeline.
            for_nest_level++;
            if (for_nest_level <= 3) {
                assert(op->for_type == ForType::Parallel);
                assert(op->device_api == DeviceAPI::Renderscript);
            }
        }
        IRMutator::visit(op);
    }

    void visit(const ProducerConsumer *op) {
        assert(!in_pipeline); // There should be only one pipeline in the test.
        for_nest_level = 0;
        in_pipeline = true;

        assert(op->produce.defined());
        assert(!op->update.defined());
        assert(op->consume.defined());

        IRMutator::visit(op);
        stmt = Evaluate::make(0);
    }

    int for_nest_level = -1;
    bool in_pipeline = false;
    int channels;

public:
    ValidateInterleavedPipeline(int _channels) : channels(_channels) {}
};

class ValidateInterleavedVectorizedPipeline: public ValidateInterleavedPipeline {
    using IRMutator::visit;
    using ValidateInterleavedPipeline::visit;

    virtual void visit(const Call *call) {
        if (in_pipeline && call->is_intrinsic(Call::image_store)) {
            assert(for_nest_level == 3); // Should be three nested for-loops before we get to the first call.

            std::map<std::string, Expr> matches;

            assert(expr_match(
                Broadcast::make(StringImm::make("result"), channels),
                call->args[0],
                matches));
            assert(expr_match(
                Broadcast::make(Variable::make(Handle(1), "result.buffer"), channels),
                call->args[1],
                matches));
            assert(expr_match(
                Broadcast::make(Variable::make(Int(32), "result.s0.x.__block_id_x") + Variable::make(Int(32), "result.min.0"), channels),
                call->args[2],
                matches));
            assert(expr_match(
                Broadcast::make(Variable::make(Int(32), "result.s0.y.__block_id_y") + Variable::make(Int(32), "result.min.1"), channels),
                call->args[3],
                matches));
            assert(expr_match(
                Ramp::make(0, 1, channels), call->args[4], matches));
            assert(expr_match(
                Call::make(
                    UInt(8, channels),
                    Call::image_load,
                    {
                        Broadcast::make(StringImm::make("input"), channels),
                        Broadcast::make(Variable::make(Handle(1), "input.buffer"), channels),
                        Broadcast::make(
                            Add::make(
                                Variable::make(Int(32), "result.s0.x.__block_id_x"),
                                Variable::make(Int(32), "result.min.0")) -
                            Variable::make(Int(32), "input.min.0"),
                            channels),
                        Broadcast::make(Variable::make(Int(32), "input.extent.0"), channels),
                        Broadcast::make(
                            Add::make(
                                Variable::make(Int(32), "result.s0.y.__block_id_y"),
                                Variable::make(Int(32), "result.min.1")) -
                            Variable::make(Int(32), "input.min.1"),
                            channels),
                        Broadcast::make(Variable::make(Int(32), "input.extent.1"), channels),
                        Ramp::make(0, 1, channels),
                        Broadcast::make(channels, channels)
                    },
                    Call::CallType::PureIntrinsic),
                call->args[5],
                matches));
        }
        IRMutator::visit(call);
    }
public:
    ValidateInterleavedVectorizedPipeline(int _channels) : ValidateInterleavedPipeline(_channels) {}
};

Image<uint8_t> make_interleaved_image(uint8_t *host, int W, int H, int channels) {
    buffer_t buf = { 0 };
    buf.host = host;
    buf.extent[0] = W;
    buf.stride[0] = channels;
    buf.extent[1] = H;
    buf.stride[1] = buf.stride[0] * buf.extent[0];
    buf.extent[2] = channels;
    buf.stride[2] = 1;
    buf.elem_size = 1;
    return Image<uint8_t>(&buf);
}

void copy_interleaved(bool vectorized = false, int channels = 4) {
    ImageParam input8(UInt(8), 3, "input");
    input8.set_stride(0, channels)
        .set_stride(1, Halide::Expr())
        .set_stride(2, 1)
        .set_bounds(2, 0, channels);  // expecting interleaved image
    uint8_t *in_buf = new uint8_t[128 * 128 * channels];
    Image<uint8_t> in = make_interleaved_image(in_buf, 128, 128, channels);
    input8.set(in);

    Var x, y, c;
    Func result("result");
    result(x, y, c) = input8(x, y, c);
    result.output_buffer()
        .set_stride(0, channels)
        .set_stride(1, Halide::Expr())
        .set_stride(2, 1)
        .set_bounds(2, 0, channels);  // expecting interleaved image

    result.bound(c, 0, channels);
    result.shader(x, y, c, DeviceAPI::Renderscript);
    if (vectorized) result.vectorize(c);

    result.add_custom_lowering_pass(
        vectorized?
            new ValidateInterleavedVectorizedPipeline(channels):
            new ValidateInterleavedPipeline(channels));

    result.compile_jit();

    delete[] in_buf;
}

int main(int argc, char **argv) {
    copy_interleaved(true, 4);
    copy_interleaved(false, 4);
    copy_interleaved(true, 3);
    copy_interleaved(false, 3);

    std::cout << "Done!" << std::endl;
    return 0;
}
<|endoftext|>


<commit_before>#include <gtest/gtest.h>

#include <fstream>
#include <iostream>
#include <sstream>

#include "ast.h"
#include "parser.h"
#include "stream_utils.h"
#include "test_utils.h"

namespace {
string str_test{"test"};
string str_1{"1"};
string str_4dot5{"4.5"};
string str_plus{"+"};
string str_mul{"*"};
string str_addition{"1+7"};
string str_add_error{"1+"};
string str_neg{"-4.2"};
string str_paren{"(1+7)"};
string str_expr_stmt{"1;"};
string str_empty_compound{"{}"};
string str_compound{"{ 1; { {} 7; }}"};
string str_if{"if(1) {}"};
string str_if_else{"if (1) {} else 5;"};
string str_if_error{"if 7;"};
string str_int{"int"};
string str_float{"  float "};
string str_decl{"int test;"};
}

TEST(Libmc, Types) {
  EXPECT_NO_MATCH(parser::type, str_plus);
  EXPECT_NO_MATCH(parser::type, str_test);
  EXPECT_MATCH_TYPED(parser::type, str_int, ast::int_type,
                     [](auto res) { return true; });
  EXPECT_MATCH_TYPED(parser::type, str_float, ast::float_type,
                     [](auto res) { return true; });
}

TEST(Libmc, Literals) {
  EXPECT_NO_MATCH(parser::int_literal, str_test);
  EXPECT_NO_MATCH(parser::int_literal, str_plus);
  EXPECT_MATCH(parser::int_literal, str_1,
               [](auto res) { return res->value == 1; });
  EXPECT_MATCH(parser::float_literal, str_4dot5,
               [](auto res) { return res->value == 4.5; });
}

TEST(Libmc, Variables) {
  EXPECT_NO_MATCH(parser::variable, str_1);
  EXPECT_NO_MATCH(parser::variable, str_4dot5);
  {
    parser::parser_state p(str_test.cbegin(), str_test.cend());
    sptr<ast::variable> var = std::make_shared<ast::variable>(
        std::make_shared<ast::int_type>(), str_test);
    p.scopes.back().declare(p, str_test, var);
    // freestanding
    auto match = parser::variable(p);
    EXPECT(match);
    EXPECT(*match == *var);

    EXPECT(p.e == str_test.cend())
    // in expression
    string str_var_expr{"test+5"};
    p.set_string(str_var_expr);
    auto match2 = parser::binary_operation(p);
    EXPECT(match2);
    EXPECT(*match2->lhs == *var);
    EXPECT(p.e == str_var_expr.cend())
  }
}

TEST(Libmc, BinOperands) {
  EXPECT_NO_MATCH(parser::binary_operand, str_1);
  EXPECT_MATCH(parser::binary_operand, str_plus,
               [](auto res) { return *res == ast::binary_operand::ADD; });
  EXPECT_MATCH(parser::binary_operand, str_mul,
               [](auto res) { return *res == ast::binary_operand::MUL; });
}

TEST(Libmc, Expressions) {
  EXPECT_NO_MATCH(parser::literal, str_test);
  EXPECT_MATCH_TYPED(parser::literal, str_1, ast::int_literal,
                     [](auto res) { return res->value == 1; });
  EXPECT_MATCH_TYPED(parser::literal, str_4dot5, ast::float_literal,
                     [](auto res) { return res->value == 4.5; });

  EXPECT_NO_MATCH(parser::binary_operation, str_test);
  EXPECT_NO_MATCH(parser::binary_operation, str_4dot5);
  EXPECT_NO_MATCH(parser::binary_operation, str_mul);
  EXPECT_MATCH(parser::binary_operation, str_addition, [](auto res) {
    return *res->op == ast::binary_operand::ADD &&
           *res->lhs == ast::int_literal{1};
  });
  EXPECT_ERROR(parser::binary_operation, str_add_error);

  EXPECT_NO_MATCH(parser::unary_operation, str_1);
  EXPECT_MATCH(parser::unary_operation, str_neg, [](auto res) {
    return *res->op == ast::unary_operand::MINUS &&
           *res->sub == ast::float_literal{4.2};
  });

  EXPECT_NO_MATCH(parser::paren_expr, str_neg);
  EXPECT_MATCH(parser::paren_expr, str_paren, [](auto res) {
    return *res->sub == *parser::parse_expr("1+7");
  });
}

TEST(Libmc, statements) {
  EXPECT_ERROR(parser::expr_stmt, str_test);
  EXPECT_MATCH(parser::expr_stmt, str_expr_stmt,
               [](auto res) { return *res->sub == ast::int_literal{1}; });

  EXPECT_NO_MATCH(parser::compound_stmt, str_test);
  EXPECT_MATCH(parser::compound_stmt, str_empty_compound,
               [](auto res) { return res->statements.empty(); });
  EXPECT_MATCH(parser::compound_stmt, str_compound,
               [](auto res) { return res->statements.size() == 2; });

  EXPECT_NO_MATCH(parser::if_stmt, str_test);
  EXPECT_MATCH(parser::if_stmt, str_if,
               [](auto res) { return *res->condition == ast::int_literal{1}; });
  EXPECT_MATCH(parser::if_stmt, str_if_else, [](auto res) {
    return *res->else_stmt ==
           ast::expr_stmt{std::make_shared<ast::int_literal>(5)};
  });
  EXPECT_MATCH(parser::decl_stmt, str_decl, [](auto res) {
    return res->var->name == "test" &&
           typeid(*res->var->var_type) == typeid(ast::int_type);
  });

  EXPECT(parser::parse(R"(
	{
		int x;
		if(1 == 2) {
			5+4*8.0;
			7;
			x = 8;
		} else {
			if(2 < 9) {
			}
		}
		2.9;
    while(1 < 3) {
      x = x + 1;
    }
	}
	)"));
}
<commit_msg>Added parser tests for array operations<commit_after>#include <gtest/gtest.h>

#include <fstream>
#include <iostream>
#include <sstream>

#include "ast.h"
#include "parser.h"
#include "stream_utils.h"
#include "test_utils.h"

namespace {
string str_test{"test"};
string str_1{"1"};
string str_4dot5{"4.5"};
string str_plus{"+"};
string str_mul{"*"};
string str_addition{"1+7"};
string str_add_error{"1+"};
string str_neg{"-4.2"};
string str_paren{"(1+7)"};
string str_expr_stmt{"1;"};
string str_empty_compound{"{}"};
string str_compound{"{ 1; { {} 7; }}"};
string str_if{"if(1) {}"};
string str_if_else{"if (1) {} else 5;"};
string str_if_error{"if 7;"};
string str_int{"int"};
string str_float{"  float "};
string str_decl{"int test;"};
string str_array_decl{"int [10] testarray;"};
string str_array_access{"testarray[5]"};
}

TEST(Libmc, Types) {
  EXPECT_NO_MATCH(parser::type, str_plus);
  EXPECT_NO_MATCH(parser::type, str_test);
  EXPECT_MATCH_TYPED(parser::type, str_int, ast::int_type,
                     [](auto res) { return true; });
  EXPECT_MATCH_TYPED(parser::type, str_float, ast::float_type,
                     [](auto res) { return true; });
}

TEST(Libmc, Literals) {
  EXPECT_NO_MATCH(parser::int_literal, str_test);
  EXPECT_NO_MATCH(parser::int_literal, str_plus);
  EXPECT_MATCH(parser::int_literal, str_1,
               [](auto res) { return res->value == 1; });
  EXPECT_MATCH(parser::float_literal, str_4dot5,
               [](auto res) { return res->value == 4.5; });
}

TEST(Libmc, Variables) {
  EXPECT_NO_MATCH(parser::variable, str_1);
  EXPECT_NO_MATCH(parser::variable, str_4dot5);
  {
    parser::parser_state p(str_test.cbegin(), str_test.cend());
    sptr<ast::variable> var = std::make_shared<ast::variable>(
        std::make_shared<ast::int_type>(), str_test);
    p.scopes.back().declare(p, str_test, var);
    // freestanding
    auto match = parser::variable(p);
    EXPECT(match);
    EXPECT(*match == *var);

    EXPECT(p.e == str_test.cend())
    // in expression
    string str_var_expr{"test+5"};
    p.set_string(str_var_expr);
    auto match2 = parser::binary_operation(p);
    EXPECT(match2);
    EXPECT(*match2->lhs == *var);
    EXPECT(p.e == str_var_expr.cend())
  }
}

TEST(Libmc, ArrayAccess) {
  EXPECT_NO_MATCH(parser::array_access, str_1);
  EXPECT_NO_MATCH(parser::array_access, str_4dot5);
  {
    parser::parser_state p(str_array_access.cbegin(), str_array_access.cend());
    sptr<ast::array> testarray = std::make_shared<ast::array>(
            std::make_shared<ast::int_type>(), std::make_shared<ast::int_literal>(5), "testarray");

    p.scopes.back().declareArray(p, "testarray", testarray);
    sptr<ast::expression> position = std::make_shared<ast::int_literal>(5);
    sptr<ast::array_access> access = std::make_shared<ast::array_access>(
            testarray, position);

    //freestanding
    auto match = parser::array_access(p);
    EXPECT(match);
    EXPECT(*match->m_array == *testarray);
    EXPECT(*match->access_expr == *position)
    EXPECT(p.e == str_array_access.cend())

    //in expression
    string str_array_access_expr{"testarray[5]+5"};
    p.set_string(str_array_access_expr);
    auto match2 = parser::binary_operation(p);
    EXPECT(match2);
    EXPECT(*match2->lhs == *access);
    EXPECT(p.e == str_array_access_expr.cend())

  }
}

TEST(Libmc, BinOperands) {
  EXPECT_NO_MATCH(parser::binary_operand, str_1);
  EXPECT_MATCH(parser::binary_operand, str_plus,
               [](auto res) { return *res == ast::binary_operand::ADD; });
  EXPECT_MATCH(parser::binary_operand, str_mul,
               [](auto res) { return *res == ast::binary_operand::MUL; });
}

TEST(Libmc, Expressions) {
  EXPECT_NO_MATCH(parser::literal, str_test);
  EXPECT_MATCH_TYPED(parser::literal, str_1, ast::int_literal,
                     [](auto res) { return res->value == 1; });
  EXPECT_MATCH_TYPED(parser::literal, str_4dot5, ast::float_literal,
                     [](auto res) { return res->value == 4.5; });

  EXPECT_NO_MATCH(parser::binary_operation, str_test);
  EXPECT_NO_MATCH(parser::binary_operation, str_4dot5);
  EXPECT_NO_MATCH(parser::binary_operation, str_mul);
  EXPECT_MATCH(parser::binary_operation, str_addition, [](auto res) {
    return *res->op == ast::binary_operand::ADD &&
           *res->lhs == ast::int_literal{1};
  });
  EXPECT_ERROR(parser::binary_operation, str_add_error);

  EXPECT_NO_MATCH(parser::unary_operation, str_1);
  EXPECT_MATCH(parser::unary_operation, str_neg, [](auto res) {
    return *res->op == ast::unary_operand::MINUS &&
           *res->sub == ast::float_literal{4.2};
  });

  EXPECT_NO_MATCH(parser::paren_expr, str_neg);
  EXPECT_MATCH(parser::paren_expr, str_paren, [](auto res) {
    return *res->sub == *parser::parse_expr("1+7");
  });
}

TEST(Libmc, statements) {
  EXPECT_ERROR(parser::expr_stmt, str_test);
  EXPECT_MATCH(parser::expr_stmt, str_expr_stmt,
               [](auto res) { return *res->sub == ast::int_literal{1}; });

  EXPECT_NO_MATCH(parser::compound_stmt, str_test);
  EXPECT_MATCH(parser::compound_stmt, str_empty_compound,
               [](auto res) { return res->statements.empty(); });
  EXPECT_MATCH(parser::compound_stmt, str_compound,
               [](auto res) { return res->statements.size() == 2; });

  EXPECT_NO_MATCH(parser::if_stmt, str_test);
  EXPECT_MATCH(parser::if_stmt, str_if,
               [](auto res) { return *res->condition == ast::int_literal{1}; });
  EXPECT_MATCH(parser::if_stmt, str_if_else, [](auto res) {
    return *res->else_stmt ==
           ast::expr_stmt{std::make_shared<ast::int_literal>(5)};
  });
  EXPECT_MATCH(parser::decl_stmt, str_decl, [](auto res) {
    return res->var->name == "test" &&
           typeid(*res->var->var_type) == typeid(ast::int_type);
  });

   EXPECT_MATCH(parser::array_decl_stmt, str_array_decl, [](auto res) {
    return res->decl_array->name == "testarray" &&
           typeid(*res->decl_array->array_type) == typeid(ast::int_type) &&
            *res->decl_array->array_size == ast::int_literal{10};
  });

  EXPECT(parser::parse(R"(
	{
		int x;
		if(1 == 2) {
			5+4*8.0;
			7;
			x = 8;
		} else {
			if(2 < 9) {
			}
		}
		2.9;
    while(1 < 3) {
      x = x + 1;
    }
	}
	)"));
}
<|endoftext|>