<commit_before>/*
 * Copyright (c) 2021 Samsung Electronics Co., Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// CLASS HEADER
#include <dali-toolkit/internal/text/rendering/vector-based/vector-blob-atlas.h>

// EXTERNAL INCLUDES
#include <dali/integration-api/debug.h>

namespace
{
#if defined(DEBUG_ENABLED)
Debug::Filter* gLogFilter = Debug::Filter::New(Debug::Concise, true, "LOG_TEXT_RENDERING");
#endif

} // namespace

namespace Dali
{
namespace Toolkit
{
namespace Text
{
static void
EncodeBlobCoordinate(unsigned int cornerX, unsigned int cornerY, unsigned int atlasX, unsigned int atlasY, unsigned int nominalWidth, unsigned int nominalHeight, BlobCoordinate* v)
{
  DALI_ASSERT_DEBUG(0 == (atlasX & ~0x7F));
  DALI_ASSERT_DEBUG(0 == (atlasY & ~0x7F));
  DALI_ASSERT_DEBUG(0 == (cornerX & ~1));
  DALI_ASSERT_DEBUG(0 == (cornerY & ~1));
  DALI_ASSERT_DEBUG(0 == (nominalWidth & ~0x3F));
  DALI_ASSERT_DEBUG(0 == (nominalHeight & ~0x3F));

  unsigned int x = (((atlasX << 6) | nominalWidth) << 1) | cornerX;
  unsigned int y = (((atlasY << 6) | nominalHeight) << 1) | cornerY;

  unsigned int encoded = (x << 16) | y;

  v->u = encoded >> 16;
  v->v = encoded & 0xFFFF;
}

VectorBlobAtlas::VectorBlobAtlas(unsigned int textureWidth,
                                 unsigned int textureHeight,
                                 unsigned int itemWidth,
                                 unsigned int itemHeightQuantum)
: mTextureWidth(textureWidth),
  mTextureHeight(textureHeight),
  mItemWidth(itemWidth),
  mItemHeightQuantum(itemHeightQuantum),
  mCursorX(0),
  mCursorY(0),
  mIsFull(false)
{
  DALI_LOG_INFO(gLogFilter, Debug::General, "Blob atlas %p size %dx%d, item width %d, height quantum %d\n", this, textureWidth, textureHeight, itemWidth, itemHeightQuantum);

  mAtlasTexture = Texture::New(TextureType::TEXTURE_2D, Pixel::RGBA8888, textureWidth, textureHeight);

  mTextureSet = TextureSet::New();
  mTextureSet.SetTexture(0, mAtlasTexture);
}

bool VectorBlobAtlas::IsFull() const
{
  return mIsFull;
}

bool VectorBlobAtlas::FindGlyph(FontId          fontId,
                                GlyphIndex      glyphIndex,
                                BlobCoordinate* coords)
{
  const unsigned int size(mItemLookup.size());

  for(unsigned int i = 0; i < size; ++i)
  {
    if(mItemLookup[i].fontId == fontId &&
       mItemLookup[i].glyphIndex == glyphIndex)
    {
      const Item& item = mItemCache[mItemLookup[i].cacheIndex];

      coords[0] = item.coords[0];
      coords[1] = item.coords[1];
      coords[2] = item.coords[2];
      coords[3] = item.coords[3];

      return true;
    }
  }

  return false;
}

bool VectorBlobAtlas::AddGlyph(unsigned int    fontId,
                               unsigned int    glyphIndex,
                               VectorBlob*     blob,
                               unsigned int    length,
                               unsigned int    nominalWidth,
                               unsigned int    nominalHeight,
                               BlobCoordinate* coords)
{
  if(mIsFull)
  {
    return false;
  }

  unsigned int w, h, x, y;

  w = mItemWidth;
  h = (length + w - 1) / w;

  if(mCursorY + h > mTextureHeight)
  {
    // Go to next column
    mCursorX += mItemWidth;
    mCursorY = 0;
  }

  if(mCursorX + w <= mTextureWidth && mCursorY + h <= mTextureHeight)
  {
    x = mCursorX;
    y = mCursorY;
    mCursorY += (h + mItemHeightQuantum - 1) & ~(mItemHeightQuantum - 1);
  }
  else
  {
    DALI_LOG_INFO(gLogFilter, Debug::General, "Blob atlas %p is now FULL\n", this);

    // The atlas is now considered to be full
    mIsFull = true;
    return false;
  }

  if(w * h == length)
  {
    TexSubImage(x, y, w, h, blob);
  }
  else
  {
    TexSubImage(x, y, w, h - 1, blob);

    // Upload the last row separately
    TexSubImage(x, y + h - 1, length - (w * (h - 1)), 1, blob + w * (h - 1));
  }

  DALI_LOG_INFO(gLogFilter, Debug::General, "Blob atlas %p capacity %d filled %d %f\%\n", this, mTextureWidth * mTextureHeight, mCursorY * mItemWidth + mCursorX * mTextureHeight, 100.0f * (float)(mCursorY * mItemWidth + mCursorX * mTextureHeight) / (float)(mTextureWidth * mTextureHeight));

  Key key;
  key.fontId     = fontId;
  key.glyphIndex = glyphIndex;
  key.cacheIndex = mItemCache.size();
  mItemLookup.push_back(key);

  x /= mItemWidth;
  y /= mItemHeightQuantum;

  Item item;
  EncodeBlobCoordinate(0, 0, x, y, nominalWidth, nominalHeight, &item.coords[0]); // BOTTOM_LEFT
  EncodeBlobCoordinate(0, 1, x, y, nominalWidth, nominalHeight, &item.coords[1]); // TOP_LEFT
  EncodeBlobCoordinate(1, 0, x, y, nominalWidth, nominalHeight, &item.coords[2]); // BOTTOM_RIGHT
  EncodeBlobCoordinate(1, 1, x, y, nominalWidth, nominalHeight, &item.coords[3]); // TOP_RIGHT
  mItemCache.push_back(item);

  coords[0] = item.coords[0];
  coords[1] = item.coords[1];
  coords[2] = item.coords[2];
  coords[3] = item.coords[3];

  return true;
}

void VectorBlobAtlas::TexSubImage(unsigned int offsetX,
                                  unsigned int offsetY,
                                  unsigned int width,
                                  unsigned int height,
                                  VectorBlob*  blob)
{
  const size_t size   = width * height * 4;
  uint8_t*     pixbuf = new uint8_t[size];

  size_t pos;
  size_t dataIndex = 0;
  for(size_t y = 0; y < height; y++)
  {
    pos = y * mTextureWidth * 4;
    for(size_t x = 0; x < width; x++)
    {
      pixbuf[pos + x * 4]     = 0xFF & blob[dataIndex].r;
      pixbuf[pos + x * 4 + 1] = 0xFF & blob[dataIndex].g;
      pixbuf[pos + x * 4 + 2] = 0xFF & blob[dataIndex].b;
      pixbuf[pos + x * 4 + 3] = 0xFF & blob[dataIndex].a;
      dataIndex++;
    }
  }

  PixelData pixelData = PixelData::New(pixbuf, size, width, height, Pixel::RGBA8888, PixelData::DELETE_ARRAY);
  mAtlasTexture.Upload(pixelData, 0u, 0u, offsetX, offsetY, width, height);
}

} // namespace Text

} // namespace Toolkit

} // namespace Dali
<commit_msg>Fixed memory scribbler in text vector blob<commit_after>/*
 * Copyright (c) 2021 Samsung Electronics Co., Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// CLASS HEADER
#include <dali-toolkit/internal/text/rendering/vector-based/vector-blob-atlas.h>

// EXTERNAL INCLUDES
#include <dali/integration-api/debug.h>

namespace
{
#if defined(DEBUG_ENABLED)
Debug::Filter* gLogFilter = Debug::Filter::New(Debug::Concise, true, "LOG_TEXT_RENDERING");
#endif

} // namespace

namespace Dali
{
namespace Toolkit
{
namespace Text
{
static void
EncodeBlobCoordinate(unsigned int cornerX, unsigned int cornerY, unsigned int atlasX, unsigned int atlasY, unsigned int nominalWidth, unsigned int nominalHeight, BlobCoordinate* v)
{
  DALI_ASSERT_DEBUG(0 == (atlasX & ~0x7F));
  DALI_ASSERT_DEBUG(0 == (atlasY & ~0x7F));
  DALI_ASSERT_DEBUG(0 == (cornerX & ~1));
  DALI_ASSERT_DEBUG(0 == (cornerY & ~1));
  DALI_ASSERT_DEBUG(0 == (nominalWidth & ~0x3F));
  DALI_ASSERT_DEBUG(0 == (nominalHeight & ~0x3F));

  unsigned int x = (((atlasX << 6) | nominalWidth) << 1) | cornerX;
  unsigned int y = (((atlasY << 6) | nominalHeight) << 1) | cornerY;

  unsigned int encoded = (x << 16) | y;

  v->u = encoded >> 16;
  v->v = encoded & 0xFFFF;
}

VectorBlobAtlas::VectorBlobAtlas(unsigned int textureWidth,
                                 unsigned int textureHeight,
                                 unsigned int itemWidth,
                                 unsigned int itemHeightQuantum)
: mTextureWidth(textureWidth),
  mTextureHeight(textureHeight),
  mItemWidth(itemWidth),
  mItemHeightQuantum(itemHeightQuantum),
  mCursorX(0),
  mCursorY(0),
  mIsFull(false)
{
  DALI_LOG_INFO(gLogFilter, Debug::General, "Blob atlas %p size %dx%d, item width %d, height quantum %d\n", this, textureWidth, textureHeight, itemWidth, itemHeightQuantum);

  mAtlasTexture = Texture::New(TextureType::TEXTURE_2D, Pixel::RGBA8888, textureWidth, textureHeight);

  mTextureSet = TextureSet::New();
  mTextureSet.SetTexture(0, mAtlasTexture);
}

bool VectorBlobAtlas::IsFull() const
{
  return mIsFull;
}

bool VectorBlobAtlas::FindGlyph(FontId          fontId,
                                GlyphIndex      glyphIndex,
                                BlobCoordinate* coords)
{
  const unsigned int size(mItemLookup.size());

  for(unsigned int i = 0; i < size; ++i)
  {
    if(mItemLookup[i].fontId == fontId &&
       mItemLookup[i].glyphIndex == glyphIndex)
    {
      const Item& item = mItemCache[mItemLookup[i].cacheIndex];

      coords[0] = item.coords[0];
      coords[1] = item.coords[1];
      coords[2] = item.coords[2];
      coords[3] = item.coords[3];

      return true;
    }
  }

  return false;
}

bool VectorBlobAtlas::AddGlyph(unsigned int    fontId,
                               unsigned int    glyphIndex,
                               VectorBlob*     blob,
                               unsigned int    length,
                               unsigned int    nominalWidth,
                               unsigned int    nominalHeight,
                               BlobCoordinate* coords)
{
  if(mIsFull)
  {
    return false;
  }

  unsigned int w, h, x, y;

  w = mItemWidth;
  h = (length + w - 1) / w;

  if(mCursorY + h > mTextureHeight)
  {
    // Go to next column
    mCursorX += mItemWidth;
    mCursorY = 0;
  }

  if(mCursorX + w <= mTextureWidth && mCursorY + h <= mTextureHeight)
  {
    x = mCursorX;
    y = mCursorY;
    mCursorY += (h + mItemHeightQuantum - 1) & ~(mItemHeightQuantum - 1);
  }
  else
  {
    DALI_LOG_INFO(gLogFilter, Debug::General, "Blob atlas %p is now FULL\n", this);

    // The atlas is now considered to be full
    mIsFull = true;
    return false;
  }

  if(w * h == length)
  {
    TexSubImage(x, y, w, h, blob);
  }
  else
  {
    TexSubImage(x, y, w, h - 1, blob);

    // Upload the last row separately
    TexSubImage(x, y + h - 1, length - (w * (h - 1)), 1, blob + w * (h - 1));
  }

  DALI_LOG_INFO(gLogFilter, Debug::General, "Blob atlas %p capacity %d filled %d %f\%\n", this, mTextureWidth * mTextureHeight, mCursorY * mItemWidth + mCursorX * mTextureHeight, 100.0f * (float)(mCursorY * mItemWidth + mCursorX * mTextureHeight) / (float)(mTextureWidth * mTextureHeight));

  Key key;
  key.fontId     = fontId;
  key.glyphIndex = glyphIndex;
  key.cacheIndex = mItemCache.size();
  mItemLookup.push_back(key);

  x /= mItemWidth;
  y /= mItemHeightQuantum;

  Item item;
  EncodeBlobCoordinate(0, 0, x, y, nominalWidth, nominalHeight, &item.coords[0]); // BOTTOM_LEFT
  EncodeBlobCoordinate(0, 1, x, y, nominalWidth, nominalHeight, &item.coords[1]); // TOP_LEFT
  EncodeBlobCoordinate(1, 0, x, y, nominalWidth, nominalHeight, &item.coords[2]); // BOTTOM_RIGHT
  EncodeBlobCoordinate(1, 1, x, y, nominalWidth, nominalHeight, &item.coords[3]); // TOP_RIGHT
  mItemCache.push_back(item);

  coords[0] = item.coords[0];
  coords[1] = item.coords[1];
  coords[2] = item.coords[2];
  coords[3] = item.coords[3];

  return true;
}

void VectorBlobAtlas::TexSubImage(unsigned int offsetX,
                                  unsigned int offsetY,
                                  unsigned int width,
                                  unsigned int height,
                                  VectorBlob*  blob)
{
  const size_t size   = width * height * 4;
  uint8_t*     pixbuf = new uint8_t[size];

  size_t pos;
  size_t dataIndex = 0;
  for(size_t y = 0; y < height; y++)
  {
    pos = y * width * 4;
    for(size_t x = 0; x < width; x++)
    {
      pixbuf[pos + x * 4]     = 0xFF & blob[dataIndex].r;
      pixbuf[pos + x * 4 + 1] = 0xFF & blob[dataIndex].g;
      pixbuf[pos + x * 4 + 2] = 0xFF & blob[dataIndex].b;
      pixbuf[pos + x * 4 + 3] = 0xFF & blob[dataIndex].a;
      dataIndex++;
    }
  }

  PixelData pixelData = PixelData::New(pixbuf, size, width, height, Pixel::RGBA8888, PixelData::DELETE_ARRAY);
  mAtlasTexture.Upload(pixelData, 0u, 0u, offsetX, offsetY, width, height);
}

} // namespace Text

} // namespace Toolkit

} // namespace Dali
<|endoftext|>


<commit_before>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// This test performs a series false positive checks using a list of URLs
// against a known set of SafeBrowsing data.
//
// It uses a normal SafeBrowsing database to create a bloom filter where it
// looks up all the URLs in the url file. A URL that has a prefix found in the
// bloom filter and found in the database is considered a hit: a valid lookup
// that will result in a gethash request. A URL that has a prefix found in the
// bloom filter but not in the database is a miss: a false positive lookup that
// will result in an unnecessary gethash request.
//
// By varying the size of the bloom filter and using a constant set of
// SafeBrowsing data, we can check a known set of URLs against the filter and
// determine the false positive rate.
//
// Usage:
//   $ ./perf_tests.exe --gtest_filter=SafeBrowsingBloomFilter.FalsePositives
//                      --filter-start=<integer>
//                      --filter-steps=<integer>
//                      --filter-verbose
//
//  --filter-start:   The filter multiplier to begin with. This represents the
//                    number of bits per prefix of memory to use in the filter.
//                    The default value is identical to the current SafeBrowsing
//                    database value.
//  --filter-steps:   The number of iterations to run, with each iteration
//                    increasing the filter multiplier by 1. The default value
//                    is 1.
//  --filter-verbose: Used to print out the hit / miss results per URL.
//  --filter-csv:     The URL file contains information about the number of
//                    unique views (the popularity) of each URL. See the format
//                    description below.
//
// Data files:
//    chrome/test/data/safe_browsing/filter/database
//    chrome/test/data/safe_browsing/filter/urls
//
// database: A normal SafeBrowsing database.
// urls:     A text file containing a list of URLs, one per line. If the option
//           --filter-csv is specified, the format of each line in the file is
//           <url>,<weight> where weight is an integer indicating the number of
//           unique views for the URL.

#include <fstream>
#include <iostream>
#include <vector>

#include "base/command_line.h"
#include "base/file_path.h"
#include "base/file_util.h"
#include "base/logging.h"
#include "base/path_service.h"
#include "base/scoped_ptr.h"
#include "base/sha2.h"
#include "base/string_util.h"
#include "chrome/browser/safe_browsing/bloom_filter.h"
#include "chrome/browser/safe_browsing/safe_browsing_util.h"
#include "chrome/common/chrome_paths.h"
#include "chrome/common/sqlite_compiled_statement.h"
#include "chrome/common/sqlite_utils.h"
#include "googleurl/src/gurl.h"
#include "testing/gtest/include/gtest/gtest.h"

namespace {

// Ensures the SafeBrowsing database is closed properly.
class ScopedPerfDatabase {
 public:
  explicit ScopedPerfDatabase(sqlite3* db) : db_(db) {}
  ~ScopedPerfDatabase() {
    sqlite3_close(db_);
  }

 private:
  sqlite3* db_;

  DISALLOW_COPY_AND_ASSIGN(ScopedPerfDatabase);
};

// Command line flags.
const wchar_t kFilterVerbose[] = L"filter-verbose";
const wchar_t kFilterStart[] = L"filter-start";
const wchar_t kFilterSteps[] = L"filter-steps";
const wchar_t kFilterCsv[] = L"filter-csv";

// Returns the path to the data used in this test, relative to the top of the
// source directory.
FilePath GetFullDataPath() {
  FilePath full_path;
  CHECK(PathService::Get(chrome::DIR_TEST_DATA, &full_path));
  full_path = full_path.Append(FILE_PATH_LITERAL("safe_browsing"));
  full_path = full_path.Append(FILE_PATH_LITERAL("filter"));
  CHECK(file_util::PathExists(full_path));
  return full_path;
}

// Constructs a bloom filter of the appropriate size from the provided prefixes.
void BuildBloomFilter(int size_multiplier,
                      const std::vector<SBPrefix>& prefixes,
                      BloomFilter** bloom_filter) {
  // Create a BloomFilter with the specified size.
  const int key_count = std::max(static_cast<int>(prefixes.size()), 250000);
  const int filter_size = key_count * size_multiplier;
  *bloom_filter = new BloomFilter(filter_size);

  // Add the prefixes to it.
  for (size_t i = 0; i < prefixes.size(); ++i)
    (*bloom_filter)->Insert(prefixes[i]);

  std::cout << "Bloom filter with prefixes: " << prefixes.size()
            << ", multiplier: " << size_multiplier
            << ", size (bytes): " << (*bloom_filter)->size()
            << std::endl;
}

// Reads the set of add prefixes contained in a SafeBrowsing database into a
// sorted array suitable for fast searching. This takes significantly less time
// to look up a given prefix than performing SQL queries.
bool ReadDatabase(const FilePath& path, std::vector<SBPrefix>* prefixes) {
  FilePath database_file = path.Append(FILE_PATH_LITERAL("database"));
  sqlite3* db = NULL;
  if (OpenSqliteDb(database_file, &db) != SQLITE_OK) {
    sqlite3_close(db);
    return false;
  }

  ScopedPerfDatabase database(db);
  scoped_ptr<SqliteStatementCache> sql_cache(new SqliteStatementCache(db));

  // Get the number of items in the add_prefix table.
  std::string sql = "SELECT COUNT(*) FROM add_prefix";
  SQLITE_UNIQUE_STATEMENT(count_statement, *sql_cache, sql.c_str());
  if (!count_statement.is_valid())
    return false;

  if (count_statement->step() != SQLITE_ROW)
    return false;

  const int count = count_statement->column_int(0);

  // Load them into a prefix vector and sort
  prefixes->reserve(count);
  sql = "SELECT prefix FROM add_prefix";
  SQLITE_UNIQUE_STATEMENT(prefix_statement, *sql_cache, sql.c_str());
  if (!prefix_statement.is_valid())
    return false;

  while (prefix_statement->step() == SQLITE_ROW)
    prefixes->push_back(prefix_statement->column_int(0));

  DCHECK(static_cast<int>(prefixes->size()) == count);
  sort(prefixes->begin(), prefixes->end());

  return true;
}

// Generates all legal SafeBrowsing prefixes for the specified URL, and returns
// the set of Prefixes that exist in the bloom filter. The function returns the
// number of host + path combinations checked.
int GeneratePrefixHits(const std::string url,
                       BloomFilter* bloom_filter,
                       std::vector<SBPrefix>* prefixes) {
  GURL url_check(url);
  std::vector<std::string> hosts;
  if (url_check.HostIsIPAddress()) {
    hosts.push_back(url_check.host());
  } else {
    safe_browsing_util::GenerateHostsToCheck(url_check, &hosts);
  }

  std::vector<std::string> paths;
  safe_browsing_util::GeneratePathsToCheck(url_check, &paths);

  for (size_t i = 0; i < hosts.size(); ++i) {
    for (size_t j = 0; j < paths.size(); ++j) {
      SBPrefix prefix;
      base::SHA256HashString(hosts[i] + paths[j], &prefix, sizeof(prefix));
      if (bloom_filter->Exists(prefix))
        prefixes->push_back(prefix);
    }
  }

  return hosts.size() * paths.size();
}

// Binary search of sorted prefixes.
bool IsPrefixInDatabase(SBPrefix prefix,
                        const std::vector<SBPrefix>& prefixes) {
  if (prefixes.empty())
    return false;

  int low = 0;
  int high = prefixes.size() - 1;
  while (low <= high) {
    int mid = ((unsigned int)low + (unsigned int)high) >> 1;
    SBPrefix prefix_mid = prefixes[mid];
    if (prefix_mid == prefix)
      return true;
    if (prefix_mid < prefix)
      low = mid + 1;
    else
      high = mid - 1;
  }

  return false;
}

// Construct a bloom filter with the given prefixes and multiplier, and test the
// false positive rate (misses) against a URL list.
void CalculateBloomFilterFalsePositives(
    int size_multiplier,
    const FilePath& data_dir,
    const std::vector<SBPrefix>& prefix_list) {
  BloomFilter* bloom_filter = NULL;
  BuildBloomFilter(size_multiplier, prefix_list, &bloom_filter);
  scoped_refptr<BloomFilter> scoped_filter(bloom_filter);

  // Read in data file line at a time.
  FilePath url_file = data_dir.Append(FILE_PATH_LITERAL("urls"));
  std::ifstream url_stream(WideToASCII(url_file.ToWStringHack()).c_str());

  // Keep track of stats
  int hits = 0;
  int misses = 0;
  int weighted_hits = 0;
  int weighted_misses = 0;
  int url_count = 0;
  int prefix_count = 0;

  // Print out volumes of data (per URL hit and miss information).
  bool verbose = CommandLine::ForCurrentProcess()->HasSwitch(kFilterVerbose);
  bool use_weights = CommandLine::ForCurrentProcess()->HasSwitch(kFilterCsv);

  std::string url;
  while (std::getline(url_stream, url)) {
    ++url_count;

    // Handle a format that contains URLs weighted by unique views.
    int weight = 1;
    if (use_weights) {
      std::string::size_type pos = url.find_last_of(",");
      if (pos != std::string::npos) {
        weight = StringToInt(std::string(url, pos + 1));
        url = url.substr(0, pos);
      }
    }

    // See if the URL is in the bloom filter.
    std::vector<SBPrefix> prefixes;
    prefix_count += GeneratePrefixHits(url, bloom_filter, &prefixes);

    // See if the prefix is actually in the database (in-memory prefix list).
    for (size_t i = 0; i < prefixes.size(); ++i) {
      if (IsPrefixInDatabase(prefixes[i], prefix_list)) {
        ++hits;
        weighted_hits += weight;
        if (verbose) {
          std::cout << "Hit for URL: " << url
                    << " (prefix = "   << prefixes[i] << ")"
                    << std::endl;
        }
      } else {
        ++misses;
        weighted_misses += weight;
        if (verbose) {
          std::cout << "Miss for URL: " << url
                    << " (prefix = "    << prefixes[i] << ")"
                    << std::endl;
        }
      }
    }
  }

  // Print out the results for this test.
  std::cout << "URLs checked: "      << url_count
            << ", prefix compares: " << prefix_count
            << ", hits: "            << hits
            << ", misses: "          << misses;
  if (use_weights) {
    std::cout << ", weighted hits: "   << weighted_hits
              << ", weighted misses: " << weighted_misses;
  }
  std::cout << std::endl;
}

}  // namespace

// This test can take several minutes to perform its calculations, so it should
// be disabled until you need to run it.
TEST(SafeBrowsingBloomFilter, FalsePositives) {
  std::vector<SBPrefix> prefix_list;
  FilePath data_dir = GetFullDataPath();
  if (!ReadDatabase(data_dir, &prefix_list))
    return;

  int start = BloomFilter::kBloomFilterSizeRatio;
  if (CommandLine::ForCurrentProcess()->HasSwitch(kFilterStart)) {
    start = StringToInt(
        CommandLine::ForCurrentProcess()->GetSwitchValue(kFilterStart));
  }

  int steps = 1;
  if (CommandLine::ForCurrentProcess()->HasSwitch(kFilterSteps)) {
    steps = StringToInt(
        CommandLine::ForCurrentProcess()->GetSwitchValue(kFilterSteps));
  }

  int stop = start + steps;

  for (int multiplier = start; multiplier < stop; ++multiplier)
    CalculateBloomFilterFalsePositives(multiplier, data_dir, prefix_list);
}

<commit_msg>Add a performance test for measuring hash implementation time.<commit_after>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// This test performs a series false positive checks using a list of URLs
// against a known set of SafeBrowsing data.
//
// It uses a normal SafeBrowsing database to create a bloom filter where it
// looks up all the URLs in the url file. A URL that has a prefix found in the
// bloom filter and found in the database is considered a hit: a valid lookup
// that will result in a gethash request. A URL that has a prefix found in the
// bloom filter but not in the database is a miss: a false positive lookup that
// will result in an unnecessary gethash request.
//
// By varying the size of the bloom filter and using a constant set of
// SafeBrowsing data, we can check a known set of URLs against the filter and
// determine the false positive rate.
//
// False positive calculation usage:
//   $ ./perf_tests.exe --gtest_filter=SafeBrowsingBloomFilter.FalsePositives
//                      --filter-start=<integer>
//                      --filter-steps=<integer>
//                      --filter-verbose
//
//  --filter-start:   The filter multiplier to begin with. This represents the
//                    number of bits per prefix of memory to use in the filter.
//                    The default value is identical to the current SafeBrowsing
//                    database value.
//  --filter-steps:   The number of iterations to run, with each iteration
//                    increasing the filter multiplier by 1. The default value
//                    is 1.
//  --filter-verbose: Used to print out the hit / miss results per URL.
//  --filter-csv:     The URL file contains information about the number of
//                    unique views (the popularity) of each URL. See the format
//                    description below.
//
//
// Hash compute time usage:
//   $ ./perf_tests.exe --gtest_filter=SafeBrowsingBloomFilter.HashTime
//                      --filter-num-checks=<integer>
//
//  --filter-num-checks: The number of hash look ups to perform on the bloom
//                       filter. The default is 10 million.
//
// Data files:
//    chrome/test/data/safe_browsing/filter/database
//    chrome/test/data/safe_browsing/filter/urls
//
// database: A normal SafeBrowsing database.
// urls:     A text file containing a list of URLs, one per line. If the option
//           --filter-csv is specified, the format of each line in the file is
//           <url>,<weight> where weight is an integer indicating the number of
//           unique views for the URL.

#include <fstream>
#include <iostream>
#include <vector>

#include "base/command_line.h"
#include "base/file_path.h"
#include "base/file_util.h"
#include "base/logging.h"
#include "base/path_service.h"
#include "base/rand_util.h"
#include "base/scoped_ptr.h"
#include "base/sha2.h"
#include "base/string_util.h"
#include "base/time.h"
#include "chrome/browser/safe_browsing/bloom_filter.h"
#include "chrome/browser/safe_browsing/safe_browsing_util.h"
#include "chrome/common/chrome_paths.h"
#include "chrome/common/sqlite_compiled_statement.h"
#include "chrome/common/sqlite_utils.h"
#include "googleurl/src/gurl.h"
#include "testing/gtest/include/gtest/gtest.h"

using base::Time;
using base::TimeDelta;

namespace {

// Ensures the SafeBrowsing database is closed properly.
class ScopedPerfDatabase {
 public:
  explicit ScopedPerfDatabase(sqlite3* db) : db_(db) {}
  ~ScopedPerfDatabase() {
    sqlite3_close(db_);
  }

 private:
  sqlite3* db_;

  DISALLOW_COPY_AND_ASSIGN(ScopedPerfDatabase);
};

// Command line flags.
const wchar_t kFilterVerbose[] = L"filter-verbose";
const wchar_t kFilterStart[] = L"filter-start";
const wchar_t kFilterSteps[] = L"filter-steps";
const wchar_t kFilterCsv[] = L"filter-csv";
const wchar_t kFilterNumChecks[] = L"filter-num-checks";

// Number of hash checks to make during performance testing.
static const int kNumHashChecks = 10000000;

// Returns the path to the data used in this test, relative to the top of the
// source directory.
FilePath GetFullDataPath() {
  FilePath full_path;
  CHECK(PathService::Get(chrome::DIR_TEST_DATA, &full_path));
  full_path = full_path.Append(FILE_PATH_LITERAL("safe_browsing"));
  full_path = full_path.Append(FILE_PATH_LITERAL("filter"));
  CHECK(file_util::PathExists(full_path));
  return full_path;
}

// Constructs a bloom filter of the appropriate size from the provided prefixes.
void BuildBloomFilter(int size_multiplier,
                      const std::vector<SBPrefix>& prefixes,
                      BloomFilter** bloom_filter) {
  // Create a BloomFilter with the specified size.
  const int key_count = std::max(static_cast<int>(prefixes.size()),
                                 BloomFilter::kBloomFilterMinSize);
  const int filter_size = key_count * size_multiplier;
  *bloom_filter = new BloomFilter(filter_size);

  // Add the prefixes to it.
  for (size_t i = 0; i < prefixes.size(); ++i)
    (*bloom_filter)->Insert(prefixes[i]);

  std::cout << "Bloom filter with prefixes: " << prefixes.size()
            << ", multiplier: " << size_multiplier
            << ", size (bytes): " << (*bloom_filter)->size()
            << std::endl;
}

// Reads the set of add prefixes contained in a SafeBrowsing database into a
// sorted array suitable for fast searching. This takes significantly less time
// to look up a given prefix than performing SQL queries.
bool ReadDatabase(const FilePath& path, std::vector<SBPrefix>* prefixes) {
  FilePath database_file = path.Append(FILE_PATH_LITERAL("database"));
  sqlite3* db = NULL;
  if (OpenSqliteDb(database_file, &db) != SQLITE_OK) {
    sqlite3_close(db);
    return false;
  }

  ScopedPerfDatabase database(db);
  scoped_ptr<SqliteStatementCache> sql_cache(new SqliteStatementCache(db));

  // Get the number of items in the add_prefix table.
  std::string sql = "SELECT COUNT(*) FROM add_prefix";
  SQLITE_UNIQUE_STATEMENT(count_statement, *sql_cache, sql.c_str());
  if (!count_statement.is_valid())
    return false;

  if (count_statement->step() != SQLITE_ROW)
    return false;

  const int count = count_statement->column_int(0);

  // Load them into a prefix vector and sort
  prefixes->reserve(count);
  sql = "SELECT prefix FROM add_prefix";
  SQLITE_UNIQUE_STATEMENT(prefix_statement, *sql_cache, sql.c_str());
  if (!prefix_statement.is_valid())
    return false;

  while (prefix_statement->step() == SQLITE_ROW)
    prefixes->push_back(prefix_statement->column_int(0));

  DCHECK(static_cast<int>(prefixes->size()) == count);
  sort(prefixes->begin(), prefixes->end());

  return true;
}

// Generates all legal SafeBrowsing prefixes for the specified URL, and returns
// the set of Prefixes that exist in the bloom filter. The function returns the
// number of host + path combinations checked.
int GeneratePrefixHits(const std::string url,
                       BloomFilter* bloom_filter,
                       std::vector<SBPrefix>* prefixes) {
  GURL url_check(url);
  std::vector<std::string> hosts;
  if (url_check.HostIsIPAddress()) {
    hosts.push_back(url_check.host());
  } else {
    safe_browsing_util::GenerateHostsToCheck(url_check, &hosts);
  }

  std::vector<std::string> paths;
  safe_browsing_util::GeneratePathsToCheck(url_check, &paths);

  for (size_t i = 0; i < hosts.size(); ++i) {
    for (size_t j = 0; j < paths.size(); ++j) {
      SBPrefix prefix;
      base::SHA256HashString(hosts[i] + paths[j], &prefix, sizeof(prefix));
      if (bloom_filter->Exists(prefix))
        prefixes->push_back(prefix);
    }
  }

  return hosts.size() * paths.size();
}

// Binary search of sorted prefixes.
bool IsPrefixInDatabase(SBPrefix prefix,
                        const std::vector<SBPrefix>& prefixes) {
  if (prefixes.empty())
    return false;

  int low = 0;
  int high = prefixes.size() - 1;
  while (low <= high) {
    int mid = ((unsigned int)low + (unsigned int)high) >> 1;
    SBPrefix prefix_mid = prefixes[mid];
    if (prefix_mid == prefix)
      return true;
    if (prefix_mid < prefix)
      low = mid + 1;
    else
      high = mid - 1;
  }

  return false;
}

// Construct a bloom filter with the given prefixes and multiplier, and test the
// false positive rate (misses) against a URL list.
void CalculateBloomFilterFalsePositives(
    int size_multiplier,
    const FilePath& data_dir,
    const std::vector<SBPrefix>& prefix_list) {
  BloomFilter* bloom_filter = NULL;
  BuildBloomFilter(size_multiplier, prefix_list, &bloom_filter);
  scoped_refptr<BloomFilter> scoped_filter(bloom_filter);

  // Read in data file line at a time.
  FilePath url_file = data_dir.Append(FILE_PATH_LITERAL("urls"));
  std::ifstream url_stream(WideToASCII(url_file.ToWStringHack()).c_str());

  // Keep track of stats
  int hits = 0;
  int misses = 0;
  int weighted_hits = 0;
  int weighted_misses = 0;
  int url_count = 0;
  int prefix_count = 0;

  // Print out volumes of data (per URL hit and miss information).
  bool verbose = CommandLine::ForCurrentProcess()->HasSwitch(kFilterVerbose);
  bool use_weights = CommandLine::ForCurrentProcess()->HasSwitch(kFilterCsv);

  std::string url;
  while (std::getline(url_stream, url)) {
    ++url_count;

    // Handle a format that contains URLs weighted by unique views.
    int weight = 1;
    if (use_weights) {
      std::string::size_type pos = url.find_last_of(",");
      if (pos != std::string::npos) {
        weight = StringToInt(std::string(url, pos + 1));
        url = url.substr(0, pos);
      }
    }

    // See if the URL is in the bloom filter.
    std::vector<SBPrefix> prefixes;
    prefix_count += GeneratePrefixHits(url, bloom_filter, &prefixes);

    // See if the prefix is actually in the database (in-memory prefix list).
    for (size_t i = 0; i < prefixes.size(); ++i) {
      if (IsPrefixInDatabase(prefixes[i], prefix_list)) {
        ++hits;
        weighted_hits += weight;
        if (verbose) {
          std::cout << "Hit for URL: " << url
                    << " (prefix = "   << prefixes[i] << ")"
                    << std::endl;
        }
      } else {
        ++misses;
        weighted_misses += weight;
        if (verbose) {
          std::cout << "Miss for URL: " << url
                    << " (prefix = "    << prefixes[i] << ")"
                    << std::endl;
        }
      }
    }
  }

  // Print out the results for this test.
  std::cout << "URLs checked: "      << url_count
            << ", prefix compares: " << prefix_count
            << ", hits: "            << hits
            << ", misses: "          << misses;
  if (use_weights) {
    std::cout << ", weighted hits: "   << weighted_hits
              << ", weighted misses: " << weighted_misses;
  }
  std::cout << std::endl;
}

}  // namespace

// This test can take several minutes to perform its calculations, so it should
// be disabled until you need to run it.
TEST(SafeBrowsingBloomFilter, FalsePositives) {
  std::vector<SBPrefix> prefix_list;
  FilePath data_dir = GetFullDataPath();
  ASSERT_TRUE(ReadDatabase(data_dir, &prefix_list));

  int start = BloomFilter::kBloomFilterSizeRatio;
  if (CommandLine::ForCurrentProcess()->HasSwitch(kFilterStart)) {
    start = StringToInt(
        CommandLine::ForCurrentProcess()->GetSwitchValue(kFilterStart));
  }

  int steps = 1;
  if (CommandLine::ForCurrentProcess()->HasSwitch(kFilterSteps)) {
    steps = StringToInt(
        CommandLine::ForCurrentProcess()->GetSwitchValue(kFilterSteps));
  }

  int stop = start + steps;

  for (int multiplier = start; multiplier < stop; ++multiplier)
    CalculateBloomFilterFalsePositives(multiplier, data_dir, prefix_list);
}

// Computes the time required for performing a number of look ups in a bloom
// filter. This is useful for measuring the performance of new hash functions.
TEST(SafeBrowsingBloomFilter, HashTime) {
  // Read the data from the database.
  std::vector<SBPrefix> prefix_list;
  FilePath data_dir = GetFullDataPath();
  ASSERT_TRUE(ReadDatabase(data_dir, &prefix_list));

  int num_checks = kNumHashChecks;
  if (CommandLine::ForCurrentProcess()->HasSwitch(kFilterNumChecks)) {
    num_checks = StringToInt(
      CommandLine::ForCurrentProcess()->GetSwitchValue(kFilterNumChecks));
  }

  // Populate the bloom filter and measure the time.
  BloomFilter* bloom_filter = NULL;
  Time populate_before = Time::Now();
  BuildBloomFilter(BloomFilter::kBloomFilterSizeRatio,
                   prefix_list, &bloom_filter);
  TimeDelta populate = Time::Now() - populate_before;

  // Check a large number of random prefixes against the filter.
  int hits = 0;
  Time check_before = Time::Now();
  for (int i = 0; i < num_checks; ++i) {
    uint32 prefix = static_cast<uint32>(base::RandUint64());
    if (bloom_filter->Exists(prefix))
      ++hits;
  }
  TimeDelta check = Time::Now() - check_before;

  int64 time_per_insert = populate.InMicroseconds() /
                          static_cast<int>(prefix_list.size());
  int64 time_per_check = check.InMicroseconds() / num_checks;

  std::cout << "Time results for checks: " << num_checks
            << ", prefixes: "              << prefix_list.size()
            << ", populate time (ms): "    << populate.InMilliseconds()
            << ", check time (ms): "       << check.InMilliseconds()
            << ", hits: "                  << hits
            << ", per-populate (us): "     << time_per_insert
            << ", per-check (us): "        << time_per_check
            << std::endl;
}

<|endoftext|>


<commit_before>/*
    Copyright 2002-2013 CEA LIST

    This file is part of LIMA.

    LIMA is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    LIMA is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with LIMA.  If not, see <http://www.gnu.org/licenses/>
*/
/***************************************************************************
 *   Copyright (C) 2004-2012 by CEA LIST                              *
 *                                                                         *
 ***************************************************************************/
#include "MorphoSyntacticData.h"
#include "MorphoSyntacticDataUtils.h"

#include "common/MediaticData/mediaticData.h"
#include "common/Data/strwstrtools.h"

#include <algorithm>

using namespace std;
using namespace Lima::Common::MediaticData;

namespace Lima
{

namespace LinguisticProcessing
{

namespace LinguisticAnalysisStructure
{

bool LinguisticElement::operator==(const LinguisticElement& le) const
{
  return ((properties==le.properties) &&
          (inflectedForm==le.inflectedForm) &&
          (lemma==le.lemma) &&
          (normalizedForm==le.normalizedForm) &&
          (type==le.type));
}

bool LinguisticElement::operator<(const LinguisticElement& le) const
{
  if (inflectedForm!=le.inflectedForm) return inflectedForm<le.inflectedForm;
  if (lemma!=le.lemma) return lemma<le.lemma;
  if (normalizedForm!=le.normalizedForm) return normalizedForm<le.normalizedForm;
  if (properties!=le.properties) return properties<le.properties;
  return type<le.type;
}

MorphoSyntacticData::MorphoSyntacticData()
{}

MorphoSyntacticData::~MorphoSyntacticData()
{}

bool MorphoSyntacticData::hasUniqueMicro(const Common::PropertyCode::PropertyAccessor& microAccessor,const std::list<LinguisticCode>& microFilter)
{
  LinguisticCode micro(0);
  for (const_iterator it=begin();
       it!=end();
       it++)
  {
    LinguisticCode tmp=microAccessor.readValue(it->properties);
    if (micro!=static_cast<LinguisticCode>(0))
    {
      if (micro!=tmp)
      {
        return false;
      }
    }
    else
    {
      micro=tmp;
    }
    bool found=false;
    for (list<LinguisticCode>::const_iterator filterItr=microFilter.begin();
         filterItr!=microFilter.end();
         filterItr++)
    {
      if (tmp==*filterItr)
      {
        found=true;
        break;
      }
    }
    if (!found)
    {
      return false;
    }
  }
  // if micro is 0, then there was no micros, return false
  return micro!=static_cast<LinguisticCode>(0);
}

uint64_t MorphoSyntacticData::countValues(const Common::PropertyCode::PropertyAccessor& propertyAccessor)
{
  set<LinguisticCode> values;
  allValues(propertyAccessor,values);
  return values.size();
}

void MorphoSyntacticData::allValues(const Common::PropertyCode::PropertyAccessor& propertyAccessor,std::set<LinguisticCode>& result) const
  {
    for (const_iterator it=begin();
         it!=end();
         it++)
    {
      if (!propertyAccessor.empty(it->properties))
      {
        result.insert(propertyAccessor.readValue(it->properties));
      }
    }
  }

void MorphoSyntacticData::outputXml(std::ostream& xmlStream,const Common::PropertyCode::PropertyCodeManager& pcm,const FsaStringsPool& sp) const
{
  xmlStream << "    <data>" << std::endl;

  if (!empty())
  {
    // trie pour avoir les donn�s group�s par strings
    MorphoSyntacticData tmp(*this);
    sort(tmp.begin(),tmp.end(),ltString());
    MorphoSyntacticType type(NO_MORPHOSYNTACTICTYPE);
    StringsPoolIndex form(0);
    StringsPoolIndex lemma(0);
    StringsPoolIndex norm(0);
    string currentType;
    bool firstEntry=true;
    for (const_iterator it=tmp.begin();
         it!=tmp.end();
         it++)
    {
      if (it->type != type)
      {
        // Changement type
        if (!firstEntry)
        {
          xmlStream << "        </form>" << std::endl;
          xmlStream << "      </" << currentType << ">" << std::endl;
        }
        type=it->type;
        switch ( type )
        {
            case SIMPLE_WORD :
            currentType = "simple_word";
            break;
            case IDIOMATIC_EXPRESSION :
            currentType = "idiomatic_expression";
            break;
            case UNKNOWN_WORD :
            currentType = "unknown_word";
            break;
            case ABBREV_ALTERNATIVE :
            currentType = "abbrev_alternative";
            break;
            case HYPHEN_ALTERNATIVE :
            currentType = "hyphen_alternative";
            break;
            case CONCATENATED_ALTERNATIVE :
            currentType = "concatenated_alternative";
            break;
            case CAPITALFIRST_WORD :
            currentType = "capitalfirst_word";
            break;
            case AGGLUTINATED_WORD:
            currentType = "agglutinated_word";
            break;
            case DESAGGLUTINATED_WORD :
            currentType = "desagglutinated_word";
            break;
            case SPECIFIC_ENTITY :
            currentType = "specific_entity";
            break;
            case HYPERWORD_ALTERNATIVE:
            currentType = "hyperwordstemmer_alternative";
            break;
            case CHINESE_SEGMENTER:
            currentType = "chinese_segmenter";
            break;
            default :
            currentType = "UNKNOWN_MORPHOSYNTACTIC_TYPE";
            break;
        }
        xmlStream << "      <" << currentType << ">" << std::endl;
      }
      if ((it->inflectedForm != form) || (it->lemma != lemma) || (it->normalizedForm != norm))
      {
        if (!firstEntry)
        {
          xmlStream << "        </form>" << std::endl;
        }
        form=it->inflectedForm;
        lemma=it->lemma;
        norm=it->normalizedForm;
        xmlStream << "      <form infl=\"" << Common::Misc::transcodeToXmlEntities(sp[form]) << "\" ";
        xmlStream << "lemma=\"" << Common::Misc::transcodeToXmlEntities(sp[lemma]) << "\" ";
        xmlStream << "norm=\"" << Common::Misc::transcodeToXmlEntities(sp[norm]) << "\">" << std::endl;
      }
      const std::map<std::string,Common::PropertyCode::PropertyManager>& managers=pcm.getPropertyManagers();
      xmlStream << "        <property>" << std::endl;
      for (std::map<std::string,Common::PropertyCode::PropertyManager>::const_iterator propItr=managers.begin();
           propItr!=managers.end();
           propItr++)
      {
        if (!propItr->second.getPropertyAccessor().empty(it->properties))
        {
          xmlStream << "          <p prop=\"" << propItr->first << "\" val=\"" << propItr->second.getPropertySymbolicValue(it->properties) << "\"/>" << std::endl;
        }
      }
      xmlStream << "        </property>" << std::endl;
      firstEntry=false;
    }
    xmlStream << "      </form>" << std::endl;
    xmlStream << "    </" << currentType << ">" << std::endl;
  }
  xmlStream << "    </data>" << std::endl;
}

std::set<StringsPoolIndex> MorphoSyntacticData::allInflectedForms() const
  {
    set<StringsPoolIndex> forms;
    for (const_iterator it=begin();
         it!=end();
         it++)
    {
      forms.insert(it->inflectedForm);
    }
    return forms;
  }

std::set<StringsPoolIndex> MorphoSyntacticData::allLemma() const
  {
    set<StringsPoolIndex> lemma;
    for (const_iterator it=begin();
         it!=end();
         it++)
    {
      lemma.insert(it->lemma);
    }
    return lemma;
  }

std::set<StringsPoolIndex> MorphoSyntacticData::allNormalizedForms() const
  {
    set<StringsPoolIndex> norms;
    for (const_iterator it=begin();
         it!=end();
         it++)
    {
      norms.insert(it->normalizedForm);
    }
    return norms;
  }

LinguisticCode MorphoSyntacticData::firstValue(const Common::PropertyCode::PropertyAccessor& propertyAccessor) const
{
  for (const_iterator it=begin();
       it!=end();
       it++)
  {
    if (!propertyAccessor.empty(it->properties))
    {
      return propertyAccessor.readValue(it->properties);
    }
  }
  return static_cast<LinguisticCode>(0);
}

} // LinguisticAnalysisStructure

} // LinguisticProcessing

} // Lima
<commit_msg>add ENCHANT_LIBRARIES for spell checking<commit_after>/*
    Copyright 2002-2013 CEA LIST

    This file is part of LIMA.

    LIMA is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    LIMA is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with LIMA.  If not, see <http://www.gnu.org/licenses/>
*/
/***************************************************************************
 *   Copyright (C) 2004-2012 by CEA LIST                              *
 *                                                                         *
 ***************************************************************************/
#include "MorphoSyntacticData.h"
#include "MorphoSyntacticDataUtils.h"

#include "common/MediaticData/mediaticData.h"
#include "common/Data/strwstrtools.h"

#include <algorithm>

using namespace std;
using namespace Lima::Common::MediaticData;

namespace Lima
{

namespace LinguisticProcessing
{

namespace LinguisticAnalysisStructure
{

bool LinguisticElement::operator==(const LinguisticElement& le) const
{
  return ((properties==le.properties) &&
          (inflectedForm==le.inflectedForm) &&
          (lemma==le.lemma) &&
          (normalizedForm==le.normalizedForm) &&
          (type==le.type));
}

bool LinguisticElement::operator<(const LinguisticElement& le) const
{
  if (inflectedForm!=le.inflectedForm) return inflectedForm<le.inflectedForm;
  if (lemma!=le.lemma) return lemma<le.lemma;
  if (normalizedForm!=le.normalizedForm) return normalizedForm<le.normalizedForm;
  if (properties!=le.properties) return properties<le.properties;
  return type<le.type;
}

MorphoSyntacticData::MorphoSyntacticData()
{}

MorphoSyntacticData::~MorphoSyntacticData()
{}

bool MorphoSyntacticData::hasUniqueMicro(const Common::PropertyCode::PropertyAccessor& microAccessor,const std::list<LinguisticCode>& microFilter)
{
  LinguisticCode micro(0);
  for (const_iterator it=begin();
       it!=end();
       it++)
  {
    LinguisticCode tmp=microAccessor.readValue(it->properties);
    if (micro!=static_cast<LinguisticCode>(0))
    {
      if (micro!=tmp)
      {
        return false;
      }
    }
    else
    {
      micro=tmp;
    }
    bool found=false;
    for (list<LinguisticCode>::const_iterator filterItr=microFilter.begin();
         filterItr!=microFilter.end();
         filterItr++)
    {
      if (tmp==*filterItr)
      {
        found=true;
        break;
      }
    }
    if (!found)
    {
      return false;
    }
  }
  // if micro is 0, then there was no micros, return false
  return micro!=static_cast<LinguisticCode>(0);
}

uint64_t MorphoSyntacticData::countValues(const Common::PropertyCode::PropertyAccessor& propertyAccessor)
{
  set<LinguisticCode> values;
  allValues(propertyAccessor,values);
  return values.size();
}

void MorphoSyntacticData::allValues(const Common::PropertyCode::PropertyAccessor& propertyAccessor,std::set<LinguisticCode>& result) const
  {
    for (const_iterator it=begin();
         it!=end();
         it++)
    {
      if (!propertyAccessor.empty(it->properties))
      {
        result.insert(propertyAccessor.readValue(it->properties));
      }
    }
  }

void MorphoSyntacticData::outputXml(std::ostream& xmlStream,const Common::PropertyCode::PropertyCodeManager& pcm,const FsaStringsPool& sp) const
{
  xmlStream << "    <data>" << std::endl;

  if (!empty())
  {
    // trie pour avoir les donn�s group�s par strings
    MorphoSyntacticData tmp(*this);
    sort(tmp.begin(),tmp.end(),ltString());
    MorphoSyntacticType type(NO_MORPHOSYNTACTICTYPE);
    StringsPoolIndex form(0);
    StringsPoolIndex lemma(0);
    StringsPoolIndex norm(0);
    string currentType;
    bool firstEntry=true;
    for (const_iterator it=tmp.begin();
         it!=tmp.end();
         it++)
    {
      if (it->type != type)
      {
        // Changement type
        if (!firstEntry)
        {
          xmlStream << "        </form>" << std::endl;
          xmlStream << "      </" << currentType << ">" << std::endl;
        }
        type=it->type;
        switch ( type )
        {
            case SIMPLE_WORD :
            currentType = "simple_word";
            break;
            case IDIOMATIC_EXPRESSION :
            currentType = "idiomatic_expression";
            break;
            case UNKNOWN_WORD :
            currentType = "unknown_word";
            break;
            case ABBREV_ALTERNATIVE :
            currentType = "abbrev_alternative";
            break;
            case HYPHEN_ALTERNATIVE :
            currentType = "hyphen_alternative";
            break;
            case CONCATENATED_ALTERNATIVE :
              currentType = "concatenated_alternative";
              break;
            case SPELLING_ALTERNATIVE :
              currentType = "spelling_alternative";
              break;
            case CAPITALFIRST_WORD :
            currentType = "capitalfirst_word";
            break;
            case AGGLUTINATED_WORD:
            currentType = "agglutinated_word";
            break;
            case DESAGGLUTINATED_WORD :
            currentType = "desagglutinated_word";
            break;
            case SPECIFIC_ENTITY :
            currentType = "specific_entity";
            break;
            case HYPERWORD_ALTERNATIVE:
            currentType = "hyperwordstemmer_alternative";
            break;
            case CHINESE_SEGMENTER:
            currentType = "chinese_segmenter";
            break;
            default :
            currentType = "UNKNOWN_MORPHOSYNTACTIC_TYPE";
            break;
        }
        xmlStream << "      <" << currentType << ">" << std::endl;
      }
      if ((it->inflectedForm != form) || (it->lemma != lemma) || (it->normalizedForm != norm))
      {
        if (!firstEntry)
        {
          xmlStream << "        </form>" << std::endl;
        }
        form=it->inflectedForm;
        lemma=it->lemma;
        norm=it->normalizedForm;
        xmlStream << "      <form infl=\"" << Common::Misc::transcodeToXmlEntities(sp[form]) << "\" ";
        xmlStream << "lemma=\"" << Common::Misc::transcodeToXmlEntities(sp[lemma]) << "\" ";
        xmlStream << "norm=\"" << Common::Misc::transcodeToXmlEntities(sp[norm]) << "\">" << std::endl;
      }
      const std::map<std::string,Common::PropertyCode::PropertyManager>& managers=pcm.getPropertyManagers();
      xmlStream << "        <property>" << std::endl;
      for (std::map<std::string,Common::PropertyCode::PropertyManager>::const_iterator propItr=managers.begin();
           propItr!=managers.end();
           propItr++)
      {
        if (!propItr->second.getPropertyAccessor().empty(it->properties))
        {
          xmlStream << "          <p prop=\"" << propItr->first << "\" val=\"" << propItr->second.getPropertySymbolicValue(it->properties) << "\"/>" << std::endl;
        }
      }
      xmlStream << "        </property>" << std::endl;
      firstEntry=false;
    }
    xmlStream << "      </form>" << std::endl;
    xmlStream << "    </" << currentType << ">" << std::endl;
  }
  xmlStream << "    </data>" << std::endl;
}

std::set<StringsPoolIndex> MorphoSyntacticData::allInflectedForms() const
  {
    set<StringsPoolIndex> forms;
    for (const_iterator it=begin();
         it!=end();
         it++)
    {
      forms.insert(it->inflectedForm);
    }
    return forms;
  }

std::set<StringsPoolIndex> MorphoSyntacticData::allLemma() const
  {
    set<StringsPoolIndex> lemma;
    for (const_iterator it=begin();
         it!=end();
         it++)
    {
      lemma.insert(it->lemma);
    }
    return lemma;
  }

std::set<StringsPoolIndex> MorphoSyntacticData::allNormalizedForms() const
  {
    set<StringsPoolIndex> norms;
    for (const_iterator it=begin();
         it!=end();
         it++)
    {
      norms.insert(it->normalizedForm);
    }
    return norms;
  }

LinguisticCode MorphoSyntacticData::firstValue(const Common::PropertyCode::PropertyAccessor& propertyAccessor) const
{
  for (const_iterator it=begin();
       it!=end();
       it++)
  {
    if (!propertyAccessor.empty(it->properties))
    {
      return propertyAccessor.readValue(it->properties);
    }
  }
  return static_cast<LinguisticCode>(0);
}

} // LinguisticAnalysisStructure

} // LinguisticProcessing

} // Lima
<|endoftext|>


<commit_before>//=================================================================================================
/*!
//  \file src/mathtest/lapack/OperationTest.cpp
//  \brief Source file for the LAPACK operation test
//
//  Copyright (C) 2013 Klaus Iglberger - All Rights Reserved
//
//  This file is part of the Blaze library. You can redistribute it and/or modify it under
//  the terms of the New (Revised) BSD License. Redistribution and use in source and binary
//  forms, with or without modification, are permitted provided that the following conditions
//  are met:
//
//  1. Redistributions of source code must retain the above copyright notice, this list of
//     conditions and the following disclaimer.
//  2. Redistributions in binary form must reproduce the above copyright notice, this list
//     of conditions and the following disclaimer in the documentation and/or other materials
//     provided with the distribution.
//  3. Neither the names of the Blaze development group nor the names of its contributors
//     may be used to endorse or promote products derived from this software without specific
//     prior written permission.
//
//  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
//  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
//  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
//  SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
//  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
//  TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
//  BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
//  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
//  ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
//  DAMAGE.
*/
//=================================================================================================

//*************************************************************************************************
// Includes
//*************************************************************************************************

#include <cstdlib>
#include <iostream>
#include <blaze/util/Complex.h>
#include <blazetest/mathtest/lapack/OperationTest.h>

namespace blazetest {

namespace mathtest {

namespace lapack {

//=================================================================================================
//
//  CONSTRUCTORS
//
//=================================================================================================

//*************************************************************************************************
/*!\brief Constructor for the OperationTest class test.
//
// \exception std::runtime_error Operation error detected.
*/
OperationTest::OperationTest()
{
/*
   testGeqrf< float                  >();
   testGeqrf< double                 >();
   testGeqrf< blaze::complex<float>  >();
   testGeqrf< blaze::complex<double> >();

   testGetrf< float                  >();
   testGetrf< double                 >();
   testGetrf< blaze::complex<float>  >();
   testGetrf< blaze::complex<double> >();

   testSytrf< float                  >();
   testSytrf< double                 >();
   testSytrf< blaze::complex<float>  >();
   testSytrf< blaze::complex<double> >();

   testHetrf< blaze::complex<float>  >();
   testHetrf< blaze::complex<double> >();

   testPotrf< float                  >();
   testPotrf< double                 >();
   testPotrf< blaze::complex<float>  >();
   testPotrf< blaze::complex<double> >();

   testGetri< float                  >();
   testGetri< double                 >();
   testGetri< blaze::complex<float>  >();
   testGetri< blaze::complex<double> >();

   testSytri< float                  >();
   testSytri< double                 >();
   testSytri< blaze::complex<float>  >();
   testSytri< blaze::complex<double> >();
   */

   //testHetri< blaze::complex<float>  >();
   testHetri< blaze::complex<double> >();

   /*
   testPotri< float                  >();
   testPotri< double                 >();
   testPotri< blaze::complex<float>  >();
   testPotri< blaze::complex<double> >();

   testTrtri< float                  >();
   testTrtri< double                 >();
   testTrtri< blaze::complex<float>  >();
   testTrtri< blaze::complex<double> >();

   testGesv< float                  >();
   testGesv< double                 >();
   testGesv< blaze::complex<float>  >();
   testGesv< blaze::complex<double> >();
   */
}
//*************************************************************************************************

} // namespace lapack

} // namespace mathtest

} // namespace blazetest

//=================================================================================================
//
//  MAIN FUNCTION
//
//=================================================================================================

//*************************************************************************************************
int main()
{
   std::cout << "   Running LAPACK operation test..." << std::endl;

   try
   {
      RUN_LAPACK_OPERATION_TEST;
   }
   catch( std::exception& ex ) {
      std::cerr << "\n\n ERROR DETECTED during LAPACK operation test:\n"
                << ex.what() << "\n";
      return EXIT_FAILURE;
   }

   return EXIT_SUCCESS;
}
//*************************************************************************************************
<commit_msg>Fix the operation test for the LAPACK wrapper functions<commit_after>//=================================================================================================
/*!
//  \file src/mathtest/lapack/OperationTest.cpp
//  \brief Source file for the LAPACK operation test
//
//  Copyright (C) 2013 Klaus Iglberger - All Rights Reserved
//
//  This file is part of the Blaze library. You can redistribute it and/or modify it under
//  the terms of the New (Revised) BSD License. Redistribution and use in source and binary
//  forms, with or without modification, are permitted provided that the following conditions
//  are met:
//
//  1. Redistributions of source code must retain the above copyright notice, this list of
//     conditions and the following disclaimer.
//  2. Redistributions in binary form must reproduce the above copyright notice, this list
//     of conditions and the following disclaimer in the documentation and/or other materials
//     provided with the distribution.
//  3. Neither the names of the Blaze development group nor the names of its contributors
//     may be used to endorse or promote products derived from this software without specific
//     prior written permission.
//
//  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
//  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
//  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
//  SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
//  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
//  TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
//  BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
//  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
//  ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
//  DAMAGE.
*/
//=================================================================================================

//*************************************************************************************************
// Includes
//*************************************************************************************************

#include <cstdlib>
#include <iostream>
#include <blaze/util/Complex.h>
#include <blazetest/mathtest/lapack/OperationTest.h>

namespace blazetest {

namespace mathtest {

namespace lapack {

//=================================================================================================
//
//  CONSTRUCTORS
//
//=================================================================================================

//*************************************************************************************************
/*!\brief Constructor for the OperationTest class test.
//
// \exception std::runtime_error Operation error detected.
*/
OperationTest::OperationTest()
{
   testGeqrf< float                  >();
   testGeqrf< double                 >();
   testGeqrf< blaze::complex<float>  >();
   testGeqrf< blaze::complex<double> >();

   testGetrf< float                  >();
   testGetrf< double                 >();
   testGetrf< blaze::complex<float>  >();
   testGetrf< blaze::complex<double> >();

   testSytrf< float                  >();
   testSytrf< double                 >();
   testSytrf< blaze::complex<float>  >();
   testSytrf< blaze::complex<double> >();

   testHetrf< blaze::complex<float>  >();
   testHetrf< blaze::complex<double> >();

   testPotrf< float                  >();
   testPotrf< double                 >();
   testPotrf< blaze::complex<float>  >();
   testPotrf< blaze::complex<double> >();

   testGetri< float                  >();
   testGetri< double                 >();
   testGetri< blaze::complex<float>  >();
   testGetri< blaze::complex<double> >();

   testSytri< float                  >();
   testSytri< double                 >();
   testSytri< blaze::complex<float>  >();
   testSytri< blaze::complex<double> >();

   testHetri< blaze::complex<float>  >();
   testHetri< blaze::complex<double> >();

   testPotri< float                  >();
   testPotri< double                 >();
   testPotri< blaze::complex<float>  >();
   testPotri< blaze::complex<double> >();

   testTrtri< float                  >();
   testTrtri< double                 >();
   testTrtri< blaze::complex<float>  >();
   testTrtri< blaze::complex<double> >();

   testGesv< float                  >();
   testGesv< double                 >();
   testGesv< blaze::complex<float>  >();
   testGesv< blaze::complex<double> >();
}
//*************************************************************************************************

} // namespace lapack

} // namespace mathtest

} // namespace blazetest

//=================================================================================================
//
//  MAIN FUNCTION
//
//=================================================================================================

//*************************************************************************************************
int main()
{
   std::cout << "   Running LAPACK operation test..." << std::endl;

   try
   {
      RUN_LAPACK_OPERATION_TEST;
   }
   catch( std::exception& ex ) {
      std::cerr << "\n\n ERROR DETECTED during LAPACK operation test:\n"
                << ex.what() << "\n";
      return EXIT_FAILURE;
   }

   return EXIT_SUCCESS;
}
//*************************************************************************************************
<|endoftext|>


<commit_before>#include "Adafruit_MCP23008.h"

//  This is a library for the MCP23008 i2c port expander

//  These displays use I2C to communicate, 2 pins are required to
//  interface
//  Adafruit invests time and resources providing this open source code,
//  please support Adafruit and open-source hardware by purchasing
//  products from Adafruit!

//  Written by Limor Fried/Ladyada for Adafruit Industries.
//  BSD license, all text above must be included in any redistribution

////////////////////////////////////////////////////////////////////////////////
// RTC_DS1307 implementation

void Adafruit_MCP23008::begin(uint8_t addr) {
  if (addr > 7) {
    addr = 7;
  }
  i2caddr = addr;

  Wire.setSpeed(CLOCK_SPEED_100KHZ);
  Wire.stretchClock(true);
  Wire.begin();
  delayMicroseconds(2);
  // set defaults!
  Wire.beginTransmission(MCP23008_ADDRESS | i2caddr);
  delayMicroseconds(2);
  Wire.write((byte)MCP23008_IODIR);
  delayMicroseconds(2);
  Wire.write((byte)0xFF);  // all inputs
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.endTransmission();
  delayMicroseconds(2);
}

void Adafruit_MCP23008::begin(void) {
  begin(0);
}

void Adafruit_MCP23008::pinMode(uint8_t p, uint8_t d) {
  uint8_t iodir;

  // only 8 bits!
  if (p > 7)
    return;

  iodir = read8(MCP23008_IODIR);

  // set the pin and direction
  if (d == INPUT) {
    iodir |= 1 << p;
  } else {
    iodir &= ~(1 << p);
  }

  // write the new IODIR
  write8(MCP23008_IODIR, iodir);
}

uint8_t Adafruit_MCP23008::readGPIO(void) {
  // read the current GPIO input
  return read8(MCP23008_GPIO);
}

void Adafruit_MCP23008::writeGPIO(uint8_t gpio) {
  write8(MCP23008_GPIO, gpio);
}

void Adafruit_MCP23008::digitalWrite(uint8_t p, uint8_t d) {
  uint8_t gpio;

  // only 8 bits!
  if (p > 7)
    return;

  // read the current GPIO output latches
  gpio = readGPIO();

  // set the pin and direction
  if (d == HIGH) {
    gpio |= 1 << p;
  } else {
    gpio &= ~(1 << p);
  }

  // write the new GPIO
  writeGPIO(gpio);
}

void Adafruit_MCP23008::pullUp(uint8_t p, uint8_t d) {
  uint8_t gppu;

  // only 8 bits!
  if (p > 7)
    return;

  gppu = read8(MCP23008_GPPU);
  // set the pin and direction
  if (d == HIGH) {
    gppu |= 1 << p;
  } else {
    gppu &= ~(1 << p);
  }
  // write the new GPIO
  write8(MCP23008_GPPU, gppu);
}

uint8_t Adafruit_MCP23008::digitalRead(uint8_t p) {
  // only 8 bits!
  if (p > 7)
    return 0;

  // read the current GPIO
  return (readGPIO() >> p) & 0x1;
}

uint8_t Adafruit_MCP23008::read8(uint8_t addr) {
  Wire.beginTransmission(MCP23008_ADDRESS | i2caddr);
  delayMicroseconds(2);
  Wire.write((byte)addr);
  delayMicroseconds(2);
  Wire.endTransmission();
  delayMicroseconds(2);
  Wire.requestFrom(MCP23008_ADDRESS | i2caddr, 1);
  delayMicroseconds(2);

  return Wire.read();
}

void Adafruit_MCP23008::write8(uint8_t addr, uint8_t data) {
  Wire.beginTransmission(MCP23008_ADDRESS | i2caddr);
  delayMicroseconds(2);
  Wire.write((byte)addr);
  delayMicroseconds(2);
  Wire.write((byte)data);
  delayMicroseconds(2);
  Wire.endTransmission();
  delayMicroseconds(2);
}
<commit_msg>modified end of transmission delay<commit_after>#include "Adafruit_MCP23008.h"

//  This is a library for the MCP23008 i2c port expander

//  These displays use I2C to communicate, 2 pins are required to
//  interface
//  Adafruit invests time and resources providing this open source code,
//  please support Adafruit and open-source hardware by purchasing
//  products from Adafruit!

//  Written by Limor Fried/Ladyada for Adafruit Industries.
//  BSD license, all text above must be included in any redistribution

////////////////////////////////////////////////////////////////////////////////
// RTC_DS1307 implementation

void Adafruit_MCP23008::begin(uint8_t addr) {
  if (addr > 7) {
    addr = 7;
  }
  i2caddr = addr;

  //Wire.setSpeed(CLOCK_SPEED_100KHZ);
  //Wire.stretchClock(true);
  Wire.begin();
  delayMicroseconds(2);
  // set defaults!
  Wire.beginTransmission(MCP23008_ADDRESS | i2caddr);
  delayMicroseconds(2);
  Wire.write((byte)MCP23008_IODIR);
  delayMicroseconds(2);
  Wire.write((byte)0xFF);  // all inputs
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.write((byte)0x00);
  delayMicroseconds(2);
  Wire.endTransmission();
  delayMicroseconds(5);
}

void Adafruit_MCP23008::begin(void) {
  begin(0);
}

void Adafruit_MCP23008::pinMode(uint8_t p, uint8_t d) {
  uint8_t iodir;

  // only 8 bits!
  if (p > 7)
    return;

  iodir = read8(MCP23008_IODIR);

  // set the pin and direction
  if (d == INPUT) {
    iodir |= 1 << p;
  } else {
    iodir &= ~(1 << p);
  }

  // write the new IODIR
  write8(MCP23008_IODIR, iodir);
}

uint8_t Adafruit_MCP23008::readGPIO(void) {
  // read the current GPIO input
  return read8(MCP23008_GPIO);
}

void Adafruit_MCP23008::writeGPIO(uint8_t gpio) {
  write8(MCP23008_GPIO, gpio);
}

void Adafruit_MCP23008::digitalWrite(uint8_t p, uint8_t d) {
  uint8_t gpio;

  // only 8 bits!
  if (p > 7)
    return;

  // read the current GPIO output latches
  gpio = readGPIO();

  // set the pin and direction
  if (d == HIGH) {
    gpio |= 1 << p;
  } else {
    gpio &= ~(1 << p);
  }

  // write the new GPIO
  writeGPIO(gpio);
}

void Adafruit_MCP23008::pullUp(uint8_t p, uint8_t d) {
  uint8_t gppu;

  // only 8 bits!
  if (p > 7)
    return;

  gppu = read8(MCP23008_GPPU);
  // set the pin and direction
  if (d == HIGH) {
    gppu |= 1 << p;
  } else {
    gppu &= ~(1 << p);
  }
  // write the new GPIO
  write8(MCP23008_GPPU, gppu);
}

uint8_t Adafruit_MCP23008::digitalRead(uint8_t p) {
  // only 8 bits!
  if (p > 7)
    return 0;

  // read the current GPIO
  return (readGPIO() >> p) & 0x1;
}

uint8_t Adafruit_MCP23008::read8(uint8_t addr) {
  Wire.beginTransmission(MCP23008_ADDRESS | i2caddr);
  delayMicroseconds(2);
  Wire.write((byte)addr);
  delayMicroseconds(2);
  Wire.endTransmission();
  delayMicroseconds(5);
  Wire.requestFrom(MCP23008_ADDRESS | i2caddr, 1);
  delayMicroseconds(2);

  return Wire.read();
}

void Adafruit_MCP23008::write8(uint8_t addr, uint8_t data) {
  Wire.beginTransmission(MCP23008_ADDRESS | i2caddr);
  delayMicroseconds(2);
  Wire.write((byte)addr);
  delayMicroseconds(2);
  Wire.write((byte)data);
  delayMicroseconds(2);
  Wire.endTransmission();
  delayMicroseconds(5);
}
<|endoftext|>


<commit_before>#include <algorithm>
#include <cmath>

#include "gain_schedule.h"
#include "MPU6050.h"

namespace control {

vector_t<state_size> StateEstimator::update(const vector_t<state_size>& x,
                                            const vector_t<input_size>& u) const
{
  return A * x + B * u;
}

vector_t<output_size> LQRController::update(const vector_t<state_size>& x) const
{
  return C * x;
}

vector_t<output_size> PIController::update(const vector_t<output_size>& x,
                                           const vector_t<output_size>& e) const
{
  return Kp * x + Ki * e;
}

bool rt_controller_t::operator<(const rt_controller_t& rhs) const
{
  return rate < rhs.rate;
}

GainSchedule::GainSchedule() : state_{{}}, pi_control_enabled_{false}
{

}

float GainSchedule::rate() const
{
  return rate_;
}

bool GainSchedule::set_sample(Sample& s)
{
  s_ = &s;
  return set_rate(s.encoder.rear_wheel_rate);
}

void GainSchedule::set_state(const vector_t<state_size>& state)
{
  state_ = state;
}

bool GainSchedule::set_rate(float rate)
{
  bool valid = true;
  rate_ = rate;

  r.rate = rate;
  auto it = std::upper_bound(schedule_.begin(), schedule_.end(), r);
  if (it == schedule_.begin() || it == schedule_.end()) {
    valid = false;
    if (it == schedule_.begin()) {
      s_->estimate.theta_R_dot_upper = it->rate;
      s_->estimate.theta_R_dot_lower = NAN;
    } else {
      s_->estimate.theta_R_dot_upper = NAN;
      s_->estimate.theta_R_dot_lower = (--it)->rate;
    }
  } else {
    valid = true;
    s_->estimate.theta_R_dot_upper = it->rate;
    ss_upper_ = const_cast<controller_t*>(&(it->controller));
    s_->estimate.theta_R_dot_lower = (--it)->rate;
    ss_lower_ = const_cast<controller_t*>(&(it->controller));

    alpha_ = (rate - s_->estimate.theta_R_dot_lower) /
             (s_->estimate.theta_R_dot_upper - s_->estimate.theta_R_dot_lower);
  }

  return valid;
}

void GainSchedule::state_estimate(float torque_prev)
{
  if (state_estimate_time_ == s_->loop_count)
    return;
  state_estimate_time_ = s_->loop_count;
  vector_t<input_size> input {{torque_prev, s_->encoder.steer,
                               hardware::MPU6050::phi_dot(*s_)}};
  auto state_lower = ss_lower_->estimator.update(state_, input);
  auto state_upper = ss_upper_->estimator.update(state_, input);
  state_ = alpha_ * (state_upper - state_lower) + state_lower;
  s_->estimate.phi = state_(0, 0);
  s_->estimate.delta = state_(1, 0);
  s_->estimate.phi_dot = state_(2, 0);
  s_->estimate.delta_dot = state_(3, 0);
}

float GainSchedule::lqr_output() const
{
  const float t0 = ss_lower_->lqr.update(state_)(0, 0);
  const float t1 = ss_upper_->lqr.update(state_)(0, 0);
  return alpha_ * (t1 - t0) + t0;
}

float GainSchedule::pi_output() const
{
  vector_t<output_size> x {{s_->yaw_rate_pi.x}};
  vector_t<output_size> e {{s_->yaw_rate_pi.e}};
  const float t0 = ss_lower_->pi.update(x, e)(0, 0);
  const float t1 = ss_upper_->pi.update(x, e)(0, 0);
  return alpha_ * (t1 - t0) + t0;
}

float GainSchedule::compute_updated_torque(float torque_prev)
{
  state_estimate(torque_prev);
  return lqr_output() + pi_output();
}

} // namespace control
<commit_msg>Always update state estimate when called.<commit_after>#include <algorithm>
#include <cmath>

#include "gain_schedule.h"
#include "MPU6050.h"

namespace control {

vector_t<state_size> StateEstimator::update(const vector_t<state_size>& x,
                                            const vector_t<input_size>& u) const
{
  return A * x + B * u;
}

vector_t<output_size> LQRController::update(const vector_t<state_size>& x) const
{
  return C * x;
}

vector_t<output_size> PIController::update(const vector_t<output_size>& x,
                                           const vector_t<output_size>& e) const
{
  return Kp * x + Ki * e;
}

bool rt_controller_t::operator<(const rt_controller_t& rhs) const
{
  return rate < rhs.rate;
}

GainSchedule::GainSchedule() : state_{{}}, pi_control_enabled_{false}
{

}

float GainSchedule::rate() const
{
  return rate_;
}

bool GainSchedule::set_sample(Sample& s)
{
  s_ = &s;
  return set_rate(s.encoder.rear_wheel_rate);
}

void GainSchedule::set_state(const vector_t<state_size>& state)
{
  state_ = state;
}

bool GainSchedule::set_rate(float rate)
{
  bool valid = true;
  rate_ = rate;

  r.rate = rate;
  auto it = std::upper_bound(schedule_.begin(), schedule_.end(), r);
  if (it == schedule_.begin() || it == schedule_.end()) {
    valid = false;
    if (it == schedule_.begin()) {
      s_->estimate.theta_R_dot_upper = it->rate;
      s_->estimate.theta_R_dot_lower = NAN;
    } else {
      s_->estimate.theta_R_dot_upper = NAN;
      s_->estimate.theta_R_dot_lower = (--it)->rate;
    }
  } else {
    valid = true;
    s_->estimate.theta_R_dot_upper = it->rate;
    ss_upper_ = const_cast<controller_t*>(&(it->controller));
    s_->estimate.theta_R_dot_lower = (--it)->rate;
    ss_lower_ = const_cast<controller_t*>(&(it->controller));

    alpha_ = (rate - s_->estimate.theta_R_dot_lower) /
             (s_->estimate.theta_R_dot_upper - s_->estimate.theta_R_dot_lower);
  }

  return valid;
}

void GainSchedule::state_estimate(float torque_prev)
{
  state_estimate_time_ = s_->loop_count;
  vector_t<input_size> input {{torque_prev, s_->encoder.steer,
                               hardware::MPU6050::phi_dot(*s_)}};
  auto state_lower = ss_lower_->estimator.update(state_, input);
  auto state_upper = ss_upper_->estimator.update(state_, input);
  state_ = alpha_ * (state_upper - state_lower) + state_lower;
  s_->estimate.phi = state_(0, 0);
  s_->estimate.delta = state_(1, 0);
  s_->estimate.phi_dot = state_(2, 0);
  s_->estimate.delta_dot = state_(3, 0);
}

float GainSchedule::lqr_output() const
{
  const float t0 = ss_lower_->lqr.update(state_)(0, 0);
  const float t1 = ss_upper_->lqr.update(state_)(0, 0);
  return alpha_ * (t1 - t0) + t0;
}

float GainSchedule::pi_output() const
{
  vector_t<output_size> x {{s_->yaw_rate_pi.x}};
  vector_t<output_size> e {{s_->yaw_rate_pi.e}};
  const float t0 = ss_lower_->pi.update(x, e)(0, 0);
  const float t1 = ss_upper_->pi.update(x, e)(0, 0);
  return alpha_ * (t1 - t0) + t0;
}

float GainSchedule::compute_updated_torque(float torque_prev)
{
  state_estimate(torque_prev);
  return lqr_output() + pi_output();
}

} // namespace control
<|endoftext|>


<commit_before>/**
 * @file uart.cpp
 *
 * @brief Contains uart access functionality
 *
 * Example usage:
 * @code
 *
 * Uart uart0(&UBRR0, &UCSR0A, &UCSR0B, &UCSR0C, &UDR0);
 * uart0.Init(9600);
 *
 * @endcode
 */

#include <stdint.h>
#include <avr/io.h>
#include <avr/interrupt.h>

#include "uart.h"
#include "util/ringbuffer.h"

/**
 * @brief Used to access uart0
 *
 * On atmega2561:<b>
 * Pin2: RXD0 <b>
 * Pin3: TXD0
 */
Uart uart0(&UBRR0, &UCSR0A, &UCSR0B, &UCSR0C, &UDR0);

/**
 * @brief Used to access uart1
 *
 * On atmega2561:<b>
 * Pin27: RXD0 <b>
 * Pin28: TXD0
 */
Uart uart1(&UBRR1, &UCSR1A, &UCSR1B, &UCSR1C, &UDR1);

/**
 * @brief Calculates the clock scale needed to run uart at the specified baud rate.
 *
 * @param baudrate  Typical values include 1200, 2400, 4800, 9600, 14400, 19200, 38400, 57600, 128000, 256000.
 *                  If not specified 9600 is used.
 */
uint16_t Uart::BaudScale(uint16_t baudrate)
{
  return (((F_CPU / (baudrate * 16UL))) - 1);
}

/**
 * @brief Initializes uart with specified baud rate.
 *
 * @param ubrr
 * @param ucsra
 * @param ucsrb
 * @param ucsrc
 * @param udr
 *
 * Example usage:
 * @code
 *
 * Uart uart0(&UBRR0, &UCSR0A, &UCSR0B, &UCSR0C, &UDR0);
 * Uart uart1(&UBRR0, &UCSR0A, &UCSR0B, &UCSR0C, &UDR0);
 *
 * @endcode
 */
Uart::Uart(volatile uint16_t *ubrr, volatile uint8_t *ucsra, volatile uint8_t *ucsrb, volatile uint8_t *ucsrc, volatile uint8_t *udr)
{
  _ubrr   = ubrr;
  _ucsra  = ucsra;
  _ucsrb  = ucsrb;
  _ucsrc  = ucsrc;
  _udr    = udr;
}

/**
 * @brief Initializes uart with specified baud rate.
 *
 * @param baudrate  Typical values include 1200, 2400, 4800, 9600, 14400, 19200, 38400, 57600, 128000, 256000.
 *                  If not specified 9600 is used.
 *
 * Example usage:
 * @code
 *
 * Uart uart0(&UBRR0, &UCSR0A, &UCSR0B, &UCSR0C, &UDR0);
 * uart0.Init(9600);
 *
 * @endcode
 */
void Uart::Init(uint16_t baudrate)
{
  // initializing uart to specified baud rate
  *_ubrr = BaudScale(baudrate);

  // setting 8 data bits, 1 stop bit, no parity
  *_ucsrc = ((0 << USBS0) | (1 << UCSZ00) | (1 << UCSZ01));

  // enabling receiver and transmitter
  *_ucsrb = ((1 << RXEN0) | (1 << TXEN0));

  // enabling uart interrupts
  *_ucsrb |= ((1 << RXCIE0) | (1 << TXCIE0));
}

/**
 * @brief Pushes byte to rx buffer. If buffer is full, nothing happens.
 *
 * @param byte  Byte to push to buffer.
 */
void _PushRx(Uart *uart, char byte)
{
  uart->_rx_buffer.Push(byte);
}

ISR(USART0_RX_vect)
{
  char rxbyte;

  // echo rx to tx
  rxbyte = UDR0;
  UDR0 = rxbyte;

  // pushing byte to rxbuff
  _PushRx(&uart0, rxbyte);
}
<commit_msg>Added loopback isr and tested for uart1<commit_after>/**
 * @file uart.cpp
 *
 * @brief Contains uart access functionality
 *
 * Example usage:
 * @code
 *
 * Uart uart0(&UBRR0, &UCSR0A, &UCSR0B, &UCSR0C, &UDR0);
 * uart0.Init(9600);
 *
 * @endcode
 */

#include <stdint.h>
#include <avr/io.h>
#include <avr/interrupt.h>

#include "uart.h"
#include "util/ringbuffer.h"

/**
 * @brief Used to access uart0
 *
 * On atmega2561:<b>
 * Pin2: RXD0 <b>
 * Pin3: TXD0
 */
Uart uart0(&UBRR0, &UCSR0A, &UCSR0B, &UCSR0C, &UDR0);

/**
 * @brief Used to access uart1
 *
 * On atmega2561:<b>
 * Pin27: RXD0 <b>
 * Pin28: TXD0
 */
Uart uart1(&UBRR1, &UCSR1A, &UCSR1B, &UCSR1C, &UDR1);

/**
 * @brief Calculates the clock scale needed to run uart at the specified baud rate.
 *
 * @param baudrate  Typical values include 1200, 2400, 4800, 9600, 14400, 19200, 38400, 57600, 128000, 256000.
 *                  If not specified 9600 is used.
 */
uint16_t Uart::BaudScale(uint16_t baudrate)
{
  return (((F_CPU / (baudrate * 16UL))) - 1);
}

/**
 * @brief Initializes uart with specified baud rate.
 *
 * @param ubrr
 * @param ucsra
 * @param ucsrb
 * @param ucsrc
 * @param udr
 *
 * Example usage:
 * @code
 *
 * Uart uart0(&UBRR0, &UCSR0A, &UCSR0B, &UCSR0C, &UDR0);
 * Uart uart1(&UBRR0, &UCSR0A, &UCSR0B, &UCSR0C, &UDR0);
 *
 * @endcode
 */
Uart::Uart(volatile uint16_t *ubrr, volatile uint8_t *ucsra, volatile uint8_t *ucsrb, volatile uint8_t *ucsrc, volatile uint8_t *udr)
{
  _ubrr   = ubrr;
  _ucsra  = ucsra;
  _ucsrb  = ucsrb;
  _ucsrc  = ucsrc;
  _udr    = udr;
}

/**
 * @brief Initializes uart with specified baud rate.
 *
 * @param baudrate  Typical values include 1200, 2400, 4800, 9600, 14400, 19200, 38400, 57600, 128000, 256000.
 *                  If not specified 9600 is used.
 *
 * Example usage:
 * @code
 *
 * Uart uart0(&UBRR0, &UCSR0A, &UCSR0B, &UCSR0C, &UDR0);
 * uart0.Init(9600);
 *
 * @endcode
 */
void Uart::Init(uint16_t baudrate)
{
  // initializing uart to specified baud rate
  *_ubrr = BaudScale(baudrate);

  // setting 8 data bits, 1 stop bit, no parity
  *_ucsrc = ((0 << USBS0) | (1 << UCSZ00) | (1 << UCSZ01));

  // enabling receiver and transmitter
  *_ucsrb = ((1 << RXEN0) | (1 << TXEN0));

  // enabling uart interrupts
  *_ucsrb |= ((1 << RXCIE0) | (1 << TXCIE0));
}

/**
 * @brief Pushes byte to rx buffer. If buffer is full, nothing happens.
 *
 * @param byte  Byte to push to buffer.
 */
void _PushRx(Uart *uart, char byte)
{
  uart->_rx_buffer.Push(byte);
}

ISR(USART0_RX_vect)
{
  char rxbyte;

  // echo rx to tx
  rxbyte = UDR0;
  UDR0 = rxbyte;

  // pushing byte to rxbuff
  _PushRx(&uart0, rxbyte);
}

ISR(USART1_RX_vect)
{
  char rxbyte;

  // echo rx to tx
  rxbyte = UDR1;
  UDR1 = rxbyte;

  // pushing byte to rxbuff
  _PushRx(&uart1, rxbyte);
}
<|endoftext|>