<commit_before>/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/cc/saved_model/loader.h"

#include <unordered_set>

#include "tensorflow/cc/saved_model/constants.h"
#include "tensorflow/cc/saved_model/reader.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/lib/monitoring/counter.h"
#include "tensorflow/core/lib/monitoring/sampler.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/lib/strings/strcat.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/protobuf_internal.h"
#include "tensorflow/core/protobuf/saved_model.pb.h"
#include "tensorflow/core/protobuf/saver.pb.h"
#include "tensorflow/core/public/session.h"
#include "tensorflow/core/public/session_options.h"
#include "tensorflow/core/util/tensor_bundle/naming.h"

namespace tensorflow {
namespace {

auto* load_attempt_count = monitoring::Counter<2>::New(
    "/tensorflow/cc/saved_model/load_attempt_count",
    "The number of times a SavedModel was successfully loaded.", "model_path",
    "status");
auto* load_latency = monitoring::Counter<1>::New(
    "/tensorflow/cc/saved_model/load_latency",
    "Latency in microseconds for SavedModels that were successfully loaded.",
    "model_path");
auto* load_latency_by_stage = monitoring::Sampler<2>::New(
    {
        "/tensorflow/cc/saved_model/load_latency_by_stage",  // metric name
        "Distribution of wall time spent (in microseconds) in each stage "
        "(restore graph from disk, run init graph op, etc) when loading the "
        "model",
        "model_path",
        "stage",
    },
    // Scale of 10, power of 1.5 with bucket count 36 (~20 minutes).
    monitoring::Buckets::Exponential(10, 1.5, 36));

constexpr char kLoadAttemptFail[] = "fail";
constexpr char kLoadAttemptSuccess[] = "success";

uint64 GetLatencyMicroseconds(const uint64 start_microseconds) {
  const uint64 end_microseconds = Env::Default()->NowMicros();
  // Avoid clock skew.
  if (end_microseconds < start_microseconds) return 0;
  return end_microseconds - start_microseconds;
}

Status LoadMetaGraphIntoSession(const MetaGraphDef& meta_graph_def,
                                const SessionOptions& session_options,
                                std::unique_ptr<Session>* session) {
  Session* session_p = nullptr;
  TF_RETURN_IF_ERROR(NewSession(session_options, &session_p));
  session->reset(session_p);
  return (*session)->Create(meta_graph_def.graph_def());
}

Tensor CreateStringTensor(const string& value) {
  Tensor tensor(DT_STRING, TensorShape({}));
  tensor.scalar<string>()() = value;
  return tensor;
}

void AddAssetsTensorsToInputs(const StringPiece export_dir,
                              const std::vector<AssetFileDef>& asset_file_defs,
                              std::vector<std::pair<string, Tensor>>* inputs) {
  if (asset_file_defs.empty()) {
    return;
  }
  for (auto& asset_file_def : asset_file_defs) {
    Tensor assets_file_path_tensor = CreateStringTensor(io::JoinPath(
        export_dir, kSavedModelAssetsDirectory, asset_file_def.filename()));
    inputs->push_back(
        {asset_file_def.tensor_info().name(), assets_file_path_tensor});
  }
}

// Like Session::Run(), but uses the Make/Run/ReleaseCallable() API to avoid
// leaving behind non-GC'ed state.
//
// Detailed motivation behind this approach, from ashankar@:
//
// Each call to Session::Run() that identifies a new subgraph (based on feeds
// and fetches) creates some datastructures that live as long as the session
// (the partitioned graph, associated executors etc.).
//
// A pathological case of this would be if say the initialization op
// (main_op/legacy_init_op) involves the use of a large constant. Then we
// allocate memory for that large constant that will just stick around till the
// session dies. With this Callable mechanism, that memory will be released
// right after ReleaseCallable returns.
//
// However, the resource manager state remains.
Status RunOnce(const RunOptions& run_options,
               const std::vector<std::pair<string, Tensor>>& inputs,
               const std::vector<string>& output_tensor_names,
               const std::vector<string>& target_node_names,
               std::vector<Tensor>* outputs, RunMetadata* run_metadata,
               Session* session) {
  CallableOptions callable_options;
  std::vector<Tensor> feed_tensors;
  *callable_options.mutable_run_options() = run_options;
  for (const auto& input : inputs) {
    const string& name = input.first;
    const Tensor& tensor = input.second;
    callable_options.add_feed(name);
    feed_tensors.push_back(tensor);
  }
  for (const string& output_tensor_name : output_tensor_names) {
    callable_options.add_fetch(output_tensor_name);
  }
  for (const string& target_node_name : target_node_names) {
    callable_options.add_target(target_node_name);
  }

  Session::CallableHandle callable_handle;
  TF_RETURN_IF_ERROR(session->MakeCallable(callable_options, &callable_handle));
  const Status run_status = session->RunCallable(callable_handle, feed_tensors,
                                                 outputs, run_metadata);
  // Be sure to call ReleaseCallable() regardless of the outcome of
  // RunCallable().
  session->ReleaseCallable(callable_handle).IgnoreError();
  return run_status;
}

// RunInitOp will return OK if the initialization op was run successfully.
// An empty init_op_name indicates that there are no init ops to run.
Status RunInitOp(const RunOptions& run_options, const string& export_dir,
                 const MetaGraphDef& meta_graph_def,
                 const std::vector<AssetFileDef>& asset_file_defs,
                 Session* session, const string& init_op_name) {
  if (!init_op_name.empty()) {
    LOG(INFO) << "Running initialization op on SavedModel bundle.";
    std::vector<std::pair<string, Tensor>> inputs;
    AddAssetsTensorsToInputs(export_dir, asset_file_defs, &inputs);
    RunMetadata run_metadata;
    return RunOnce(run_options, inputs, {}, {init_op_name},
                   nullptr /* outputs */, &run_metadata, session);
  }
  return Status::OK();
}

// A SavedModel may store the name of the initialization op to run in the
// in the SignatureDef (v2) or a collection (v1). If an init_op collection
// exists, then the collection must contain exactly one op.
Status GetInitOp(const string& export_dir, const MetaGraphDef& meta_graph_def,
                 string* init_op_name) {
  const auto& sig_def_map = meta_graph_def.signature_def();
  const auto& init_op_sig_it =
      meta_graph_def.signature_def().find(kSavedModelInitOpSignatureKey);
  if (init_op_sig_it != sig_def_map.end()) {
    *init_op_name = init_op_sig_it->second.outputs()
                        .find(kSavedModelInitOpSignatureKey)
                        ->second.name();
    return Status::OK();
  }

  const auto& collection_def_map = meta_graph_def.collection_def();
  string init_op_collection_key;
  if (collection_def_map.find(kSavedModelMainOpKey) !=
      collection_def_map.end()) {
    init_op_collection_key = kSavedModelMainOpKey;
  } else {
    init_op_collection_key = kSavedModelLegacyInitOpKey;
  }

  const auto init_op_it = collection_def_map.find(init_op_collection_key);
  if (init_op_it != collection_def_map.end()) {
    if (init_op_it->second.node_list().value_size() != 1) {
      return errors::FailedPrecondition(
          strings::StrCat("Expected exactly one main op in : ", export_dir));
    }
    *init_op_name = init_op_it->second.node_list().value(0);
  }
  return Status::OK();
}

Status RunRestore(const RunOptions& run_options, const string& export_dir,
                  const StringPiece restore_op_name,
                  const StringPiece variable_filename_const_op_name,
                  const std::vector<AssetFileDef>& asset_file_defs,
                  Session* session) {
  LOG(INFO) << "Restoring SavedModel bundle.";
  // Find path to variables to be restored in export directory.
  const string variables_directory =
      io::JoinPath(export_dir, kSavedModelVariablesDirectory);
  // Check for saver checkpoints in v2 format. Models exported in the checkpoint
  // v2 format will have a variables.index file. The corresponding
  // variables are stored in the variables.data-?????-of-????? files.
  const string variables_index_path = io::JoinPath(
      variables_directory, MetaFilename(kSavedModelVariablesFilename));
  if (!Env::Default()->FileExists(variables_index_path).ok()) {
    LOG(INFO) << "The specified SavedModel has no variables; no checkpoints "
                 "were restored. File does not exist: "
              << variables_index_path;
    return Status::OK();
  }
  const string variables_path =
      io::JoinPath(variables_directory, kSavedModelVariablesFilename);

  // Add variables to the graph.
  Tensor variables_path_tensor(DT_STRING, TensorShape({}));
  variables_path_tensor.scalar<string>()() = variables_path;

  std::vector<std::pair<string, Tensor>> inputs = {
      {string(variable_filename_const_op_name), variables_path_tensor}};

  AddAssetsTensorsToInputs(export_dir, asset_file_defs, &inputs);

  RunMetadata run_metadata;
  return RunOnce(run_options, inputs, {}, {string(restore_op_name)},
                 nullptr /* outputs */, &run_metadata, session);
}

Status GetAssetFileDefs(const MetaGraphDef& meta_graph_def,
                        std::vector<AssetFileDef>* asset_file_defs) {
  // With SavedModel v2, we write asset file def into metagraph instead of
  // collection, so read from metagraph first.
  if (meta_graph_def.asset_file_def_size() > 0) {
    for (const auto& asset : meta_graph_def.asset_file_def()) {
      asset_file_defs->push_back(asset);
    }
    return Status::OK();
  }
  // Fall back to read from collection to be backward compatible with v1.
  const auto& collection_def_map = meta_graph_def.collection_def();
  const auto assets_it = collection_def_map.find(kSavedModelAssetsKey);
  if (assets_it == collection_def_map.end()) {
    return Status::OK();
  }
  const auto& any_assets = assets_it->second.any_list().value();
  for (const auto& any_asset : any_assets) {
    AssetFileDef asset_file_def;
    TF_RETURN_IF_ERROR(
        ParseAny(any_asset, &asset_file_def, "tensorflow.AssetFileDef"));
    asset_file_defs->push_back(asset_file_def);
  }
  return Status::OK();
}

Status LoadSavedModelInternal(const SessionOptions& session_options,
                              const RunOptions& run_options,
                              const string& export_dir,
                              const std::unordered_set<string>& tags,
                              SavedModelBundle* const bundle) {
  const uint64 read_start_microseconds = Env::Default()->NowMicros();
  TF_RETURN_IF_ERROR(ReadMetaGraphDefFromSavedModel(export_dir, tags,
                                                    &bundle->meta_graph_def));

  TF_RETURN_IF_ERROR(LoadMetaGraphIntoSession(
      bundle->meta_graph_def, session_options, &bundle->session));

  std::vector<AssetFileDef> asset_file_defs;
  TF_RETURN_IF_ERROR(
      GetAssetFileDefs(bundle->meta_graph_def, &asset_file_defs));
  TF_RETURN_IF_ERROR(
      RunRestore(run_options, export_dir,
                 bundle->meta_graph_def.saver_def().restore_op_name(),
                 bundle->meta_graph_def.saver_def().filename_tensor_name(),
                 asset_file_defs, bundle->session.get()));
  // Record walltime spent in restoring graph from disk, but postpone metric
  // increments until graph init finishes.
  const uint64 restore_graph_walltime =
      GetLatencyMicroseconds(read_start_microseconds);

  const uint64 graph_init_start_microseconds = Env::Default()->NowMicros();
  string init_op_name;
  TF_RETURN_IF_ERROR(
      GetInitOp(export_dir, bundle->meta_graph_def, &init_op_name));
  TF_RETURN_IF_ERROR(RunInitOp(run_options, export_dir, bundle->meta_graph_def,
                               asset_file_defs, bundle->session.get(),
                               init_op_name));
  load_latency_by_stage->GetCell(export_dir, "restore_graph")
      ->Add(restore_graph_walltime);
  // Record wall time spent in init op.
  load_latency_by_stage->GetCell(export_dir, "init_graph")
      ->Add(GetLatencyMicroseconds(graph_init_start_microseconds));
  return Status::OK();
}

}  // namespace

Status LoadSavedModel(const SessionOptions& session_options,
                      const RunOptions& run_options, const string& export_dir,
                      const std::unordered_set<string>& tags,
                      SavedModelBundle* const bundle) {
  // TODO(robson): Add tests for the counters.
  const uint64 start_microseconds = Env::Default()->NowMicros();
  const Status status = LoadSavedModelInternal(session_options, run_options,
                                               export_dir, tags, bundle);
  auto log_and_count = [&](const string& status_str) {
    LOG(INFO) << "SavedModel load for tags { " << str_util::Join(tags, " ")
              << " }; Status: " << status_str << ". Took "
              << GetLatencyMicroseconds(start_microseconds) << " microseconds.";
    load_attempt_count->GetCell(export_dir, status_str)->IncrementBy(1);
  };
  if (status.ok()) {
    log_and_count(kLoadAttemptSuccess);
  } else {
    log_and_count(kLoadAttemptFail);
  }
  load_latency->GetCell(export_dir)
      ->IncrementBy(GetLatencyMicroseconds(start_microseconds));
  return status;
}

bool MaybeSavedModelDirectory(const string& export_dir) {
  const string saved_model_pb_path =
      io::JoinPath(export_dir, kSavedModelFilenamePb);
  const string saved_model_pbtxt_path =
      io::JoinPath(export_dir, kSavedModelFilenamePbTxt);
  return Env::Default()->FileExists(saved_model_pb_path).ok() ||
         Env::Default()->FileExists(saved_model_pbtxt_path).ok();
}

}  // namespace tensorflow
<commit_msg>Use Sampler to record warm up latency distribution (by status).<commit_after>/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/cc/saved_model/loader.h"

#include <unordered_set>

#include "tensorflow/cc/saved_model/constants.h"
#include "tensorflow/cc/saved_model/reader.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/lib/monitoring/counter.h"
#include "tensorflow/core/lib/monitoring/sampler.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/lib/strings/strcat.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/protobuf_internal.h"
#include "tensorflow/core/protobuf/saved_model.pb.h"
#include "tensorflow/core/protobuf/saver.pb.h"
#include "tensorflow/core/public/session.h"
#include "tensorflow/core/public/session_options.h"
#include "tensorflow/core/util/tensor_bundle/naming.h"

namespace tensorflow {
namespace {

auto* load_attempt_count = monitoring::Counter<2>::New(
    "/tensorflow/cc/saved_model/load_attempt_count",
    "The number of times a SavedModel was successfully loaded.", "model_path",
    "status");
auto* load_latency = monitoring::Counter<1>::New(
    "/tensorflow/cc/saved_model/load_latency",
    "Latency in microseconds for SavedModels that were successfully loaded.",
    "model_path");
auto* load_latency_by_stage = monitoring::Sampler<2>::New(
    {
        "/tensorflow/cc/saved_model/load_latency_by_stage",  // metric name
        "Distribution of wall time spent (in microseconds) in each stage "
        "(restore graph from disk, run init graph op, etc) when loading the "
        "model",
        "model_path",
        "stage",
    },
    // Scale of 10, power of 1.8 with bucket count 33 (~20 minutes).
    monitoring::Buckets::Exponential(10, 1.8, 33));

constexpr char kLoadAttemptFail[] = "fail";
constexpr char kLoadAttemptSuccess[] = "success";

uint64 GetLatencyMicroseconds(const uint64 start_microseconds) {
  const uint64 end_microseconds = Env::Default()->NowMicros();
  // Avoid clock skew.
  if (end_microseconds < start_microseconds) return 0;
  return end_microseconds - start_microseconds;
}

Status LoadMetaGraphIntoSession(const MetaGraphDef& meta_graph_def,
                                const SessionOptions& session_options,
                                std::unique_ptr<Session>* session) {
  Session* session_p = nullptr;
  TF_RETURN_IF_ERROR(NewSession(session_options, &session_p));
  session->reset(session_p);
  return (*session)->Create(meta_graph_def.graph_def());
}

Tensor CreateStringTensor(const string& value) {
  Tensor tensor(DT_STRING, TensorShape({}));
  tensor.scalar<string>()() = value;
  return tensor;
}

void AddAssetsTensorsToInputs(const StringPiece export_dir,
                              const std::vector<AssetFileDef>& asset_file_defs,
                              std::vector<std::pair<string, Tensor>>* inputs) {
  if (asset_file_defs.empty()) {
    return;
  }
  for (auto& asset_file_def : asset_file_defs) {
    Tensor assets_file_path_tensor = CreateStringTensor(io::JoinPath(
        export_dir, kSavedModelAssetsDirectory, asset_file_def.filename()));
    inputs->push_back(
        {asset_file_def.tensor_info().name(), assets_file_path_tensor});
  }
}

// Like Session::Run(), but uses the Make/Run/ReleaseCallable() API to avoid
// leaving behind non-GC'ed state.
//
// Detailed motivation behind this approach, from ashankar@:
//
// Each call to Session::Run() that identifies a new subgraph (based on feeds
// and fetches) creates some datastructures that live as long as the session
// (the partitioned graph, associated executors etc.).
//
// A pathological case of this would be if say the initialization op
// (main_op/legacy_init_op) involves the use of a large constant. Then we
// allocate memory for that large constant that will just stick around till the
// session dies. With this Callable mechanism, that memory will be released
// right after ReleaseCallable returns.
//
// However, the resource manager state remains.
Status RunOnce(const RunOptions& run_options,
               const std::vector<std::pair<string, Tensor>>& inputs,
               const std::vector<string>& output_tensor_names,
               const std::vector<string>& target_node_names,
               std::vector<Tensor>* outputs, RunMetadata* run_metadata,
               Session* session) {
  CallableOptions callable_options;
  std::vector<Tensor> feed_tensors;
  *callable_options.mutable_run_options() = run_options;
  for (const auto& input : inputs) {
    const string& name = input.first;
    const Tensor& tensor = input.second;
    callable_options.add_feed(name);
    feed_tensors.push_back(tensor);
  }
  for (const string& output_tensor_name : output_tensor_names) {
    callable_options.add_fetch(output_tensor_name);
  }
  for (const string& target_node_name : target_node_names) {
    callable_options.add_target(target_node_name);
  }

  Session::CallableHandle callable_handle;
  TF_RETURN_IF_ERROR(session->MakeCallable(callable_options, &callable_handle));
  const Status run_status = session->RunCallable(callable_handle, feed_tensors,
                                                 outputs, run_metadata);
  // Be sure to call ReleaseCallable() regardless of the outcome of
  // RunCallable().
  session->ReleaseCallable(callable_handle).IgnoreError();
  return run_status;
}

// RunInitOp will return OK if the initialization op was run successfully.
// An empty init_op_name indicates that there are no init ops to run.
Status RunInitOp(const RunOptions& run_options, const string& export_dir,
                 const MetaGraphDef& meta_graph_def,
                 const std::vector<AssetFileDef>& asset_file_defs,
                 Session* session, const string& init_op_name) {
  if (!init_op_name.empty()) {
    LOG(INFO) << "Running initialization op on SavedModel bundle.";
    std::vector<std::pair<string, Tensor>> inputs;
    AddAssetsTensorsToInputs(export_dir, asset_file_defs, &inputs);
    RunMetadata run_metadata;
    return RunOnce(run_options, inputs, {}, {init_op_name},
                   nullptr /* outputs */, &run_metadata, session);
  }
  return Status::OK();
}

// A SavedModel may store the name of the initialization op to run in the
// in the SignatureDef (v2) or a collection (v1). If an init_op collection
// exists, then the collection must contain exactly one op.
Status GetInitOp(const string& export_dir, const MetaGraphDef& meta_graph_def,
                 string* init_op_name) {
  const auto& sig_def_map = meta_graph_def.signature_def();
  const auto& init_op_sig_it =
      meta_graph_def.signature_def().find(kSavedModelInitOpSignatureKey);
  if (init_op_sig_it != sig_def_map.end()) {
    *init_op_name = init_op_sig_it->second.outputs()
                        .find(kSavedModelInitOpSignatureKey)
                        ->second.name();
    return Status::OK();
  }

  const auto& collection_def_map = meta_graph_def.collection_def();
  string init_op_collection_key;
  if (collection_def_map.find(kSavedModelMainOpKey) !=
      collection_def_map.end()) {
    init_op_collection_key = kSavedModelMainOpKey;
  } else {
    init_op_collection_key = kSavedModelLegacyInitOpKey;
  }

  const auto init_op_it = collection_def_map.find(init_op_collection_key);
  if (init_op_it != collection_def_map.end()) {
    if (init_op_it->second.node_list().value_size() != 1) {
      return errors::FailedPrecondition(
          strings::StrCat("Expected exactly one main op in : ", export_dir));
    }
    *init_op_name = init_op_it->second.node_list().value(0);
  }
  return Status::OK();
}

Status RunRestore(const RunOptions& run_options, const string& export_dir,
                  const StringPiece restore_op_name,
                  const StringPiece variable_filename_const_op_name,
                  const std::vector<AssetFileDef>& asset_file_defs,
                  Session* session) {
  LOG(INFO) << "Restoring SavedModel bundle.";
  // Find path to variables to be restored in export directory.
  const string variables_directory =
      io::JoinPath(export_dir, kSavedModelVariablesDirectory);
  // Check for saver checkpoints in v2 format. Models exported in the checkpoint
  // v2 format will have a variables.index file. The corresponding
  // variables are stored in the variables.data-?????-of-????? files.
  const string variables_index_path = io::JoinPath(
      variables_directory, MetaFilename(kSavedModelVariablesFilename));
  if (!Env::Default()->FileExists(variables_index_path).ok()) {
    LOG(INFO) << "The specified SavedModel has no variables; no checkpoints "
                 "were restored. File does not exist: "
              << variables_index_path;
    return Status::OK();
  }
  const string variables_path =
      io::JoinPath(variables_directory, kSavedModelVariablesFilename);

  // Add variables to the graph.
  Tensor variables_path_tensor(DT_STRING, TensorShape({}));
  variables_path_tensor.scalar<string>()() = variables_path;

  std::vector<std::pair<string, Tensor>> inputs = {
      {string(variable_filename_const_op_name), variables_path_tensor}};

  AddAssetsTensorsToInputs(export_dir, asset_file_defs, &inputs);

  RunMetadata run_metadata;
  return RunOnce(run_options, inputs, {}, {string(restore_op_name)},
                 nullptr /* outputs */, &run_metadata, session);
}

Status GetAssetFileDefs(const MetaGraphDef& meta_graph_def,
                        std::vector<AssetFileDef>* asset_file_defs) {
  // With SavedModel v2, we write asset file def into metagraph instead of
  // collection, so read from metagraph first.
  if (meta_graph_def.asset_file_def_size() > 0) {
    for (const auto& asset : meta_graph_def.asset_file_def()) {
      asset_file_defs->push_back(asset);
    }
    return Status::OK();
  }
  // Fall back to read from collection to be backward compatible with v1.
  const auto& collection_def_map = meta_graph_def.collection_def();
  const auto assets_it = collection_def_map.find(kSavedModelAssetsKey);
  if (assets_it == collection_def_map.end()) {
    return Status::OK();
  }
  const auto& any_assets = assets_it->second.any_list().value();
  for (const auto& any_asset : any_assets) {
    AssetFileDef asset_file_def;
    TF_RETURN_IF_ERROR(
        ParseAny(any_asset, &asset_file_def, "tensorflow.AssetFileDef"));
    asset_file_defs->push_back(asset_file_def);
  }
  return Status::OK();
}

Status LoadSavedModelInternal(const SessionOptions& session_options,
                              const RunOptions& run_options,
                              const string& export_dir,
                              const std::unordered_set<string>& tags,
                              SavedModelBundle* const bundle) {
  const uint64 read_start_microseconds = Env::Default()->NowMicros();
  TF_RETURN_IF_ERROR(ReadMetaGraphDefFromSavedModel(export_dir, tags,
                                                    &bundle->meta_graph_def));

  TF_RETURN_IF_ERROR(LoadMetaGraphIntoSession(
      bundle->meta_graph_def, session_options, &bundle->session));

  std::vector<AssetFileDef> asset_file_defs;
  TF_RETURN_IF_ERROR(
      GetAssetFileDefs(bundle->meta_graph_def, &asset_file_defs));
  TF_RETURN_IF_ERROR(
      RunRestore(run_options, export_dir,
                 bundle->meta_graph_def.saver_def().restore_op_name(),
                 bundle->meta_graph_def.saver_def().filename_tensor_name(),
                 asset_file_defs, bundle->session.get()));
  // Record walltime spent in restoring graph from disk, but postpone metric
  // increments until graph init finishes.
  const uint64 restore_graph_walltime =
      GetLatencyMicroseconds(read_start_microseconds);

  const uint64 graph_init_start_microseconds = Env::Default()->NowMicros();
  string init_op_name;
  TF_RETURN_IF_ERROR(
      GetInitOp(export_dir, bundle->meta_graph_def, &init_op_name));
  TF_RETURN_IF_ERROR(RunInitOp(run_options, export_dir, bundle->meta_graph_def,
                               asset_file_defs, bundle->session.get(),
                               init_op_name));
  load_latency_by_stage->GetCell(export_dir, "restore_graph")
      ->Add(restore_graph_walltime);
  // Record wall time spent in init op.
  load_latency_by_stage->GetCell(export_dir, "init_graph")
      ->Add(GetLatencyMicroseconds(graph_init_start_microseconds));
  return Status::OK();
}

}  // namespace

Status LoadSavedModel(const SessionOptions& session_options,
                      const RunOptions& run_options, const string& export_dir,
                      const std::unordered_set<string>& tags,
                      SavedModelBundle* const bundle) {
  // TODO(robson): Add tests for the counters.
  const uint64 start_microseconds = Env::Default()->NowMicros();
  const Status status = LoadSavedModelInternal(session_options, run_options,
                                               export_dir, tags, bundle);
  auto log_and_count = [&](const string& status_str) {
    LOG(INFO) << "SavedModel load for tags { " << str_util::Join(tags, " ")
              << " }; Status: " << status_str << ". Took "
              << GetLatencyMicroseconds(start_microseconds) << " microseconds.";
    load_attempt_count->GetCell(export_dir, status_str)->IncrementBy(1);
  };
  if (status.ok()) {
    log_and_count(kLoadAttemptSuccess);
  } else {
    log_and_count(kLoadAttemptFail);
  }
  load_latency->GetCell(export_dir)
      ->IncrementBy(GetLatencyMicroseconds(start_microseconds));
  return status;
}

bool MaybeSavedModelDirectory(const string& export_dir) {
  const string saved_model_pb_path =
      io::JoinPath(export_dir, kSavedModelFilenamePb);
  const string saved_model_pbtxt_path =
      io::JoinPath(export_dir, kSavedModelFilenamePbTxt);
  return Env::Default()->FileExists(saved_model_pb_path).ok() ||
         Env::Default()->FileExists(saved_model_pbtxt_path).ok();
}

}  // namespace tensorflow
<|endoftext|>


<commit_before>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/framework/common_shape_fns.h"
#include "tensorflow/core/framework/numeric_op.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/shape_inference.h"

namespace tensorflow {

using shape_inference::DimensionHandle;
using shape_inference::InferenceContext;
using shape_inference::ShapeHandle;

REGISTER_OP("FFT")
    .Input("input: complex64")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) {
      return shape_inference::UnchangedShapeWithRankAtLeast(c, 1);
    })
    .Doc(R"doc(
Compute the 1-dimensional discrete Fourier Transform over the inner-most
dimension of `input`.

input: A complex64 tensor.
output: A complex64 tensor of the same shape as `input`. The inner-most
  dimension of `input` is replaced with its 1D Fourier Transform.

@compatibility(numpy)
Equivalent to np.fft.fft
@end_compatibility
)doc");

REGISTER_OP("IFFT")
    .Input("input: complex64")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) {
      return shape_inference::UnchangedShapeWithRankAtLeast(c, 1);
    })
    .Doc(R"doc(
Compute the inverse 1-dimensional discrete Fourier Transform over the inner-most
dimension of `input`.

input: A complex64 tensor.
output: A complex64 tensor of the same shape as `input`. The inner-most
  dimension of `input` is replaced with its inverse 1D Fourier Transform.

@compatibility(numpy)
Equivalent to np.fft.ifft
@end_compatibility
)doc");

REGISTER_OP("FFT2D")
    .Input("input: complex64")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) {
      return shape_inference::UnchangedShapeWithRankAtLeast(c, 2);
    })
    .Doc(R"doc(
Compute the 2-dimensional discrete Fourier Transform over the inner-most
2 dimensions of `input`.

input: A complex64 tensor.
output: A complex64 tensor of the same shape as `input`. The inner-most 2
  dimensions of `input` are replaced with their 2D Fourier Transform.

@compatibility(numpy)
Equivalent to np.fft.fft2
@end_compatibility
)doc");

REGISTER_OP("IFFT2D")
    .Input("input: complex64")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) {
      return shape_inference::UnchangedShapeWithRankAtLeast(c, 2);
    })
    .Doc(R"doc(
Compute the inverse 2-dimensional discrete Fourier Transform over the inner-most
2 dimensions of `input`.

input: A complex64 tensor.
output: A complex64 tensor of the same shape as `input`. The inner-most 2
  dimensions of `input` are replaced with their inverse 2D Fourier Transform.

@compatibility(numpy)
Equivalent to np.fft.ifft2
@end_compatibility
)doc");

REGISTER_OP("FFT3D")
    .Input("input: complex64")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) {
      return shape_inference::UnchangedShapeWithRankAtLeast(c, 3);
    })
    .Doc(R"doc(
Compute the 3-dimensional discrete Fourier Transform over the inner-most 3
dimensions of `input`.

input: A complex64 tensor.
output: A complex64 tensor of the same shape as `input`. The inner-most 3
  dimensions of `input` are replaced with their 3D Fourier Transform.

@compatibility(numpy)
Equivalent to np.fft.fftn with 3 dimensions.
@end_compatibility
)doc");

REGISTER_OP("IFFT3D")
    .Input("input: complex64")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) {
      return shape_inference::UnchangedShapeWithRankAtLeast(c, 3);
    })
    .Doc(R"doc(
Compute the inverse 3-dimensional discrete Fourier Transform over the inner-most
3 dimensions of `input`.

input: A complex64 tensor.
output: A complex64 tensor of the same shape as `input`. The inner-most 3
  dimensions of `input` are replaced with their inverse 3D Fourier Transform.

@compatibility(numpy)
Equivalent to np.fft.ifftn with 3 dimensions.
@end_compatibility
)doc");

Status RFFTShape(InferenceContext* c, const bool forward, const int rank) {
  ShapeHandle out;
  TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), rank, &out));

  // Check that fft_length has shape [rank].
  ShapeHandle unused_shape;
  DimensionHandle unused_dim;
  ShapeHandle fft_length_input = c->input(1);
  TF_RETURN_IF_ERROR(c->WithRank(fft_length_input, 1, &unused_shape));
  TF_RETURN_IF_ERROR(
      c->WithValue(c->Dim(fft_length_input, 0), rank, &unused_dim));
  const Tensor* fft_length_tensor = c->input_tensor(1);

  // If fft_length is unknown at graph creation time, we can't predict the
  // output size.
  if (fft_length_tensor == nullptr) {
    // We can't know the dimension of any of the rank inner dimensions of the
    // output without knowing fft_length.
    for (int i = 0; i < rank; ++i) {
      TF_RETURN_IF_ERROR(c->ReplaceDim(out, -rank + i, c->UnknownDim(), &out));
    }
  } else {
    auto fft_length_as_vec = fft_length_tensor->vec<int32>();
    for (int i = 0; i < rank; ++i) {
      // For RFFT, replace the last dimension with fft_length/2 + 1.
      auto dim = forward && i == rank - 1 && fft_length_as_vec(i) != 0
                     ? fft_length_as_vec(i) / 2 + 1
                     : fft_length_as_vec(i);
      TF_RETURN_IF_ERROR(c->ReplaceDim(out, -rank + i, c->MakeDim(dim), &out));
    }
  }

  c->set_output(0, out);
  return Status::OK();
}

REGISTER_OP("RFFT")
    .Input("input: float")
    .Input("fft_length: int32")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) { return RFFTShape(c, true, 1); })
    .Doc(R"doc(
Compute the 1-dimensional discrete Fourier Transform of a real-valued signal
over the inner-most dimension of `input`.

Since the DFT of a real signal is Hermitian-symmetric, `RFFT` only returns the
`fft_length / 2 + 1` unique components of the FFT: the zero-frequency term,
followed by the `fft_length / 2` positive-frequency terms.

input: A float32 tensor.
fft_length: An int32 tensor of shape [1]. The FFT length.
output: A complex64 tensor of the same rank as `input`. The inner-most
  dimension of `input` is replaced with the `fft_length / 2 + 1` unique
  frequency components of its 1D Fourier Transform.

@compatibility(numpy)
Equivalent to np.fft.rfft
@end_compatibility
)doc");

REGISTER_OP("IRFFT")
    .Input("input: complex64")
    .Input("fft_length: int32")
    .Output("output: float")
    .SetShapeFn([](InferenceContext* c) { return RFFTShape(c, false, 1); })
    .Doc(R"doc(
Compute the inverse 1-dimensional discrete Fourier Transform of a real-valued
signal over the inner-most dimension of `input`.

The inner-most dimension of `input` is assumed to be the result of `RFFT`: the
`fft_length / 2 + 1` unique components of the DFT of a real-valued signal. If
`fft_length` is not provided, it is computed from the size of the inner-most
dimension of `input` (`fft_length = 2 * (inner - 1)`). If the FFT length used to
compute `input` is odd, it should be provided since it cannot be inferred
properly.

input: A complex64 tensor.
fft_length: An int32 tensor of shape [1]. The FFT length.
output: A float32 tensor of the same rank as `input`. The inner-most
  dimension of `input` is replaced with the `fft_length` samples of its inverse
  1D Fourier Transform.

@compatibility(numpy)
Equivalent to np.fft.irfft
@end_compatibility
)doc");

REGISTER_OP("RFFT2D")
    .Input("input: float")
    .Input("fft_length: int32")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) { return RFFTShape(c, true, 2); })
    .Doc(R"doc(
Compute the 2-dimensional discrete Fourier Transform of a real-valued signal
over the inner-most 2 dimensions of `input`.

Since the DFT of a real signal is Hermitian-symmetric, `RFFT2D` only returns the
`fft_length / 2 + 1` unique components of the FFT for the inner-most dimension
of `output`: the zero-frequency term, followed by the `fft_length / 2`
positive-frequency terms.

input: A float32 tensor.
fft_length: An int32 tensor of shape [2]. The FFT length for each dimension.
output: A complex64 tensor of the same rank as `input`. The inner-most 2
  dimensions of `input` are replaced with their 2D Fourier Transform. The
  inner-most dimension contains `fft_length / 2 + 1` unique frequency
  components.

@compatibility(numpy)
Equivalent to np.fft.rfft2
@end_compatibility
)doc");

REGISTER_OP("IRFFT2D")
    .Input("input: complex64")
    .Input("fft_length: int32")
    .Output("output: float")
    .SetShapeFn([](InferenceContext* c) { return RFFTShape(c, false, 2); })
    .Doc(R"doc(
Compute the inverse 2-dimensional discrete Fourier Transform of a real-valued
signal over the inner-most 2 dimensions of `input`.

The inner-most 2 dimensions of `input` are assumed to be the result of `RFFT2D`:
The inner-most dimension contains the `fft_length / 2 + 1` unique components of
the DFT of a real-valued signal. If `fft_length` is not provided, it is computed
from the size of the inner-most 2 dimensions of `input`. If the FFT length used
to compute `input` is odd, it should be provided since it cannot be inferred
properly.

input: A complex64 tensor.
fft_length: An int32 tensor of shape [2]. The FFT length for each dimension.
output: A float32 tensor of the same rank as `input`. The inner-most 2
  dimensions of `input` are replaced with the `fft_length` samples of their
  inverse 2D Fourier Transform.

@compatibility(numpy)
Equivalent to np.fft.irfft2
@end_compatibility
)doc");

REGISTER_OP("RFFT3D")
    .Input("input: float")
    .Input("fft_length: int32")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) { return RFFTShape(c, true, 3); })
    .Doc(R"doc(
Compute the 3-dimensional discrete Fourier Transform of a real-valued signal
over the inner-most 3 dimensions of `input`.

Since the DFT of a real signal is Hermitian-symmetric, `RFFT3D` only returns the
`fft_length / 2 + 1` unique components of the FFT for the inner-most dimension
of `output`: the zero-frequency term, followed by the `fft_length / 2`
positive-frequency terms.

input: A float32 tensor.
fft_length: An int32 tensor of shape [3]. The FFT length for each dimension.
output: A complex64 tensor of the same rank as `input`. The inner-most 3
  dimensions of `input` are replaced with the their 3D Fourier Transform. The
  inner-most dimension contains `fft_length / 2 + 1` unique frequency
  components.

@compatibility(numpy)
Equivalent to np.fft.rfftn with 3 dimensions.
@end_compatibility
)doc");

REGISTER_OP("IRFFT3D")
    .Input("input: complex64")
    .Input("fft_length: int32")
    .Output("output: float")
    .SetShapeFn([](InferenceContext* c) { return RFFTShape(c, false, 3); })
    .Doc(R"doc(
Compute the inverse 3-dimensional discrete Fourier Transform of a real-valued
signal over the inner-most 3 dimensions of `input`.

The inner-most 3 dimensions of `input` are assumed to be the result of `RFFT3D`:
The inner-most dimension contains the `fft_length / 2 + 1` unique components of
the DFT of a real-valued signal. If `fft_length` is not provided, it is computed
from the size of the inner-most 3 dimensions of `input`. If the FFT length used
to compute `input` is odd, it should be provided since it cannot be inferred
properly.

input: A complex64 tensor.
fft_length: An int32 tensor of shape [3]. The FFT length for each dimension.
output: A float32 tensor of the same rank as `input`. The inner-most 3
  dimensions of `input` are replaced with the `fft_length` samples of their
  inverse 3D real Fourier Transform.

@compatibility(numpy)
Equivalent to np.irfftn with 3 dimensions.
@end_compatibility
)doc");

// Deprecated ops:
REGISTER_OP("BatchFFT")
    .Input("input: complex64")
    .Output("output: complex64")
    .Deprecated(15, "Use FFT");
REGISTER_OP("BatchIFFT")
    .Input("input: complex64")
    .Output("output: complex64")
    .Deprecated(15, "Use IFFT");
REGISTER_OP("BatchFFT2D")
    .Input("input: complex64")
    .Output("output: complex64")
    .Deprecated(15, "Use FFT2D");
REGISTER_OP("BatchIFFT2D")
    .Input("input: complex64")
    .Output("output: complex64")
    .Deprecated(15, "Use IFFT2D");
REGISTER_OP("BatchFFT3D")
    .Input("input: complex64")
    .Output("output: complex64")
    .Deprecated(15, "Use FFT3D");
REGISTER_OP("BatchIFFT3D")
    .Input("input: complex64")
    .Output("output: complex64")
    .Deprecated(15, "Use IFFT3D");

}  // namespace tensorflow
<commit_msg>Remove extra blank lines from auto-generated Python documentation. Change: 152046128<commit_after>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/framework/common_shape_fns.h"
#include "tensorflow/core/framework/numeric_op.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/shape_inference.h"

namespace tensorflow {

using shape_inference::DimensionHandle;
using shape_inference::InferenceContext;
using shape_inference::ShapeHandle;

REGISTER_OP("FFT")
    .Input("input: complex64")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) {
      return shape_inference::UnchangedShapeWithRankAtLeast(c, 1);
    })
    .Doc(R"doc(
Fast Fourier transform.

Computes the 1-dimensional discrete Fourier transform over the inner-most
dimension of `input`.

input: A complex64 tensor.
output: A complex64 tensor of the same shape as `input`. The inner-most
  dimension of `input` is replaced with its 1D Fourier transform.

@compatibility(numpy)
Equivalent to np.fft.fft
@end_compatibility
)doc");

REGISTER_OP("IFFT")
    .Input("input: complex64")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) {
      return shape_inference::UnchangedShapeWithRankAtLeast(c, 1);
    })
    .Doc(R"doc(
Inverse fast Fourier transform.

Computes the inverse 1-dimensional discrete Fourier transform over the
inner-most dimension of `input`.

input: A complex64 tensor.
output: A complex64 tensor of the same shape as `input`. The inner-most
  dimension of `input` is replaced with its inverse 1D Fourier transform.

@compatibility(numpy)
Equivalent to np.fft.ifft
@end_compatibility
)doc");

REGISTER_OP("FFT2D")
    .Input("input: complex64")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) {
      return shape_inference::UnchangedShapeWithRankAtLeast(c, 2);
    })
    .Doc(R"doc(
2D fast Fourier transform.

Computes the 2-dimensional discrete Fourier transform over the inner-most
2 dimensions of `input`.

input: A complex64 tensor.
output: A complex64 tensor of the same shape as `input`. The inner-most 2
  dimensions of `input` are replaced with their 2D Fourier transform.

@compatibility(numpy)
Equivalent to np.fft.fft2
@end_compatibility
)doc");

REGISTER_OP("IFFT2D")
    .Input("input: complex64")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) {
      return shape_inference::UnchangedShapeWithRankAtLeast(c, 2);
    })
    .Doc(R"doc(
Inverse 2D fast Fourier transform.

Computes the inverse 2-dimensional discrete Fourier transform over the
inner-most 2 dimensions of `input`.

input: A complex64 tensor.
output: A complex64 tensor of the same shape as `input`. The inner-most 2
  dimensions of `input` are replaced with their inverse 2D Fourier transform.

@compatibility(numpy)
Equivalent to np.fft.ifft2
@end_compatibility
)doc");

REGISTER_OP("FFT3D")
    .Input("input: complex64")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) {
      return shape_inference::UnchangedShapeWithRankAtLeast(c, 3);
    })
    .Doc(R"doc(
3D fast Fourier transform.

Computes the 3-dimensional discrete Fourier transform over the inner-most 3
dimensions of `input`.

input: A complex64 tensor.
output: A complex64 tensor of the same shape as `input`. The inner-most 3
  dimensions of `input` are replaced with their 3D Fourier transform.

@compatibility(numpy)
Equivalent to np.fft.fftn with 3 dimensions.
@end_compatibility
)doc");

REGISTER_OP("IFFT3D")
    .Input("input: complex64")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) {
      return shape_inference::UnchangedShapeWithRankAtLeast(c, 3);
    })
    .Doc(R"doc(
Inverse 3D fast Fourier transform.

Computes the inverse 3-dimensional discrete Fourier transform over the
inner-most 3 dimensions of `input`.

input: A complex64 tensor.
output: A complex64 tensor of the same shape as `input`. The inner-most 3
  dimensions of `input` are replaced with their inverse 3D Fourier transform.

@compatibility(numpy)
Equivalent to np.fft.ifftn with 3 dimensions.
@end_compatibility
)doc");

Status RFFTShape(InferenceContext* c, const bool forward, const int rank) {
  ShapeHandle out;
  TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), rank, &out));

  // Check that fft_length has shape [rank].
  ShapeHandle unused_shape;
  DimensionHandle unused_dim;
  ShapeHandle fft_length_input = c->input(1);
  TF_RETURN_IF_ERROR(c->WithRank(fft_length_input, 1, &unused_shape));
  TF_RETURN_IF_ERROR(
      c->WithValue(c->Dim(fft_length_input, 0), rank, &unused_dim));
  const Tensor* fft_length_tensor = c->input_tensor(1);

  // If fft_length is unknown at graph creation time, we can't predict the
  // output size.
  if (fft_length_tensor == nullptr) {
    // We can't know the dimension of any of the rank inner dimensions of the
    // output without knowing fft_length.
    for (int i = 0; i < rank; ++i) {
      TF_RETURN_IF_ERROR(c->ReplaceDim(out, -rank + i, c->UnknownDim(), &out));
    }
  } else {
    auto fft_length_as_vec = fft_length_tensor->vec<int32>();
    for (int i = 0; i < rank; ++i) {
      // For RFFT, replace the last dimension with fft_length/2 + 1.
      auto dim = forward && i == rank - 1 && fft_length_as_vec(i) != 0
                     ? fft_length_as_vec(i) / 2 + 1
                     : fft_length_as_vec(i);
      TF_RETURN_IF_ERROR(c->ReplaceDim(out, -rank + i, c->MakeDim(dim), &out));
    }
  }

  c->set_output(0, out);
  return Status::OK();
}

REGISTER_OP("RFFT")
    .Input("input: float")
    .Input("fft_length: int32")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) { return RFFTShape(c, true, 1); })
    .Doc(R"doc(
Real-valued fast Fourier transform.

Computes the 1-dimensional discrete Fourier transform of a real-valued signal
over the inner-most dimension of `input`.

Since the DFT of a real signal is Hermitian-symmetric, `RFFT` only returns the
`fft_length / 2 + 1` unique components of the FFT: the zero-frequency term,
followed by the `fft_length / 2` positive-frequency terms.

input: A float32 tensor.
fft_length: An int32 tensor of shape [1]. The FFT length.
output: A complex64 tensor of the same rank as `input`. The inner-most
  dimension of `input` is replaced with the `fft_length / 2 + 1` unique
  frequency components of its 1D Fourier transform.

@compatibility(numpy)
Equivalent to np.fft.rfft
@end_compatibility
)doc");

REGISTER_OP("IRFFT")
    .Input("input: complex64")
    .Input("fft_length: int32")
    .Output("output: float")
    .SetShapeFn([](InferenceContext* c) { return RFFTShape(c, false, 1); })
    .Doc(R"doc(
Inverse real-valued fast Fourier transform.

Computes the inverse 1-dimensional discrete Fourier transform of a real-valued
signal over the inner-most dimension of `input`.

The inner-most dimension of `input` is assumed to be the result of `RFFT`: the
`fft_length / 2 + 1` unique components of the DFT of a real-valued signal. If
`fft_length` is not provided, it is computed from the size of the inner-most
dimension of `input` (`fft_length = 2 * (inner - 1)`). If the FFT length used to
compute `input` is odd, it should be provided since it cannot be inferred
properly.

input: A complex64 tensor.
fft_length: An int32 tensor of shape [1]. The FFT length.
output: A float32 tensor of the same rank as `input`. The inner-most
  dimension of `input` is replaced with the `fft_length` samples of its inverse
  1D Fourier transform.

@compatibility(numpy)
Equivalent to np.fft.irfft
@end_compatibility
)doc");

REGISTER_OP("RFFT2D")
    .Input("input: float")
    .Input("fft_length: int32")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) { return RFFTShape(c, true, 2); })
    .Doc(R"doc(
2D real-valued fast Fourier transform.

Computes the 2-dimensional discrete Fourier transform of a real-valued signal
over the inner-most 2 dimensions of `input`.

Since the DFT of a real signal is Hermitian-symmetric, `RFFT2D` only returns the
`fft_length / 2 + 1` unique components of the FFT for the inner-most dimension
of `output`: the zero-frequency term, followed by the `fft_length / 2`
positive-frequency terms.

input: A float32 tensor.
fft_length: An int32 tensor of shape [2]. The FFT length for each dimension.
output: A complex64 tensor of the same rank as `input`. The inner-most 2
  dimensions of `input` are replaced with their 2D Fourier transform. The
  inner-most dimension contains `fft_length / 2 + 1` unique frequency
  components.

@compatibility(numpy)
Equivalent to np.fft.rfft2
@end_compatibility
)doc");

REGISTER_OP("IRFFT2D")
    .Input("input: complex64")
    .Input("fft_length: int32")
    .Output("output: float")
    .SetShapeFn([](InferenceContext* c) { return RFFTShape(c, false, 2); })
    .Doc(R"doc(
Inverse 2D real-valued fast Fourier transform.

Computes the inverse 2-dimensional discrete Fourier transform of a real-valued
signal over the inner-most 2 dimensions of `input`.

The inner-most 2 dimensions of `input` are assumed to be the result of `RFFT2D`:
The inner-most dimension contains the `fft_length / 2 + 1` unique components of
the DFT of a real-valued signal. If `fft_length` is not provided, it is computed
from the size of the inner-most 2 dimensions of `input`. If the FFT length used
to compute `input` is odd, it should be provided since it cannot be inferred
properly.

input: A complex64 tensor.
fft_length: An int32 tensor of shape [2]. The FFT length for each dimension.
output: A float32 tensor of the same rank as `input`. The inner-most 2
  dimensions of `input` are replaced with the `fft_length` samples of their
  inverse 2D Fourier transform.

@compatibility(numpy)
Equivalent to np.fft.irfft2
@end_compatibility
)doc");

REGISTER_OP("RFFT3D")
    .Input("input: float")
    .Input("fft_length: int32")
    .Output("output: complex64")
    .SetShapeFn([](InferenceContext* c) { return RFFTShape(c, true, 3); })
    .Doc(R"doc(
3D real-valued fast Fourier transform.

Computes the 3-dimensional discrete Fourier transform of a real-valued signal
over the inner-most 3 dimensions of `input`.

Since the DFT of a real signal is Hermitian-symmetric, `RFFT3D` only returns the
`fft_length / 2 + 1` unique components of the FFT for the inner-most dimension
of `output`: the zero-frequency term, followed by the `fft_length / 2`
positive-frequency terms.

input: A float32 tensor.
fft_length: An int32 tensor of shape [3]. The FFT length for each dimension.
output: A complex64 tensor of the same rank as `input`. The inner-most 3
  dimensions of `input` are replaced with the their 3D Fourier transform. The
  inner-most dimension contains `fft_length / 2 + 1` unique frequency
  components.

@compatibility(numpy)
Equivalent to np.fft.rfftn with 3 dimensions.
@end_compatibility
)doc");

REGISTER_OP("IRFFT3D")
    .Input("input: complex64")
    .Input("fft_length: int32")
    .Output("output: float")
    .SetShapeFn([](InferenceContext* c) { return RFFTShape(c, false, 3); })
    .Doc(R"doc(
Inverse 3D real-valued fast Fourier transform.

Computes the inverse 3-dimensional discrete Fourier transform of a real-valued
signal over the inner-most 3 dimensions of `input`.

The inner-most 3 dimensions of `input` are assumed to be the result of `RFFT3D`:
The inner-most dimension contains the `fft_length / 2 + 1` unique components of
the DFT of a real-valued signal. If `fft_length` is not provided, it is computed
from the size of the inner-most 3 dimensions of `input`. If the FFT length used
to compute `input` is odd, it should be provided since it cannot be inferred
properly.

input: A complex64 tensor.
fft_length: An int32 tensor of shape [3]. The FFT length for each dimension.
output: A float32 tensor of the same rank as `input`. The inner-most 3
  dimensions of `input` are replaced with the `fft_length` samples of their
  inverse 3D real Fourier transform.

@compatibility(numpy)
Equivalent to np.irfftn with 3 dimensions.
@end_compatibility
)doc");

// Deprecated ops:
REGISTER_OP("BatchFFT")
    .Input("input: complex64")
    .Output("output: complex64")
    .Deprecated(15, "Use FFT");
REGISTER_OP("BatchIFFT")
    .Input("input: complex64")
    .Output("output: complex64")
    .Deprecated(15, "Use IFFT");
REGISTER_OP("BatchFFT2D")
    .Input("input: complex64")
    .Output("output: complex64")
    .Deprecated(15, "Use FFT2D");
REGISTER_OP("BatchIFFT2D")
    .Input("input: complex64")
    .Output("output: complex64")
    .Deprecated(15, "Use IFFT2D");
REGISTER_OP("BatchFFT3D")
    .Input("input: complex64")
    .Output("output: complex64")
    .Deprecated(15, "Use FFT3D");
REGISTER_OP("BatchIFFT3D")
    .Input("input: complex64")
    .Output("output: complex64")
    .Deprecated(15, "Use IFFT3D");

}  // namespace tensorflow
<|endoftext|>


<commit_before>// SUBS-04.WeirdCombinations.cpp
// https://judge.softuni.bg/Contests/37/CSharp-Basics-Exam-7-November-2014
#include <iostream>
#include <stack>
#include <string>
using namespace std;

const int numSysBase = 5 ;

int main ()
{ // numeral system based solution. The author uses nested loops.
	string s ;
	int n,
		qtyDigits ;
	stack<int> digits ;
	char result [numSysBase] ;
	cin >>s >>n;

	do
	{	// moves to stack a last digit
		digits.push(n%numSysBase) ;
		n = n/numSysBase ; // remove last digit
	} while (n != 0) ;

	qtyDigits = digits.size() ;
	if (qtyDigits > numSysBase)
	{
		cout <<"No" ;
		return 0 ;
	} ;

	for (int k=0; k<numSysBase-qtyDigits; k++)
		cout <<s[0] ; // prints leading zeros if they are available

	while (! digits.empty())
	{ // to print a last remainder
		cout <<s[digits.top()];
		digits.pop(); // to remove and to be ready for a next remainder
	}
}

<commit_msg>minor<commit_after>// SUBS-04.WeirdCombinations.cpp
// https://judge.softuni.bg/Contests/37/CSharp-Basics-Exam-7-November-2014
#include <iostream>
// https://bg.wikipedia.org/wiki/%D0%A1%D1%82%D0%B5%D0%BA_(%D1%81%D1%82%D1%80%D1%83%D0%BA%D1%82%D1%83%D1%80%D0%B0_%D0%BE%D1%82_%D0%B4%D0%B0%D0%BD%D0%BD%D0%B8)
// https://www.youtube.com/watch?v=5UjENioK8tw
#include <stack>
#include <string> // Koala Press - SIMVOLNI NIZOVE - p 123.
using namespace std;

const int numSysBase = 5 ;

int main ()
{ // numeral system based solution. The author uses nested loops.
	string s ;
	int n,
		qtyDigits ;
	stack<int> digits ;
	char result [numSysBase] ;
	cin >>s >>n;

	do
	{	// moves to stack a last digit
		digits.push(n%numSysBase) ;
		n = n/numSysBase ; // remove last digit
	} while (n != 0) ;

	qtyDigits = digits.size() ;
	if (qtyDigits > numSysBase)
	{
		cout <<"No" ;
		return 0 ;
	} ;

	for (int k=0; k<numSysBase-qtyDigits; k++)
		cout <<s[0] ; // prints leading zeros if they are available

	while (! digits.empty())
	{ // to print a last remainder
		cout <<s[digits.top()];
		digits.pop(); // to remove and to be ready for a next remainder
	}
}

<|endoftext|>


<commit_before>#include "s3.h"
using namespace himan;

#ifdef HAVE_S3
#include "debug.h"
#include "timer.h"
#include "util.h"
#include <iostream>
#include <libs3.h>
#include <mutex>
#include <string.h>  // memcpy

namespace
{
static std::once_flag oflag;

const char* access_key = 0;
const char* secret_key = 0;
const char* security_token = 0;
S3Protocol protocol = S3ProtocolHTTP;

thread_local S3Status statusG = S3StatusOK;

void CheckS3Error(S3Status errarg, const char* file, const int line);

#define S3_CHECK(errarg) CheckS3Error(errarg, __FILE__, __LINE__)

void HandleS3Error(himan::logger& logr, const std::string& host, const std::string& object, const std::string& op)
{
	logr.Error(fmt::format("{} operation to/from host={} object=s3://{} failed", op, host, object));

	switch (statusG)
	{
		case S3StatusInternalError:
			logr.Error(fmt::format("{}: is there a proxy blocking the connection?", S3_get_status_name(statusG)));
			throw himan::kFileDataNotFound;
		case S3StatusFailedToConnect:
			logr.Error(fmt::format("{}: is proxy required but not set?", S3_get_status_name(statusG)));
			throw himan::kFileDataNotFound;
		case S3StatusErrorInvalidAccessKeyId:
			logr.Error(fmt::format(
			    "{}: are Temporary Security Credentials used without security token (env: S3_SESSION_TOKEN)?",
			    S3_get_status_name(statusG)));
			throw himan::kFileDataNotFound;
		default:
			logr.Error(S3_get_status_name(statusG));
			throw himan::kFileDataNotFound;
	}
}

std::vector<std::string> GetBucketAndFileName(const std::string& fullFileName)
{
	std::vector<std::string> ret;

	auto fileName = fullFileName;

	// strip protocol from string if it's there
	const auto pos = fullFileName.find("s3://");

	if (pos != std::string::npos)
	{
		fileName = fileName.erase(pos, 5);
	}

	// erase forward slash if exists (s3 buckets can't start with /)
	if (fileName[0] == '/')
	{
		fileName = fileName.erase(0, 1);
	}

	auto tokens = util::Split(fileName, "/");

	ret.push_back(tokens[0]);
	tokens.erase(std::begin(tokens), std::begin(tokens) + 1);

	std::string key;
	for (const auto& piece : tokens)
	{
		if (!key.empty())
		{
			key += "/";
		}
		key += piece;
	}

	ret.push_back(key);
	return ret;
}

inline void CheckS3Error(S3Status errarg, const char* file, const int line)
{
	if (errarg)
	{
		std::cerr << "Error at " << file << "(" << line << "): " << S3_get_status_name(errarg) << std::endl;
		himan::Abort();
	}
}

S3Status responsePropertiesCallback(const S3ResponseProperties* properties, void* callbackData)
{
	return S3StatusOK;
}

static void responseCompleteCallback(S3Status status, const S3ErrorDetails* error, void* callbackData)
{
	statusG = status;
	return;
}

thread_local S3ResponseHandler responseHandler = {&responsePropertiesCallback, &responseCompleteCallback};

static S3Status getObjectDataCallback(int bufferSize, const char* buffer, void* callbackData)
{
	himan::buffer* ret = static_cast<himan::buffer*>(callbackData);

	ret->data = static_cast<unsigned char*>(realloc(ret->data, ret->length + bufferSize));
	memcpy(ret->data + ret->length, buffer, bufferSize);
	ret->length += bufferSize;

	return S3StatusOK;
}

void Initialize()
{
	call_once(oflag, [&]() {
		access_key = getenv("S3_ACCESS_KEY_ID");
		secret_key = getenv("S3_SECRET_ACCESS_KEY");
		security_token = getenv("S3_SESSION_TOKEN");

		logger logr("s3");

		if (!access_key)
		{
			logr.Info("Environment variable S3_ACCESS_KEY_ID not defined");
		}

		if (!secret_key)
		{
			logr.Info("Environment variable S3_SECRET_ACCESS_KEY not defined");
		}

		try
		{
			const auto envproto = himan::util::GetEnv("S3_PROTOCOL");
			if (envproto == "https")
			{
				protocol = S3ProtocolHTTPS;
			}
			else if (envproto == "http")
			{
				protocol = S3ProtocolHTTP;
			}
			else
			{
				logr.Warning(fmt::format("Unrecognized value found from env variable S3_PROTOCOL: '{}'", envproto));
			}
		}
		catch (const std::invalid_argument& e)
		{
		}

		S3_CHECK(S3_initialize("s3", S3_INIT_ALL, NULL));
	});
}

std::string ReadAWSRegionFromHostname(const std::string& hostname)
{
	if (hostname.find("amazonaws.com") != std::string::npos)
	{
		// extract region name from host name, assuming aws hostname like
		// s3.us-east-1.amazonaws.com

		auto tokens = util::Split(hostname, ".");

		logger logr("s3");

		if (tokens.size() != 4)
		{
			logr.Fatal("Hostname does not follow pattern s3.<regionname>.amazonaws.com");
		}
		else
		{
			logr.Trace(fmt::format("s3 authentication hostname: {}", tokens[1]));
			return tokens[1];
		}
	}

	return "";
}

struct write_data
{
	himan::buffer buffer;
	size_t write_ptr;
};

static int putObjectDataCallback(int bufferSize, char* buffer, void* callbackData)
{
	write_data* data = static_cast<write_data*>(callbackData);
	int bytesWritten = 0;

	if (data->buffer.length)
	{
		bytesWritten =
		    static_cast<int>((static_cast<int>(data->buffer.length) > bufferSize) ? bufferSize : data->buffer.length);
		memcpy(buffer, data->buffer.data + data->write_ptr, bytesWritten);
		data->write_ptr += bytesWritten;
		data->buffer.length -= bytesWritten;
	}

	return bytesWritten;
}

}  // namespace

buffer s3::ReadFile(const file_information& fileInformation)
{
	Initialize();
	logger logr("s3");

	S3GetObjectHandler getObjectHandler = {responseHandler, &getObjectDataCallback};

	const auto bucketAndFileName = GetBucketAndFileName(fileInformation.file_location);
	const auto bucket = bucketAndFileName[0];
	const auto key = bucketAndFileName[1];

	buffer ret;

#ifdef S3_DEFAULT_REGION

	std::string region = ReadAWSRegionFromHostname(fileInformation.file_server);

	// clang-format off

        S3BucketContext bucketContext =
        {
                fileInformation.file_server.c_str(),
                bucket.c_str(),
                protocol,
                S3UriStylePath,
                access_key,
                secret_key,
                security_token,
                region.c_str()
        };
#else

	// clang-format off

	S3BucketContext bucketContext =
	{
		fileInformation.file_server.c_str(),
		bucket.c_str(),
		protocol,
		S3UriStylePath,
		access_key,
		secret_key,
		security_token
	};

	// clang-format on
#endif

	int count = 0;
	do
	{
		if (count > 0)
		{
			sleep(2 * count);
		}
		const unsigned long offset = fileInformation.offset.get();
		const unsigned long length = fileInformation.length.get();

#ifdef S3_DEFAULT_REGION
		S3_get_object(&bucketContext, key.c_str(), NULL, offset, length, NULL, 0, &getObjectHandler, &ret);
#else
		S3_get_object(&bucketContext, key.c_str(), NULL, offset, length, NULL, &getObjectHandler, &ret);
#endif
		count++;
	} while (S3_status_is_retryable(statusG) && count < 3);

	switch (statusG)
	{
		case S3StatusOK:
			break;
		default:
			HandleS3Error(logr, fileInformation.file_server, fmt::format("{}/{}", bucket, key), "Read");
			break;
	}

	if (ret.length == 0)
	{
		throw himan::kFileDataNotFound;
	}

	return ret;
}

void s3::WriteObject(const std::string& objectName, const buffer& buff)
{
	Initialize();

	const auto bucketAndFileName = GetBucketAndFileName(objectName);
	const auto bucket = bucketAndFileName[0];
	const auto key = bucketAndFileName[1];

	const char* host = getenv("S3_HOSTNAME");

	logger logr("s3");

	if (!host)
	{
		logr.Fatal("Environment variable S3_HOSTNAME not defined");
		himan::Abort();
	}

#ifdef S3_DEFAULT_REGION

	std::string region = ReadAWSRegionFromHostname(std::string(host));

	// clang-format off

        S3BucketContext bucketContext =
        {
                host,
                bucket.c_str(),
                protocol,
                S3UriStylePath,
                access_key,
                secret_key,
                security_token,
                region.c_str()
        };
#else

	// clang-format off

	S3BucketContext bucketContext =
	{
		host,
		bucket.c_str(),
		protocol,
		S3UriStylePath,
		access_key,
		secret_key,
		security_token
	};
#endif

	// clang-format on

	S3PutObjectHandler putObjectHandler = {responseHandler, &putObjectDataCallback};

	write_data data;
	data.buffer.data = buff.data;
	data.buffer.length = buff.length;
	data.write_ptr = 0;

	timer t(true);

	int count = 0;
	do
	{
		if (count > 0)
		{
			sleep(2 * count);
		}

#ifdef S3_DEFAULT_REGION
		S3_put_object(&bucketContext, key.c_str(), buff.length, NULL, NULL, 0, &putObjectHandler, &data);
#else
		S3_put_object(&bucketContext, key.c_str(), buff.length, NULL, NULL, &putObjectHandler, &data);
#endif

		count++;
	} while (S3_status_is_retryable(statusG) && count < 3);

	// remove pointer to original buff so that double free doesn't occur
	data.buffer.data = 0;

	switch (statusG)
	{
		case S3StatusOK:
		{
			t.Stop();
			const double time = static_cast<double>(t.GetTime());
			const double size = util::round(static_cast<double>(buff.length) / 1024. / 1024., 1);
			logr.Info(fmt::format("Wrote {} MB in {} ms ({:.1f} MBps) to file s3://{}/{}", size, time, size / time,
			                      bucket, key));
		}
		break;
		default:
			HandleS3Error(logr, host, fmt::format("{}/{}", bucket, key), "Write");
			break;
	}
}

#else
buffer s3::ReadFile(const file_information& fileInformation)
{
	throw std::runtime_error("S3 support not compiled");
}
void s3::WriteObject(const std::string& objectName, const buffer& buff)
{
	throw std::runtime_error("S3 support not compiled");
}
#endif
<commit_msg>Correct calculation of transmission speed<commit_after>#include "s3.h"
using namespace himan;

#ifdef HAVE_S3
#include "debug.h"
#include "timer.h"
#include "util.h"
#include <iostream>
#include <libs3.h>
#include <mutex>
#include <string.h>  // memcpy

namespace
{
static std::once_flag oflag;

const char* access_key = 0;
const char* secret_key = 0;
const char* security_token = 0;
S3Protocol protocol = S3ProtocolHTTP;

thread_local S3Status statusG = S3StatusOK;

void CheckS3Error(S3Status errarg, const char* file, const int line);

#define S3_CHECK(errarg) CheckS3Error(errarg, __FILE__, __LINE__)

void HandleS3Error(himan::logger& logr, const std::string& host, const std::string& object, const std::string& op)
{
	logr.Error(fmt::format("{} operation to/from host={} object=s3://{} failed", op, host, object));

	switch (statusG)
	{
		case S3StatusInternalError:
			logr.Error(fmt::format("{}: is there a proxy blocking the connection?", S3_get_status_name(statusG)));
			throw himan::kFileDataNotFound;
		case S3StatusFailedToConnect:
			logr.Error(fmt::format("{}: is proxy required but not set?", S3_get_status_name(statusG)));
			throw himan::kFileDataNotFound;
		case S3StatusErrorInvalidAccessKeyId:
			logr.Error(fmt::format(
			    "{}: are Temporary Security Credentials used without security token (env: S3_SESSION_TOKEN)?",
			    S3_get_status_name(statusG)));
			throw himan::kFileDataNotFound;
		default:
			logr.Error(S3_get_status_name(statusG));
			throw himan::kFileDataNotFound;
	}
}

std::vector<std::string> GetBucketAndFileName(const std::string& fullFileName)
{
	std::vector<std::string> ret;

	auto fileName = fullFileName;

	// strip protocol from string if it's there
	const auto pos = fullFileName.find("s3://");

	if (pos != std::string::npos)
	{
		fileName = fileName.erase(pos, 5);
	}

	// erase forward slash if exists (s3 buckets can't start with /)
	if (fileName[0] == '/')
	{
		fileName = fileName.erase(0, 1);
	}

	auto tokens = util::Split(fileName, "/");

	ret.push_back(tokens[0]);
	tokens.erase(std::begin(tokens), std::begin(tokens) + 1);

	std::string key;
	for (const auto& piece : tokens)
	{
		if (!key.empty())
		{
			key += "/";
		}
		key += piece;
	}

	ret.push_back(key);
	return ret;
}

inline void CheckS3Error(S3Status errarg, const char* file, const int line)
{
	if (errarg)
	{
		std::cerr << "Error at " << file << "(" << line << "): " << S3_get_status_name(errarg) << std::endl;
		himan::Abort();
	}
}

S3Status responsePropertiesCallback(const S3ResponseProperties* properties, void* callbackData)
{
	return S3StatusOK;
}

static void responseCompleteCallback(S3Status status, const S3ErrorDetails* error, void* callbackData)
{
	statusG = status;
	return;
}

thread_local S3ResponseHandler responseHandler = {&responsePropertiesCallback, &responseCompleteCallback};

static S3Status getObjectDataCallback(int bufferSize, const char* buffer, void* callbackData)
{
	himan::buffer* ret = static_cast<himan::buffer*>(callbackData);

	ret->data = static_cast<unsigned char*>(realloc(ret->data, ret->length + bufferSize));
	memcpy(ret->data + ret->length, buffer, bufferSize);
	ret->length += bufferSize;

	return S3StatusOK;
}

void Initialize()
{
	call_once(oflag, [&]() {
		access_key = getenv("S3_ACCESS_KEY_ID");
		secret_key = getenv("S3_SECRET_ACCESS_KEY");
		security_token = getenv("S3_SESSION_TOKEN");

		logger logr("s3");

		if (!access_key)
		{
			logr.Info("Environment variable S3_ACCESS_KEY_ID not defined");
		}

		if (!secret_key)
		{
			logr.Info("Environment variable S3_SECRET_ACCESS_KEY not defined");
		}

		try
		{
			const auto envproto = himan::util::GetEnv("S3_PROTOCOL");
			if (envproto == "https")
			{
				protocol = S3ProtocolHTTPS;
			}
			else if (envproto == "http")
			{
				protocol = S3ProtocolHTTP;
			}
			else
			{
				logr.Warning(fmt::format("Unrecognized value found from env variable S3_PROTOCOL: '{}'", envproto));
			}
		}
		catch (const std::invalid_argument& e)
		{
		}

		S3_CHECK(S3_initialize("s3", S3_INIT_ALL, NULL));
	});
}

std::string ReadAWSRegionFromHostname(const std::string& hostname)
{
	if (hostname.find("amazonaws.com") != std::string::npos)
	{
		// extract region name from host name, assuming aws hostname like
		// s3.us-east-1.amazonaws.com

		auto tokens = util::Split(hostname, ".");

		logger logr("s3");

		if (tokens.size() != 4)
		{
			logr.Fatal("Hostname does not follow pattern s3.<regionname>.amazonaws.com");
		}
		else
		{
			logr.Trace(fmt::format("s3 authentication hostname: {}", tokens[1]));
			return tokens[1];
		}
	}

	return "";
}

struct write_data
{
	himan::buffer buffer;
	size_t write_ptr;
};

static int putObjectDataCallback(int bufferSize, char* buffer, void* callbackData)
{
	write_data* data = static_cast<write_data*>(callbackData);
	int bytesWritten = 0;

	if (data->buffer.length)
	{
		bytesWritten =
		    static_cast<int>((static_cast<int>(data->buffer.length) > bufferSize) ? bufferSize : data->buffer.length);
		memcpy(buffer, data->buffer.data + data->write_ptr, bytesWritten);
		data->write_ptr += bytesWritten;
		data->buffer.length -= bytesWritten;
	}

	return bytesWritten;
}

}  // namespace

buffer s3::ReadFile(const file_information& fileInformation)
{
	Initialize();
	logger logr("s3");

	S3GetObjectHandler getObjectHandler = {responseHandler, &getObjectDataCallback};

	const auto bucketAndFileName = GetBucketAndFileName(fileInformation.file_location);
	const auto bucket = bucketAndFileName[0];
	const auto key = bucketAndFileName[1];

	buffer ret;

#ifdef S3_DEFAULT_REGION

	std::string region = ReadAWSRegionFromHostname(fileInformation.file_server);

	// clang-format off

        S3BucketContext bucketContext =
        {
                fileInformation.file_server.c_str(),
                bucket.c_str(),
                protocol,
                S3UriStylePath,
                access_key,
                secret_key,
                security_token,
                region.c_str()
        };
#else

	// clang-format off

	S3BucketContext bucketContext =
	{
		fileInformation.file_server.c_str(),
		bucket.c_str(),
		protocol,
		S3UriStylePath,
		access_key,
		secret_key,
		security_token
	};

	// clang-format on
#endif

	int count = 0;
	do
	{
		if (count > 0)
		{
			sleep(2 * count);
		}
		const unsigned long offset = fileInformation.offset.get();
		const unsigned long length = fileInformation.length.get();

#ifdef S3_DEFAULT_REGION
		S3_get_object(&bucketContext, key.c_str(), NULL, offset, length, NULL, 0, &getObjectHandler, &ret);
#else
		S3_get_object(&bucketContext, key.c_str(), NULL, offset, length, NULL, &getObjectHandler, &ret);
#endif
		count++;
	} while (S3_status_is_retryable(statusG) && count < 3);

	switch (statusG)
	{
		case S3StatusOK:
			break;
		default:
			HandleS3Error(logr, fileInformation.file_server, fmt::format("{}/{}", bucket, key), "Read");
			break;
	}

	if (ret.length == 0)
	{
		throw himan::kFileDataNotFound;
	}

	return ret;
}

void s3::WriteObject(const std::string& objectName, const buffer& buff)
{
	Initialize();

	const auto bucketAndFileName = GetBucketAndFileName(objectName);
	const auto bucket = bucketAndFileName[0];
	const auto key = bucketAndFileName[1];

	const char* host = getenv("S3_HOSTNAME");

	logger logr("s3");

	if (!host)
	{
		logr.Fatal("Environment variable S3_HOSTNAME not defined");
		himan::Abort();
	}

#ifdef S3_DEFAULT_REGION

	std::string region = ReadAWSRegionFromHostname(std::string(host));

	// clang-format off

        S3BucketContext bucketContext =
        {
                host,
                bucket.c_str(),
                protocol,
                S3UriStylePath,
                access_key,
                secret_key,
                security_token,
                region.c_str()
        };
#else

	// clang-format off

	S3BucketContext bucketContext =
	{
		host,
		bucket.c_str(),
		protocol,
		S3UriStylePath,
		access_key,
		secret_key,
		security_token
	};
#endif

	// clang-format on

	S3PutObjectHandler putObjectHandler = {responseHandler, &putObjectDataCallback};

	write_data data;
	data.buffer.data = buff.data;
	data.buffer.length = buff.length;
	data.write_ptr = 0;

	timer t(true);

	int count = 0;
	do
	{
		if (count > 0)
		{
			sleep(2 * count);
		}

#ifdef S3_DEFAULT_REGION
		S3_put_object(&bucketContext, key.c_str(), buff.length, NULL, NULL, 0, &putObjectHandler, &data);
#else
		S3_put_object(&bucketContext, key.c_str(), buff.length, NULL, NULL, &putObjectHandler, &data);
#endif

		count++;
	} while (S3_status_is_retryable(statusG) && count < 3);

	// remove pointer to original buff so that double free doesn't occur
	data.buffer.data = 0;

	switch (statusG)
	{
		case S3StatusOK:
		{
			t.Stop();
			const double time = static_cast<double>(t.GetTime());
			const double size = util::round(static_cast<double>(buff.length) / 1024. / 1024., 1);
			logr.Info(fmt::format("Wrote {} MB in {} ms ({:.1f} MBps) to file s3://{}/{}", size, time,
			                      size / (time * 0.001), bucket, key));
		}
		break;
		default:
			HandleS3Error(logr, host, fmt::format("{}/{}", bucket, key), "Write");
			break;
	}
}

#else
buffer s3::ReadFile(const file_information& fileInformation)
{
	throw std::runtime_error("S3 support not compiled");
}
void s3::WriteObject(const std::string& objectName, const buffer& buff)
{
	throw std::runtime_error("S3 support not compiled");
}
#endif
<|endoftext|>


<commit_before>/// \file
/// \ingroup tutorial_tdataframe
/// \notebook -draw
/// This tutorial shows how VecOps can be used to slim down the programming
/// model typically adopted in HEP for analysis.
/// In this case we have a dataset containing the kinematic properties of
/// particles stored in individual arrays.
/// We want to plot the transverse momentum of these particles if the energy is
/// greater than 100.
///
/// \macro_code
///
/// \date March 2018
/// \author Danilo Piparo, Andre Vieira Silva

R__LOAD_LIBRARY(libVecOps);

auto filename = gROOT->GetTutorialDir() + "/dataframe/tdf017_vecOpsHEP.root";
auto treename = "myDataset";
using doubles = ROOT::Experimental::VecOps::TVec<double>;
using TDF = ROOT::Experimental::TDataFrame;

void WithTTreeReader()
{
   TFile f(filename);
   TTreeReader tr(treename, &f);
   TTreeReaderArray<double> px(tr, "px");
   TTreeReaderArray<double> py(tr, "py");
   TTreeReaderArray<double> E(tr, "E");

   TH1F h("pt", "pt", 16, 0, 4);

   while (tr.Next()) {
      for (auto i=0U;i < px.GetSize(); ++i) {
         if (E[i] > 100) h.Fill(sqrt(px[i]*px[i] + py[i]*py[i]));
      }
   }
   h.DrawCopy();
}

void WithTDataFrame()
{
  TDF f(treename, filename.Data());
   auto CalcPt = [](doubles &px, doubles &py, doubles &E) {
      doubles v;
      for (auto i=0U;i < px.size(); ++i) {
         if (E[i] > 100) {
            v.emplace_back(sqrt(px[i]*px[i] + py[i]*py[i]));
         }
      }
      return v;
   };
   f.Define("pt", CalcPt, {"px", "py", "E"})
    .Histo1D<doubles>({"pt", "pt", 16, 0, 4}, "pt")->DrawCopy();
}

void WithTDataFrameVecOps()
{
   TDF f(treename, filename.Data());
   auto CalcPt = [](doubles &px, doubles &py, doubles &E) {
      auto pt = sqrt(px*px + py*py);
      return pt[E>100];
   };
   f.Define("good_pt", CalcPt, {"px", "py", "E"})
    .Histo1D<doubles>({"pt", "pt", 16, 0, 4}, "good_pt")->DrawCopy();
}

void WithTDataFrameVecOpsJit()
{
   TDF f(treename, filename.Data());
   f.Define("good_pt", "sqrt(px*px + py*py)[E>100]")
    .Histo1D({"pt", "pt", 16, 0, 4}, "good_pt")->DrawCopy();
}

void tdf017_vecOpsHEP()
{
   // We plot four times the same quantity, the key is to look into the implementation
   // of the functions above
   auto c = new TCanvas();
   c->Divide(2,2);
   c->cd(1);
   WithTTreeReader();
   c->cd(2);
   WithTDataFrame();
   c->cd(3);
   WithTDataFrameVecOps();
   c->cd(4);
   WithTDataFrameVecOpsJit();
}
<commit_msg>[VecOps] Remove workaround for loading manually the library<commit_after>/// \file
/// \ingroup tutorial_tdataframe
/// \notebook -draw
/// This tutorial shows how VecOps can be used to slim down the programming
/// model typically adopted in HEP for analysis.
/// In this case we have a dataset containing the kinematic properties of
/// particles stored in individual arrays.
/// We want to plot the transverse momentum of these particles if the energy is
/// greater than 100.
///
/// \macro_code
///
/// \date March 2018
/// \author Danilo Piparo, Andre Vieira Silva

auto filename = gROOT->GetTutorialDir() + "/dataframe/tdf017_vecOpsHEP.root";
auto treename = "myDataset";
using doubles = ROOT::Experimental::VecOps::TVec<double>;
using TDF = ROOT::Experimental::TDataFrame;

void WithTTreeReader()
{
   TFile f(filename);
   TTreeReader tr(treename, &f);
   TTreeReaderArray<double> px(tr, "px");
   TTreeReaderArray<double> py(tr, "py");
   TTreeReaderArray<double> E(tr, "E");

   TH1F h("pt", "pt", 16, 0, 4);

   while (tr.Next()) {
      for (auto i=0U;i < px.GetSize(); ++i) {
         if (E[i] > 100) h.Fill(sqrt(px[i]*px[i] + py[i]*py[i]));
      }
   }
   h.DrawCopy();
}

void WithTDataFrame()
{
  TDF f(treename, filename.Data());
   auto CalcPt = [](doubles &px, doubles &py, doubles &E) {
      doubles v;
      for (auto i=0U;i < px.size(); ++i) {
         if (E[i] > 100) {
            v.emplace_back(sqrt(px[i]*px[i] + py[i]*py[i]));
         }
      }
      return v;
   };
   f.Define("pt", CalcPt, {"px", "py", "E"})
    .Histo1D<doubles>({"pt", "pt", 16, 0, 4}, "pt")->DrawCopy();
}

void WithTDataFrameVecOps()
{
   TDF f(treename, filename.Data());
   auto CalcPt = [](doubles &px, doubles &py, doubles &E) {
      auto pt = sqrt(px*px + py*py);
      return pt[E>100];
   };
   f.Define("good_pt", CalcPt, {"px", "py", "E"})
    .Histo1D<doubles>({"pt", "pt", 16, 0, 4}, "good_pt")->DrawCopy();
}

void WithTDataFrameVecOpsJit()
{
   TDF f(treename, filename.Data());
   f.Define("good_pt", "sqrt(px*px + py*py)[E>100]")
    .Histo1D({"pt", "pt", 16, 0, 4}, "good_pt")->DrawCopy();
}

void tdf017_vecOpsHEP()
{
   // We plot four times the same quantity, the key is to look into the implementation
   // of the functions above
   auto c = new TCanvas();
   c->Divide(2,2);
   c->cd(1);
   WithTTreeReader();
   c->cd(2);
   WithTDataFrame();
   c->cd(3);
   WithTDataFrameVecOps();
   c->cd(4);
   WithTDataFrameVecOpsJit();
}
<|endoftext|>


<commit_before>void MdApi::OnDisconnected(int reason)
{
	Task task = Task();
	task.task_name = ONDISCONNECTED;
	task.task_extra = reason;
	this->task_queue.push(task);
};

void MdApi::OnError(XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONERROR;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnSubMarketData(XTPST *ticker, XTPRI *error_info, bool is_last)
{
	Task task = Task();
	task.task_name = ONSUBMARKETDATA;
	if (ticker)
	{
		XTPST *task_data = new XTPST();
		*task_data = *ticker;
		task.task_data = task_data;
	}
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	task.task_last = is_last;
	this->task_queue.push(task);
};

void MdApi::OnUnSubMarketData(XTPST *ticker, XTPRI *error_info, bool is_last)
{
	Task task = Task();
	task.task_name = ONUNSUBMARKETDATA;
	if (ticker)
	{
		XTPST *task_data = new XTPST();
		*task_data = *ticker;
		task.task_data = task_data;
	}
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	task.task_last = is_last;
	this->task_queue.push(task);
};

void MdApi::OnDepthMarketData(XTPMD *market_data, int bid1_qty int bid1_count, int max_bid1_count, int ask1_qty int ask1_count, int max_ask1_count)
{
	Task task = Task();
	task.task_name = ONDEPTHMARKETDATA;
	if (market_data)
	{
		XTPMD *task_data = new XTPMD();
		*task_data = *market_data;
		task.task_data = task_data;
	}
	task.task_extra = bid1_qty;
	task.task_extra = bid1_count;
	task.task_extra = max_bid1_count;
	task.task_extra = ask1_qty;
	task.task_extra = ask1_count;
	task.task_extra = max_ask1_count;
	this->task_queue.push(task);
};

void MdApi::OnSubOrderBook(XTPST *ticker, XTPRI *error_info, bool is_last)
{
	Task task = Task();
	task.task_name = ONSUBORDERBOOK;
	if (ticker)
	{
		XTPST *task_data = new XTPST();
		*task_data = *ticker;
		task.task_data = task_data;
	}
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	task.task_last = is_last;
	this->task_queue.push(task);
};

void MdApi::OnUnSubOrderBook(XTPST *ticker, XTPRI *error_info, bool is_last)
{
	Task task = Task();
	task.task_name = ONUNSUBORDERBOOK;
	if (ticker)
	{
		XTPST *task_data = new XTPST();
		*task_data = *ticker;
		task.task_data = task_data;
	}
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	task.task_last = is_last;
	this->task_queue.push(task);
};

void MdApi::OnOrderBook(XTPOB *order_book)
{
	Task task = Task();
	task.task_name = ONORDERBOOK;
	if (order_book)
	{
		XTPOB *task_data = new XTPOB();
		*task_data = *order_book;
		task.task_data = task_data;
	}
	this->task_queue.push(task);
};

void MdApi::OnSubTickByTick(XTPST *ticker, XTPRI *error_info, bool is_last)
{
	Task task = Task();
	task.task_name = ONSUBTICKBYTICK;
	if (ticker)
	{
		XTPST *task_data = new XTPST();
		*task_data = *ticker;
		task.task_data = task_data;
	}
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	task.task_last = is_last;
	this->task_queue.push(task);
};

void MdApi::OnUnSubTickByTick(XTPST *ticker, XTPRI *error_info, bool is_last)
{
	Task task = Task();
	task.task_name = ONUNSUBTICKBYTICK;
	if (ticker)
	{
		XTPST *task_data = new XTPST();
		*task_data = *ticker;
		task.task_data = task_data;
	}
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	task.task_last = is_last;
	this->task_queue.push(task);
};

void MdApi::OnTickByTick(XTPTBT *tbt_data)
{
	Task task = Task();
	task.task_name = ONTICKBYTICK;
	if (tbt_data)
	{
		XTPTBT *task_data = new XTPTBT();
		*task_data = *tbt_data;
		task.task_data = task_data;
	}
	this->task_queue.push(task);
};

void MdApi::OnSubscribeAllMarketData(int exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONSUBSCRIBEALLMARKETDATA;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnUnSubscribeAllMarketData(int exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONUNSUBSCRIBEALLMARKETDATA;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnSubscribeAllOrderBook(int exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONSUBSCRIBEALLORDERBOOK;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnUnSubscribeAllOrderBook(int exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONUNSUBSCRIBEALLORDERBOOK;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnSubscribeAllTickByTick(int exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONSUBSCRIBEALLTICKBYTICK;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnUnSubscribeAllTickByTick(int exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONUNSUBSCRIBEALLTICKBYTICK;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnQueryAllTickers(XTPQSI* ticker_info, XTPRI *error_info, bool is_last)
{
	Task task = Task();
	task.task_name = ONQUERYALLTICKERS;
	if (ticker_info)
	{
		XTPQSI *task_data = new XTPQSI();
		*task_data = *ticker_info;
		task.task_data = task_data;
	}
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	task.task_last = is_last;
	this->task_queue.push(task);
};

void MdApi::OnQueryTickersPriceInfo(XTPTPI* ticker_info, XTPRI *error_info, bool is_last)
{
	Task task = Task();
	task.task_name = ONQUERYTICKERSPRICEINFO;
	if (ticker_info)
	{
		XTPTPI *task_data = new XTPTPI();
		*task_data = *ticker_info;
		task.task_data = task_data;
	}
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	task.task_last = is_last;
	this->task_queue.push(task);
};

void MdApi::OnSubscribeAllOptionMarketData(int exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONSUBSCRIBEALLOPTIONMARKETDATA;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnUnSubscribeAllOptionMarketData(int exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONUNSUBSCRIBEALLOPTIONMARKETDATA;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnSubscribeAllOptionOrderBook(int exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONSUBSCRIBEALLOPTIONORDERBOOK;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnUnSubscribeAllOptionOrderBook(int exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONUNSUBSCRIBEALLOPTIONORDERBOOK;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnSubscribeAllOptionTickByTick(int exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONSUBSCRIBEALLOPTIONTICKBYTICK;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnUnSubscribeAllOptionTickByTick(int exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONUNSUBSCRIBEALLOPTIONTICKBYTICK;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

<commit_msg>Update xtp_md_source_task.cpp<commit_after>void MdApi::OnDisconnected(int reason)
{
	Task task = Task();
	task.task_name = ONDISCONNECTED;
	task.task_extra = reason;
	this->task_queue.push(task);
};

void MdApi::OnError(XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONERROR;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnSubMarketData(XTPST *ticker, XTPRI *error_info, bool is_last)
{
	Task task = Task();
	task.task_name = ONSUBMARKETDATA;
	if (ticker)
	{
		XTPST *task_data = new XTPST();
		*task_data = *ticker;
		task.task_data = task_data;
	}
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	task.task_last = is_last;
	this->task_queue.push(task);
};

void MdApi::OnUnSubMarketData(XTPST *ticker, XTPRI *error_info, bool is_last)
{
	Task task = Task();
	task.task_name = ONUNSUBMARKETDATA;
	if (ticker)
	{
		XTPST *task_data = new XTPST();
		*task_data = *ticker;
		task.task_data = task_data;
	}
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	task.task_last = is_last;
	this->task_queue.push(task);
};

void MdApi::OnDepthMarketData(XTPMD *market_data, int64_t bid1_qty[], int32_t bid1_count, int32_t max_bid1_count, int64_t ask1_qty[], int32_t ask1_count, int32_t max_ask1_count)
{
	Task task = Task();
	task.task_name = ONDEPTHMARKETDATA;
	if (market_data)
	{
		XTPMD *task_data = new XTPMD();
		*task_data = *market_data;
		task.task_data = task_data;
	}
	task.task_extra = bid1_qty;
	task.task_extra = bid1_count;
	task.task_extra = max_bid1_count;
	task.task_extra = ask1_qty;
	task.task_extra = ask1_count;
	task.task_extra = max_ask1_count;
	this->task_queue.push(task);
};

void MdApi::OnSubOrderBook(XTPST *ticker, XTPRI *error_info, bool is_last)
{
	Task task = Task();
	task.task_name = ONSUBORDERBOOK;
	if (ticker)
	{
		XTPST *task_data = new XTPST();
		*task_data = *ticker;
		task.task_data = task_data;
	}
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	task.task_last = is_last;
	this->task_queue.push(task);
};

void MdApi::OnUnSubOrderBook(XTPST *ticker, XTPRI *error_info, bool is_last)
{
	Task task = Task();
	task.task_name = ONUNSUBORDERBOOK;
	if (ticker)
	{
		XTPST *task_data = new XTPST();
		*task_data = *ticker;
		task.task_data = task_data;
	}
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	task.task_last = is_last;
	this->task_queue.push(task);
};

void MdApi::OnOrderBook(XTPOB *order_book)
{
	Task task = Task();
	task.task_name = ONORDERBOOK;
	if (order_book)
	{
		XTPOB *task_data = new XTPOB();
		*task_data = *order_book;
		task.task_data = task_data;
	}
	this->task_queue.push(task);
};

void MdApi::OnSubTickByTick(XTPST *ticker, XTPRI *error_info, bool is_last)
{
	Task task = Task();
	task.task_name = ONSUBTICKBYTICK;
	if (ticker)
	{
		XTPST *task_data = new XTPST();
		*task_data = *ticker;
		task.task_data = task_data;
	}
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	task.task_last = is_last;
	this->task_queue.push(task);
};

void MdApi::OnUnSubTickByTick(XTPST *ticker, XTPRI *error_info, bool is_last)
{
	Task task = Task();
	task.task_name = ONUNSUBTICKBYTICK;
	if (ticker)
	{
		XTPST *task_data = new XTPST();
		*task_data = *ticker;
		task.task_data = task_data;
	}
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	task.task_last = is_last;
	this->task_queue.push(task);
};

void MdApi::OnTickByTick(XTPTBT *tbt_data)
{
	Task task = Task();
	task.task_name = ONTICKBYTICK;
	if (tbt_data)
	{
		XTPTBT *task_data = new XTPTBT();
		*task_data = *tbt_data;
		task.task_data = task_data;
	}
	this->task_queue.push(task);
};

void MdApi::OnSubscribeAllMarketData(XTP_EXCHANGE_TYPE exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONSUBSCRIBEALLMARKETDATA;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnUnSubscribeAllMarketData(XTP_EXCHANGE_TYPE exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONUNSUBSCRIBEALLMARKETDATA;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnSubscribeAllOrderBook(XTP_EXCHANGE_TYPE exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONSUBSCRIBEALLORDERBOOK;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnUnSubscribeAllOrderBook(XTP_EXCHANGE_TYPE exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONUNSUBSCRIBEALLORDERBOOK;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnSubscribeAllTickByTick(XTP_EXCHANGE_TYPE exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONSUBSCRIBEALLTICKBYTICK;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnUnSubscribeAllTickByTick(XTP_EXCHANGE_TYPE exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONUNSUBSCRIBEALLTICKBYTICK;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnQueryAllTickers(XTPQSI* ticker_info, XTPRI *error_info, bool is_last)
{
	Task task = Task();
	task.task_name = ONQUERYALLTICKERS;
	if (ticker_info)
	{
		XTPQSI *task_data = new XTPQSI();
		*task_data = *ticker_info;
		task.task_data = task_data;
	}
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	task.task_last = is_last;
	this->task_queue.push(task);
};

void MdApi::OnQueryTickersPriceInfo(XTPTPI* ticker_info, XTPRI *error_info, bool is_last)
{
	Task task = Task();
	task.task_name = ONQUERYTICKERSPRICEINFO;
	if (ticker_info)
	{
		XTPTPI *task_data = new XTPTPI();
		*task_data = *ticker_info;
		task.task_data = task_data;
	}
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	task.task_last = is_last;
	this->task_queue.push(task);
};

void MdApi::OnSubscribeAllOptionMarketData(XTP_EXCHANGE_TYPE exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONSUBSCRIBEALLOPTIONMARKETDATA;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnUnSubscribeAllOptionMarketData(XTP_EXCHANGE_TYPE exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONUNSUBSCRIBEALLOPTIONMARKETDATA;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnSubscribeAllOptionOrderBook(XTP_EXCHANGE_TYPE exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONSUBSCRIBEALLOPTIONORDERBOOK;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnUnSubscribeAllOptionOrderBook(XTP_EXCHANGE_TYPE exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONUNSUBSCRIBEALLOPTIONORDERBOOK;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnSubscribeAllOptionTickByTick(XTP_EXCHANGE_TYPE exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONSUBSCRIBEALLOPTIONTICKBYTICK;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

void MdApi::OnUnSubscribeAllOptionTickByTick(XTP_EXCHANGE_TYPE exchange_id, XTPRI *error_info)
{
	Task task = Task();
	task.task_name = ONUNSUBSCRIBEALLOPTIONTICKBYTICK;
	task.task_extra = (int) exchange_id;
	if (error_info)
	{
		XTPRI *task_error = new XTPRI();
		*task_error = *error_info;
		task.task_error = task_error;
	}
	this->task_queue.push(task);
};

<|endoftext|>