"""Various utility functions to use throughout the project."""

from __future__ import print_function

import sys
import inspect

import tensorflow as tf


def print_bytes(str_as_bytes, stream=None):
  """Prints a string viewed as bytes.

  This function calls ``decode()`` depending on the output stream encoding.

  Args:
    str_as_bytes: The bytes to print.
    stream: The stream to print to (``sys.stdout`` if not set).
  """
  encoding = None
  if stream is not None:
    encoding = stream.encoding
  if encoding is None:
    encoding = sys.getdefaultencoding()
  text = str_as_bytes.decode(encoding) if encoding != "ascii" else str_as_bytes
  print(text, file=stream)
  if stream is not None:
    stream.flush()

def item_or_tuple(x):
  """Returns :obj:`x` as a tuple or its single element."""
  x = tuple(x)
  if len(x) == 1:
    return x[0]
  else:
    return x

def count_lines(filename):
  """Returns the number of lines of the file :obj:`filename`."""
  with open(filename) as f:
    i = 0
    for i, _ in enumerate(f):
      pass
    return i + 1

def get_classnames_in_module(module):
  """Returns a list of classnames exposed by a module."""
  names = []
  for symbol in dir(module):
    if inspect.isclass(getattr(module, symbol)):
      names.append(symbol)
  return names

def count_parameters():
  """Returns the total number of trainable parameters."""
  total = 0
  for variable in tf.trainable_variables():
    shape = variable.get_shape()
    count = 1
    for dim in shape:
      count *= dim.value
    total += count
  return total

def extract_prefixed_keys(dictionary, prefix):
  """Returns a dictionary with all keys from :obj:`dictionary` that are prefixed
  with :obj:`prefix`.
  """
  sub_dict = {}
  for key, value in dictionary.items():
    if key.startswith(prefix):
      original_key = key[len(prefix):]
      sub_dict[original_key] = value
  return sub_dict

def extract_batches(tensors):
  """Returns a generator to iterate on each batch of a Numpy array or dict of
  Numpy arrays."""
  if not isinstance(tensors, dict):
    for tensor in tensors:
      yield tensor
  else:
    batch_size = None
    for _, value in tensors.items():
      batch_size = batch_size or value.shape[0]
    for b in range(batch_size):
      yield {
          key: value[b] for key, value in tensors.items()
      }


# The next 2 functions come with the following license and copyright:

# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

def add_dict_to_collection(collection_name, dict_):
  """Adds a dictionary to a graph collection.

  Args:
    collection_name: The name of the collection to add the dictionary to
    dict_: A dictionary of string keys to tensor values
  """
  key_collection = collection_name + "_keys"
  value_collection = collection_name + "_values"
  for key, value in dict_.items():
    tf.add_to_collection(key_collection, key)
    tf.add_to_collection(value_collection, value)

def get_dict_from_collection(collection_name):
  """Gets a dictionary from a graph collection.

  Args:
    collection_name: A collection name to read a dictionary from

  Returns:
    A dictionary with string keys and tensor values
  """
  key_collection = collection_name + "_keys"
  value_collection = collection_name + "_values"
  keys = tf.get_collection(key_collection)
  values = tf.get_collection(value_collection)
  return dict(zip(keys, values))

opennmt/utils/misc.py
"""Various utility functions to use throughout the project."""

from __future__ import print_function

import sys
import inspect

import tensorflow as tf


def print_bytes(str_as_bytes, stream=None):
  """Prints a string viewed as bytes.

  This function calls ``decode()`` depending on the output stream encoding.

  Args:
    str_as_bytes: The bytes to print.
    stream: The stream to print to (``sys.stdout`` if not set).
  """
  encoding = None
  if stream is not None:
    encoding = stream.encoding
  if encoding is None:
    encoding = sys.getdefaultencoding()
  text = str_as_bytes.decode(encoding) if encoding != "ascii" else str_as_bytes
  print(text, file=stream)
  if stream is not None:
    stream.flush()

def item_or_tuple(x):
  """Returns :obj:`x` as a tuple or its single element."""
  x = tuple(x)
  if len(x) == 1:
    return x[0]
  else:
    return x

def count_lines(filename):
  """Returns the number of lines of the file :obj:`filename`."""
  with open(filename, "rb") as f:
    i = 0
    for i, _ in enumerate(f):
      pass
    return i + 1

def get_classnames_in_module(module):
  """Returns a list of classnames exposed by a module."""
  names = []
  for symbol in dir(module):
    if inspect.isclass(getattr(module, symbol)):
      names.append(symbol)
  return names

def count_parameters():
  """Returns the total number of trainable parameters."""
  total = 0
  for variable in tf.trainable_variables():
    shape = variable.get_shape()
    count = 1
    for dim in shape:
      count *= dim.value
    total += count
  return total

def extract_prefixed_keys(dictionary, prefix):
  """Returns a dictionary with all keys from :obj:`dictionary` that are prefixed
  with :obj:`prefix`.
  """
  sub_dict = {}
  for key, value in dictionary.items():
    if key.startswith(prefix):
      original_key = key[len(prefix):]
      sub_dict[original_key] = value
  return sub_dict

def extract_batches(tensors):
  """Returns a generator to iterate on each batch of a Numpy array or dict of
  Numpy arrays."""
  if not isinstance(tensors, dict):
    for tensor in tensors:
      yield tensor
  else:
    batch_size = None
    for _, value in tensors.items():
      batch_size = batch_size or value.shape[0]
    for b in range(batch_size):
      yield {
          key: value[b] for key, value in tensors.items()
      }


# The next 2 functions come with the following license and copyright:

# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

def add_dict_to_collection(collection_name, dict_):
  """Adds a dictionary to a graph collection.

  Args:
    collection_name: The name of the collection to add the dictionary to
    dict_: A dictionary of string keys to tensor values
  """
  key_collection = collection_name + "_keys"
  value_collection = collection_name + "_values"
  for key, value in dict_.items():
    tf.add_to_collection(key_collection, key)
    tf.add_to_collection(value_collection, value)

def get_dict_from_collection(collection_name):
  """Gets a dictionary from a graph collection.

  Args:
    collection_name: A collection name to read a dictionary from

  Returns:
    A dictionary with string keys and tensor values
  """
  key_collection = collection_name + "_keys"
  value_collection = collection_name + "_values"
  keys = tf.get_collection(key_collection)
  values = tf.get_collection(value_collection)
  return dict(zip(keys, values))

opennmt/utils/misc.py
diff --git a/opennmt/utils/misc.py b/opennmt/utils/misc.py
index 20a410b4f..66c2db197 100644
--- a/opennmt/utils/misc.py
+++ b/opennmt/utils/misc.py
@@ -37,7 +37,7 @@ def item_or_tuple(x):
 
 def count_lines(filename):
   """Returns the number of lines of the file :obj:`filename`."""
-  with open(filename) as f:
+  with open(filename, "rb") as f:
     i = 0
     for i, _ in enumerate(f):
       pass

File reading unicode error
When trying the quickstart example, I faced an error which is regarding file opening in
`utils\misc.py`
It got resolved once I changed
```python
line 40: with open(filename) as f:
to
line 40:  with open(filename, encoding="utf8") as f:
``` 
I'll open a pull request with the fix.
**Windows, py3.6, tf1.4**
`python -m bin.main train --model config/models/nmt_small.
py --config config/opennmt-defaults.yml config/data/toy-ende.yml`

```bash
INFO:tensorflow:Using config: {'_model_dir': 'toy-ende', '_tf_random_seed': None, '_save_sum
mary_steps': 50, '_save_checkpoints_steps': 5000, '_save_checkpoints_secs': None, '_session_
config': gpu_options {
}
, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps
': 50, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec
 object at 0x000002213F038F60>, '_task_type': 'worker', '_task_id': 0, '_master': '', '_is_c
hief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}
INFO:tensorflow:Running training and evaluation locally (non-distributed).
INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after 18000 secs (ev
al_spec.throttle_secs) or training is finished.
Traceback (most recent call last):
  File "C:\Users\Ayush\AppData\Local\Programs\Python\Python36\lib\runpy.py", line 193, in _r
un_module_as_main
    "__main__", mod_spec)
  File "C:\Users\Ayush\AppData\Local\Programs\Python\Python36\lib\runpy.py", line 85, in _ru
n_code
    exec(code, run_globals)
  File "C:\Users\Ayush\Projects\OpenNMT-tf\bin\main.py", line 308, in <module>
    main()
  File "C:\Users\Ayush\Projects\OpenNMT-tf\bin\main.py", line 290, in main
    train(estimator, model, config)
  File "C:\Users\Ayush\Projects\OpenNMT-tf\bin\main.py", line 135, in train
    tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
  File "C:\Users\Ayush\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\p
ython\estimator\training.py", line 430, in train_and_evaluate
    executor.run_local()
  File "C:\Users\Ayush\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\p
ython\estimator\training.py", line 609, in run_local
    hooks=train_hooks)
  File "C:\Users\Ayush\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\p
ython\estimator\estimator.py", line 302, in train
    loss = self._train_model(input_fn, hooks, saving_listeners)
  File "C:\Users\Ayush\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\p
ython\estimator\estimator.py", line 708, in _train_model
    input_fn, model_fn_lib.ModeKeys.TRAIN)
  File "C:\Users\Ayush\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\p
ython\estimator\estimator.py", line 577, in _get_features_and_labels_from_input_fn
    result = self._call_input_fn(input_fn, mode)
  File "C:\Users\Ayush\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\p
ython\estimator\estimator.py", line 663, in _call_input_fn
    return input_fn(**kwargs)
  File "C:\Users\Ayush\Projects\OpenNMT-tf\opennmt\models\model.py", line 515, in <lambda>
    maximum_labels_length=maximum_labels_length)
  File "C:\Users\Ayush\Projects\OpenNMT-tf\opennmt\models\model.py", line 374, in _input_fn_
impl
    self._initialize(metadata)
  File "C:\Users\Ayush\Projects\OpenNMT-tf\opennmt\models\sequence_to_sequence.py", line 93,
 in _initialize
    self.source_inputter.initialize(metadata)
  File "C:\Users\Ayush\Projects\OpenNMT-tf\opennmt\inputters\text_inputter.py", line 304, in
 initialize
    self.vocabulary_size = count_lines(self.vocabulary_file) + self.num_oov_buckets
  File "C:\Users\Ayush\Projects\OpenNMT-tf\opennmt\utils\misc.py", line 42, in count_lines
    for i, _ in enumerate(f):
  File "C:\Users\Ayush\AppData\Local\Programs\Python\Python36\lib\encodings\cp1252.py", line
 23, in decode
    return codecs.charmap_decode(input,self.errors,decoding_table)[0]
UnicodeDecodeError: 'charmap' codec can't decode byte 0x81 in position 5597: character maps
to <undefined>```
