# coding=utf-8
# Copyright 2017 The Tensor2Tensor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Data generators for the CNN and Daily Mail datasets."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import tarfile

# Dependency imports

import six
from tensor2tensor.data_generators import generator_utils
from tensor2tensor.data_generators import problem
from tensor2tensor.data_generators import text_encoder
from tensor2tensor.utils import registry

import tensorflow as tf


# Links to data from http://cs.nyu.edu/~kcho/DMQA/
_CNN_STORIES_DRIVE_URL = "https://drive.google.com/uc?export=download&id=0BwmD_VLjROrfTHk4NFg2SndKcjQ"

_DAILYMAIL_STORIES_DRIVE_URL = "https://drive.google.com/uc?export=download&id=0BwmD_VLjROrfM1BxdkxVaTY2bWs"


# End-of-sentence marker.
EOS = text_encoder.EOS_ID


def _maybe_download_corpora(tmp_dir):
  """Download corpora if necessary and unzip them.

  Args:
    tmp_dir: directory containing dataset.

  Returns:
    filepath of the downloaded corpus file.
  """
  cnn_filename = "cnn_stories.tgz"
  cnn_finalpath = os.path.join(tmp_dir, "cnn/stories/")
  dailymail_filename = "dailymail_stories.tgz"
  dailymail_finalpath = os.path.join(tmp_dir, "dailymail/stories/")
  if not tf.gfile.Exists(cnn_finalpath):
    cnn_file = generator_utils.maybe_download_from_drive(
        tmp_dir, cnn_filename, _CNN_STORIES_DRIVE_URL)
    with tarfile.open(cnn_file, "r:gz") as cnn_tar:
      cnn_tar.extractall(tmp_dir)
  if not tf.gfile.Exists(dailymail_finalpath):
    dailymail_file = generator_utils.maybe_download_from_drive(
        tmp_dir, dailymail_filename, _DAILYMAIL_STORIES_DRIVE_URL)
    with tarfile.open(dailymail_file, "r:gz") as dailymail_tar:
      dailymail_tar.extractall(tmp_dir)
  return [cnn_finalpath, dailymail_finalpath]


def story_generator(tmp_dir):
  paths = _maybe_download_corpora(tmp_dir)
  for path in paths:
    for story_file in tf.gfile.Glob(path + "*"):
      story = u""
      for line in tf.gfile.Open(story_file):
        line = unicode(line, "utf-8") if six.PY2 else line.decode("utf-8")
        story += line
      yield story


def _story_summary_split(story):
  end_pos = story.find("\n\n")  # Upto first empty line.
  assert end_pos != -1
  return story[:end_pos], story[end_pos:].strip()


@registry.register_problem
class SummarizeCnnDailymail32k(problem.Text2TextProblem):
  """Summarize CNN and Daily Mail articles to their first paragraph."""

  @property
  def is_character_level(self):
    return False

  @property
  def has_inputs(self):
    return True

  @property
  def input_space_id(self):
    return problem.SpaceID.EN_TOK

  @property
  def target_space_id(self):
    return problem.SpaceID.EN_TOK

  @property
  def num_shards(self):
    return 100

  @property
  def vocab_name(self):
    return "vocab.cnndailymail"

  @property
  def use_subword_tokenizer(self):
    return True

  @property
  def targeted_vocab_size(self):
    return 2**15  # 32768

  @property
  def use_train_shards_for_dev(self):
    return True

  def generator(self, data_dir, tmp_dir, _):
    encoder = generator_utils.get_or_generate_vocab_inner(
        data_dir, self.vocab_file, self.targeted_vocab_size,
        story_generator(tmp_dir))
    for story in story_generator(tmp_dir):
      summary, rest = _story_summary_split(story)
      encoded_summary = encoder.encode(summary) + [EOS]
      encoded_story = encoder.encode(rest) + [EOS]
      yield {"inputs": encoded_story, "targets": encoded_summary}

tensor2tensor/data_generators/cnn_dailymail.py
# coding=utf-8
# Copyright 2017 The Tensor2Tensor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Data generators for the CNN and Daily Mail datasets."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import tarfile

# Dependency imports

import six
from tensor2tensor.data_generators import generator_utils
from tensor2tensor.data_generators import problem
from tensor2tensor.data_generators import text_encoder
from tensor2tensor.utils import registry

import tensorflow as tf


# Links to data from http://cs.nyu.edu/~kcho/DMQA/
_CNN_STORIES_DRIVE_URL = "https://drive.google.com/uc?export=download&id=0BwmD_VLjROrfTHk4NFg2SndKcjQ"

_DAILYMAIL_STORIES_DRIVE_URL = "https://drive.google.com/uc?export=download&id=0BwmD_VLjROrfM1BxdkxVaTY2bWs"


# End-of-sentence marker.
EOS = text_encoder.EOS_ID


def _maybe_download_corpora(tmp_dir):
  """Download corpora if necessary and unzip them.

  Args:
    tmp_dir: directory containing dataset.

  Returns:
    filepath of the downloaded corpus file.
  """
  cnn_filename = "cnn_stories.tgz"
  cnn_finalpath = os.path.join(tmp_dir, "cnn/stories/")
  dailymail_filename = "dailymail_stories.tgz"
  dailymail_finalpath = os.path.join(tmp_dir, "dailymail/stories/")
  if not tf.gfile.Exists(cnn_finalpath):
    cnn_file = generator_utils.maybe_download_from_drive(
        tmp_dir, cnn_filename, _CNN_STORIES_DRIVE_URL)
    with tarfile.open(cnn_file, "r:gz") as cnn_tar:
      cnn_tar.extractall(tmp_dir)
  if not tf.gfile.Exists(dailymail_finalpath):
    dailymail_file = generator_utils.maybe_download_from_drive(
        tmp_dir, dailymail_filename, _DAILYMAIL_STORIES_DRIVE_URL)
    with tarfile.open(dailymail_file, "r:gz") as dailymail_tar:
      dailymail_tar.extractall(tmp_dir)
  return [cnn_finalpath, dailymail_finalpath]


def story_generator(tmp_dir):
  paths = _maybe_download_corpora(tmp_dir)
  for path in paths:
    for story_file in tf.gfile.Glob(path + "*"):
      story = u""
      for line in tf.gfile.Open(story_file, 'rb'):
        line = unicode(line, "utf-8") if six.PY2 else line.decode("utf-8")
        story += line
      yield story


def _story_summary_split(story):
  end_pos = story.find("\n\n")  # Upto first empty line.
  assert end_pos != -1
  return story[:end_pos], story[end_pos:].strip()


@registry.register_problem
class SummarizeCnnDailymail32k(problem.Text2TextProblem):
  """Summarize CNN and Daily Mail articles to their first paragraph."""

  @property
  def is_character_level(self):
    return False

  @property
  def has_inputs(self):
    return True

  @property
  def input_space_id(self):
    return problem.SpaceID.EN_TOK

  @property
  def target_space_id(self):
    return problem.SpaceID.EN_TOK

  @property
  def num_shards(self):
    return 100

  @property
  def vocab_name(self):
    return "vocab.cnndailymail"

  @property
  def use_subword_tokenizer(self):
    return True

  @property
  def targeted_vocab_size(self):
    return 2**15  # 32768

  @property
  def use_train_shards_for_dev(self):
    return True

  def generator(self, data_dir, tmp_dir, _):
    encoder = generator_utils.get_or_generate_vocab_inner(
        data_dir, self.vocab_file, self.targeted_vocab_size,
        story_generator(tmp_dir))
    for story in story_generator(tmp_dir):
      summary, rest = _story_summary_split(story)
      encoded_summary = encoder.encode(summary) + [EOS]
      encoded_story = encoder.encode(rest) + [EOS]
      yield {"inputs": encoded_story, "targets": encoded_summary}

tensor2tensor/data_generators/cnn_dailymail.py
diff --git a/tensor2tensor/data_generators/cnn_dailymail.py b/tensor2tensor/data_generators/cnn_dailymail.py
index 2f8e9cf30..8fa1e52d0 100644
--- a/tensor2tensor/data_generators/cnn_dailymail.py
+++ b/tensor2tensor/data_generators/cnn_dailymail.py
@@ -74,7 +74,7 @@ def story_generator(tmp_dir):
   for path in paths:
     for story_file in tf.gfile.Glob(path + "*"):
       story = u""
-      for line in tf.gfile.Open(story_file):
+      for line in tf.gfile.Open(story_file, 'rb'):
         line = unicode(line, "utf-8") if six.PY2 else line.decode("utf-8")
         story += line
       yield story

Error with `.decode` on `str` object when generating  `summarize_cnn_dailymail32k` data
## CMD
```
t2t-trainer \
  --generate_data \
  --data_dir="$data" \
  --problems=summarize_cnn_dailymail32k \
  --model=transformer \
  --hparams_set=transformer_base_single_gpu \
  --output_dir="$root"
```

## OUTPUT
```
INFO:tensorflow:Generating data for summarize_cnn_dailymail32k
INFO:tensorflow:Generating vocab file: t2t_data/vocab.cnndailymail.32768
INFO:tensorflow:Downloading https://drive.google.com/uc?export=download&id=0BwmD_VLjROrfTHk4NFg2SndKcjQ&confirm=ak1U to /tmp/t2t_datagen/cnn_stories.tgz

INFO:tensorflow:Succesfully downloaded cnn_stories.tgz, 158577824 bytes.
INFO:tensorflow:Downloading https://drive.google.com/uc?export=download&id=0BwmD_VLjROrfM1BxdkxVaTY2bWs&confirm=1N53 to /tmp/t2t_datagen/dailymail_stories.tgz

INFO:tensorflow:Succesfully downloaded dailymail_stories.tgz, 375893739 bytes.
Traceback (most recent call last):
  File "/home/pltrdy/anaconda3/bin/t2t-trainer", line 96, in <module>
    tf.app.run()
  File "/home/pltrdy/anaconda3/lib/python3.6/site-packages/tensorflow/python/platform/app.py", line 48, in run
    _sys.exit(main(_sys.argv[:1] + flags_passthrough))
  File "/home/pltrdy/anaconda3/bin/t2t-trainer", line 83, in main
    problem.generate_data(data_dir, tmp_dir)
  File "/home/pltrdy/anaconda3/lib/python3.6/site-packages/tensor2tensor/data_generators/problem.py", line 625, in generate_data
    self.generator(data_dir, tmp_dir, True), all_paths)
  File "/home/pltrdy/anaconda3/lib/python3.6/site-packages/tensor2tensor/data_generators/generator_utils.py", line 143, in generate_files
    for case in generator:
  File "/home/pltrdy/anaconda3/lib/python3.6/site-packages/tensor2tensor/data_generators/cnn_dailymail.py", line 132, in generator
    story_generator(tmp_dir))
  File "/home/pltrdy/anaconda3/lib/python3.6/site-packages/tensor2tensor/data_generators/generator_utils.py", line 328, in get_or_generate_vocab_inner
    for item in generator:
  File "/home/pltrdy/anaconda3/lib/python3.6/site-packages/tensor2tensor/data_generators/cnn_dailymail.py", line 78, in story_generator
    line = unicode(line, "utf-8") if six.PY2 else line.decode("utf-8")
AttributeError: 'str' object has no attribute 'decode'
```

## CONFIG
* Python version: `Python 3.6.1 |Anaconda custom (64-bit)| (default, May 11 2017, 13:09:58)`
* Tensor2Tensor version:
```
$ pip show tensor2tensor
Name: tensor2tensor
Version: 1.2.4
Summary: Tensor2Tensor
Home-page: http://github.com/tensorflow/tensor2tensor
Author: Google Inc.
Author-email: no-reply@google.com
License: Apache 2.0
Location: /home/pltrdy/anaconda3/lib/python3.6/site-packages
Requires: sympy, numpy, requests, six, future, bz2file
```

## QUICKFIX
```
--- a/tensor2tensor/data_generators/cnn_dailymail.py
+++ b/tensor2tensor/data_generators/cnn_dailymail.py
@@ -74,7 +74,7 @@ def story_generator(tmp_dir):
   for path in paths:
     for story_file in tf.gfile.Glob(path + "*"):
       story = u""
-      for line in tf.gfile.Open(story_file):
+      for line in tf.gfile.Open(story_file, 'rb'):

```
