We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Rename self.local_data to self._local_data in class DataGenerator.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `deep_speech_2/data_utils/data.py`
Content:
```
1 """Contains data generator for orgnaizing various audio data preprocessing
2 pipeline and offering data reader interface of PaddlePaddle requirements.
3 """
4 from __future__ import absolute_import
5 from __future__ import division
6 from __future__ import print_function
7 
8 import random
9 import tarfile
10 import multiprocessing
11 import numpy as np
12 import paddle.v2 as paddle
13 from threading import local
14 from data_utils import utils
15 from data_utils.augmentor.augmentation import AugmentationPipeline
16 from data_utils.featurizer.speech_featurizer import SpeechFeaturizer
17 from data_utils.speech import SpeechSegment
18 from data_utils.normalizer import FeatureNormalizer
19 
20 
21 class DataGenerator(object):
22     """
23     DataGenerator provides basic audio data preprocessing pipeline, and offers
24     data reader interfaces of PaddlePaddle requirements.
25 
26     :param vocab_filepath: Vocabulary filepath for indexing tokenized
27                            transcripts.
28     :type vocab_filepath: basestring
29     :param mean_std_filepath: File containing the pre-computed mean and stddev.
30     :type mean_std_filepath: None|basestring
31     :param augmentation_config: Augmentation configuration in json string.
32                                 Details see AugmentationPipeline.__doc__.
33     :type augmentation_config: str
34     :param max_duration: Audio with duration (in seconds) greater than
35                          this will be discarded.
36     :type max_duration: float
37     :param min_duration: Audio with duration (in seconds) smaller than
38                          this will be discarded.
39     :type min_duration: float
40     :param stride_ms: Striding size (in milliseconds) for generating frames.
41     :type stride_ms: float
42     :param window_ms: Window size (in milliseconds) for generating frames.
43     :type window_ms: float
44     :param max_freq: Used when specgram_type is 'linear', only FFT bins
45                      corresponding to frequencies between [0, max_freq] are
46                      returned.
47     :types max_freq: None|float
48     :param specgram_type: Specgram feature type. Options: 'linear'.
49     :type specgram_type: str
50     :param use_dB_normalization: Whether to normalize the audio to -20 dB
51                                 before extracting the features.
52     :type use_dB_normalization: bool
53     :param num_threads: Number of CPU threads for processing data.
54     :type num_threads: int
55     :param random_seed: Random seed.
56     :type random_seed: int
57     """
58 
59     def __init__(self,
60                  vocab_filepath,
61                  mean_std_filepath,
62                  augmentation_config='{}',
63                  max_duration=float('inf'),
64                  min_duration=0.0,
65                  stride_ms=10.0,
66                  window_ms=20.0,
67                  max_freq=None,
68                  specgram_type='linear',
69                  use_dB_normalization=True,
70                  num_threads=multiprocessing.cpu_count() // 2,
71                  random_seed=0):
72         self._max_duration = max_duration
73         self._min_duration = min_duration
74         self._normalizer = FeatureNormalizer(mean_std_filepath)
75         self._augmentation_pipeline = AugmentationPipeline(
76             augmentation_config=augmentation_config, random_seed=random_seed)
77         self._speech_featurizer = SpeechFeaturizer(
78             vocab_filepath=vocab_filepath,
79             specgram_type=specgram_type,
80             stride_ms=stride_ms,
81             window_ms=window_ms,
82             max_freq=max_freq,
83             use_dB_normalization=use_dB_normalization)
84         self._num_threads = num_threads
85         self._rng = random.Random(random_seed)
86         self._epoch = 0
87         # for caching tar files info
88         self.local_data = local()
89         self.local_data.tar2info = {}
90         self.local_data.tar2object = {}
91 
92     def process_utterance(self, filename, transcript):
93         """Load, augment, featurize and normalize for speech data.
94 
95         :param filename: Audio filepath
96         :type filename: basestring | file
97         :param transcript: Transcription text.
98         :type transcript: basestring
99         :return: Tuple of audio feature tensor and list of token ids for
100                  transcription.
101         :rtype: tuple of (2darray, list)
102         """
103         speech_segment = SpeechSegment.from_file(filename, transcript)
104         self._augmentation_pipeline.transform_audio(speech_segment)
105         specgram, text_ids = self._speech_featurizer.featurize(speech_segment)
106         specgram = self._normalizer.apply(specgram)
107         return specgram, text_ids
108 
109     def batch_reader_creator(self,
110                              manifest_path,
111                              batch_size,
112                              min_batch_size=1,
113                              padding_to=-1,
114                              flatten=False,
115                              sortagrad=False,
116                              shuffle_method="batch_shuffle"):
117         """
118         Batch data reader creator for audio data. Return a callable generator
119         function to produce batches of data.
120 
121         Audio features within one batch will be padded with zeros to have the
122         same shape, or a user-defined shape.
123 
124         :param manifest_path: Filepath of manifest for audio files.
125         :type manifest_path: basestring
126         :param batch_size: Number of instances in a batch.
127         :type batch_size: int
128         :param min_batch_size: Any batch with batch size smaller than this will
129                                be discarded. (To be deprecated in the future.)
130         :type min_batch_size: int
131         :param padding_to:  If set -1, the maximun shape in the batch
132                             will be used as the target shape for padding.
133                             Otherwise, `padding_to` will be the target shape.
134         :type padding_to: int
135         :param flatten: If set True, audio features will be flatten to 1darray.
136         :type flatten: bool
137         :param sortagrad: If set True, sort the instances by audio duration
138                           in the first epoch for speed up training.
139         :type sortagrad: bool
140         :param shuffle_method: Shuffle method. Options:
141                                 '' or None: no shuffle.
142                                 'instance_shuffle': instance-wise shuffle.
143                                 'batch_shuffle': similarly-sized instances are
144                                                  put into batches, and then
145                                                  batch-wise shuffle the batches.
146                                                  For more details, please see
147                                                  ``_batch_shuffle.__doc__``.
148                                 'batch_shuffle_clipped': 'batch_shuffle' with
149                                                          head shift and tail
150                                                          clipping. For more
151                                                          details, please see
152                                                          ``_batch_shuffle``.
153                               If sortagrad is True, shuffle is disabled
154                               for the first epoch.
155         :type shuffle_method: None|str
156         :return: Batch reader function, producing batches of data when called.
157         :rtype: callable
158         """
159 
160         def batch_reader():
161             # read manifest
162             manifest = utils.read_manifest(
163                 manifest_path=manifest_path,
164                 max_duration=self._max_duration,
165                 min_duration=self._min_duration)
166             # sort (by duration) or batch-wise shuffle the manifest
167             if self._epoch == 0 and sortagrad:
168                 manifest.sort(key=lambda x: x["duration"])
169             else:
170                 if shuffle_method == "batch_shuffle":
171                     manifest = self._batch_shuffle(
172                         manifest, batch_size, clipped=False)
173                 elif shuffle_method == "batch_shuffle_clipped":
174                     manifest = self._batch_shuffle(
175                         manifest, batch_size, clipped=True)
176                 elif shuffle_method == "instance_shuffle":
177                     self._rng.shuffle(manifest)
178                 elif shuffle_method == None:
179                     pass
180                 else:
181                     raise ValueError("Unknown shuffle method %s." %
182                                      shuffle_method)
183             # prepare batches
184             instance_reader = self._instance_reader_creator(manifest)
185             batch = []
186             for instance in instance_reader():
187                 batch.append(instance)
188                 if len(batch) == batch_size:
189                     yield self._padding_batch(batch, padding_to, flatten)
190                     batch = []
191             if len(batch) >= min_batch_size:
192                 yield self._padding_batch(batch, padding_to, flatten)
193             self._epoch += 1
194 
195         return batch_reader
196 
197     @property
198     def feeding(self):
199         """Returns data reader's feeding dict.
200 
201         :return: Data feeding dict.
202         :rtype: dict
203         """
204         return {"audio_spectrogram": 0, "transcript_text": 1}
205 
206     @property
207     def vocab_size(self):
208         """Return the vocabulary size.
209 
210         :return: Vocabulary size.
211         :rtype: int
212         """
213         return self._speech_featurizer.vocab_size
214 
215     @property
216     def vocab_list(self):
217         """Return the vocabulary in list.
218 
219         :return: Vocabulary in list.
220         :rtype: list
221         """
222         return self._speech_featurizer.vocab_list
223 
224     def _parse_tar(self, file):
225         """Parse a tar file to get a tarfile object
226         and a map containing tarinfoes
227         """
228         result = {}
229         f = tarfile.open(file)
230         for tarinfo in f.getmembers():
231             result[tarinfo.name] = tarinfo
232         return f, result
233 
234     def _get_file_object(self, file):
235         """Get file object by file path.
236 
237         If file startwith tar, it will return a tar file object
238         and cached tar file info for next reading request.
239         It will return file directly, if the type of file is not str.
240         """
241         if file.startswith('tar:'):
242             tarpath, filename = file.split(':', 1)[1].split('#', 1)
243             if 'tar2info' not in self.local_data.__dict__:
244                 self.local_data.tar2info = {}
245             if 'tar2object' not in self.local_data.__dict__:
246                 self.local_data.tar2object = {}
247             if tarpath not in self.local_data.tar2info:
248                 object, infoes = self._parse_tar(tarpath)
249                 self.local_data.tar2info[tarpath] = infoes
250                 self.local_data.tar2object[tarpath] = object
251             return self.local_data.tar2object[tarpath].extractfile(
252                 self.local_data.tar2info[tarpath][filename])
253         else:
254             return open(file, 'r')
255 
256     def _instance_reader_creator(self, manifest):
257         """
258         Instance reader creator. Create a callable function to produce
259         instances of data.
260 
261         Instance: a tuple of ndarray of audio spectrogram and a list of
262         token indices for transcript.
263         """
264 
265         def reader():
266             for instance in manifest:
267                 yield instance
268 
269         def mapper(instance):
270             return self.process_utterance(
271                 self._get_file_object(instance["audio_filepath"]),
272                 instance["text"])
273 
274         return paddle.reader.xmap_readers(
275             mapper, reader, self._num_threads, 1024, order=True)
276 
277     def _padding_batch(self, batch, padding_to=-1, flatten=False):
278         """
279         Padding audio features with zeros to make them have the same shape (or
280         a user-defined shape) within one bach.
281 
282         If ``padding_to`` is -1, the maximun shape in the batch will be used
283         as the target shape for padding. Otherwise, `padding_to` will be the
284         target shape (only refers to the second axis).
285 
286         If `flatten` is True, features will be flatten to 1darray.
287         """
288         new_batch = []
289         # get target shape
290         max_length = max([audio.shape[1] for audio, text in batch])
291         if padding_to != -1:
292             if padding_to < max_length:
293                 raise ValueError("If padding_to is not -1, it should be larger "
294                                  "than any instance's shape in the batch")
295             max_length = padding_to
296         # padding
297         for audio, text in batch:
298             padded_audio = np.zeros([audio.shape[0], max_length])
299             padded_audio[:, :audio.shape[1]] = audio
300             if flatten:
301                 padded_audio = padded_audio.flatten()
302             new_batch.append((padded_audio, text))
303         return new_batch
304 
305     def _batch_shuffle(self, manifest, batch_size, clipped=False):
306         """Put similarly-sized instances into minibatches for better efficiency
307         and make a batch-wise shuffle.
308 
309         1. Sort the audio clips by duration.
310         2. Generate a random number `k`, k in [0, batch_size).
311         3. Randomly shift `k` instances in order to create different batches
312            for different epochs. Create minibatches.
313         4. Shuffle the minibatches.
314 
315         :param manifest: Manifest contents. List of dict.
316         :type manifest: list
317         :param batch_size: Batch size. This size is also used for generate
318                            a random number for batch shuffle.
319         :type batch_size: int
320         :param clipped: Whether to clip the heading (small shift) and trailing
321                         (incomplete batch) instances.
322         :type clipped: bool
323         :return: Batch shuffled mainifest.
324         :rtype: list
325         """
326         manifest.sort(key=lambda x: x["duration"])
327         shift_len = self._rng.randint(0, batch_size - 1)
328         batch_manifest = zip(*[iter(manifest[shift_len:])] * batch_size)
329         self._rng.shuffle(batch_manifest)
330         batch_manifest = list(sum(batch_manifest, ()))
331         if not clipped:
332             res_len = len(manifest) - shift_len - len(batch_manifest)
333             batch_manifest.extend(manifest[-res_len:])
334             batch_manifest.extend(manifest[0:shift_len])
335         return batch_manifest
336
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/deep_speech_2/data_utils/data.py b/deep_speech_2/data_utils/data.py
--- a/deep_speech_2/data_utils/data.py
+++ b/deep_speech_2/data_utils/data.py
@@ -85,9 +85,9 @@
         self._rng = random.Random(random_seed)
         self._epoch = 0
         # for caching tar files info
-        self.local_data = local()
-        self.local_data.tar2info = {}
-        self.local_data.tar2object = {}
+        self._local_data = local()
+        self._local_data.tar2info = {}
+        self._local_data.tar2object = {}
 
     def process_utterance(self, filename, transcript):
         """Load, augment, featurize and normalize for speech data.
@@ -240,16 +240,16 @@
         """
         if file.startswith('tar:'):
             tarpath, filename = file.split(':', 1)[1].split('#', 1)
-            if 'tar2info' not in self.local_data.__dict__:
-                self.local_data.tar2info = {}
-            if 'tar2object' not in self.local_data.__dict__:
-                self.local_data.tar2object = {}
-            if tarpath not in self.local_data.tar2info:
+            if 'tar2info' not in self._local_data.__dict__:
+                self._local_data.tar2info = {}
+            if 'tar2object' not in self._local_data.__dict__:
+                self._local_data.tar2object = {}
+            if tarpath not in self._local_data.tar2info:
                 object, infoes = self._parse_tar(tarpath)
-                self.local_data.tar2info[tarpath] = infoes
-                self.local_data.tar2object[tarpath] = object
-            return self.local_data.tar2object[tarpath].extractfile(
-                self.local_data.tar2info[tarpath][filename])
+                self._local_data.tar2info[tarpath] = infoes
+                self._local_data.tar2object[tarpath] = object
+            return self._local_data.tar2object[tarpath].extractfile(
+                self._local_data.tar2info[tarpath][filename])
         else:
             return open(file, 'r')

{"golden_diff": "diff --git a/deep_speech_2/data_utils/data.py b/deep_speech_2/data_utils/data.py\n--- a/deep_speech_2/data_utils/data.py\n+++ b/deep_speech_2/data_utils/data.py\n@@ -85,9 +85,9 @@\n         self._rng = random.Random(random_seed)\n         self._epoch = 0\n         # for caching tar files info\n-        self.local_data = local()\n-        self.local_data.tar2info = {}\n-        self.local_data.tar2object = {}\n+        self._local_data = local()\n+        self._local_data.tar2info = {}\n+        self._local_data.tar2object = {}\n \n     def process_utterance(self, filename, transcript):\n         \"\"\"Load, augment, featurize and normalize for speech data.\n@@ -240,16 +240,16 @@\n         \"\"\"\n         if file.startswith('tar:'):\n             tarpath, filename = file.split(':', 1)[1].split('#', 1)\n-            if 'tar2info' not in self.local_data.__dict__:\n-                self.local_data.tar2info = {}\n-            if 'tar2object' not in self.local_data.__dict__:\n-                self.local_data.tar2object = {}\n-            if tarpath not in self.local_data.tar2info:\n+            if 'tar2info' not in self._local_data.__dict__:\n+                self._local_data.tar2info = {}\n+            if 'tar2object' not in self._local_data.__dict__:\n+                self._local_data.tar2object = {}\n+            if tarpath not in self._local_data.tar2info:\n                 object, infoes = self._parse_tar(tarpath)\n-                self.local_data.tar2info[tarpath] = infoes\n-                self.local_data.tar2object[tarpath] = object\n-            return self.local_data.tar2object[tarpath].extractfile(\n-                self.local_data.tar2info[tarpath][filename])\n+                self._local_data.tar2info[tarpath] = infoes\n+                self._local_data.tar2object[tarpath] = object\n+            return self._local_data.tar2object[tarpath].extractfile(\n+                self._local_data.tar2info[tarpath][filename])\n         else:\n             return open(file, 'r')\n", "issue": "Rename self.local_data to self._local_data in class DataGenerator.\n\n", "before_files": [{"content": "\"\"\"Contains data generator for orgnaizing various audio data preprocessing\npipeline and offering data reader interface of PaddlePaddle requirements.\n\"\"\"\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport random\nimport tarfile\nimport multiprocessing\nimport numpy as np\nimport paddle.v2 as paddle\nfrom threading import local\nfrom data_utils import utils\nfrom data_utils.augmentor.augmentation import AugmentationPipeline\nfrom data_utils.featurizer.speech_featurizer import SpeechFeaturizer\nfrom data_utils.speech import SpeechSegment\nfrom data_utils.normalizer import FeatureNormalizer\n\n\nclass DataGenerator(object):\n    \"\"\"\n    DataGenerator provides basic audio data preprocessing pipeline, and offers\n    data reader interfaces of PaddlePaddle requirements.\n\n    :param vocab_filepath: Vocabulary filepath for indexing tokenized\n                           transcripts.\n    :type vocab_filepath: basestring\n    :param mean_std_filepath: File containing the pre-computed mean and stddev.\n    :type mean_std_filepath: None|basestring\n    :param augmentation_config: Augmentation configuration in json string.\n                                Details see AugmentationPipeline.__doc__.\n    :type augmentation_config: str\n    :param max_duration: Audio with duration (in seconds) greater than\n                         this will be discarded.\n    :type max_duration: float\n    :param min_duration: Audio with duration (in seconds) smaller than\n                         this will be discarded.\n    :type min_duration: float\n    :param stride_ms: Striding size (in milliseconds) for generating frames.\n    :type stride_ms: float\n    :param window_ms: Window size (in milliseconds) for generating frames.\n    :type window_ms: float\n    :param max_freq: Used when specgram_type is 'linear', only FFT bins\n                     corresponding to frequencies between [0, max_freq] are\n                     returned.\n    :types max_freq: None|float\n    :param specgram_type: Specgram feature type. Options: 'linear'.\n    :type specgram_type: str\n    :param use_dB_normalization: Whether to normalize the audio to -20 dB\n                                before extracting the features.\n    :type use_dB_normalization: bool\n    :param num_threads: Number of CPU threads for processing data.\n    :type num_threads: int\n    :param random_seed: Random seed.\n    :type random_seed: int\n    \"\"\"\n\n    def __init__(self,\n                 vocab_filepath,\n                 mean_std_filepath,\n                 augmentation_config='{}',\n                 max_duration=float('inf'),\n                 min_duration=0.0,\n                 stride_ms=10.0,\n                 window_ms=20.0,\n                 max_freq=None,\n                 specgram_type='linear',\n                 use_dB_normalization=True,\n                 num_threads=multiprocessing.cpu_count() // 2,\n                 random_seed=0):\n        self._max_duration = max_duration\n        self._min_duration = min_duration\n        self._normalizer = FeatureNormalizer(mean_std_filepath)\n        self._augmentation_pipeline = AugmentationPipeline(\n            augmentation_config=augmentation_config, random_seed=random_seed)\n        self._speech_featurizer = SpeechFeaturizer(\n            vocab_filepath=vocab_filepath,\n            specgram_type=specgram_type,\n            stride_ms=stride_ms,\n            window_ms=window_ms,\n            max_freq=max_freq,\n            use_dB_normalization=use_dB_normalization)\n        self._num_threads = num_threads\n        self._rng = random.Random(random_seed)\n        self._epoch = 0\n        # for caching tar files info\n        self.local_data = local()\n        self.local_data.tar2info = {}\n        self.local_data.tar2object = {}\n\n    def process_utterance(self, filename, transcript):\n        \"\"\"Load, augment, featurize and normalize for speech data.\n\n        :param filename: Audio filepath\n        :type filename: basestring | file\n        :param transcript: Transcription text.\n        :type transcript: basestring\n        :return: Tuple of audio feature tensor and list of token ids for\n                 transcription.\n        :rtype: tuple of (2darray, list)\n        \"\"\"\n        speech_segment = SpeechSegment.from_file(filename, transcript)\n        self._augmentation_pipeline.transform_audio(speech_segment)\n        specgram, text_ids = self._speech_featurizer.featurize(speech_segment)\n        specgram = self._normalizer.apply(specgram)\n        return specgram, text_ids\n\n    def batch_reader_creator(self,\n                             manifest_path,\n                             batch_size,\n                             min_batch_size=1,\n                             padding_to=-1,\n                             flatten=False,\n                             sortagrad=False,\n                             shuffle_method=\"batch_shuffle\"):\n        \"\"\"\n        Batch data reader creator for audio data. Return a callable generator\n        function to produce batches of data.\n\n        Audio features within one batch will be padded with zeros to have the\n        same shape, or a user-defined shape.\n\n        :param manifest_path: Filepath of manifest for audio files.\n        :type manifest_path: basestring\n        :param batch_size: Number of instances in a batch.\n        :type batch_size: int\n        :param min_batch_size: Any batch with batch size smaller than this will\n                               be discarded. (To be deprecated in the future.)\n        :type min_batch_size: int\n        :param padding_to:  If set -1, the maximun shape in the batch\n                            will be used as the target shape for padding.\n                            Otherwise, `padding_to` will be the target shape.\n        :type padding_to: int\n        :param flatten: If set True, audio features will be flatten to 1darray.\n        :type flatten: bool\n        :param sortagrad: If set True, sort the instances by audio duration\n                          in the first epoch for speed up training.\n        :type sortagrad: bool\n        :param shuffle_method: Shuffle method. Options:\n                                '' or None: no shuffle.\n                                'instance_shuffle': instance-wise shuffle.\n                                'batch_shuffle': similarly-sized instances are\n                                                 put into batches, and then\n                                                 batch-wise shuffle the batches.\n                                                 For more details, please see\n                                                 ``_batch_shuffle.__doc__``.\n                                'batch_shuffle_clipped': 'batch_shuffle' with\n                                                         head shift and tail\n                                                         clipping. For more\n                                                         details, please see\n                                                         ``_batch_shuffle``.\n                              If sortagrad is True, shuffle is disabled\n                              for the first epoch.\n        :type shuffle_method: None|str\n        :return: Batch reader function, producing batches of data when called.\n        :rtype: callable\n        \"\"\"\n\n        def batch_reader():\n            # read manifest\n            manifest = utils.read_manifest(\n                manifest_path=manifest_path,\n                max_duration=self._max_duration,\n                min_duration=self._min_duration)\n            # sort (by duration) or batch-wise shuffle the manifest\n            if self._epoch == 0 and sortagrad:\n                manifest.sort(key=lambda x: x[\"duration\"])\n            else:\n                if shuffle_method == \"batch_shuffle\":\n                    manifest = self._batch_shuffle(\n                        manifest, batch_size, clipped=False)\n                elif shuffle_method == \"batch_shuffle_clipped\":\n                    manifest = self._batch_shuffle(\n                        manifest, batch_size, clipped=True)\n                elif shuffle_method == \"instance_shuffle\":\n                    self._rng.shuffle(manifest)\n                elif shuffle_method == None:\n                    pass\n                else:\n                    raise ValueError(\"Unknown shuffle method %s.\" %\n                                     shuffle_method)\n            # prepare batches\n            instance_reader = self._instance_reader_creator(manifest)\n            batch = []\n            for instance in instance_reader():\n                batch.append(instance)\n                if len(batch) == batch_size:\n                    yield self._padding_batch(batch, padding_to, flatten)\n                    batch = []\n            if len(batch) >= min_batch_size:\n                yield self._padding_batch(batch, padding_to, flatten)\n            self._epoch += 1\n\n        return batch_reader\n\n    @property\n    def feeding(self):\n        \"\"\"Returns data reader's feeding dict.\n\n        :return: Data feeding dict.\n        :rtype: dict\n        \"\"\"\n        return {\"audio_spectrogram\": 0, \"transcript_text\": 1}\n\n    @property\n    def vocab_size(self):\n        \"\"\"Return the vocabulary size.\n\n        :return: Vocabulary size.\n        :rtype: int\n        \"\"\"\n        return self._speech_featurizer.vocab_size\n\n    @property\n    def vocab_list(self):\n        \"\"\"Return the vocabulary in list.\n\n        :return: Vocabulary in list.\n        :rtype: list\n        \"\"\"\n        return self._speech_featurizer.vocab_list\n\n    def _parse_tar(self, file):\n        \"\"\"Parse a tar file to get a tarfile object\n        and a map containing tarinfoes\n        \"\"\"\n        result = {}\n        f = tarfile.open(file)\n        for tarinfo in f.getmembers():\n            result[tarinfo.name] = tarinfo\n        return f, result\n\n    def _get_file_object(self, file):\n        \"\"\"Get file object by file path.\n\n        If file startwith tar, it will return a tar file object\n        and cached tar file info for next reading request.\n        It will return file directly, if the type of file is not str.\n        \"\"\"\n        if file.startswith('tar:'):\n            tarpath, filename = file.split(':', 1)[1].split('#', 1)\n            if 'tar2info' not in self.local_data.__dict__:\n                self.local_data.tar2info = {}\n            if 'tar2object' not in self.local_data.__dict__:\n                self.local_data.tar2object = {}\n            if tarpath not in self.local_data.tar2info:\n                object, infoes = self._parse_tar(tarpath)\n                self.local_data.tar2info[tarpath] = infoes\n                self.local_data.tar2object[tarpath] = object\n            return self.local_data.tar2object[tarpath].extractfile(\n                self.local_data.tar2info[tarpath][filename])\n        else:\n            return open(file, 'r')\n\n    def _instance_reader_creator(self, manifest):\n        \"\"\"\n        Instance reader creator. Create a callable function to produce\n        instances of data.\n\n        Instance: a tuple of ndarray of audio spectrogram and a list of\n        token indices for transcript.\n        \"\"\"\n\n        def reader():\n            for instance in manifest:\n                yield instance\n\n        def mapper(instance):\n            return self.process_utterance(\n                self._get_file_object(instance[\"audio_filepath\"]),\n                instance[\"text\"])\n\n        return paddle.reader.xmap_readers(\n            mapper, reader, self._num_threads, 1024, order=True)\n\n    def _padding_batch(self, batch, padding_to=-1, flatten=False):\n        \"\"\"\n        Padding audio features with zeros to make them have the same shape (or\n        a user-defined shape) within one bach.\n\n        If ``padding_to`` is -1, the maximun shape in the batch will be used\n        as the target shape for padding. Otherwise, `padding_to` will be the\n        target shape (only refers to the second axis).\n\n        If `flatten` is True, features will be flatten to 1darray.\n        \"\"\"\n        new_batch = []\n        # get target shape\n        max_length = max([audio.shape[1] for audio, text in batch])\n        if padding_to != -1:\n            if padding_to < max_length:\n                raise ValueError(\"If padding_to is not -1, it should be larger \"\n                                 \"than any instance's shape in the batch\")\n            max_length = padding_to\n        # padding\n        for audio, text in batch:\n            padded_audio = np.zeros([audio.shape[0], max_length])\n            padded_audio[:, :audio.shape[1]] = audio\n            if flatten:\n                padded_audio = padded_audio.flatten()\n            new_batch.append((padded_audio, text))\n        return new_batch\n\n    def _batch_shuffle(self, manifest, batch_size, clipped=False):\n        \"\"\"Put similarly-sized instances into minibatches for better efficiency\n        and make a batch-wise shuffle.\n\n        1. Sort the audio clips by duration.\n        2. Generate a random number `k`, k in [0, batch_size).\n        3. Randomly shift `k` instances in order to create different batches\n           for different epochs. Create minibatches.\n        4. Shuffle the minibatches.\n\n        :param manifest: Manifest contents. List of dict.\n        :type manifest: list\n        :param batch_size: Batch size. This size is also used for generate\n                           a random number for batch shuffle.\n        :type batch_size: int\n        :param clipped: Whether to clip the heading (small shift) and trailing\n                        (incomplete batch) instances.\n        :type clipped: bool\n        :return: Batch shuffled mainifest.\n        :rtype: list\n        \"\"\"\n        manifest.sort(key=lambda x: x[\"duration\"])\n        shift_len = self._rng.randint(0, batch_size - 1)\n        batch_manifest = zip(*[iter(manifest[shift_len:])] * batch_size)\n        self._rng.shuffle(batch_manifest)\n        batch_manifest = list(sum(batch_manifest, ()))\n        if not clipped:\n            res_len = len(manifest) - shift_len - len(batch_manifest)\n            batch_manifest.extend(manifest[-res_len:])\n            batch_manifest.extend(manifest[0:shift_len])\n        return batch_manifest\n", "path": "deep_speech_2/data_utils/data.py"}], "after_files": [{"content": "\"\"\"Contains data generator for orgnaizing various audio data preprocessing\npipeline and offering data reader interface of PaddlePaddle requirements.\n\"\"\"\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport random\nimport tarfile\nimport multiprocessing\nimport numpy as np\nimport paddle.v2 as paddle\nfrom threading import local\nfrom data_utils import utils\nfrom data_utils.augmentor.augmentation import AugmentationPipeline\nfrom data_utils.featurizer.speech_featurizer import SpeechFeaturizer\nfrom data_utils.speech import SpeechSegment\nfrom data_utils.normalizer import FeatureNormalizer\n\n\nclass DataGenerator(object):\n    \"\"\"\n    DataGenerator provides basic audio data preprocessing pipeline, and offers\n    data reader interfaces of PaddlePaddle requirements.\n\n    :param vocab_filepath: Vocabulary filepath for indexing tokenized\n                           transcripts.\n    :type vocab_filepath: basestring\n    :param mean_std_filepath: File containing the pre-computed mean and stddev.\n    :type mean_std_filepath: None|basestring\n    :param augmentation_config: Augmentation configuration in json string.\n                                Details see AugmentationPipeline.__doc__.\n    :type augmentation_config: str\n    :param max_duration: Audio with duration (in seconds) greater than\n                         this will be discarded.\n    :type max_duration: float\n    :param min_duration: Audio with duration (in seconds) smaller than\n                         this will be discarded.\n    :type min_duration: float\n    :param stride_ms: Striding size (in milliseconds) for generating frames.\n    :type stride_ms: float\n    :param window_ms: Window size (in milliseconds) for generating frames.\n    :type window_ms: float\n    :param max_freq: Used when specgram_type is 'linear', only FFT bins\n                     corresponding to frequencies between [0, max_freq] are\n                     returned.\n    :types max_freq: None|float\n    :param specgram_type: Specgram feature type. Options: 'linear'.\n    :type specgram_type: str\n    :param use_dB_normalization: Whether to normalize the audio to -20 dB\n                                before extracting the features.\n    :type use_dB_normalization: bool\n    :param num_threads: Number of CPU threads for processing data.\n    :type num_threads: int\n    :param random_seed: Random seed.\n    :type random_seed: int\n    \"\"\"\n\n    def __init__(self,\n                 vocab_filepath,\n                 mean_std_filepath,\n                 augmentation_config='{}',\n                 max_duration=float('inf'),\n                 min_duration=0.0,\n                 stride_ms=10.0,\n                 window_ms=20.0,\n                 max_freq=None,\n                 specgram_type='linear',\n                 use_dB_normalization=True,\n                 num_threads=multiprocessing.cpu_count() // 2,\n                 random_seed=0):\n        self._max_duration = max_duration\n        self._min_duration = min_duration\n        self._normalizer = FeatureNormalizer(mean_std_filepath)\n        self._augmentation_pipeline = AugmentationPipeline(\n            augmentation_config=augmentation_config, random_seed=random_seed)\n        self._speech_featurizer = SpeechFeaturizer(\n            vocab_filepath=vocab_filepath,\n            specgram_type=specgram_type,\n            stride_ms=stride_ms,\n            window_ms=window_ms,\n            max_freq=max_freq,\n            use_dB_normalization=use_dB_normalization)\n        self._num_threads = num_threads\n        self._rng = random.Random(random_seed)\n        self._epoch = 0\n        # for caching tar files info\n        self._local_data = local()\n        self._local_data.tar2info = {}\n        self._local_data.tar2object = {}\n\n    def process_utterance(self, filename, transcript):\n        \"\"\"Load, augment, featurize and normalize for speech data.\n\n        :param filename: Audio filepath\n        :type filename: basestring | file\n        :param transcript: Transcription text.\n        :type transcript: basestring\n        :return: Tuple of audio feature tensor and list of token ids for\n                 transcription.\n        :rtype: tuple of (2darray, list)\n        \"\"\"\n        speech_segment = SpeechSegment.from_file(filename, transcript)\n        self._augmentation_pipeline.transform_audio(speech_segment)\n        specgram, text_ids = self._speech_featurizer.featurize(speech_segment)\n        specgram = self._normalizer.apply(specgram)\n        return specgram, text_ids\n\n    def batch_reader_creator(self,\n                             manifest_path,\n                             batch_size,\n                             min_batch_size=1,\n                             padding_to=-1,\n                             flatten=False,\n                             sortagrad=False,\n                             shuffle_method=\"batch_shuffle\"):\n        \"\"\"\n        Batch data reader creator for audio data. Return a callable generator\n        function to produce batches of data.\n\n        Audio features within one batch will be padded with zeros to have the\n        same shape, or a user-defined shape.\n\n        :param manifest_path: Filepath of manifest for audio files.\n        :type manifest_path: basestring\n        :param batch_size: Number of instances in a batch.\n        :type batch_size: int\n        :param min_batch_size: Any batch with batch size smaller than this will\n                               be discarded. (To be deprecated in the future.)\n        :type min_batch_size: int\n        :param padding_to:  If set -1, the maximun shape in the batch\n                            will be used as the target shape for padding.\n                            Otherwise, `padding_to` will be the target shape.\n        :type padding_to: int\n        :param flatten: If set True, audio features will be flatten to 1darray.\n        :type flatten: bool\n        :param sortagrad: If set True, sort the instances by audio duration\n                          in the first epoch for speed up training.\n        :type sortagrad: bool\n        :param shuffle_method: Shuffle method. Options:\n                                '' or None: no shuffle.\n                                'instance_shuffle': instance-wise shuffle.\n                                'batch_shuffle': similarly-sized instances are\n                                                 put into batches, and then\n                                                 batch-wise shuffle the batches.\n                                                 For more details, please see\n                                                 ``_batch_shuffle.__doc__``.\n                                'batch_shuffle_clipped': 'batch_shuffle' with\n                                                         head shift and tail\n                                                         clipping. For more\n                                                         details, please see\n                                                         ``_batch_shuffle``.\n                              If sortagrad is True, shuffle is disabled\n                              for the first epoch.\n        :type shuffle_method: None|str\n        :return: Batch reader function, producing batches of data when called.\n        :rtype: callable\n        \"\"\"\n\n        def batch_reader():\n            # read manifest\n            manifest = utils.read_manifest(\n                manifest_path=manifest_path,\n                max_duration=self._max_duration,\n                min_duration=self._min_duration)\n            # sort (by duration) or batch-wise shuffle the manifest\n            if self._epoch == 0 and sortagrad:\n                manifest.sort(key=lambda x: x[\"duration\"])\n            else:\n                if shuffle_method == \"batch_shuffle\":\n                    manifest = self._batch_shuffle(\n                        manifest, batch_size, clipped=False)\n                elif shuffle_method == \"batch_shuffle_clipped\":\n                    manifest = self._batch_shuffle(\n                        manifest, batch_size, clipped=True)\n                elif shuffle_method == \"instance_shuffle\":\n                    self._rng.shuffle(manifest)\n                elif shuffle_method == None:\n                    pass\n                else:\n                    raise ValueError(\"Unknown shuffle method %s.\" %\n                                     shuffle_method)\n            # prepare batches\n            instance_reader = self._instance_reader_creator(manifest)\n            batch = []\n            for instance in instance_reader():\n                batch.append(instance)\n                if len(batch) == batch_size:\n                    yield self._padding_batch(batch, padding_to, flatten)\n                    batch = []\n            if len(batch) >= min_batch_size:\n                yield self._padding_batch(batch, padding_to, flatten)\n            self._epoch += 1\n\n        return batch_reader\n\n    @property\n    def feeding(self):\n        \"\"\"Returns data reader's feeding dict.\n\n        :return: Data feeding dict.\n        :rtype: dict\n        \"\"\"\n        return {\"audio_spectrogram\": 0, \"transcript_text\": 1}\n\n    @property\n    def vocab_size(self):\n        \"\"\"Return the vocabulary size.\n\n        :return: Vocabulary size.\n        :rtype: int\n        \"\"\"\n        return self._speech_featurizer.vocab_size\n\n    @property\n    def vocab_list(self):\n        \"\"\"Return the vocabulary in list.\n\n        :return: Vocabulary in list.\n        :rtype: list\n        \"\"\"\n        return self._speech_featurizer.vocab_list\n\n    def _parse_tar(self, file):\n        \"\"\"Parse a tar file to get a tarfile object\n        and a map containing tarinfoes\n        \"\"\"\n        result = {}\n        f = tarfile.open(file)\n        for tarinfo in f.getmembers():\n            result[tarinfo.name] = tarinfo\n        return f, result\n\n    def _get_file_object(self, file):\n        \"\"\"Get file object by file path.\n\n        If file startwith tar, it will return a tar file object\n        and cached tar file info for next reading request.\n        It will return file directly, if the type of file is not str.\n        \"\"\"\n        if file.startswith('tar:'):\n            tarpath, filename = file.split(':', 1)[1].split('#', 1)\n            if 'tar2info' not in self._local_data.__dict__:\n                self._local_data.tar2info = {}\n            if 'tar2object' not in self._local_data.__dict__:\n                self._local_data.tar2object = {}\n            if tarpath not in self._local_data.tar2info:\n                object, infoes = self._parse_tar(tarpath)\n                self._local_data.tar2info[tarpath] = infoes\n                self._local_data.tar2object[tarpath] = object\n            return self._local_data.tar2object[tarpath].extractfile(\n                self._local_data.tar2info[tarpath][filename])\n        else:\n            return open(file, 'r')\n\n    def _instance_reader_creator(self, manifest):\n        \"\"\"\n        Instance reader creator. Create a callable function to produce\n        instances of data.\n\n        Instance: a tuple of ndarray of audio spectrogram and a list of\n        token indices for transcript.\n        \"\"\"\n\n        def reader():\n            for instance in manifest:\n                yield instance\n\n        def mapper(instance):\n            return self.process_utterance(\n                self._get_file_object(instance[\"audio_filepath\"]),\n                instance[\"text\"])\n\n        return paddle.reader.xmap_readers(\n            mapper, reader, self._num_threads, 1024, order=True)\n\n    def _padding_batch(self, batch, padding_to=-1, flatten=False):\n        \"\"\"\n        Padding audio features with zeros to make them have the same shape (or\n        a user-defined shape) within one bach.\n\n        If ``padding_to`` is -1, the maximun shape in the batch will be used\n        as the target shape for padding. Otherwise, `padding_to` will be the\n        target shape (only refers to the second axis).\n\n        If `flatten` is True, features will be flatten to 1darray.\n        \"\"\"\n        new_batch = []\n        # get target shape\n        max_length = max([audio.shape[1] for audio, text in batch])\n        if padding_to != -1:\n            if padding_to < max_length:\n                raise ValueError(\"If padding_to is not -1, it should be larger \"\n                                 \"than any instance's shape in the batch\")\n            max_length = padding_to\n        # padding\n        for audio, text in batch:\n            padded_audio = np.zeros([audio.shape[0], max_length])\n            padded_audio[:, :audio.shape[1]] = audio\n            if flatten:\n                padded_audio = padded_audio.flatten()\n            new_batch.append((padded_audio, text))\n        return new_batch\n\n    def _batch_shuffle(self, manifest, batch_size, clipped=False):\n        \"\"\"Put similarly-sized instances into minibatches for better efficiency\n        and make a batch-wise shuffle.\n\n        1. Sort the audio clips by duration.\n        2. Generate a random number `k`, k in [0, batch_size).\n        3. Randomly shift `k` instances in order to create different batches\n           for different epochs. Create minibatches.\n        4. Shuffle the minibatches.\n\n        :param manifest: Manifest contents. List of dict.\n        :type manifest: list\n        :param batch_size: Batch size. This size is also used for generate\n                           a random number for batch shuffle.\n        :type batch_size: int\n        :param clipped: Whether to clip the heading (small shift) and trailing\n                        (incomplete batch) instances.\n        :type clipped: bool\n        :return: Batch shuffled mainifest.\n        :rtype: list\n        \"\"\"\n        manifest.sort(key=lambda x: x[\"duration\"])\n        shift_len = self._rng.randint(0, batch_size - 1)\n        batch_manifest = zip(*[iter(manifest[shift_len:])] * batch_size)\n        self._rng.shuffle(batch_manifest)\n        batch_manifest = list(sum(batch_manifest, ()))\n        if not clipped:\n            res_len = len(manifest) - shift_len - len(batch_manifest)\n            batch_manifest.extend(manifest[-res_len:])\n            batch_manifest.extend(manifest[0:shift_len])\n        return batch_manifest\n", "path": "deep_speech_2/data_utils/data.py"}]}