text: """This script is for uploading data for DeepSpeech2 training on paddlecloud.
  
  Steps:
  1. Read original manifests and extract local sound files.
  2. Tar all local sound files into multiple tar files and upload them.
  3. Modify original manifests with updated paths in cloud filesystem.
  """
  from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  
  import json
  import os
  import tarfile
  import sys
  import argparse
  import shutil
  from subprocess import call
  import _init_paths
  from data_utils.utils import read_manifest
  
  parser = argparse.ArgumentParser(description=__doc__)
  parser.add_argument(
      "--in_manifest_paths",
      default=[
          "../datasets/manifest.train", "../datasets/manifest.dev",
          "../datasets/manifest.test"
      ],
      type=str,
      nargs='+',
      help="Local filepaths of input manifests to load, pack and upload."
      "(default: %(default)s)")
  parser.add_argument(
      "--out_manifest_paths",
      default=[
          "./cloud.manifest.train", "./cloud.manifest.dev",
          "./cloud.manifest.test"
      ],
      type=str,
      nargs='+',
      help="Local filepaths of modified manifests to write to. "
      "(default: %(default)s)")
  parser.add_argument(
      "--cloud_data_dir",
      required=True,
      type=str,
      help="Destination directory on paddlecloud to upload data to.")
  parser.add_argument(
      "--num_shards",
      default=10,
      type=int,
      help="Number of parts to split data to. (default: %(default)s)")
  parser.add_argument(
      "--local_tmp_dir",
      default="./tmp/",
      type=str,
      help="Local directory for storing temporary data. (default: %(default)s)")
  args = parser.parse_args()
  
  
  def upload_data(in_manifest_path_list, out_manifest_path_list, local_tmp_dir,
                  upload_tar_dir, num_shards):
      """Extract and pack sound files listed in the manifest files into multple
      tar files and upload them to padldecloud. Besides, generate new manifest
      files with updated paths in paddlecloud.
      """
      # compute total audio number
      total_line = 0
      for manifest_path in in_manifest_path_list:
          with open(manifest_path, 'r') as f:
              total_line += len(f.readlines())
      line_per_tar = (total_line // num_shards) + 1
  
      # pack and upload shard by shard
      line_count, tar_file = 0, None
      for manifest_path, out_manifest_path in zip(in_manifest_path_list,
                                                  out_manifest_path_list):
          manifest = read_manifest(manifest_path)
          out_manifest = []
          for json_data in manifest:
              sound_filepath = json_data['audio_filepath']
              sound_filename = os.path.basename(sound_filepath)
              if line_count % line_per_tar == 0:
                  if tar_file != None:
                      tar_file.close()
                      pcloud_cp(tar_path, upload_tar_dir)
                      os.remove(tar_path)
                  tar_name = 'part-%s-of-%s.tar' % (
                      str(line_count // line_per_tar).zfill(5),
                      str(num_shards).zfill(5))
                  tar_path = os.path.join(local_tmp_dir, tar_name)
                  tar_file = tarfile.open(tar_path, 'w')
              tar_file.add(sound_filepath, arcname=sound_filename)
              line_count += 1
              json_data['audio_filepath'] = "tar:%s#%s" % (
                  os.path.join(upload_tar_dir, tar_name), sound_filename)
              out_manifest.append("%s\n" % json.dumps(json_data))
          with open(out_manifest_path, 'w') as f:
              f.writelines(out_manifest)
          pcloud_cp(out_manifest_path, upload_tar_dir)
      tar_file.close()
      pcloud_cp(tar_path, upload_tar_dir)
      os.remove(tar_path)
  
  
  def pcloud_mkdir(dir):
      """Make directory in PaddleCloud filesystem.
      """
      if call(['paddlecloud', 'mkdir', dir]) != 0:
          raise IOError("PaddleCloud mkdir failed: %s." % dir)
  
  
  def pcloud_cp(src, dst):
      """Copy src from local filesytem to dst in PaddleCloud filesystem,
      or downlowd src from PaddleCloud filesystem to dst in local filesystem.
      """
      if call(['paddlecloud', 'cp', src, dst]) != 0:
          raise IOError("PaddleCloud cp failed: from [%s] to [%s]." % (src, dst))
  
  
  if __name__ == '__main__':
      if not os.path.exists(args.local_tmp_dir):
          os.makedirs(args.local_tmp_dir)
      pcloud_mkdir(args.cloud_data_dir)
  
      upload_data(args.in_manifest_paths, args.out_manifest_paths,
                  args.local_tmp_dir, args.cloud_data_dir, args.num_shards)
  
      shutil.rmtree(args.local_tmp_dir)
  