s4cmd.py
def download(self, source, target, mpi=None, pos=0, chunk=0, part=0):
    '''Thread worker for download operation.'''
    s3url = S3URL(source)
    obj = self.lookup(s3url)
    if obj is None:
      raise Failure('The obj "%s" does not exists.' % (s3url.path,))

    # Initialization: Set up multithreaded downloads.
    if not mpi:
      # optional checks
      if self.opt.dry_run:
        message('%s => %s', source, target)
        return
      elif self.opt.sync_check and self.sync_check(LocalMD5Cache(target), obj):
        message('%s => %s (synced)', source, target)
        return
      elif not self.opt.force and os.path.exists(target):
        raise Failure('File already exists: %s' % target)

      fsize = int(obj['ContentLength'])

      # Small file optimization.
      if fsize < self.opt.max_singlepart_download_size:
        # Create a single part to chain back main download operation.
        mpi = ThreadUtil.MultipartItem(tempfile_get(target))
        mpi.total = 1
        pos = 0
        chunk = fsize
        # Continue as one part download.
      else:
        # Here we use temp filename as the id of mpi.
        for args in self.get_file_splits(tempfile_get(target), source, target, fsize, self.opt.multipart_split_size):
          self.pool.download(*args)
        return

    tempfile = mpi.id
    if self.opt.recursive:
      self.mkdirs(tempfile)

    # Download part of the file, range is inclusive.
    response = self.s3.get_object(Bucket=s3url.bucket, Key=s3url.path, Range='bytes=%d-%d' % (pos, pos + chunk - 1))
    self.write_file_chunk(tempfile, pos, chunk, response['Body'])

    # Finalize
    if mpi.complete({'PartNumber': part}):
      try:
        self.update_privilege(obj, tempfile)
        self._verify_file_size(obj, tempfile)
        tempfile_set(tempfile, target)
        message('%s => %s', source, target)
      except Exception as e:
        # Note that we don't retry in this case, because
        # We are going to remove the temp file, and if we
        # retry here with original parameters (wrapped in
        # the task item), it would fail anyway
        tempfile_set(tempfile, None)
        raise Failure('Download Failure: %s, Source: %s.' % (e.message, source))
def download(self, source, target, mpi=None, pos=0, chunk=0, part=0):
    '''Thread worker for download operation.'''
    s3url = S3URL(source)
    obj = self.lookup(s3url)
    if obj is None:
      raise Failure('The obj "%s" does not exists.' % (s3url.path,))

    # Initialization: Set up multithreaded downloads.
    if not mpi:
      # optional checks
      if self.opt.dry_run:
        message('%s => %s', source, target)
        return
      elif self.opt.sync_check and self.sync_check(LocalMD5Cache(target), obj):
        message('%s => %s (synced)', source, target)
        return
      elif not self.opt.force and os.path.exists(target):
        raise Failure('File already exists: %s' % target)

      fsize = int(obj['ContentLength'])

      # Small file optimization.
      if fsize < self.opt.max_singlepart_download_size:
        # Create a single part to chain back main download operation.
        mpi = ThreadUtil.MultipartItem(tempfile_get(target))
        mpi.total = 1
        pos = 0
        chunk = fsize
        # Continue as one part download.
      else:
        # Here we use temp filename as the id of mpi.
        for args in self.get_file_splits(tempfile_get(target), source, target, fsize, self.opt.multipart_split_size):
          self.pool.download(*args)
        return

    tempfile = mpi.id
    if self.opt.recursive:
      self.mkdirs(tempfile)

    # Download part of the file, range is inclusive.
    response = self.s3.get_object(Bucket=s3url.bucket, Key=s3url.path, Range='bytes=%d-%d' % (pos, pos + chunk - 1))
    self.write_file_chunk(tempfile, pos, chunk, response['Body'])

    # Finalize
    if mpi.complete({'PartNumber': part}):
      try:
        self.update_privilege(obj, tempfile)
        self._verify_file_size(obj, tempfile)
        tempfile_set(tempfile, target)
        message('%s => %s', source, target)
      except Exception as e:
        # Note that we don't retry in this case, because
        # We are going to remove the temp file, and if we
        # retry here with original parameters (wrapped in
        # the task item), it would fail anyway
        tempfile_set(tempfile, None)
        raise Failure('Download Failure: %s, Source: %s.' % (e.message, source))
def download ( self , source , target , mpi = None , pos = 0 , chunk = 0 , part = 0 ) : s3url = S3URL ( source ) obj = self . lookup ( s3url ) if obj is None : raise Failure ( 'The obj "%s" does not exists.' % ( s3url . path , ) ) # Initialization: Set up multithreaded downloads. if not mpi : # optional checks if self . opt . dry_run : message ( '%s => %s' , source , target ) return elif self . opt . sync_check and self . sync_check ( LocalMD5Cache ( target ) , obj ) : message ( '%s => %s (synced)' , source , target ) return elif not self . opt . force and os . path . exists ( target ) : raise Failure ( 'File already exists: %s' % target ) fsize = int ( obj [ 'ContentLength' ] ) # Small file optimization. if fsize < self . opt . max_singlepart_download_size : # Create a single part to chain back main download operation. mpi = ThreadUtil . MultipartItem ( tempfile_get ( target ) ) mpi . total = 1 pos = 0 chunk = fsize # Continue as one part download. else : # Here we use temp filename as the id of mpi. for args in self . get_file_splits ( tempfile_get ( target ) , source , target , fsize , self . opt . multipart_split_size ) : self . pool . download ( * args ) return tempfile = mpi . id if self . opt . recursive : self . mkdirs ( tempfile ) # Download part of the file, range is inclusive. response = self . s3 . get_object ( Bucket = s3url . bucket , Key = s3url . path , Range = 'bytes=%d-%d' % ( pos , pos + chunk - 1 ) ) self . write_file_chunk ( tempfile , pos , chunk , response [ 'Body' ] ) # Finalize if mpi . complete ( { 'PartNumber' : part } ) : try : self . update_privilege ( obj , tempfile ) self . _verify_file_size ( obj , tempfile ) tempfile_set ( tempfile , target ) message ( '%s => %s' , source , target ) except Exception as e : # Note that we don't retry in this case, because # We are going to remove the temp file, and if we # retry here with original parameters (wrapped in # the task item), it would fail anyway tempfile_set ( tempfile , None ) raise Failure ( 'Download Failure: %s, Source: %s.' % ( e . message , source ) )
Thread worker for download operation.
Thread worker for download operation .