import sys
import time
from azure.storage import BlobService
from azure import WindowsAzureMissingResourceError

from CREDENTIALS import account_name, account_key
db = BlobService(account_name=account_name, account_key=account_key)

###
bucket = 'crawl-data'
in_progress = set()
#
existing = set([x.name for x in db.list_blobs(bucket, 'common-crawl/crawl-data/CC-MAIN-2014-23/')])
todo = [x.rstrip() for x in sys.stdin.readlines()]
todo = [x for x in todo if x not in existing]
###
while todo or in_progress:
  new_progress = set()
  for path in in_progress:
    props = db.get_blob_properties(bucket, path)
    if props['x-ms-copy-status'] == 'success':
      print '\t%s completed' % path
      continue
    new_progress.add(path)
    time.sleep(0.25)
  print 'Task queue length is %d' % len(new_progress)
  print 'TODO queue length is %d' % len(todo)
  # Populate the queue
  while todo and len(new_progress) < 256:
    path = todo.pop()
    # If it exists, skip it -- only add if it's missing
    try:
      resp = db.get_blob_properties(bucket, path)
    except WindowsAzureMissingResourceError:
      db.copy_blob(bucket, path, 'https://aws-publicdatasets.s3.amazonaws.com/' + path)
      new_progress.add(path)
  in_progress = new_progress
  time.sleep(300)

Add a script that moves files to Azure storage using Copy Blob API
Add a script that moves files to Azure storage using Copy Blob API
