scripts/check_md5s.py
def validate_all_objects_in_directory(gs_dir):
    """Validate files with MD5s in the provided gs directory"""
    backend = hb.ServiceBackend(
        billing_project=os.getenv('HAIL_BILLING_PROJECT'),
        bucket=os.getenv('HAIL_BUCKET'),
    )
    b = hb.Batch('validate_md5s', backend=backend)
    client = storage.Client()

    if not gs_dir.startswith('gs://'):
        raise ValueError(f'Expected GS directory, got: {gs_dir}')

    bucket_name, *components = gs_dir[5:].split('/')

    blobs = client.list_blobs(bucket_name, prefix='/'.join(components))
    files: Set[str] = {f'gs://{bucket_name}/{blob.name}' for blob in blobs}
    for obj in files:
        if obj.endswith('.md5'):
            continue
        if f'{obj}.md5' not in files:
            continue

        job = b.new_job(f'validate_{os.path.basename(obj)}')
        job.image(DRIVER_IMAGE)
        validate_md5(job, obj)

    b.run(wait=False)
Validate files with MD5s in the provided gs directory
Validate files with MD5s in the provided gs directory
Validate files with MD5s in the provided gs directory
def validate_all_objects_in_directory(gs_dir):
    backend = hb.ServiceBackend(
        billing_project=os.getenv('HAIL_BILLING_PROJECT'),
        bucket=os.getenv('HAIL_BUCKET'),
    )
    b = hb.Batch('validate_md5s', backend=backend)
    client = storage.Client()
    if not gs_dir.startswith('gs://'):
        raise ValueError(f'Expected GS directory, got: {gs_dir}')
    bucket_name, *components = gs_dir[5:].split('/')
    blobs = client.list_blobs(bucket_name, prefix='/'.join(components))
    files: Set[str] = {f'gs://{bucket_name}/{blob.name}' for blob in blobs}
    for obj in files:
        if obj.endswith('.md5'):
            continue
        if f'{obj}.md5' not in files:
            continue
        job = b.new_job(f'validate_{os.path.basename(obj)}')
        job.image(DRIVER_IMAGE)
        validate_md5(job, obj)
    b.run(wait=False)
def validate_all_objects_in_directory ( gs_dir ) : backend = hb . ServiceBackend ( billing_project = os . getenv ( 'HAIL_BILLING_PROJECT' ) , bucket = os . getenv ( 'HAIL_BUCKET' ) , ) b = hb . Batch ( 'validate_md5s' , backend = backend ) client = storage . Client ( ) if not gs_dir . startswith ( 'gs://' ) : raise ValueError ( f'Expected GS directory, got: {gs_dir}' ) bucket_name , * components = gs_dir [ 5 : ] . split ( '/' ) blobs = client . list_blobs ( bucket_name , prefix = '/' . join ( components ) ) files : Set [ str ] = { f'gs://{bucket_name}/{blob.name}' for blob in blobs } for obj in files : if obj . endswith ( '.md5' ) : continue if f'{obj}.md5' not in files : continue job = b . new_job ( f'validate_{os.path.basename(obj)}' ) job . image ( DRIVER_IMAGE ) validate_md5 ( job , obj ) b . run ( wait = False )
Validate files with MD5s in the provided gs directory
Validate files with MD5s in the provided gs directory
"""Validate files with MD5s in the provided gs directory"""
gs_dir