# coding: utf-8
# Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.

import filecmp
import json
import pytest
import oci
import services.object_storage.src.oci_cli_object_storage as oci_cli_object_storage
import os
import random
import shutil
import six
import string
from tests import util
from tests import test_config_container
from mimetypes import guess_type


OBJECTS_TO_CREATE_IN_BUCKET_FOR_BULK_GET = 100
OBJECTS_TO_CREATE_IN_FOLDER_FOR_BULK_PUT = 20
CONTENT_STRING_LENGTH = 5000
MID_SIZED_FILE_IN_MEBIBTYES = 20
LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES = 150  # Default multipart is 128MiB


# Holds the objects we create and their content so that we can verify results
bulk_get_object_to_content = {}
bulk_get_prefix_to_object = {
    'a/b/c/d': [],
    'a/b/c': [],
    'a/b': [],
    '/a': [],
    '': []
}
bulk_get_bucket_name = None

bulk_put_large_files = set()
bulk_put_mid_sized_files = set()
root_bulk_put_folder = None
bulk_put_bucket_name = None


@pytest.fixture
def vcr_fixture(request):
    with test_config_container.create_vcr(cassette_library_dir='services/object_storage/tests/cassettes').use_cassette('object_storage_bulk_operations_{name}.yml'.format(name=request.function.__name__)):
        yield


# Generate test data for different operations:
#
#    Bulk Get: create a new bucket and populate it with some objects, then tear it all down afterwards
#    Bulk Put: create a folder structure containing small and large files, then tear it all down afterwards
#    Bulk Delete: uses the folders and files generated for bulk put
@pytest.fixture(scope='module', autouse=True)
def generate_test_data(object_storage_client):
    global bulk_get_object_to_content, bulk_get_bucket_name, root_bulk_put_folder, bulk_put_large_files, bulk_put_mid_sized_files, bulk_put_bucket_name

    # Create a test bucket
    create_bucket_request = oci.object_storage.models.CreateBucketDetails()
    create_bucket_request.name = 'ObjectStorageBulkGetTest_{}'.format(util.random_number_string())
    create_bucket_request.compartment_id = util.COMPARTMENT_ID
    util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, create_bucket_request.name)
    object_storage_client.create_bucket(util.NAMESPACE, create_bucket_request)

    bulk_get_bucket_name = create_bucket_request.name

    # Create items at various heirarchy levels (to be surfaced as different directories on disk)
    for i in range(OBJECTS_TO_CREATE_IN_BUCKET_FOR_BULK_GET):
        if i % 5 == 4:
            object_name = 'a/b/c/d/Object_{}'.format(i)
            bulk_get_prefix_to_object['a/b/c/d'].append(object_name)
        elif i % 5 == 3:
            object_name = 'a/b/c/Object_{}'.format(i)
            bulk_get_prefix_to_object['a/b/c'].append(object_name)
        elif i % 5 == 2:
            object_name = 'a/b/Object_{}'.format(i)
            bulk_get_prefix_to_object['a/b'].append(object_name)
        elif i % 5 == 1:
            # This is equivalent to a/ on the file system because we drop the leading slash (we drop path separators from the front to avoid unexpected results)
            object_name = '/a/Object_{}'.format(i)
            bulk_get_prefix_to_object['/a'].append(object_name)
        else:
            # At the root of the bucket
            object_name = 'Object_{}'.format(i)
            bulk_get_prefix_to_object[''].append(object_name)

        object_content = generate_random_string(CONTENT_STRING_LENGTH)
        object_storage_client.put_object(util.NAMESPACE, create_bucket_request.name, object_name, object_content)
        bulk_get_object_to_content[object_name] = object_content

    # makedirs creates all subfolders recursively
    root_bulk_put_folder = 'tests/temp/bulk_put_{}'.format(util.random_number_string())
    bulk_put_folder_leaf = '{}/subfolder1/subfolder2/subfolder3'.format(root_bulk_put_folder)
    if not os.path.exists(bulk_put_folder_leaf):
        os.makedirs(bulk_put_folder_leaf)

    create_bucket_request = oci.object_storage.models.CreateBucketDetails()
    create_bucket_request.name = 'ObjectStorageBulkPutTest_{}'.format(util.random_number_string())
    create_bucket_request.compartment_id = util.COMPARTMENT_ID
    util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, create_bucket_request.name)
    object_storage_client.create_bucket(util.NAMESPACE, create_bucket_request)

    bulk_put_bucket_name = create_bucket_request.name

    subfolders = ['', 'subfolder1', 'subfolder1/subfolder2', 'subfolder1/subfolder2/subfolder3']
    for subfolder in subfolders:
        if subfolder == '':
            full_folder = root_bulk_put_folder
        else:
            full_folder = os.path.join(root_bulk_put_folder, subfolder)

        for i in range(OBJECTS_TO_CREATE_IN_FOLDER_FOR_BULK_PUT + 1):
            file_path = '{}/object_{}'.format(full_folder, i)
            if i != 0 and i % OBJECTS_TO_CREATE_IN_FOLDER_FOR_BULK_PUT == 0:
                # Put in one big file per subfolder
                util.create_large_file(file_path, LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES)
                bulk_put_large_files.add(file_path)
            elif i != 0 and i % 10 == 0:
                # Put in the occasional file with a reasonable size so that we can force multipart
                util.create_large_file(file_path, MID_SIZED_FILE_IN_MEBIBTYES)
                bulk_put_mid_sized_files.add(file_path)
            else:
                with open(file_path, 'w') as f:
                    f.write(generate_random_string(CONTENT_STRING_LENGTH))

    yield

    # Tear down stuff by deleting all the things and then deleting the buckets
    delete_bucket_and_all_items(object_storage_client, bulk_get_bucket_name)
    delete_bucket_and_all_items(object_storage_client, bulk_put_bucket_name)

    # Remove all directories recursively
    shutil.rmtree(root_bulk_put_folder)


@util.skip_while_rerecording
def test_normalize_object_name_path():
    assert '/this/is/a/path' == oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage('/this/is/a/path')
    assert '/this/is/a/path' == oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage('/this/is/a/path', '/')
    assert '/this/is/a/path' == oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage('\\this\\is\\a\\path', '\\')
    assert '/this/is/a/path' == oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage('\\this/is/a\\path', '\\')

    assert 'thisisapath' == oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage('thisisapath')
    assert 'thisisapath' == oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage('thisisapath', '/')
    assert 'thisisapath' == oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage('thisisapath', '\\')


@util.skip_while_rerecording
def test_get_all_objects_in_bucket(vcr_fixture):
    download_folder = 'tests/temp/get_all_{}'.format(bulk_get_bucket_name)
    result = invoke(['os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--download-dir', download_folder])
    print(result.output)

    # Ensure that content matches
    for object_name in bulk_get_object_to_content:
        if object_name[0] == '/' or object_name[0] == '\\':
            file_path = os.path.join(download_folder, object_name[1:])
        else:
            file_path = os.path.join(download_folder, object_name)

        with open(file_path, 'r') as content_file:
            content = content_file.read()
            assert content == bulk_get_object_to_content[object_name]

    assert len(bulk_get_object_to_content) == get_count_of_files_in_folder_and_subfolders(download_folder)

    shutil.rmtree(download_folder)


@util.skip_while_rerecording
def test_get_directory_and_subdirectories(vcr_fixture):
    download_folder = 'tests/temp/get_directory_and_subdirectories_{}'.format(bulk_get_bucket_name)

    # This should get us a/b/<object>, a/b/c/<object> and a/b/c/d/<object>
    invoke(['os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--download-dir', download_folder, '--prefix', 'a/b'])

    for object_name in bulk_get_prefix_to_object['a/b']:
        file_path = os.path.join(download_folder, object_name)
        with open(file_path, 'r') as content_file:
            content = content_file.read()
            assert content == bulk_get_object_to_content[object_name]

    for object_name in bulk_get_prefix_to_object['a/b/c']:
        file_path = os.path.join(download_folder, object_name)
        with open(file_path, 'r') as content_file:
            content = content_file.read()
            assert content == bulk_get_object_to_content[object_name]

    for object_name in bulk_get_prefix_to_object['a/b/c/d']:
        file_path = os.path.join(download_folder, object_name)
        with open(file_path, 'r') as content_file:
            content = content_file.read()
            assert content == bulk_get_object_to_content[object_name]

    assert len(bulk_get_prefix_to_object['a/b']) + len(bulk_get_prefix_to_object['a/b/c']) + len(bulk_get_prefix_to_object['a/b/c/d']) == get_count_of_files_in_folder_and_subfolders(download_folder)

    shutil.rmtree(download_folder)


@util.skip_while_rerecording
def test_get_directory_no_subdirectory(vcr_fixture):
    download_folder = 'tests/temp/get_directory_only_{}'.format(bulk_get_bucket_name)
    invoke(['os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--download-dir', download_folder, '--prefix', 'a/b/c/', '--delimiter', '/'])

    for object_name in bulk_get_prefix_to_object['a/b/c']:
        file_path = os.path.join(download_folder, object_name)
        with open(file_path, 'r') as content_file:
            content = content_file.read()
            assert content == bulk_get_object_to_content[object_name]

    assert len(bulk_get_prefix_to_object['a/b/c']) == get_count_of_files_in_folder_and_subfolders(download_folder)

    shutil.rmtree(download_folder)


@util.skip_while_rerecording
def test_get_files_skipped():
    download_folder = 'tests/temp/skip_and_replace_{}'.format(bulk_get_bucket_name)
    invoke(['os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--download-dir', download_folder])

    # Sanity check
    assert len(bulk_get_object_to_content) == get_count_of_files_in_folder_and_subfolders(download_folder)

    # We should skip over all objects since there is no --overwrite. There should be prompts
    result = invoke(['os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--download-dir', download_folder])
    parsed_result = parse_json_response_from_mixed_output(result.output)
    assert 'Are you sure you want to overwrite it?' in result.output
    assert len(parsed_result['skipped-objects']) == len(bulk_get_object_to_content)

    # We should skip over all objects since we say --no-overwrite. Additionally there should be no prompts
    result = invoke(['os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--download-dir', download_folder, '--no-overwrite'])
    parsed_result = parse_json_response_from_mixed_output(result.output)
    assert 'Are you sure you want to overwrite it?' not in result.output
    assert len(parsed_result['skipped-objects']) == len(bulk_get_object_to_content)

    # We should skip over no objects since we --overwrite
    result = invoke(['os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--download-dir', download_folder, '--overwrite'])
    parsed_result = parse_json_response_from_mixed_output(result.output)
    assert len(parsed_result['skipped-objects']) == 0

    shutil.rmtree(download_folder)


@util.skip_while_rerecording
def test_get_no_objects(vcr_fixture):
    download_folder = 'tests/temp/no_objects_{}'.format(bulk_get_bucket_name)
    invoke(['os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--download-dir', download_folder, '--prefix', 'batman'])

    assert 0 == get_count_of_files_in_folder_and_subfolders(download_folder)

    shutil.rmtree(download_folder)


@util.skip_while_rerecording
def test_get_multipart(object_storage_client):
    create_bucket_request = oci.object_storage.models.CreateBucketDetails()
    create_bucket_request.name = 'ObjectStorageBulkGetMultipartsTest_{}'.format(util.random_number_string())
    create_bucket_request.compartment_id = util.COMPARTMENT_ID
    util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, create_bucket_request.name)
    object_storage_client.create_bucket(util.NAMESPACE, create_bucket_request)

    large_file_root_dir = os.path.join('tests', 'temp', 'multipart_get_large_files')
    if not os.path.exists(large_file_root_dir):
        os.makedirs(large_file_root_dir)
    util.create_large_file(os.path.join(large_file_root_dir, '1.bin'), LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES)
    util.create_large_file(os.path.join(large_file_root_dir, '2.bin'), LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES)
    util.create_large_file(os.path.join(large_file_root_dir, '3.bin'), LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES)
    util.create_large_file(os.path.join(large_file_root_dir, '4.bin'), LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES)
    util.create_large_file(os.path.join(large_file_root_dir, '5.bin'), LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES)
    util.create_large_file(os.path.join(large_file_root_dir, '6.bin'), 1)  # Creates a 1 MiB file for variety

    invoke([
        'os', 'object', 'bulk-upload',
        '--namespace', util.NAMESPACE,
        '--bucket-name', create_bucket_request.name,
        '--src-dir', large_file_root_dir
    ])

    large_file_verify_dir = os.path.join('tests', 'temp', 'multipart_get_large_files_verify')

    invoke(['os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name, '--download-dir', large_file_verify_dir, '--multipart-download-threshold', '128'])

    assert get_count_of_files_in_folder_and_subfolders(large_file_verify_dir) == 6
    assert filecmp.cmp(os.path.join(large_file_root_dir, '1.bin'), os.path.join(large_file_verify_dir, '1.bin'))
    assert filecmp.cmp(os.path.join(large_file_root_dir, '2.bin'), os.path.join(large_file_verify_dir, '2.bin'))
    assert filecmp.cmp(os.path.join(large_file_root_dir, '3.bin'), os.path.join(large_file_verify_dir, '3.bin'))
    assert filecmp.cmp(os.path.join(large_file_root_dir, '4.bin'), os.path.join(large_file_verify_dir, '4.bin'))
    assert filecmp.cmp(os.path.join(large_file_root_dir, '5.bin'), os.path.join(large_file_verify_dir, '5.bin'))
    assert filecmp.cmp(os.path.join(large_file_root_dir, '6.bin'), os.path.join(large_file_verify_dir, '6.bin'))

    shutil.rmtree(large_file_root_dir)
    shutil.rmtree(large_file_verify_dir)

    delete_bucket_and_all_items(object_storage_client, create_bucket_request.name)


# Since we've created a reasonable number of objects in this test suite, it's a good opportunity to test using the --all and --limit parameters
@util.skip_while_rerecording
def test_list_all_objects_operations(vcr_fixture):
    result = invoke(['os', 'object', 'list', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--all'])
    parsed_result = json.loads(result.output)
    assert len(parsed_result['data']) == OBJECTS_TO_CREATE_IN_BUCKET_FOR_BULK_GET
    assert 'next-start-with' not in result.output

    result = invoke(['os', 'object', 'list', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--all', '--page-size', '20'])
    parsed_result = json.loads(result.output)
    assert len(parsed_result['data']) == OBJECTS_TO_CREATE_IN_BUCKET_FOR_BULK_GET
    assert 'next-start-with' not in result.output

    result = invoke(['os', 'object', 'list', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--limit', '47'])
    parsed_result = json.loads(result.output)
    assert len(parsed_result['data']) == 47
    assert 'next-start-with' in result.output

    result = invoke(['os', 'object', 'list', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--limit', '33', '--page-size', '3'])
    parsed_result = json.loads(result.output)
    assert len(parsed_result['data']) == 33
    assert 'next-start-with' in result.output


# Bulk puts objects, uses multipart where appropriate (when we breach the default of 128MiB)
@util.skip_while_rerecording
def test_bulk_put_default_options():
    result = invoke(['os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--src-dir', root_bulk_put_folder])

    # No failures or skips and we uploaded everything
    parsed_result = parse_json_response_from_mixed_output(result.output)
    assert parsed_result['skipped-objects'] == []
    assert parsed_result['upload-failures'] == {}
    assert len(parsed_result['uploaded-objects']) == get_count_of_files_in_folder_and_subfolders(root_bulk_put_folder)

    # Pull everything down and verify that the files match (everything in source appears in destination and they are equal)
    download_folder = 'tests/temp/verify_files_{}'.format(bulk_put_bucket_name)
    invoke(['os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--download-dir', download_folder])
    object_name_set = set()
    for dir_name, subdir_list, file_list in os.walk(root_bulk_put_folder):
        for file in file_list:
            source_file_path = os.path.join(dir_name, file)
            downloaded_file_path = source_file_path.replace(root_bulk_put_folder, download_folder)

            assert os.path.exists(downloaded_file_path)
            assert filecmp.cmp(source_file_path, downloaded_file_path, shallow=False)

            # Sanity check that we're reporting back that we uploaded the right files
            assert get_object_name_from_path(root_bulk_put_folder, source_file_path) in parsed_result['uploaded-objects']
            object_name_set.add(get_object_name_from_path(root_bulk_put_folder, source_file_path))

    # If we try and put it in the same bucket without --overwrite then everything should be skipped. There should be prompts
    result = invoke(['os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--src-dir', root_bulk_put_folder])
    parsed_result = parse_json_response_from_mixed_output(result.output)
    assert 'Are you sure you want to overwrite it?' in result.output
    assert set(parsed_result['skipped-objects']) == object_name_set
    assert parsed_result['upload-failures'] == {}
    assert parsed_result['uploaded-objects'] == {}

    # If we say to --no-overwrite then everything should be skipped. There should be no prompts
    result = invoke(['os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--src-dir', root_bulk_put_folder, '--no-overwrite'])
    parsed_result = parse_json_response_from_mixed_output(result.output)
    assert 'Are you sure you want to overwrite it?' not in result.output
    assert set(parsed_result['skipped-objects']) == object_name_set
    assert parsed_result['upload-failures'] == {}
    assert parsed_result['uploaded-objects'] == {}

    # Now we force it
    result = invoke(['os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--src-dir', root_bulk_put_folder, '--overwrite'])
    parsed_result = parse_json_response_from_mixed_output(result.output)
    assert parsed_result['skipped-objects'] == []
    assert parsed_result['upload-failures'] == {}
    assert len(parsed_result['uploaded-objects']) == len(object_name_set)
    for object_name in object_name_set:
        assert object_name in parsed_result['uploaded-objects']

    shutil.rmtree(download_folder)


# Bulk puts objects with --content-type as auto
@util.skip_while_rerecording
def test_bulk_put_auto_content_type():
    result = invoke(['os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--src-dir', root_bulk_put_folder, '--content-type', 'auto', '--overwrite'])

    # No failures or skips and we uploaded everything
    parsed_result = parse_json_response_from_mixed_output(result.output)
    assert parsed_result['skipped-objects'] == []
    assert parsed_result['upload-failures'] == {}
    assert len(parsed_result['uploaded-objects']) == get_count_of_files_in_folder_and_subfolders(root_bulk_put_folder)

    # Pull everything down and verify that the files match (everything in source appears in destination and they are equal)
    download_folder = 'tests/temp/verify_files_{}'.format(bulk_put_bucket_name)
    invoke(['os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--download-dir', download_folder])
    object_name_set = set()
    for dir_name, subdir_list, file_list in os.walk(root_bulk_put_folder):
        for file in file_list:
            source_file_path = os.path.join(dir_name, file)
            downloaded_file_path = source_file_path.replace(root_bulk_put_folder, download_folder)

            assert os.path.exists(downloaded_file_path)
            assert filecmp.cmp(source_file_path, downloaded_file_path, shallow=False)
            assert guess_type(source_file_path) == guess_type(downloaded_file_path)

            # Sanity check that we're reporting back that we uploaded the right files
            assert get_object_name_from_path(root_bulk_put_folder, source_file_path) in parsed_result['uploaded-objects']
            object_name_set.add(get_object_name_from_path(root_bulk_put_folder, source_file_path))

    shutil.rmtree(download_folder)


# Tests that multipart params are applied:
#
#   - Try to upload with a part size of 10MiB (this will force the large and mid-sized files to be multipart uploaded)
#   - Try to upload with multipart disabled
@util.skip_while_rerecording
def test_bulk_put_with_multipart_params(object_storage_client):
    create_bucket_request = oci.object_storage.models.CreateBucketDetails()
    create_bucket_request.name = 'ObjectStorageBulkPutMultipartsTest_{}'.format(util.random_number_string())
    create_bucket_request.compartment_id = util.COMPARTMENT_ID
    util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, create_bucket_request.name)
    object_storage_client.create_bucket(util.NAMESPACE, create_bucket_request)

    result = invoke([
        'os', 'object', 'bulk-upload',
        '--namespace', util.NAMESPACE,
        '--bucket-name', create_bucket_request.name,
        '--src-dir', root_bulk_put_folder,
        '--part-size', '10'
    ])
    parsed_result = parse_json_response_from_mixed_output(result.output)
    assert parsed_result['skipped-objects'] == []
    assert parsed_result['upload-failures'] == {}
    assert len(parsed_result['uploaded-objects']) == get_count_of_files_in_folder_and_subfolders(root_bulk_put_folder)

    result = invoke([
        'os', 'object', 'bulk-upload',
        '--namespace', util.NAMESPACE,
        '--bucket-name', create_bucket_request.name,
        '--src-dir', root_bulk_put_folder,
        '--no-multipart',
        '--overwrite'
    ])
    parsed_result = parse_json_response_from_mixed_output(result.output)
    assert parsed_result['skipped-objects'] == []
    assert parsed_result['upload-failures'] == {}
    assert len(parsed_result['uploaded-objects']) == get_count_of_files_in_folder_and_subfolders(root_bulk_put_folder)

    delete_bucket_and_all_items(object_storage_client, create_bucket_request.name)


@util.skip_while_rerecording
def test_bulk_put_with_prefix():
    result = invoke(['os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--src-dir', root_bulk_put_folder, '--object-prefix', 'bulk_put_prefix_test/'])

    # No failures or skips and we uploaded everything
    parsed_result = parse_json_response_from_mixed_output(result.output)
    assert parsed_result['skipped-objects'] == []
    assert parsed_result['upload-failures'] == {}
    assert len(parsed_result['uploaded-objects']) == get_count_of_files_in_folder_and_subfolders(root_bulk_put_folder)

    download_folder = 'tests/temp/verify_files_bulk_put_prefix_{}'.format(bulk_put_bucket_name)
    invoke(['os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--download-dir', download_folder, '--prefix', 'bulk_put_prefix_test/'])

    actual_download_folder = os.path.join(download_folder, 'bulk_put_prefix_test')
    for dir_name, subdir_list, file_list in os.walk(root_bulk_put_folder):
        for file in file_list:
            source_file_path = os.path.join(dir_name, file)
            downloaded_file_path = source_file_path.replace(root_bulk_put_folder, actual_download_folder)

            assert os.path.exists(downloaded_file_path)
            assert filecmp.cmp(source_file_path, downloaded_file_path, shallow=False)

            # Sanity check that we're reporting back that we uploaded the right files
            assert 'bulk_put_prefix_test/{}'.format(get_object_name_from_path(root_bulk_put_folder, source_file_path)) in parsed_result['uploaded-objects']

    shutil.rmtree(download_folder)


@util.skip_while_rerecording
def test_bulk_put_with_non_existent_folder():
    fake_directory = 'tests/folder/not/exist'
    result = invoke(['os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--src-dir', fake_directory])

    assert 'UsageError' in result.output
    assert 'The specified --src-dir {} (expanded to: {}) does not exist'.format(fake_directory, fake_directory) in result.output


@util.skip_while_rerecording
def test_bulk_put_get_delete_with_inclusions(object_storage_client):
    inclusion_test_folder = os.path.join('tests', 'temp', 'os_bulk_upload_inclusion_test')
    if not os.path.exists(inclusion_test_folder):
        os.makedirs(inclusion_test_folder)

    # Make some files for include/exclude
    folders_to_files = {
        '': ['test_file1.txt', 'test_file2.png'],
        'subfolder': ['blah.pdf', 'hello.txt', 'testfile3.png'],
        'subfolder/subfolder2': ['xyz.jpg', 'blag.txt', 'byz.jpg', 'testfile4.png']
    }
    for folder, files in six.iteritems(folders_to_files):
        folder_path = os.path.join(inclusion_test_folder, folder)
        if not os.path.exists(folder_path):
            os.makedirs(folder_path)

        for file in files:
            file_path = os.path.join(folder_path, file)
            with open(file_path, 'w') as f:
                # For non-text extension types this won't create a valid file, but for testing is probably OK
                f.write(generate_random_string(CONTENT_STRING_LENGTH))

    result = invoke([
        'os',
        'object',
        'bulk-upload',
        '--namespace', util.NAMESPACE,
        '--bucket-name', bulk_put_bucket_name,
        '--src-dir', inclusion_test_folder,
        '--object-prefix', 'inclusion_test/',
        '--include', '*.txt',  # Matches test_file1.txt, subfolder/hello.txt, subfolder/subfolder2/blag.txt
        '--include', 'subfolder/*.png',  # Matches subfolder/testfile3.png, subfolder/subfolder2/testfile4.png
        '--include', 'subfolder/[b]lah.pdf',  # Matches subfolder/blah.pdf
        '--include', '*/[ax]yz.jpg'  # Matches subfolder/subfolder2/xyz.jpg
    ])
    parsed_result = parse_json_response_from_mixed_output(result.output)
    assert parsed_result['skipped-objects'] == []
    assert parsed_result['upload-failures'] == {}

    expected_uploaded_files = [
        '{}{}'.format('inclusion_test/', 'test_file1.txt'),
        '{}{}'.format('inclusion_test/', 'subfolder/hello.txt'),
        '{}{}'.format('inclusion_test/', 'subfolder/subfolder2/blag.txt'),
        '{}{}'.format('inclusion_test/', 'subfolder/testfile3.png'),
        '{}{}'.format('inclusion_test/', 'subfolder/subfolder2/testfile4.png'),
        '{}{}'.format('inclusion_test/', 'subfolder/blah.pdf'),
        '{}{}'.format('inclusion_test/', 'subfolder/subfolder2/xyz.jpg')
    ]

    # Check that we uploaded what we said we did
    assert len(parsed_result['uploaded-objects']) == len(expected_uploaded_files)
    for f in expected_uploaded_files:
        assert f in parsed_result['uploaded-objects']

    download_folder_base = os.path.join('tests', 'temp', 'verify_os_bulk_upload_inclusion_test')
    verify_downloaded_folders_for_inclusion_exclusion_tests(
        expected_uploaded_files=expected_uploaded_files,
        source_folder=inclusion_test_folder,
        download_folder=download_folder_base,
        download_prefix_no_slash='inclusion_test'
    )

    # Download objects with inclusions to make sure that works
    target_download_folder = os.path.join(download_folder_base, 'get_with_include')
    invoke([
        'os', 'object', 'bulk-download',
        '--namespace', util.NAMESPACE,
        '--bucket-name', bulk_put_bucket_name,
        '--download-dir', target_download_folder,
        '--prefix', 'inclusion_test/',
        '--include', '*.txt',
        '--include', 'subfolder/*.png',
        '--include', 'subfolder/blah.pdf',
    ])
    expected_uploaded_files.remove('{}{}'.format('inclusion_test/', 'subfolder/subfolder2/xyz.jpg'))  # This is not in our --include switches
    assert not os.path.exists(os.path.join(target_download_folder, 'inclusion_test', 'subfolder', 'subfolder2', 'xyz.jpg'))
    for expected_file in expected_uploaded_files:
        target_file = os.path.join(target_download_folder, expected_file)
        original_file = target_file.replace(os.path.join(target_download_folder, 'inclusion_test'), inclusion_test_folder)

        assert os.path.exists(target_file)
        assert filecmp.cmp(original_file, target_file, shallow=False)

    # Download a specific object with inclusions
    invoke([
        'os', 'object', 'bulk-download',
        '--namespace', util.NAMESPACE,
        '--bucket-name', bulk_put_bucket_name,
        '--download-dir', target_download_folder,
        '--prefix', 'inclusion_test/',
        '--include', 'subfolder/subfolder2/xyz.jpg'
    ])
    assert os.path.exists(os.path.join(target_download_folder, 'inclusion_test', 'subfolder', 'subfolder2', 'xyz.jpg'))

    # Delete objects with inclusions
    result = invoke([
        'os', 'object', 'bulk-delete',
        '--namespace', util.NAMESPACE,
        '--bucket-name', bulk_put_bucket_name,
        '--prefix', 'inclusion_test/',
        '--include', '*.txt',
        '--include', 'subfolder/blah.pdf',
        '--dry-run'
    ])
    parsed_dry_run_result = parse_json_response_from_mixed_output(result.output)
    assert len(parsed_dry_run_result['deleted-objects']) == 4

    result = invoke([
        'os', 'object', 'bulk-delete',
        '--namespace', util.NAMESPACE,
        '--bucket-name', bulk_put_bucket_name,
        '--prefix', 'inclusion_test/',
        '--include', '*.txt',
        '--include', 'subfolder/blah.pdf',
        '--force'
    ])
    parsed_result = parse_json_response_from_mixed_output(result.output)
    assert parsed_result['delete-failures'] == {}
    assert set(parsed_result['deleted-objects']) == set(parsed_dry_run_result['deleted-objects'])

    list_objects_responses = oci_cli_object_storage.objectstorage_cli_extended.retrying_list_objects(
        client=object_storage_client,
        request_id=None,
        namespace=util.NAMESPACE,
        bucket_name=bulk_put_bucket_name,
        prefix='inclusion_test/',
        start=None,
        end=None,
        limit=1000,
        delimiter=None,
        fields='name',
        retrieve_all=True
    )
    remaining_objects = []
    for response in list_objects_responses:
        remaining_objects.extend(map(lambda obj: obj.name, response.data.objects))
    assert len(remaining_objects) == 3
    assert '{}{}'.format('inclusion_test/', 'subfolder/testfile3.png') in remaining_objects
    assert '{}{}'.format('inclusion_test/', 'subfolder/subfolder2/testfile4.png') in remaining_objects
    assert '{}{}'.format('inclusion_test/', 'subfolder/subfolder2/xyz.jpg') in remaining_objects

    shutil.rmtree(target_download_folder)
    shutil.rmtree(inclusion_test_folder)


@util.skip_while_rerecording
def test_bulk_put_get_delete_with_exclusions(object_storage_client):
    exclusion_test_folder = os.path.join('tests', 'temp', 'os_bulk_upload_exclusion_test')
    if not os.path.exists(exclusion_test_folder):
        os.makedirs(exclusion_test_folder)

    # Make some files for include/exclude
    folders_to_files = {
        '': ['test_file1.txt', 'test_file2.png'],
        'subfolder': ['blah.pdf', 'hello.txt', 'testfile3.png'],
        'subfolder/subfolder2': ['xyz.jpg', 'blag.txt', 'byz.jpg', 'testfile4.png']
    }
    for folder, files in six.iteritems(folders_to_files):
        folder_path = os.path.join(exclusion_test_folder, folder)
        if not os.path.exists(folder_path):
            os.makedirs(folder_path)

        for file in files:
            file_path = os.path.join(folder_path, file)
            with open(file_path, 'w') as f:
                # For non-text extension types this won't create a valid file, but for testing is probably OK
                f.write(generate_random_string(CONTENT_STRING_LENGTH))

    result = invoke([
        'os',
        'object',
        'bulk-upload',
        '--namespace', util.NAMESPACE,
        '--bucket-name', bulk_put_bucket_name,
        '--src-dir', exclusion_test_folder,
        '--object-prefix', 'exclusion_test/',
        '--exclude', '*.txt',
        '--exclude', '*.ps1',  # Shouldn't match anything
        '--exclude', 'subfolder/subfolder2/xyz.jpg',
        '--exclude', 'subfolder/[spqr]lah.pdf'  # blah.pdf should still be included because it's not slah.pdf, plah.pdf, qlah.pdf or rlah.pdf
    ])
    parsed_result = parse_json_response_from_mixed_output(result.output)
    assert parsed_result['skipped-objects'] == []
    assert parsed_result['upload-failures'] == {}

    expected_uploaded_files = [
        '{}{}'.format('exclusion_test/', 'test_file2.png'),
        '{}{}'.format('exclusion_test/', 'subfolder/blah.pdf'),
        '{}{}'.format('exclusion_test/', 'subfolder/testfile3.png'),
        '{}{}'.format('exclusion_test/', 'subfolder/subfolder2/byz.jpg'),
        '{}{}'.format('exclusion_test/', 'subfolder/subfolder2/testfile4.png')
    ]

    # Check that we uploaded what we said we did
    assert len(parsed_result['uploaded-objects']) == len(expected_uploaded_files)
    for f in expected_uploaded_files:
        assert f in parsed_result['uploaded-objects']

    download_folder_base = os.path.join('tests', 'temp', 'verify_os_bulk_upload_exclusion_test')
    verify_downloaded_folders_for_inclusion_exclusion_tests(
        expected_uploaded_files=expected_uploaded_files,
        source_folder=exclusion_test_folder,
        download_folder=download_folder_base,
        download_prefix_no_slash='exclusion_test'
    )

    # Download objects with exclusions to make sure that works
    target_download_folder = os.path.join(download_folder_base, 'get_with_exclude')
    invoke([
        'os', 'object', 'bulk-download',
        '--namespace', util.NAMESPACE,
        '--bucket-name', bulk_put_bucket_name,
        '--download-dir', target_download_folder,
        '--prefix', 'exclusion_test/',
        '--exclude', '*.jpg',
        '--exclude', 'subfolder/subfolder2/*.png',
        '--exclude', 'subfolder/blah.pdf',
    ])

    assert not os.path.exists(os.path.join(target_download_folder, 'exclusion_test', 'subfolder', 'blah.pdf'))
    assert not os.path.exists(os.path.join(target_download_folder, 'exclusion_test', 'subfolder', 'subfolder2', 'byz.jpg'))
    assert not os.path.exists(os.path.join(target_download_folder, 'exclusion_test', 'subfolder', 'subfolder2', 'testfile4.png'))

    assert get_count_of_files_in_folder_and_subfolders(target_download_folder) == 2
    assert os.path.exists(os.path.join(target_download_folder, 'exclusion_test', 'test_file2.png'))
    assert os.path.exists(os.path.join(target_download_folder, 'exclusion_test', 'subfolder', 'testfile3.png'))

    assert filecmp.cmp(
        os.path.join(exclusion_test_folder, 'test_file2.png'),
        os.path.join(target_download_folder, 'exclusion_test', 'test_file2.png')
    )
    assert filecmp.cmp(
        os.path.join(exclusion_test_folder, 'subfolder', 'testfile3.png'),
        os.path.join(target_download_folder, 'exclusion_test', 'subfolder', 'testfile3.png')
    )

    # Delete objects with exclusions
    result = invoke([
        'os', 'object', 'bulk-delete',
        '--namespace', util.NAMESPACE,
        '--bucket-name', bulk_put_bucket_name,
        '--prefix', 'exclusion_test/',
        '--exclude', '*.jpg',
        '--exclude', 'subfolder/blah.pdf',
        '--dry-run'
    ])
    parsed_dry_run_result = parse_json_response_from_mixed_output(result.output)
    assert len(parsed_dry_run_result['deleted-objects']) == 3

    result = invoke([
        'os', 'object', 'bulk-delete',
        '--namespace', util.NAMESPACE,
        '--bucket-name', bulk_put_bucket_name,
        '--prefix', 'exclusion_test/',
        '--exclude', '*.jpg',
        '--exclude', 'subfolder/blah.pdf',
        '--force'
    ])
    parsed_result = parse_json_response_from_mixed_output(result.output)
    assert parsed_result['delete-failures'] == {}
    assert set(parsed_result['deleted-objects']) == set(parsed_dry_run_result['deleted-objects'])

    list_objects_responses = oci_cli_object_storage.objectstorage_cli_extended.retrying_list_objects(
        client=object_storage_client,
        request_id=None,
        namespace=util.NAMESPACE,
        bucket_name=bulk_put_bucket_name,
        prefix='exclusion_test/',
        start=None,
        end=None,
        limit=1000,
        delimiter=None,
        fields='name',
        retrieve_all=True
    )
    remaining_objects = []
    for response in list_objects_responses:
        remaining_objects.extend(map(lambda obj: obj.name, response.data.objects))
    assert len(remaining_objects) == 2
    assert '{}{}'.format('exclusion_test/', 'subfolder/blah.pdf') in remaining_objects
    assert '{}{}'.format('exclusion_test/', 'subfolder/subfolder2/byz.jpg') in remaining_objects

    shutil.rmtree(target_download_folder)
    shutil.rmtree(exclusion_test_folder)


@util.skip_while_rerecording
def test_delete_when_no_objects_in_bucket(vcr_fixture, object_storage_client):
    create_bucket_request = oci.object_storage.models.CreateBucketDetails()
    create_bucket_request.name = 'ObjectStorageBulkDelete_{}'.format(util.random_number_string())
    create_bucket_request.compartment_id = util.COMPARTMENT_ID
    object_storage_client.create_bucket(util.NAMESPACE, create_bucket_request)

    result = invoke(['os', 'object', 'bulk-delete', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name])
    assert 'There are no objects to delete in {}'.format(create_bucket_request.name) in result.output

    delete_bucket_and_all_items(object_storage_client, create_bucket_request.name)


@util.skip_while_rerecording
def test_delete_dry_run(vcr_fixture):
    # Dry-run against entire bucket
    result = invoke(['os', 'object', 'bulk-delete', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--dry-run'])
    parsed_result = json.loads(result.output)
    assert set(parsed_result['deleted-objects']) == set(bulk_get_object_to_content.keys())

    # Dry-run against a folder and all subfolders
    result = invoke(['os', 'object', 'bulk-delete', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--prefix', 'a/b/', '--dry-run'])
    parsed_result = json.loads(result.output)
    expected_objects = set().union(bulk_get_prefix_to_object['a/b'], bulk_get_prefix_to_object['a/b/c'], bulk_get_prefix_to_object['a/b/c/d'])
    assert set(parsed_result['deleted-objects']) == expected_objects

    # Dry-run against a folder and no subfolders
    result = invoke(['os', 'object', 'bulk-delete', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--prefix', 'a/b/', '--delimiter', '/', '--dry-run'])
    parsed_result = json.loads(result.output)
    assert set(parsed_result['deleted-objects']) == set(bulk_get_prefix_to_object['a/b'])


@util.skip_while_rerecording
def test_delete(object_storage_client):
    create_bucket_request = oci.object_storage.models.CreateBucketDetails()
    create_bucket_request.name = 'ObjectStorageBulkDelete_{}'.format(random.randint(0, 1000000))
    create_bucket_request.compartment_id = util.COMPARTMENT_ID
    util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, create_bucket_request.name)
    object_storage_client.create_bucket(util.NAMESPACE, create_bucket_request)

    invoke(['os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name, '--src-dir', root_bulk_put_folder])
    num_objects_to_delete = get_count_of_files_in_folder_and_subfolders(root_bulk_put_folder)

    # Sanity check that the bucket has things in it
    assert get_number_of_objects_in_bucket(object_storage_client, create_bucket_request.name) > 0

    result = invoke(['os', 'object', 'bulk-delete', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name])
    if num_objects_to_delete >= 1000:
        confirm_prompt = 'WARNING: This command will delete at least {} objects. Are you sure you wish to continue?'.format(num_objects_to_delete)
    else:
        confirm_prompt = 'WARNING: This command will delete {} objects. Are you sure you wish to continue?'.format(num_objects_to_delete)
    assert confirm_prompt in result.output

    result = invoke(['os', 'object', 'bulk-delete', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name, '--force'])
    parsed_result = parse_json_response_from_mixed_output(result.output)
    assert parsed_result['delete-failures'] == {}
    assert len(parsed_result['deleted-objects']) == num_objects_to_delete

    # Check that the bucket is now empty
    assert get_number_of_objects_in_bucket(object_storage_client, create_bucket_request.name) == 0

    delete_bucket_and_all_items(object_storage_client, create_bucket_request.name)


@util.skip_while_rerecording
def test_bulk_operation_table_output_query(object_storage_client):
    create_bucket_request = oci.object_storage.models.CreateBucketDetails()
    create_bucket_request.name = 'ObjectStorageTableOutput_{}'.format(util.random_number_string())
    create_bucket_request.compartment_id = util.COMPARTMENT_ID
    util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, create_bucket_request.name)
    object_storage_client.create_bucket(util.NAMESPACE, create_bucket_request)

    result = invoke(['os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name, '--src-dir', root_bulk_put_folder, '--output', 'table', '--query', "[?action=='Uploaded'].{file: file, \"opc-content-md5\": \"opc-content-md5\"}"])
    assert 'file' in result.output
    assert 'opc-content-md5' in result.output
    assert 'etag' not in result.output

    result = invoke(['os', 'object', 'bulk-delete', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--dry-run', '--output', 'table'])
    assert 'action' in result.output
    assert 'object' in result.output
    assert '/a/Object_1' in result.output

    result = invoke(['os', 'object', 'bulk-delete', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--dry-run', '--output', 'table', '--query', "[?object=='Object_0'][object]"])
    assert 'action' not in result.output
    assert '/a/Object_1' not in result.output
    assert 'Object_0' in result.output

    target_download_folder = os.path.join('tests', 'temp', create_bucket_request.name)
    result = invoke([
        'os', 'object', 'bulk-download',
        '--namespace', util.NAMESPACE,
        '--bucket-name', create_bucket_request.name,
        '--download-dir', target_download_folder,
        '--output', 'table',
    ])

    delete_bucket_and_all_items(object_storage_client, create_bucket_request.name)

    shutil.rmtree(target_download_folder)


def invoke(commands, debug=False, ** args):
    if debug is True:
        commands = ['--debug'] + commands
    return util.invoke_command(commands, ** args)


def get_count_of_files_in_folder_and_subfolders(directory):
    file_count = 0
    for dir_name, subdir_list, file_list in os.walk(directory):
        file_count = file_count + len(file_list)

    return file_count


def generate_random_string(length):
    if test_config_container.using_vcr_with_mock_responses():
        return 'a' * length
    else:
        return ''.join(random.choice(string.ascii_lowercase) for i in range(length))


# Pull JSON data out of output which may have stuff other than JSON in it. Assumes that nothing
# comes after the JSON data
def parse_json_response_from_mixed_output(output):
    lines = output.split('\n')
    json_str = ''
    object_begun = False
    for line in lines:
        if object_begun or line.startswith('{'):
            object_begun = True
            json_str += line

    return json.loads(json_str)


# For the bulk operations, object names are taken from the file path of the thing we uploaded. Normalize to
# / in the paths (Windows can go both ways) then chop the front bit off
def get_object_name_from_path(path_root, full_path):
    return full_path.replace(os.sep, '/').replace(path_root + '/', '')


def delete_bucket_and_all_items(object_storage_client, bucket_name):
    list_object_responses = oci_cli_object_storage.objectstorage_cli_extended.retrying_list_objects(
        client=object_storage_client,
        request_id=None,
        namespace=util.NAMESPACE,
        bucket_name=bucket_name,
        prefix=None,
        start=None,
        end=None,
        limit=1000,
        delimiter=None,
        fields='name',
        retrieve_all=True
    )
    for response in list_object_responses:
        for obj in response.data.objects:
            object_storage_client.delete_object(util.NAMESPACE, bucket_name, obj.name)
    object_storage_client.delete_bucket(util.NAMESPACE, bucket_name)


def get_number_of_objects_in_bucket(object_storage_client, bucket_name):
    list_object_responses = oci_cli_object_storage.objectstorage_cli_extended.retrying_list_objects(
        client=object_storage_client,
        request_id=None,
        namespace=util.NAMESPACE,
        bucket_name=bucket_name,
        prefix=None,
        start=None,
        end=None,
        limit=1000,
        delimiter=None,
        fields='name',
        retrieve_all=True
    )
    num_objects_in_bucket = 0
    for response in list_object_responses:
        num_objects_in_bucket = num_objects_in_bucket + len(response.data.objects)

    return num_objects_in_bucket


def verify_downloaded_folders_for_inclusion_exclusion_tests(expected_uploaded_files, source_folder, download_folder, download_prefix_no_slash):
    # Download uploaded files and check they are the same
    invoke(['os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--download-dir', download_folder, '--prefix', download_prefix_no_slash + '/'])

    # The strings in the expected_uploaded_files array have a "/" in them, but this doesn't match with paths on Windows. Using normpath converts these of
    # "\" on Windows and so our matching/comparison works. For Linux/Unix/macOS this doesn't appear to have an impact
    normalized_expected_uploaded_files = []
    for euf in expected_uploaded_files:
        normalized_expected_uploaded_files.append(os.path.normpath(euf))

    actual_download_folder = os.path.join(download_folder, download_prefix_no_slash)
    files_compared = 0
    for dir_name, subdir_list, file_list in os.walk(source_folder):
        for file in file_list:
            source_file_path = os.path.join(dir_name, file)
            downloaded_file_path = source_file_path.replace(source_folder, actual_download_folder)

            if downloaded_file_path.replace(actual_download_folder, download_prefix_no_slash) in normalized_expected_uploaded_files:
                files_compared += 1
                assert os.path.exists(downloaded_file_path)
                assert filecmp.cmp(source_file_path, downloaded_file_path, shallow=False)
    assert files_compared == len(expected_uploaded_files)

    shutil.rmtree(actual_download_folder)

[((54, 1, 54, 45), 'pytest.fixture', 'pytest.fixture', (), '', False, 'import pytest\n'), ((59, 28, 59, 75), 'oci.object_storage.models.CreateBucketDetails', 'oci.object_storage.models.CreateBucketDetails', ({}, {}), '()', False, 'import oci\n'), ((62, 4, 62, 112), 'tests.util.clear_test_data', 'util.clear_test_data', ({(62, 25, 62, 46): 'object_storage_client', (62, 48, 62, 62): 'util.NAMESPACE', (62, 64, 62, 83): 'util.COMPARTMENT_ID', (62, 85, 62, 111): 'create_bucket_request.name'}, {}), '(object_storage_client, util.NAMESPACE, util.\n    COMPARTMENT_ID, create_bucket_request.name)', False, 'from tests import util\n'), ((97, 28, 97, 75), 'oci.object_storage.models.CreateBucketDetails', 'oci.object_storage.models.CreateBucketDetails', ({}, {}), '()', False, 'import oci\n'), ((100, 4, 100, 112), 'tests.util.clear_test_data', 'util.clear_test_data', ({(100, 25, 100, 46): 'object_storage_client', (100, 48, 100, 62): 'util.NAMESPACE', (100, 64, 100, 83): 'util.COMPARTMENT_ID', (100, 85, 100, 111): 'create_bucket_request.name'}, {}), '(object_storage_client, util.NAMESPACE, util.\n    COMPARTMENT_ID, create_bucket_request.name)', False, 'from tests import util\n'), ((133, 4, 133, 39), 'shutil.rmtree', 'shutil.rmtree', ({(133, 18, 133, 38): 'root_bulk_put_folder'}, {}), '(root_bulk_put_folder)', False, 'import shutil\n'), ((167, 4, 167, 34), 'shutil.rmtree', 'shutil.rmtree', ({(167, 18, 167, 33): 'download_folder'}, {}), '(download_folder)', False, 'import shutil\n'), ((197, 4, 197, 34), 'shutil.rmtree', 'shutil.rmtree', ({(197, 18, 197, 33): 'download_folder'}, {}), '(download_folder)', False, 'import shutil\n'), ((213, 4, 213, 34), 'shutil.rmtree', 'shutil.rmtree', ({(213, 18, 213, 33): 'download_folder'}, {}), '(download_folder)', False, 'import shutil\n'), ((241, 4, 241, 34), 'shutil.rmtree', 'shutil.rmtree', ({(241, 18, 241, 33): 'download_folder'}, {}), '(download_folder)', False, 'import shutil\n'), ((251, 4, 251, 34), 'shutil.rmtree', 'shutil.rmtree', ({(251, 18, 251, 33): 'download_folder'}, {}), '(download_folder)', False, 'import shutil\n'), ((256, 28, 256, 75), 'oci.object_storage.models.CreateBucketDetails', 'oci.object_storage.models.CreateBucketDetails', ({}, {}), '()', False, 'import oci\n'), ((259, 4, 259, 112), 'tests.util.clear_test_data', 'util.clear_test_data', ({(259, 25, 259, 46): 'object_storage_client', (259, 48, 259, 62): 'util.NAMESPACE', (259, 64, 259, 83): 'util.COMPARTMENT_ID', (259, 85, 259, 111): 'create_bucket_request.name'}, {}), '(object_storage_client, util.NAMESPACE, util.\n    COMPARTMENT_ID, create_bucket_request.name)', False, 'from tests import util\n'), ((262, 26, 262, 84), 'os.path.join', 'os.path.join', ({(262, 39, 262, 46): '"""tests"""', (262, 48, 262, 54): '"""temp"""', (262, 56, 262, 83): '"""multipart_get_large_files"""'}, {}), "('tests', 'temp', 'multipart_get_large_files')", False, 'import os\n'), ((279, 28, 279, 93), 'os.path.join', 'os.path.join', ({(279, 41, 279, 48): '"""tests"""', (279, 50, 279, 56): '"""temp"""', (279, 58, 279, 92): '"""multipart_get_large_files_verify"""'}, {}), "('tests', 'temp', 'multipart_get_large_files_verify')", False, 'import os\n'), ((291, 4, 291, 38), 'shutil.rmtree', 'shutil.rmtree', ({(291, 18, 291, 37): 'large_file_root_dir'}, {}), '(large_file_root_dir)', False, 'import shutil\n'), ((292, 4, 292, 40), 'shutil.rmtree', 'shutil.rmtree', ({(292, 18, 292, 39): 'large_file_verify_dir'}, {}), '(large_file_verify_dir)', False, 'import shutil\n'), ((301, 20, 301, 45), 'json.loads', 'json.loads', ({(301, 31, 301, 44): 'result.output'}, {}), '(result.output)', False, 'import json\n'), ((306, 20, 306, 45), 'json.loads', 'json.loads', ({(306, 31, 306, 44): 'result.output'}, {}), '(result.output)', False, 'import json\n'), ((311, 20, 311, 45), 'json.loads', 'json.loads', ({(311, 31, 311, 44): 'result.output'}, {}), '(result.output)', False, 'import json\n'), ((316, 20, 316, 45), 'json.loads', 'json.loads', ({(316, 31, 316, 44): 'result.output'}, {}), '(result.output)', False, 'import json\n'), ((336, 44, 336, 73), 'os.walk', 'os.walk', ({(336, 52, 336, 72): 'root_bulk_put_folder'}, {}), '(root_bulk_put_folder)', False, 'import os\n'), ((373, 4, 373, 34), 'shutil.rmtree', 'shutil.rmtree', ({(373, 18, 373, 33): 'download_folder'}, {}), '(download_folder)', False, 'import shutil\n'), ((391, 44, 391, 73), 'os.walk', 'os.walk', ({(391, 52, 391, 72): 'root_bulk_put_folder'}, {}), '(root_bulk_put_folder)', False, 'import os\n'), ((404, 4, 404, 34), 'shutil.rmtree', 'shutil.rmtree', ({(404, 18, 404, 33): 'download_folder'}, {}), '(download_folder)', False, 'import shutil\n'), ((413, 28, 413, 75), 'oci.object_storage.models.CreateBucketDetails', 'oci.object_storage.models.CreateBucketDetails', ({}, {}), '()', False, 'import oci\n'), ((416, 4, 416, 112), 'tests.util.clear_test_data', 'util.clear_test_data', ({(416, 25, 416, 46): 'object_storage_client', (416, 48, 416, 62): 'util.NAMESPACE', (416, 64, 416, 83): 'util.COMPARTMENT_ID', (416, 85, 416, 111): 'create_bucket_request.name'}, {}), '(object_storage_client, util.NAMESPACE, util.\n    COMPARTMENT_ID, create_bucket_request.name)', False, 'from tests import util\n'), ((460, 29, 460, 82), 'os.path.join', 'os.path.join', ({(460, 42, 460, 57): 'download_folder', (460, 59, 460, 81): '"""bulk_put_prefix_test"""'}, {}), "(download_folder, 'bulk_put_prefix_test')", False, 'import os\n'), ((461, 44, 461, 73), 'os.walk', 'os.walk', ({(461, 52, 461, 72): 'root_bulk_put_folder'}, {}), '(root_bulk_put_folder)', False, 'import os\n'), ((472, 4, 472, 34), 'shutil.rmtree', 'shutil.rmtree', ({(472, 18, 472, 33): 'download_folder'}, {}), '(download_folder)', False, 'import shutil\n'), ((486, 28, 486, 90), 'os.path.join', 'os.path.join', ({(486, 41, 486, 48): '"""tests"""', (486, 50, 486, 56): '"""temp"""', (486, 58, 486, 89): '"""os_bulk_upload_inclusion_test"""'}, {}), "('tests', 'temp', 'os_bulk_upload_inclusion_test')", False, 'import os\n'), ((496, 25, 496, 56), 'six.iteritems', 'six.iteritems', ({(496, 39, 496, 55): 'folders_to_files'}, {}), '(folders_to_files)', False, 'import six\n'), ((539, 27, 539, 96), 'os.path.join', 'os.path.join', ({(539, 40, 539, 47): '"""tests"""', (539, 49, 539, 55): '"""temp"""', (539, 57, 539, 95): '"""verify_os_bulk_upload_inclusion_test"""'}, {}), "('tests', 'temp', 'verify_os_bulk_upload_inclusion_test')", False, 'import os\n'), ((548, 29, 548, 83), 'os.path.join', 'os.path.join', ({(548, 42, 548, 62): 'download_folder_base', (548, 64, 548, 82): '"""get_with_include"""'}, {}), "(download_folder_base, 'get_with_include')", False, 'import os\n'), ((605, 29, 617, 5), 'services.object_storage.src.oci_cli_object_storage.objectstorage_cli_extended.retrying_list_objects', 'oci_cli_object_storage.objectstorage_cli_extended.retrying_list_objects', (), '', True, 'import services.object_storage.src.oci_cli_object_storage as oci_cli_object_storage\n'), ((626, 4, 626, 41), 'shutil.rmtree', 'shutil.rmtree', ({(626, 18, 626, 40): 'target_download_folder'}, {}), '(target_download_folder)', False, 'import shutil\n'), ((627, 4, 627, 40), 'shutil.rmtree', 'shutil.rmtree', ({(627, 18, 627, 39): 'inclusion_test_folder'}, {}), '(inclusion_test_folder)', False, 'import shutil\n'), ((632, 28, 632, 90), 'os.path.join', 'os.path.join', ({(632, 41, 632, 48): '"""tests"""', (632, 50, 632, 56): '"""temp"""', (632, 58, 632, 89): '"""os_bulk_upload_exclusion_test"""'}, {}), "('tests', 'temp', 'os_bulk_upload_exclusion_test')", False, 'import os\n'), ((642, 25, 642, 56), 'six.iteritems', 'six.iteritems', ({(642, 39, 642, 55): 'folders_to_files'}, {}), '(folders_to_files)', False, 'import six\n'), ((683, 27, 683, 96), 'os.path.join', 'os.path.join', ({(683, 40, 683, 47): '"""tests"""', (683, 49, 683, 55): '"""temp"""', (683, 57, 683, 95): '"""verify_os_bulk_upload_exclusion_test"""'}, {}), "('tests', 'temp', 'verify_os_bulk_upload_exclusion_test')", False, 'import os\n'), ((692, 29, 692, 83), 'os.path.join', 'os.path.join', ({(692, 42, 692, 62): 'download_folder_base', (692, 64, 692, 82): '"""get_with_exclude"""'}, {}), "(download_folder_base, 'get_with_exclude')", False, 'import os\n'), ((747, 29, 759, 5), 'services.object_storage.src.oci_cli_object_storage.objectstorage_cli_extended.retrying_list_objects', 'oci_cli_object_storage.objectstorage_cli_extended.retrying_list_objects', (), '', True, 'import services.object_storage.src.oci_cli_object_storage as oci_cli_object_storage\n'), ((767, 4, 767, 41), 'shutil.rmtree', 'shutil.rmtree', ({(767, 18, 767, 40): 'target_download_folder'}, {}), '(target_download_folder)', False, 'import shutil\n'), ((768, 4, 768, 40), 'shutil.rmtree', 'shutil.rmtree', ({(768, 18, 768, 39): 'exclusion_test_folder'}, {}), '(exclusion_test_folder)', False, 'import shutil\n'), ((773, 28, 773, 75), 'oci.object_storage.models.CreateBucketDetails', 'oci.object_storage.models.CreateBucketDetails', ({}, {}), '()', False, 'import oci\n'), ((788, 20, 788, 45), 'json.loads', 'json.loads', ({(788, 31, 788, 44): 'result.output'}, {}), '(result.output)', False, 'import json\n'), ((793, 20, 793, 45), 'json.loads', 'json.loads', ({(793, 31, 793, 44): 'result.output'}, {}), '(result.output)', False, 'import json\n'), ((799, 20, 799, 45), 'json.loads', 'json.loads', ({(799, 31, 799, 44): 'result.output'}, {}), '(result.output)', False, 'import json\n'), ((805, 28, 805, 75), 'oci.object_storage.models.CreateBucketDetails', 'oci.object_storage.models.CreateBucketDetails', ({}, {}), '()', False, 'import oci\n'), ((808, 4, 808, 112), 'tests.util.clear_test_data', 'util.clear_test_data', ({(808, 25, 808, 46): 'object_storage_client', (808, 48, 808, 62): 'util.NAMESPACE', (808, 64, 808, 83): 'util.COMPARTMENT_ID', (808, 85, 808, 111): 'create_bucket_request.name'}, {}), '(object_storage_client, util.NAMESPACE, util.\n    COMPARTMENT_ID, create_bucket_request.name)', False, 'from tests import util\n'), ((837, 28, 837, 75), 'oci.object_storage.models.CreateBucketDetails', 'oci.object_storage.models.CreateBucketDetails', ({}, {}), '()', False, 'import oci\n'), ((840, 4, 840, 112), 'tests.util.clear_test_data', 'util.clear_test_data', ({(840, 25, 840, 46): 'object_storage_client', (840, 48, 840, 62): 'util.NAMESPACE', (840, 64, 840, 83): 'util.COMPARTMENT_ID', (840, 85, 840, 111): 'create_bucket_request.name'}, {}), '(object_storage_client, util.NAMESPACE, util.\n    COMPARTMENT_ID, create_bucket_request.name)', False, 'from tests import util\n'), ((858, 29, 858, 86), 'os.path.join', 'os.path.join', ({(858, 42, 858, 49): '"""tests"""', (858, 51, 858, 57): '"""temp"""', (858, 59, 858, 85): 'create_bucket_request.name'}, {}), "('tests', 'temp', create_bucket_request.name)", False, 'import os\n'), ((869, 4, 869, 41), 'shutil.rmtree', 'shutil.rmtree', ({(869, 18, 869, 40): 'target_download_folder'}, {}), '(target_download_folder)', False, 'import shutil\n'), ((875, 11, 875, 49), 'tests.util.invoke_command', 'util.invoke_command', ({(875, 31, 875, 39): 'commands'}, {}), '(commands, **args)', False, 'from tests import util\n'), ((880, 44, 880, 62), 'os.walk', 'os.walk', ({(880, 52, 880, 61): 'directory'}, {}), '(directory)', False, 'import os\n'), ((887, 7, 887, 60), 'tests.test_config_container.using_vcr_with_mock_responses', 'test_config_container.using_vcr_with_mock_responses', ({}, {}), '()', False, 'from tests import test_config_container\n'), ((904, 11, 904, 31), 'json.loads', 'json.loads', ({(904, 22, 904, 30): 'json_str'}, {}), '(json_str)', False, 'import json\n'), ((914, 28, 926, 5), 'services.object_storage.src.oci_cli_object_storage.objectstorage_cli_extended.retrying_list_objects', 'oci_cli_object_storage.objectstorage_cli_extended.retrying_list_objects', (), '', True, 'import services.object_storage.src.oci_cli_object_storage as oci_cli_object_storage\n'), ((934, 28, 946, 5), 'services.object_storage.src.oci_cli_object_storage.objectstorage_cli_extended.retrying_list_objects', 'oci_cli_object_storage.objectstorage_cli_extended.retrying_list_objects', (), '', True, 'import services.object_storage.src.oci_cli_object_storage as oci_cli_object_storage\n'), ((964, 29, 964, 84), 'os.path.join', 'os.path.join', ({(964, 42, 964, 57): 'download_folder', (964, 59, 964, 83): 'download_prefix_no_slash'}, {}), '(download_folder, download_prefix_no_slash)', False, 'import os\n'), ((966, 44, 966, 66), 'os.walk', 'os.walk', ({(966, 52, 966, 65): 'source_folder'}, {}), '(source_folder)', False, 'import os\n'), ((977, 4, 977, 41), 'shutil.rmtree', 'shutil.rmtree', ({(977, 18, 977, 40): 'actual_download_folder'}, {}), '(actual_download_folder)', False, 'import shutil\n'), ((60, 70, 60, 97), 'tests.util.random_number_string', 'util.random_number_string', ({}, {}), '()', False, 'from tests import util\n'), ((92, 59, 92, 86), 'tests.util.random_number_string', 'util.random_number_string', ({}, {}), '()', False, 'from tests import util\n'), ((94, 11, 94, 47), 'os.path.exists', 'os.path.exists', ({(94, 26, 94, 46): 'bulk_put_folder_leaf'}, {}), '(bulk_put_folder_leaf)', False, 'import os\n'), ((95, 8, 95, 41), 'os.makedirs', 'os.makedirs', ({(95, 20, 95, 40): 'bulk_put_folder_leaf'}, {}), '(bulk_put_folder_leaf)', False, 'import os\n'), ((98, 70, 98, 97), 'tests.util.random_number_string', 'util.random_number_string', ({}, {}), '()', False, 'from tests import util\n'), ((138, 32, 138, 146), 'services.object_storage.src.oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage', 'oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage', ({(138, 128, 138, 145): '"""/this/is/a/path"""'}, {}), "(\n    '/this/is/a/path')", True, 'import services.object_storage.src.oci_cli_object_storage as oci_cli_object_storage\n'), ((139, 32, 139, 151), 'services.object_storage.src.oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage', 'oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage', ({(139, 128, 139, 145): '"""/this/is/a/path"""', (139, 147, 139, 150): '"""/"""'}, {}), "(\n    '/this/is/a/path', '/')", True, 'import services.object_storage.src.oci_cli_object_storage as oci_cli_object_storage\n'), ((140, 32, 140, 156), 'services.object_storage.src.oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage', 'oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage', ({(140, 128, 140, 149): '"""\\\\this\\\\is\\\\a\\\\path"""', (140, 151, 140, 155): '"""\\\\"""'}, {}), "(\n    '\\\\this\\\\is\\\\a\\\\path', '\\\\')", True, 'import services.object_storage.src.oci_cli_object_storage as oci_cli_object_storage\n'), ((141, 32, 141, 154), 'services.object_storage.src.oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage', 'oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage', ({(141, 128, 141, 147): '"""\\\\this/is/a\\\\path"""', (141, 149, 141, 153): '"""\\\\"""'}, {}), "(\n    '\\\\this/is/a\\\\path', '\\\\')", True, 'import services.object_storage.src.oci_cli_object_storage as oci_cli_object_storage\n'), ((143, 28, 143, 138), 'services.object_storage.src.oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage', 'oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage', ({(143, 124, 143, 137): '"""thisisapath"""'}, {}), "(\n    'thisisapath')", True, 'import services.object_storage.src.oci_cli_object_storage as oci_cli_object_storage\n'), ((144, 28, 144, 143), 'services.object_storage.src.oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage', 'oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage', ({(144, 124, 144, 137): '"""thisisapath"""', (144, 139, 144, 142): '"""/"""'}, {}), "(\n    'thisisapath', '/')", True, 'import services.object_storage.src.oci_cli_object_storage as oci_cli_object_storage\n'), ((145, 28, 145, 144), 'services.object_storage.src.oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage', 'oci_cli_object_storage.objectstorage_cli_extended.normalize_object_name_path_for_object_storage', ({(145, 124, 145, 137): '"""thisisapath"""', (145, 139, 145, 143): '"""\\\\"""'}, {}), "(\n    'thisisapath', '\\\\')", True, 'import services.object_storage.src.oci_cli_object_storage as oci_cli_object_storage\n'), ((178, 20, 178, 62), 'os.path.join', 'os.path.join', ({(178, 33, 178, 48): 'download_folder', (178, 50, 178, 61): 'object_name'}, {}), '(download_folder, object_name)', False, 'import os\n'), ((184, 20, 184, 62), 'os.path.join', 'os.path.join', ({(184, 33, 184, 48): 'download_folder', (184, 50, 184, 61): 'object_name'}, {}), '(download_folder, object_name)', False, 'import os\n'), ((190, 20, 190, 62), 'os.path.join', 'os.path.join', ({(190, 33, 190, 48): 'download_folder', (190, 50, 190, 61): 'object_name'}, {}), '(download_folder, object_name)', False, 'import os\n'), ((206, 20, 206, 62), 'os.path.join', 'os.path.join', ({(206, 33, 206, 48): 'download_folder', (206, 50, 206, 61): 'object_name'}, {}), '(download_folder, object_name)', False, 'import os\n'), ((257, 80, 257, 107), 'tests.util.random_number_string', 'util.random_number_string', ({}, {}), '()', False, 'from tests import util\n'), ((263, 11, 263, 46), 'os.path.exists', 'os.path.exists', ({(263, 26, 263, 45): 'large_file_root_dir'}, {}), '(large_file_root_dir)', False, 'import os\n'), ((264, 8, 264, 40), 'os.makedirs', 'os.makedirs', ({(264, 20, 264, 39): 'large_file_root_dir'}, {}), '(large_file_root_dir)', False, 'import os\n'), ((265, 27, 265, 69), 'os.path.join', 'os.path.join', ({(265, 40, 265, 59): 'large_file_root_dir', (265, 61, 265, 68): '"""1.bin"""'}, {}), "(large_file_root_dir, '1.bin')", False, 'import os\n'), ((266, 27, 266, 69), 'os.path.join', 'os.path.join', ({(266, 40, 266, 59): 'large_file_root_dir', (266, 61, 266, 68): '"""2.bin"""'}, {}), "(large_file_root_dir, '2.bin')", False, 'import os\n'), ((267, 27, 267, 69), 'os.path.join', 'os.path.join', ({(267, 40, 267, 59): 'large_file_root_dir', (267, 61, 267, 68): '"""3.bin"""'}, {}), "(large_file_root_dir, '3.bin')", False, 'import os\n'), ((268, 27, 268, 69), 'os.path.join', 'os.path.join', ({(268, 40, 268, 59): 'large_file_root_dir', (268, 61, 268, 68): '"""4.bin"""'}, {}), "(large_file_root_dir, '4.bin')", False, 'import os\n'), ((269, 27, 269, 69), 'os.path.join', 'os.path.join', ({(269, 40, 269, 59): 'large_file_root_dir', (269, 61, 269, 68): '"""5.bin"""'}, {}), "(large_file_root_dir, '5.bin')", False, 'import os\n'), ((270, 27, 270, 69), 'os.path.join', 'os.path.join', ({(270, 40, 270, 59): 'large_file_root_dir', (270, 61, 270, 68): '"""6.bin"""'}, {}), "(large_file_root_dir, '6.bin')", False, 'import os\n'), ((284, 23, 284, 65), 'os.path.join', 'os.path.join', ({(284, 36, 284, 55): 'large_file_root_dir', (284, 57, 284, 64): '"""1.bin"""'}, {}), "(large_file_root_dir, '1.bin')", False, 'import os\n'), ((284, 67, 284, 111), 'os.path.join', 'os.path.join', ({(284, 80, 284, 101): 'large_file_verify_dir', (284, 103, 284, 110): '"""1.bin"""'}, {}), "(large_file_verify_dir, '1.bin')", False, 'import os\n'), ((285, 23, 285, 65), 'os.path.join', 'os.path.join', ({(285, 36, 285, 55): 'large_file_root_dir', (285, 57, 285, 64): '"""2.bin"""'}, {}), "(large_file_root_dir, '2.bin')", False, 'import os\n'), ((285, 67, 285, 111), 'os.path.join', 'os.path.join', ({(285, 80, 285, 101): 'large_file_verify_dir', (285, 103, 285, 110): '"""2.bin"""'}, {}), "(large_file_verify_dir, '2.bin')", False, 'import os\n'), ((286, 23, 286, 65), 'os.path.join', 'os.path.join', ({(286, 36, 286, 55): 'large_file_root_dir', (286, 57, 286, 64): '"""3.bin"""'}, {}), "(large_file_root_dir, '3.bin')", False, 'import os\n'), ((286, 67, 286, 111), 'os.path.join', 'os.path.join', ({(286, 80, 286, 101): 'large_file_verify_dir', (286, 103, 286, 110): '"""3.bin"""'}, {}), "(large_file_verify_dir, '3.bin')", False, 'import os\n'), ((287, 23, 287, 65), 'os.path.join', 'os.path.join', ({(287, 36, 287, 55): 'large_file_root_dir', (287, 57, 287, 64): '"""4.bin"""'}, {}), "(large_file_root_dir, '4.bin')", False, 'import os\n'), ((287, 67, 287, 111), 'os.path.join', 'os.path.join', ({(287, 80, 287, 101): 'large_file_verify_dir', (287, 103, 287, 110): '"""4.bin"""'}, {}), "(large_file_verify_dir, '4.bin')", False, 'import os\n'), ((288, 23, 288, 65), 'os.path.join', 'os.path.join', ({(288, 36, 288, 55): 'large_file_root_dir', (288, 57, 288, 64): '"""5.bin"""'}, {}), "(large_file_root_dir, '5.bin')", False, 'import os\n'), ((288, 67, 288, 111), 'os.path.join', 'os.path.join', ({(288, 80, 288, 101): 'large_file_verify_dir', (288, 103, 288, 110): '"""5.bin"""'}, {}), "(large_file_verify_dir, '5.bin')", False, 'import os\n'), ((289, 23, 289, 65), 'os.path.join', 'os.path.join', ({(289, 36, 289, 55): 'large_file_root_dir', (289, 57, 289, 64): '"""6.bin"""'}, {}), "(large_file_root_dir, '6.bin')", False, 'import os\n'), ((289, 67, 289, 111), 'os.path.join', 'os.path.join', ({(289, 80, 289, 101): 'large_file_verify_dir', (289, 103, 289, 110): '"""6.bin"""'}, {}), "(large_file_verify_dir, '6.bin')", False, 'import os\n'), ((414, 80, 414, 107), 'tests.util.random_number_string', 'util.random_number_string', ({}, {}), '()', False, 'from tests import util\n'), ((487, 11, 487, 48), 'os.path.exists', 'os.path.exists', ({(487, 26, 487, 47): 'inclusion_test_folder'}, {}), '(inclusion_test_folder)', False, 'import os\n'), ((488, 8, 488, 42), 'os.makedirs', 'os.makedirs', ({(488, 20, 488, 41): 'inclusion_test_folder'}, {}), '(inclusion_test_folder)', False, 'import os\n'), ((497, 22, 497, 65), 'os.path.join', 'os.path.join', ({(497, 35, 497, 56): 'inclusion_test_folder', (497, 58, 497, 64): 'folder'}, {}), '(inclusion_test_folder, folder)', False, 'import os\n'), ((562, 22, 562, 73), 'os.path.join', 'os.path.join', ({(562, 35, 562, 57): 'target_download_folder', (562, 59, 562, 72): 'expected_file'}, {}), '(target_download_folder, expected_file)', False, 'import os\n'), ((565, 15, 565, 42), 'os.path.exists', 'os.path.exists', ({(565, 30, 565, 41): 'target_file'}, {}), '(target_file)', False, 'import os\n'), ((566, 15, 566, 69), 'filecmp.cmp', 'filecmp.cmp', (), '', False, 'import filecmp\n'), ((577, 26, 577, 118), 'os.path.join', 'os.path.join', ({(577, 39, 577, 61): 'target_download_folder', (577, 63, 577, 79): '"""inclusion_test"""', (577, 81, 577, 92): '"""subfolder"""', (577, 94, 577, 106): '"""subfolder2"""', (577, 108, 577, 117): '"""xyz.jpg"""'}, {}), "(target_download_folder, 'inclusion_test', 'subfolder',\n    'subfolder2', 'xyz.jpg')", False, 'import os\n'), ((633, 11, 633, 48), 'os.path.exists', 'os.path.exists', ({(633, 26, 633, 47): 'exclusion_test_folder'}, {}), '(exclusion_test_folder)', False, 'import os\n'), ((634, 8, 634, 42), 'os.makedirs', 'os.makedirs', ({(634, 20, 634, 41): 'exclusion_test_folder'}, {}), '(exclusion_test_folder)', False, 'import os\n'), ((643, 22, 643, 65), 'os.path.join', 'os.path.join', ({(643, 35, 643, 56): 'exclusion_test_folder', (643, 58, 643, 64): 'folder'}, {}), '(exclusion_test_folder, folder)', False, 'import os\n'), ((709, 26, 709, 98), 'os.path.join', 'os.path.join', ({(709, 39, 709, 61): 'target_download_folder', (709, 63, 709, 79): '"""exclusion_test"""', (709, 81, 709, 97): '"""test_file2.png"""'}, {}), "(target_download_folder, 'exclusion_test', 'test_file2.png')", False, 'import os\n'), ((710, 26, 710, 110), 'os.path.join', 'os.path.join', ({(710, 39, 710, 61): 'target_download_folder', (710, 63, 710, 79): '"""exclusion_test"""', (710, 81, 710, 92): '"""subfolder"""', (710, 94, 710, 109): '"""testfile3.png"""'}, {}), "(target_download_folder, 'exclusion_test', 'subfolder',\n    'testfile3.png')", False, 'import os\n'), ((713, 8, 713, 61), 'os.path.join', 'os.path.join', ({(713, 21, 713, 42): 'exclusion_test_folder', (713, 44, 713, 60): '"""test_file2.png"""'}, {}), "(exclusion_test_folder, 'test_file2.png')", False, 'import os\n'), ((714, 8, 714, 80), 'os.path.join', 'os.path.join', ({(714, 21, 714, 43): 'target_download_folder', (714, 45, 714, 61): '"""exclusion_test"""', (714, 63, 714, 79): '"""test_file2.png"""'}, {}), "(target_download_folder, 'exclusion_test', 'test_file2.png')", False, 'import os\n'), ((717, 8, 717, 73), 'os.path.join', 'os.path.join', ({(717, 21, 717, 42): 'exclusion_test_folder', (717, 44, 717, 55): '"""subfolder"""', (717, 57, 717, 72): '"""testfile3.png"""'}, {}), "(exclusion_test_folder, 'subfolder', 'testfile3.png')", False, 'import os\n'), ((718, 8, 718, 92), 'os.path.join', 'os.path.join', ({(718, 21, 718, 43): 'target_download_folder', (718, 45, 718, 61): '"""exclusion_test"""', (718, 63, 718, 74): '"""subfolder"""', (718, 76, 718, 91): '"""testfile3.png"""'}, {}), "(target_download_folder, 'exclusion_test', 'subfolder',\n    'testfile3.png')", False, 'import os\n'), ((774, 69, 774, 96), 'tests.util.random_number_string', 'util.random_number_string', ({}, {}), '()', False, 'from tests import util\n'), ((806, 69, 806, 95), 'random.randint', 'random.randint', ({(806, 84, 806, 85): '0', (806, 87, 806, 94): '1000000'}, {}), '(0, 1000000)', False, 'import random\n'), ((838, 70, 838, 97), 'tests.util.random_number_string', 'util.random_number_string', ({}, {}), '()', False, 'from tests import util\n'), ((110, 26, 110, 71), 'os.path.join', 'os.path.join', ({(110, 39, 110, 59): 'root_bulk_put_folder', (110, 61, 110, 70): 'subfolder'}, {}), '(root_bulk_put_folder, subfolder)', False, 'import os\n'), ((157, 24, 157, 70), 'os.path.join', 'os.path.join', ({(157, 37, 157, 52): 'download_folder', (157, 54, 157, 69): 'object_name[1:]'}, {}), '(download_folder, object_name[1:])', False, 'import os\n'), ((159, 24, 159, 66), 'os.path.join', 'os.path.join', ({(159, 37, 159, 52): 'download_folder', (159, 54, 159, 65): 'object_name'}, {}), '(download_folder, object_name)', False, 'import os\n'), ((338, 31, 338, 59), 'os.path.join', 'os.path.join', ({(338, 44, 338, 52): 'dir_name', (338, 54, 338, 58): 'file'}, {}), '(dir_name, file)', False, 'import os\n'), ((341, 19, 341, 55), 'os.path.exists', 'os.path.exists', ({(341, 34, 341, 54): 'downloaded_file_path'}, {}), '(downloaded_file_path)', False, 'import os\n'), ((342, 19, 342, 85), 'filecmp.cmp', 'filecmp.cmp', (), '', False, 'import filecmp\n'), ((393, 31, 393, 59), 'os.path.join', 'os.path.join', ({(393, 44, 393, 52): 'dir_name', (393, 54, 393, 58): 'file'}, {}), '(dir_name, file)', False, 'import os\n'), ((396, 19, 396, 55), 'os.path.exists', 'os.path.exists', ({(396, 34, 396, 54): 'downloaded_file_path'}, {}), '(downloaded_file_path)', False, 'import os\n'), ((397, 19, 397, 85), 'filecmp.cmp', 'filecmp.cmp', (), '', False, 'import filecmp\n'), ((463, 31, 463, 59), 'os.path.join', 'os.path.join', ({(463, 44, 463, 52): 'dir_name', (463, 54, 463, 58): 'file'}, {}), '(dir_name, file)', False, 'import os\n'), ((466, 19, 466, 55), 'os.path.exists', 'os.path.exists', ({(466, 34, 466, 54): 'downloaded_file_path'}, {}), '(downloaded_file_path)', False, 'import os\n'), ((467, 19, 467, 85), 'filecmp.cmp', 'filecmp.cmp', (), '', False, 'import filecmp\n'), ((498, 15, 498, 42), 'os.path.exists', 'os.path.exists', ({(498, 30, 498, 41): 'folder_path'}, {}), '(folder_path)', False, 'import os\n'), ((499, 12, 499, 36), 'os.makedirs', 'os.makedirs', ({(499, 24, 499, 35): 'folder_path'}, {}), '(folder_path)', False, 'import os\n'), ((502, 24, 502, 55), 'os.path.join', 'os.path.join', ({(502, 37, 502, 48): 'folder_path', (502, 50, 502, 54): 'file'}, {}), '(folder_path, file)', False, 'import os\n'), ((560, 30, 560, 122), 'os.path.join', 'os.path.join', ({(560, 43, 560, 65): 'target_download_folder', (560, 67, 560, 83): '"""inclusion_test"""', (560, 85, 560, 96): '"""subfolder"""', (560, 98, 560, 110): '"""subfolder2"""', (560, 112, 560, 121): '"""xyz.jpg"""'}, {}), "(target_download_folder, 'inclusion_test', 'subfolder',\n    'subfolder2', 'xyz.jpg')", False, 'import os\n'), ((563, 44, 563, 98), 'os.path.join', 'os.path.join', ({(563, 57, 563, 79): 'target_download_folder', (563, 81, 563, 97): '"""inclusion_test"""'}, {}), "(target_download_folder, 'inclusion_test')", False, 'import os\n'), ((644, 15, 644, 42), 'os.path.exists', 'os.path.exists', ({(644, 30, 644, 41): 'folder_path'}, {}), '(folder_path)', False, 'import os\n'), ((645, 12, 645, 36), 'os.makedirs', 'os.makedirs', ({(645, 24, 645, 35): 'folder_path'}, {}), '(folder_path)', False, 'import os\n'), ((648, 24, 648, 55), 'os.path.join', 'os.path.join', ({(648, 37, 648, 48): 'folder_path', (648, 50, 648, 54): 'file'}, {}), '(folder_path, file)', False, 'import os\n'), ((704, 30, 704, 109), 'os.path.join', 'os.path.join', ({(704, 43, 704, 65): 'target_download_folder', (704, 67, 704, 83): '"""exclusion_test"""', (704, 85, 704, 96): '"""subfolder"""', (704, 98, 704, 108): '"""blah.pdf"""'}, {}), "(target_download_folder, 'exclusion_test', 'subfolder', 'blah.pdf')", False, 'import os\n'), ((705, 30, 705, 122), 'os.path.join', 'os.path.join', ({(705, 43, 705, 65): 'target_download_folder', (705, 67, 705, 83): '"""exclusion_test"""', (705, 85, 705, 96): '"""subfolder"""', (705, 98, 705, 110): '"""subfolder2"""', (705, 112, 705, 121): '"""byz.jpg"""'}, {}), "(target_download_folder, 'exclusion_test', 'subfolder',\n    'subfolder2', 'byz.jpg')", False, 'import os\n'), ((706, 30, 706, 128), 'os.path.join', 'os.path.join', ({(706, 43, 706, 65): 'target_download_folder', (706, 67, 706, 83): '"""exclusion_test"""', (706, 85, 706, 96): '"""subfolder"""', (706, 98, 706, 110): '"""subfolder2"""', (706, 112, 706, 127): '"""testfile4.png"""'}, {}), "(target_download_folder, 'exclusion_test', 'subfolder',\n    'subfolder2', 'testfile4.png')", False, 'import os\n'), ((962, 50, 962, 71), 'os.path.normpath', 'os.path.normpath', ({(962, 67, 962, 70): 'euf'}, {}), '(euf)', False, 'import os\n'), ((968, 31, 968, 59), 'os.path.join', 'os.path.join', ({(968, 44, 968, 52): 'dir_name', (968, 54, 968, 58): 'file'}, {}), '(dir_name, file)', False, 'import os\n'), ((45, 9, 45, 105), 'tests.test_config_container.create_vcr', 'test_config_container.create_vcr', (), '', False, 'from tests import test_config_container\n'), ((116, 16, 116, 87), 'tests.util.create_large_file', 'util.create_large_file', ({(116, 39, 116, 48): 'file_path', (116, 50, 116, 86): 'LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES'}, {}), '(file_path, LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES)', False, 'from tests import util\n'), ((398, 19, 398, 47), 'mimetypes.guess_type', 'guess_type', ({(398, 30, 398, 46): 'source_file_path'}, {}), '(source_file_path)', False, 'from mimetypes import guess_type\n'), ((398, 51, 398, 83), 'mimetypes.guess_type', 'guess_type', ({(398, 62, 398, 82): 'downloaded_file_path'}, {}), '(downloaded_file_path)', False, 'from mimetypes import guess_type\n'), ((890, 23, 890, 60), 'random.choice', 'random.choice', ({(890, 37, 890, 59): 'string.ascii_lowercase'}, {}), '(string.ascii_lowercase)', False, 'import random\n'), ((973, 23, 973, 59), 'os.path.exists', 'os.path.exists', ({(973, 38, 973, 58): 'downloaded_file_path'}, {}), '(downloaded_file_path)', False, 'import os\n'), ((974, 23, 974, 89), 'filecmp.cmp', 'filecmp.cmp', (), '', False, 'import filecmp\n'), ((120, 16, 120, 78), 'tests.util.create_large_file', 'util.create_large_file', ({(120, 39, 120, 48): 'file_path', (120, 50, 120, 77): 'MID_SIZED_FILE_IN_MEBIBTYES'}, {}), '(file_path, MID_SIZED_FILE_IN_MEBIBTYES)', False, 'from tests import util\n')]