from google.colab import auth

auth.authenticate_user()

"""
Explanation: <a href="https://colab.research.google.com/github/probml/probml-notebooks/blob/main/notebooks/GCS_demo_v2.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>
Authenticate in order to access the GCS
We don't need authentication to access public data in GCloud but
in order to access protected data or write to a protected bucket, you need to set up credentials (authenticate)
End of explanation
"""


project_id = "your_project_id"

!gcloud config set project {project_id}

"""
Explanation: Setting a project
We need to choose a project inorder to work with buckets, if you dont have any, create a project in Gcloud Console 
First we need to set a default project using the project_id so that you are able to use commands which unless require specifying the project.
glcoud can be used to set the default project.
End of explanation
"""


import uuid

bucket_name = "colab-sample-bucket-" + str(uuid.uuid1())

"""
Explanation: Using gsutil- CLI for GCloud
Docs
Create
command to create a bucket
gsutil mb [-b (on|off)] [-c &lt;class&gt;] [-l &lt;location&gt;] [-p &lt;proj_id&gt;]
          [--retention &lt;time&gt;] [--pap &lt;setting&gt;] gs://&lt;bucket_name&gt;...
End of explanation
"""


!gsutil mb gs://{bucket_name}

"""
Explanation: This will create a bucket with bucket_name with default configurations.
End of explanation
"""


!mkdir /tmp/test_folder

with open("/tmp/test_folder/test_file.txt", "w") as f:
    f.write("this file get saved in the test_folder")

"""
Explanation: Creating a local folder with a test file to upload to the bucket
End of explanation
"""


!gsutil cp -r /tmp/test_folder gs://{bucket_name}

# @markdown Once the upload has finished, the data will appear in the Cloud Console storage browser for your project:
print("https://console.cloud.google.com/storage/browser?project=" + project_id)

"""
Explanation: Uploading the folder to the bucket
The object  gs://{bucket_name}/test_folder/test_file.txt is created.
End of explanation
"""


!gsutil cat gs://{bucket_name}/test_folder/test_file.txt

"""
Explanation: Read
The contents of the uploaded file in the bucket can be read in this way
End of explanation
"""


!gsutil cp -r gs://{bucket_name}/test_folder /content/
!gsutil cp  gs://{bucket_name}/test_folder/test_file.txt /content/

"""
Explanation: The whole folder/file from the bucket can be downloaded in this way.
End of explanation
"""


with open("/tmp/test_folder/test_file.txt", "a") as f:
    f.write(" this new string is added later")
!gsutil cp  /tmp/test_folder/test_file.txt gs://{bucket_name}/test_folder

!gsutil cat gs://{bucket_name}/test_folder/test_file.txt

"""
Explanation: Update
Updating a file
Edit the local copy and overwrite the file in the bucket.
End of explanation
"""


!rm /tmp/test_folder/test_file.txt

"""
Explanation: If you want to update a folder in the bucket to be in sync with a local copy of the folder, use rsync
Making some changes to the test_folder
End of explanation
"""


with open("/tmp/test_folder/test_file2.txt", "w") as f:
    f.write("this is a new file named test_file2")

"""
Explanation: adding a new file test_file2.txt
End of explanation
"""


!gsutil ls gs://{bucket_name}/test_folder

!gsutil rsync -d /tmp/test_folder gs://{bucket_name}/test_folder

"""
Explanation: You can check the contents of your bucket at any level using the ls cmd.
bucket contents before updating.
End of explanation
"""


!gsutil ls gs://{bucket_name}/test_folder

"""
Explanation: bucket contents after updating.
End of explanation
"""


!gsutil rm -r gs://{bucket_name}/test_folder

!gsutil ls gs://{bucket_name}

"""
Explanation: Delete
Contents of a bucket can be deleted using rm command
End of explanation
"""


!gsutil rb gs://{bucket_name}

"""
Explanation: Deleting a bucket
The rb command deletes a bucket. Buckets must be empty before you can delete them.
End of explanation
"""


import tensorflow_datasets as tfds

source_dir = "gs://gsoc_bucket/ILSVRC2012/"
dest_dir = "gs://gsoc_bucket/ILSVRC2012_tf_records"
tfds.builder("imagenet2012", data_dir=dest_dir).download_and_prepare(
    download_config=tfds.download.DownloadConfig(manual_dir=source_dir)
)

"""
Explanation: Making TFDS records
End of explanation
"""


bucket_name = "colab-sample-bucket-" + str(uuid.uuid1())

# Imports the Google Cloud client library
from google.cloud import storage

# Instantiates a client
storage_client = storage.Client(project=project_id)

# Creates the new bucket
bucket = storage_client.create_bucket(bucket_name)

print("Bucket {} created.".format(bucket.name))

!mkdir /tmp/test_api_folder

with open("/tmp/test_api_folder/test_api_file.txt", "w") as f:
    f.write("this file get saved in the test_api_folder")

"""
Explanation: Using GC python API
Create
Creating a bucket
End of explanation
"""


destination_blob_name = "test_api_folder/test_api_file.txt"
source_file_name = "/tmp/test_api_folder/test_api_file.txt"

blob = bucket.blob(destination_blob_name)

blob.upload_from_filename(source_file_name)

print("File {} uploaded to {}.".format(source_file_name, destination_blob_name))

# @markdown Once the upload has finished, the data will appear in the Cloud Console storage browser for your project:
print("https://console.cloud.google.com/storage/browser?project=" + project_id)

"""
Explanation: Uploading a object (a file with its path)
End of explanation
"""


source_blob_name = "test_api_folder/test_api_file.txt"
destination_file_name = "/content/downloaded_test_api.txt"
source_blob = bucket.blob(source_blob_name)
source_blob.download_to_filename(destination_file_name)
print("Blob {} downloaded to {}.".format(source_blob_name, destination_file_name))

!cat /content/downloaded_test_api.txt

"""
Explanation: Read
End of explanation
"""


with open("/tmp/test_api_folder/test_api_file.txt", "a") as f:
    f.write(" this is an appended string")

source_file_name = "/tmp/test_api_folder/test_api_file.txt"
destination_blob_name = "test_api_folder/test_api_file.txt"
destination_file_name = "/content/downloaded_test_api.txt"

blob = bucket.blob(destination_blob_name)
blob.upload_from_filename(source_file_name)
print("File {} uploaded to {}.".format(source_file_name, destination_blob_name))
blob.download_to_filename(destination_file_name)
print("Blob {} downloaded to {}.".format(destination_blob_name, destination_file_name))

!cat /content/downloaded_test_api.txt

"""
Explanation: To easily download all objects in a bucket or subdirectory, use the gsutil cp command.
Update
Updating a file/object
Its simply overwriting the existing copy of the object.
End of explanation
"""


blob_name = "test_api_folder/test_api_file.txt"
blob = bucket.blob(blob_name)
blob.delete()
print("Blob {} deleted.".format(blob_name))

!gsutil ls gs://{bucket_name}

"""
Explanation: Delete
objects can be deleted easily by using blob.delete()
When a folder is empty it will be vanished.
End of explanation
"""


bucket.delete()
print("Bucket {} deleted".format(bucket.name))

"""
Explanation: Deleting a bucket
End of explanation
"""


import uuid

bucket_name = "colab-sample-bucket-" + str(uuid.uuid1())
!gsutil mb gs://{bucket_name}

"""
Explanation: Using gcsfuse(mount) and bash
Another way is to mount the GCS bucket to the current session of colab and you can make any regular(CRUD) operations using bash commands(just like on any other directory)
In order to use this, you should first create a bucket you can do this using any of the methods described above)
gsutil mb is used here
Create
End of explanation
"""


!echo "deb http://packages.cloud.google.com/apt gcsfuse-bionic main" > /etc/apt/sources.list.d/gcsfuse.list
!curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
!apt -qq update
!apt -qq install gcsfuse
!mkdir $bucket_name
!gcsfuse $bucket_name /content/$bucket_name

cd /content/$bucket_name

"""
Explanation: Mounting the bucket to /content/{bucket_name}
End of explanation
"""


!mkdir test_mount_folder

"""
Explanation: Creating a folder
End of explanation
"""


with open("./test_mount_folder/test_mount_file.txt", "w") as f:
    f.write("this file get saved in the test_folder you just created")

# @markdown Once the upload has finished, the data will appear in the Cloud Console storage browser for your project:
print("https://console.cloud.google.com/storage/browser?project=" + project_id)

"""
Explanation: Creating a file/object in the folder just created.
End of explanation
"""


from google.colab import files

files.view(f"/content/{bucket_name}/test_mount_folder/test_mount_file.txt")

!cat ./test_mount_folder/test_mount_file.txt

"""
Explanation: Read
As the bucket is already mounted you can just access files by opening them using files or just double clicking the file
End of explanation
"""


with open("./test_mount_folder/test_mount_file.txt", "a") as f:
    f.write(" this new string is added later")
files.view(f"/content/{bucket_name}/test_mount_folder/test_mount_file.txt")

"""
Explanation: Update
Update to file
Update can be done normally either by editing or by using python.
End of explanation
"""


!echo "this is a second file to test_folder in the bucket" >> ./test_mount_folder/test_mount_file2.txt

"""
Explanation: Update to folder
New files can be added either by colab's UI or by using python or any other way that works for regular dirs.
End of explanation
"""


!cat ./test_mount_folder/test_mount_file2.txt

"""
Explanation: 
End of explanation
"""


!rm ./test_mount_folder/test_mount_file.txt

"""
Explanation: Delete
rm cmd can be used to delete the objects of the bucket using the mounted path.
Deleting the file {bucket_name}/test_mount_folder/test_mount_file.txt
End of explanation
"""


!rm -r ./test_mount_folder

"""
Explanation: to delete the folder we can sue rm -r or -rm -rf to force remove
End of explanation
"""


cd ..

!fusermount -u /content/$bucket_name

"""
Explanation: Deleting a bucket
you can't delete a bucekt using gcsfuse, but you can unmount the mounted path and then can delete using any of the above mentioned ways.
Unmount the bucket
After unmounting, any changes in the local path will not be rendered in the cloud bucket.
End of explanation
"""


!gsutil rb gs://{bucket_name}

"""
Explanation: deleting the bucket using gsutil.
End of explanation
"""


# @markdown Go to your project's browser:
print("https://console.cloud.google.com/storage/browser?project=" + project_id)

"""
Explanation: Using GC UI
Bucketa can be also be accessed directly through Gcloud Consoles's UI.
End of explanation
"""


# Pick a bucket name, they should be unique,
# for the demo purpose we can create a bucket name in this way
import uuid

bucket_name = "colab-sample-bucket-" + str(uuid.uuid1())
bucket_name

"""
Explanation: Create
Bucket can be created in the following way.
End of explanation
"""


