# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
# SKIP
# public code - not needed
import oauth2 as oauth

consumer_key = '<KEY>'
consumer_secret = '67yn9cpuzesrme7m'
consumer = oauth.Consumer(consumer_key, consumer_secret)
request_url = "http://previews.7digital.com/clip/14259160"
#request_url = "http://previews.7digital.com/clip"

req = oauth.Request(method="GET", url=request_url,is_form_encoded=True)

req['oauth_timestamp'] = oauth.Request.make_timestamp()
req['oauth_nonce'] = oauth.Request.make_nonce()
req['trackId'] = '14259160'
req['country'] = 'ww'

sig_method = oauth.SignatureMethod_HMAC_SHA1()

req.sign_request(sig_method, consumer, token=None)

print(req.to_url())

# +
# SKIP
# Get release ids
import oauth2 as oauth

request_url = "http://api.7digital.com/1.2/track/details"

req = oauth.Request(method="GET", url=request_url,is_form_encoded=True)
req['trackId'] = '123456'
req['oauth_consumer_key'] = '7digital.com'
req['country'] = 'GB'
req['usageTypes'] = 'download'

trckDetailsURL = req.to_url()
#trackDeatilsXML = 

# example :
#"http://api.7digital.com/1.2/track/details?trackid=123456&oauth_consumer_key=7digital.com&country=GB&usageTypes=download"
# -

import pandas as pd

# +
# python 3
import pickle

#with open('/Users/dmarcous/Downloads/EN_to_7D.pkl', 'rb') as f:
with open('/Users/dmarcous/git/wavenetmusicrecommendation/data/EN_to_7D.pkl', 'rb') as f:
    idlist = pickle.load(f)
# -

len(idlist)

msdTo7digMap = {k: v for k, v in idlist.items() if len(v) == 1 and len(v[0]) == 2}
msdTo7digMap = {k: [vi.replace(":", "") for vi in v[0]] for k, v in msdTo7digMap.items()}
#idlist["SOAAADD12AB018A9DD"]
idlist["SOAAADD12AB018A9DD"][0]

len(msdTo7digMap)
#177740

# +
import os
audio_dir = '/Users/dmarcous/git/wavenetmusicrecommendation/data/audio_clips/'
downloaded_clip_list = os.listdir(audio_dir)
downloadedClipNameList = [ fname.split(".")[0] for fname in downloaded_clip_list]
cleanClipMap = {k: msdTo7digMap[k] for k in msdTo7digMap if k not in downloadedClipNameList}

print(downloadedClipNameList[0])
print(msdTo7digMap[downloadedClipNameList[0]])
print(downloadedClipNameList[0] in cleanClipMap)
# -

len(cleanClipMap)

# +
# crawl functions
import urllib
import sys
from urllib.request import urlretrieve

def generateDownloadClipURL(releaseId, trackId):
    base_url = "https://us.7digital.com/stream/release/"
    download_url = base_url + releaseId + '/track/' + trackId + '/mp3'
    return(download_url)

def downloadClip(trackId_msd, releaseId_7dig, trackId_7dig):
    download_url = generateDownloadClipURL(releaseId_7dig, trackId_7dig)
    local_dir = '/Users/dmarcous/git/wavenetmusicrecommendation/data/audio_clips/'
    local_file = trackId_msd + '.mp3'
    download_path = local_dir + local_file
    print(download_url)
    try:
        urlretrieve(download_url, download_path)
    except urllib.error.HTTPError as e:
        print("Track not found! HTTP error({0}): {1}".format(e.errno, e.strerror))
    except:
        print("Unexpected error:")#, sys.exc_info()[0])
        #raise



# -

# python 3
index = 1
for key in list(cleanClipMap):
    value = cleanClipMap[key]
    trackId_msd = key
    trackId_7dig = value[0]
    releaseId_7dig = value[1]
    print(str(index) + ': downloading song ' + trackId_msd) 
    downloadClip(trackId_msd, releaseId_7dig, trackId_7dig)
    del cleanClipMap[key]
    index = index + 1
    #break
# 41370

#list(cleanClipMap.items())[0][]
#msdTo7digMap['SODPBKY12A6702169C'][0]
cleanClipMap['SODPBKY12A6702169C'][0]

print(1)



StarcoderdataJupyterScriptsDedupFiltered