# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import requests as rq
import json
import urllib.parse as urlparse
from urllib.parse import parse_qs
from urllib.parse import unquote
from datetime import datetime


# Reference page
# https://docs.microsoft.com/en-us/rest/api/cost-management/retail-prices/azure-retail-prices

# +
# Helper method for extracting the number of the next skip value
def extract_skip(url):
    decoded = unquote(url)
    parsed = urlparse.urlparse(decoded)
    result = parse_qs(parsed.query)['$skip'][0]
    return int(result)

def read_page(items, url):
    reponse = rq.get(url)
    page = json.loads(reponse.text)
    items += page['Items'] 
    next_page = page['NextPageLink']
    return items, next_page


# -

def read_and_extract(items, next_page):
    while next_page:
        items, next_page = read_page(items, next_page)    
        if not next_page:
            break
    
        # only print for every 1000 pages
        skip = extract_skip(next_page)
        if skip % 1000 == 0 :
            now = datetime.now()
            dt_string = now.strftime('%H:%M:%S')
            print (dt_string, ': ', next_page)
    
    print('===== Done ======')
    return items


# ### Compute

compute_url = "https://prices.azure.com/api/retail/prices?currencyCode='USD'&$filter=serviceFamily%20eq%20'Compute'"
items = []   
items, next_page = read_page(items, compute_url)
items = read_and_extract(items, next_page)

# pickle
azure_data = pd.DataFrame(items)
azure_data.to_pickle('./data/azure_src.compute.xz.pkl', compression='xz')

azure_src = './data/azure_src.compute.xz.pkl' 
azure_data = pd.read_pickle(azure_src, compression="xz")

azure_data

# ### Storage

storage_url = "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20%27Storage%27"
items = []   
items, next_page = read_page(items, storage_url)
items = read_and_extract(items, next_page)

# pickle
storage_data = pd.DataFrame(items)
storage_data.to_pickle('./data/azure.storage.xz.pkl', compression='xz')

storage_data.head(10)



StarcoderdataJupyterScriptsDedupFiltered