Get data from JSON values I have been working on where I want to improve my knowledge to do a safe JSON scraping. By that is that I could pretty much do a ugly workaround and wrap it into a try-except but that is not the case. I would like to know where it goes wrong and how I can improve the code I have been working with:
import json
from json import JSONDecodeError

import requests

payload = {
    &quot;name&quot;: None,
    &quot;price&quot;: None,
    &quot;sizes&quot;: None
}

headers = {'origin': 'https://www.aplace.com', 'Content-Type': 'application/json'}
with requests.get(&quot;http://api.aplace.com/sv/dam/aaltoili-iso-mehu-gren-red-sand&quot;, headers=headers) as response:
    if not response.ok:
        print(f&quot;Requests NOT ok -&gt; {payload}&quot;)

    try:
        # They added extra comma to fail the JSON. Fixed it by removing duplicate commas
        doc = json.loads(response.text.replace(&quot;,,&quot;, &quot;,&quot;)).get(&quot;productData&quot;, {})
    except (JSONDecodeError, KeyError, ValueError) as err:
        print(f&quot;Not able to parse json -&gt; {err} -&gt; Payload: {payload}&quot;)
        raise

if doc:

    product_name = doc.get('name', {})
    product_brand = doc.get('brandName', {})
    product_price = doc.get('markets', {}).get('6', {}).get('pricesByPricelist', {}).get('3', {}).get('price', {})

    # Added exception incase the [0] is not able to get any value from 'media'
    try:
        product_image = doc.get('media', {})[0].get('sources', {}).get('full', {}).get('url', {})
    except KeyError:
        product_image = None

    product_size = doc.get('items', {})

    try:

        if product_name and product_brand:
            payload['name'] = f&quot;{product_brand} {product_name}&quot;

        if product_price:
            payload['price'] = product_price.replace('\xa0', '')

        if product_image:
            payload['image'] = product_image

        if product_size:

            count = 0
            sizes = []
            sizesWithStock = []
            for value in product_size.values():
                if value.get('stockByMarket', {}).get('6', {}):
                    count += value['stockByMarket']['6']
                    sizes.append(value['name'])
                    sizesWithStock.append(f&quot;{value['name']} - ({value['stockByMarket']['6']})&quot;)

            payload['sizes'] = sizes
            payload['sizesWithStock'] = sizesWithStock
            payload['stock'] = count

    except Exception as err:
        print(&quot;Create notification if we hit here later on&quot;)
        raise

print(f&quot;Finished payload -&gt; {payload}&quot;)

My goal here is to cover all the values and if we do not find it inside the json then we use the if statements to see if we have caught a value or not.
Things that annoys me:

Long nested dicts but im not sure if there is any other way, is there a better way?
try-except for product_image, because its a list and I just want to grab the first in the list, is it possible to skip the try-except?
If unexpected error happens, print to the exception but am I doing it correctly?

Hopefully I can get new knowledge from here :)
You need functions.
Functions help to split up your code and make understanding how everything fits together easier.
You should have a function for:

Getting the page.

Parsing the JSON with your helpful error message.

Getting the payload.

A main function for calling all the other functions.
We should only call the main function if the script is the entry point.
We can do so with the following code:
if __name__ == &quot;__main__&quot;:
    main()




You should add a function to walk paths.

product_price = doc.get('markets', {}).get('6', {}).get('pricesByPricelist', {}).get('3', {}).get('price', {})



# Added exception incase the [0] is not able to get any value from 'media'
try:
    product_image = doc.get('media', {})[0].get('sources', {}).get('full', {}).get('url', {})
except KeyError:
    product_image = None


Things would be so much simpler if we could just do:
price = get(data, &quot;markets&quot;, &quot;6&quot;, &quot;pricesByPricelist&quot;, &quot;3&quot;, &quot;price&quot;)
image = get(data, &quot;media&quot;, 0, &quot;sources&quot;, &quot;full&quot;, &quot;url&quot;)

To build get is simple.

We take an object, a path and a default value.
For each segment of the path we index the object.
If the object doesn't have the value (__getitem__ raises LookupError) we return the default value.
The default value should be to raise an error.
We can ensure anything the user passes to the function is returned.
By making a new instance of an object we can then check if the default is the instance.
For convenience we can build the get function which defaults to None.

_SENTINEL = object()

def walk(obj, path, default=_SENTINEL):
    try:
        for segment in path:
            obj = obj[segment]
        return obj
    except LookupError:
        if default is _SENTINEL:
            raise LookupError(f&quot;couldn't walk path; {path}&quot;) from None
        else:
            return default

def get(obj, *path):
    return walk(obj, path, default=None)


I'd highly recommend using exceptions for control flow in Python.
We can subclass BaseException to make an exception that prints to the user and then exits the system.
By wrapping the main() call in a try we can check if our custom error is raised print and then exit with a status of 1.
try:
    main()
except PrettyExitException as e:
    print(e)
    raise SystemExit(1) from None



import json
from json import JSONDecodeError

import requests

_SENTINEL = object()


class PrettyExitException(BaseException):
    pass


def get_page():
    headers = {'origin': 'https://www.aplace.com', 'Content-Type': 'application/json'}
    with requests.get(&quot;http://api.aplace.com/sv/dam/aaltoili-iso-mehu-gren-red-sand&quot;, headers=headers) as response:
        if not response.ok:
            raise PrettyExitException(f&quot;Requests NOT ok -&gt; {payload}&quot;)
        return response.text


def parse_json(raw_json):
    try:
        return json.loads(raw_json)
    except (JSONDecodeError, KeyError, ValueError) as err:
        raise PrettyExitException(f&quot;Not able to parse JSON -&gt; {err} -&gt; Payload: {payload}&quot;) from None


def walk(obj, path, default=_SENTINEL):
    try:
        for segment in path:
            obj = obj[segment]
        return obj
    except LookupError:
        if default is _SENTINEL:
            raise LookupError(f&quot;couldn't walk path; {path}&quot;) from None
        else:
            return default


def get(obj, *path):
    return walk(obj, path, default=None)


def build_payload(data):
    name = get(data, &quot;name&quot;)
    brand = get(data, &quot;brandName&quot;)
    price = get(data, &quot;markets&quot;, &quot;6&quot;, &quot;pricesByPricelist&quot;, &quot;3&quot;, &quot;price&quot;)
    image = get(data, &quot;media&quot;, 0, &quot;sources&quot;, &quot;full&quot;, &quot;url&quot;)
    size = get(data, &quot;items&quot;)
    count = 0
    sizes = []
    sizes_with_stock = []

    if name is not None and brand is not None:
        name = f&quot;{brand} {name}&quot;
    else:
        name = None

    if price is not None:
        price = price.replace(&quot;\xa0&quot;, &quot;&quot;)

    if size is not None:
        for value in size.values():
            stock = get(value, &quot;stockByMarket&quot;, &quot;6&quot;)
            if stock is not None:
                _name = get(value, &quot;name&quot;)
                count += stock
                sizes.append(_name)
                sizes_with_stock.append(f&quot;{_name} - ({stock})&quot;)

    return {
        &quot;name&quot;: name,
        &quot;price&quot;: price,
        &quot;image&quot;: image,
        &quot;sizes&quot;: sizes,
        &quot;sizesWithStock&quot;: sizes_with_stock,
        &quot;stock&quot;: count,
    }


def main():
    raw_data = get_page()
    data = parse_json(raw_data.replace(&quot;,,&quot;, &quot;,&quot;))
    payload = build_payload(data.get(&quot;productData&quot;, {}))
    print(f&quot;Finished payload -&gt; {payload}&quot;)


if __name__ == &quot;__main__&quot;:
    try:
        main()
    except PrettyExitException as e:
        print(e)
        raise SystemExit(1) from None