Converting an unknown data structure to a known data structure in Python
Question: I recently used an API to pull in some data from a platform I use.

But the problem is, the data that I pulled, is not a recognized data
structure.

It's almost a list of dictionaries, with some extra stuff.

I need to know how can I convert it to a recognized data structure. I don't
necessarily need the code to do so, just a north on what I need to study would
be helpful already. I'm new to Python and I don't have a lot of coding
experience.

I have a file with this data in each line, this is a sample line from the
file:

    
    
    [<OrderProducts at 0x24333f0, {'price_ex_tax': '99.0000', 'event_date': '', 'wrapping_name': '', 'price_tax': '0.0000', 'id': 3, 'cost_price_tax': '0.0000', 'bin_picking_number': '', 'ebay_transaction_id': '', 'wrapping_cost_ex_tax': '0.0000', 'base_total': '99.0000', 'quantity': 1, 'ebay_item_id': '', 'type': 'physical', 'product_id': 83, 'price_inc_tax': '99.0000', 'base_wrapping_cost': '0.0000', 'parent_order_product_id': None, 'option_set_id': 15, 'wrapping_message': '', 'weight': '3.0000', 'refund_amount': '0.0000', 'applied_discounts': [{'amount': 99, 'id': 'total-coupon'}], 'event_name': None, 'cost_price_ex_tax': '0.0000', 'base_price': '99.0000', 'wrapping_cost_tax': '0.0000', 'total_inc_tax': '99.0000', 'total_ex_tax': '99.0000', 'quantity_shipped': 0, 'fixed_shipping_cost': '0.0000', 'total_tax': '0.0000', 'sku': 'S-TIM-BAC-STD', 'return_id': 0, 'wrapping_cost_inc_tax': '0.0000', 'cost_price_inc_tax': '0.0000', 'name': 'University of Timbuktu Bachelor Set', 'is_bundled_product ': False, 'order_id': 614534, 'configurable_fields': [], 'order_address_id': 2, 'is_refunded': False, 'product_options': [{'display_style': 'Pick list', 'type': 'Product list', 'product_option_id': 95, 'display_value': 'Cambridge-Style Bachelor Gown, Size L', 'id': 2, 'option_id': 19, 'value': '77', 'display_name': 'Gown size', 'name': 'Bachelor gown size', 'order_product_id': 3}, {'display_style': 'Pick list', 'type': 'Product list', 'product_option_id': 97, 'display_value': 'Bachelor and Masters Trencher, Size L', 'id': 3, 'option_id': 20, 'value': '80', 'display_name': 'Trencher size', 'name': 'Trencher size', 'order_product_id': 3}], 'base_cost_price': '0.0000'}>, <OrderProducts at 0x2433420, {'price_ex_tax': '0.0000', 'event_date': '', 'wrapping_name': '', 'price_tax': '0.0000', 'id': 4, 'cost_price_tax': '0.0000', 'bin_picking_number': '', 'ebay_transaction_id': '', 'wrapping_cost_ex_tax': '0.0000', 'base_total': '0.0000', 'quantity': 1, 'ebay_item_id': '', 'type': 'physical', 'product_id': 80, 'price_inc_tax': '0.0000', 'base_wrapping_cost': '0.0000', 'parent_order_product_id': 3, 'option_set_id': None, 'wrapping_message': '', 'weight': '0.0000', 'refund_amount': '0.0000', 'applied_discounts': [], 'event_name': None, 'cost_price_ex_tax': '0.0000', 'base_price': '0.0000', 'wrapping_cost_tax': '0.0000', 'total_inc_tax': '0.0000', 'total_ex_tax': '0.0000', 'quantity_shipped': 0, 'fixed_shipping_cost': '0.0000', 'total_tax': '0.0000', 'sku': 'G-CAM-BAC-L', 'return_id': 0, 'wrapping_cost_inc_tax': '0.0000', 'cost_price_inc_tax': '0.0000', 'name': 'Cambridge-Style Bachelor Gown, Size L', 'is_bundled_product ': True, 'order_id': 614534, 'configurable_fields': [], 'order_address_id': 2, 'is_refunded': False, 'product_options': [], 'base_cost_price': '0.0000'}>, <OrderProducts at 0x2433450, {'price_ex_tax': '0.0000', 'event_date': '', 'wrapping_name': '', 'price_tax': '0.0000', 'id': 5, 'cost_price_tax': '0.0000', 'bin_picking_number': '', 'ebay_transaction_id': '', 'wrapping_cost_ex_tax': '0.0000', 'base_total': '0.0000', 'quantity': 1, 'ebay_item_id': '', 'type': 'physical', 'product_id': 87, 'price_inc_tax': '0.0000', 'base_wrapping_cost': '0.0000', 'parent_order_product_id': 3, 'option_set_id': None, 'wrapping_message': '', 'weight': '0.0000', 'refund_amount': '0.0000', 'applied_discounts': [], 'event_name': None, 'cost_price_ex_tax': '0.0000', 'base_price': '0.0000', 'wrapping_cost_tax': '0.0000', 'total_inc_tax': '0.0000', 'total_ex_tax': '0.0000', 'quantity_shipped': 0, 'fixed_shipping_cost': '0.0000', 'total_tax': '0.0000', 'sku': 'C-STD-B&M-L', 'return_id': 0, 'wrapping_cost_inc_tax': '0.0000', 'cost_price_inc_tax': '0.0000', 'name': 'Bachelor and Masters Trencher, Size L', 'is_bundled_product ': True, 'order_id': 614534, 'configurable_fields': [], 'order_address_id': 2, 'is_refunded': False, 'product_options': [], 'base_cost_price': '0.0000'}>]
    

EDIT: This is the code I got:

    
    
    import ast
    import re
    
    order_item = re.compile("<OrderProducts at 0x[\da-f]+, ({.*?})>", re.I)
    
     with open('allOrderProducts2') as inf:
         for line in inf:
            order = [ast.literal_eval(op) for op in re.findall(order_item, line)]
            # ta-da! Now do something with the order
            f = open("test", "w", encoding='utf-8')
            f.write("\n".join(map(lambda x: str(x), order)))
            f.close()
    


Answer: What you have is a list (`[ ... ]`) containing three `OrderProducts` objects
which, when printed, represent themselves as dictionaries (`{ key1: value1,
key2: value2 }`).

* * *

**Edit:** ok, you have the **string representation** of three OrderProducts
etc.

So the first order of business is to convert to actual Python data structures,
like so:

    
    
    import ast
    import re
    
    order_items = re.compile("<OrderProducts at 0x[\da-f]+, ({.*?})>", re.I).findall
    
    with open(FILENAME) as inf:
        for line in inf:
            order = [ast.literal_eval(op) for op in order_items(line)]
            # ta-da! Now do something with the order
    

then continue as before:

* * *

**Edit2:**

Cleaned up a bit:

    
    
    import re
    
    DATA = "allOrderProducts2"
    RESULT = "test"
    
    order_items = re.compile("<OrderProducts at 0x[\da-f]+, ({.*?})>", re.I).findall
    
    with open(DATA) as inf, open(RESULT, "w", "utf-8") as outf:
        # Instead of reading each line separately,
        # we can just parse the whole file in one gulp
        for item_str in order_items(inf.read()):
            # Also no need to convert the data
            # just to cast it back to a string again
            outf.write(item_str + "\n")
    

Then, when reading the RESULT file back in, you can pass each line to
`ast.literal_eval` to turn it back into a dict.

* * *

Looking at the `id` and `parent_order_product_id` fields, it looks like what
you have is a "University of Timbuktu Bachelor Set" consisting of a
"Cambridge-Style Bachelor Gown, Size L" and "Bachelor and Masters Trencher,
Size L" with a package price of 99.0 (units unknown) and no tax.

I wrote a quick script to figure out what a default OrderProduct looks like:

    
    
    from collections import Counter
    from pprint import pprint as pp
    
    data = [
        {'price_ex_tax': '99.0000', 'event_date': '', 'wrapping_name': '', 'price_tax': '0.0000', 'id': 3, 'cost_price_tax': '0.0000', 'bin_picking_number': '', 'ebay_transaction_id': '', 'wrapping_cost_ex_tax': '0.0000', 'base_total': '99.0000', 'quantity': 1, 'ebay_item_id': '', 'type': 'physical', 'product_id': 83, 'price_inc_tax': '99.0000', 'base_wrapping_cost': '0.0000', 'parent_order_product_id': None, 'option_set_id': 15, 'wrapping_message': '', 'weight': '3.0000', 'refund_amount': '0.0000', 'applied_discounts': [{'amount': 99, 'id': 'total-coupon'}], 'event_name': None, 'cost_price_ex_tax': '0.0000', 'base_price': '99.0000', 'wrapping_cost_tax': '0.0000', 'total_inc_tax': '99.0000', 'total_ex_tax': '99.0000', 'quantity_shipped': 0, 'fixed_shipping_cost': '0.0000', 'total_tax': '0.0000', 'sku': 'S-TIM-BAC-STD', 'return_id': 0, 'wrapping_cost_inc_tax': '0.0000', 'cost_price_inc_tax': '0.0000', 'name': 'University of Timbuktu Bachelor Set', 'is_bundled_product ': False, 'order_id': 614534, 'configurable_fields': [], 'order_address_id': 2, 'is_refunded': False, 'product_options': [{'display_style': 'Pick list', 'type': 'Product list', 'product_option_id': 95, 'display_value': 'Cambridge-Style Bachelor Gown, Size L', 'id': 2, 'option_id': 19, 'value': '77', 'display_name': 'Gown size', 'name': 'Bachelor gown size', 'order_product_id': 3}, {'display_style': 'Pick list', 'type': 'Product list', 'product_option_id': 97, 'display_value': 'Bachelor and Masters Trencher, Size L', 'id': 3, 'option_id': 20, 'value': '80', 'display_name': 'Trencher size', 'name': 'Trencher size', 'order_product_id': 3}], 'base_cost_price': '0.0000'},
        {'price_ex_tax': '0.0000', 'event_date': '', 'wrapping_name': '', 'price_tax': '0.0000', 'id': 4, 'cost_price_tax': '0.0000', 'bin_picking_number': '', 'ebay_transaction_id': '', 'wrapping_cost_ex_tax': '0.0000', 'base_total': '0.0000', 'quantity': 1, 'ebay_item_id': '', 'type': 'physical', 'product_id': 80, 'price_inc_tax': '0.0000', 'base_wrapping_cost': '0.0000', 'parent_order_product_id': 3, 'option_set_id': None, 'wrapping_message': '', 'weight': '0.0000', 'refund_amount': '0.0000', 'applied_discounts': [], 'event_name': None, 'cost_price_ex_tax': '0.0000', 'base_price': '0.0000', 'wrapping_cost_tax': '0.0000', 'total_inc_tax': '0.0000', 'total_ex_tax': '0.0000', 'quantity_shipped': 0, 'fixed_shipping_cost': '0.0000', 'total_tax': '0.0000', 'sku': 'G-CAM-BAC-L', 'return_id': 0, 'wrapping_cost_inc_tax': '0.0000', 'cost_price_inc_tax': '0.0000', 'name': 'Cambridge-Style Bachelor Gown, Size L', 'is_bundled_product ': True, 'order_id': 614534, 'configurable_fields': [], 'order_address_id': 2, 'is_refunded': False, 'product_options': [], 'base_cost_price': '0.0000'}, 
        {'price_ex_tax': '0.0000', 'event_date': '', 'wrapping_name': '', 'price_tax': '0.0000', 'id': 5, 'cost_price_tax': '0.0000', 'bin_picking_number': '', 'ebay_transaction_id': '', 'wrapping_cost_ex_tax': '0.0000', 'base_total': '0.0000', 'quantity': 1, 'ebay_item_id': '', 'type': 'physical', 'product_id': 87, 'price_inc_tax': '0.0000', 'base_wrapping_cost': '0.0000', 'parent_order_product_id': 3, 'option_set_id': None, 'wrapping_message': '', 'weight': '0.0000', 'refund_amount': '0.0000', 'applied_discounts': [], 'event_name': None, 'cost_price_ex_tax': '0.0000', 'base_price': '0.0000', 'wrapping_cost_tax': '0.0000', 'total_inc_tax': '0.0000', 'total_ex_tax': '0.0000', 'quantity_shipped': 0, 'fixed_shipping_cost': '0.0000', 'total_tax': '0.0000', 'sku': 'C-STD-B&M-L', 'return_id': 0, 'wrapping_cost_inc_tax': '0.0000', 'cost_price_inc_tax': '0.0000', 'name': 'Bachelor and Masters Trencher, Size L', 'is_bundled_product ': True, 'order_id': 614534, 'configurable_fields': [], 'order_address_id': 2, 'is_refunded': False, 'product_options': [], 'base_cost_price': '0.0000'}
    ]
    
    def get_defaults(lst_of_dct):
        defaults = {}
        majority = (len(data) + 1) // 2
        for key in lst_of_dct[0]:
            try:
                ctr = Counter(d[key] for d in lst_of_dct)
                value,count = ctr.most_common(1)[0]
                defaults[key] = value if count >= majority else ""
            except TypeError:
                # Counter doesn't like unhashable type ie lists
                defaults[key] = []
        return defaults
    
    defaults = get_defaults(data)
    pp(defaults)
    

which gives

    
    
    {'applied_discounts': [],
     'base_cost_price': '0.0000',
     'base_price': '0.0000',
     'base_total': '0.0000',
     'base_wrapping_cost': '0.0000',
     'bin_picking_number': '',
     'configurable_fields': [],
     'cost_price_ex_tax': '0.0000',
     'cost_price_inc_tax': '0.0000',
     'cost_price_tax': '0.0000',
     'ebay_item_id': '',
     'ebay_transaction_id': '',
     'event_date': '',
     'event_name': None,
     'fixed_shipping_cost': '0.0000',
     'id': '',
     'is_bundled_product ': True,
     'is_refunded': False,
     'name': '',
     'option_set_id': None,
     'order_address_id': 2,           # should be 0
     'order_id': 614534,              # should be 0
     'parent_order_product_id': 3,    # should be 0
     'price_ex_tax': '0.0000',
     'price_inc_tax': '0.0000',
     'price_tax': '0.0000',
     'product_id': '',
     'product_options': [],
     'quantity': 1,                   # should be 0
     'quantity_shipped': 0,
     'refund_amount': '0.0000',
     'return_id': 0,
     'sku': '',
     'total_ex_tax': '0.0000',
     'total_inc_tax': '0.0000',
     'total_tax': '0.0000',
     'type': 'physical',
     'weight': '0.0000',
     'wrapping_cost_ex_tax': '0.0000',
     'wrapping_cost_inc_tax': '0.0000',
     'wrapping_cost_tax': '0.0000',
     'wrapping_message': '',
     'wrapping_name': ''}
    

After manually checking and fixing some bad default values,

    
    
    def strip_defaults(dct, defaults):
        return {key:value for key,value in dct.items() if value != defaults[key]}
    
    res = [strip_defaults(d, defaults) for d in data]
    pp(res)
    

which removes all default-valued fields and gives us a slightly more readable
version:

    
    
    [{'applied_discounts': [{'amount': 99, 'id': 'total-coupon'}],
      'base_price': '99.0000',
      'base_total': '99.0000',
      'id': 3,
      'is_bundled_product ': False,
      'name': 'University of Timbuktu Bachelor Set',
      'option_set_id': 15,
      'order_address_id': 2,
      'order_id': 614534,
      'price_ex_tax': '99.0000',
      'price_inc_tax': '99.0000',
      'product_id': 83,
      'product_options': [{'display_name': 'Gown size',
                           'display_style': 'Pick list',
                           'display_value': 'Cambridge-Style Bachelor Gown, '
                                            'Size L',
                           'id': 2,
                           'name': 'Bachelor gown size',
                           'option_id': 19,
                           'order_product_id': 3,
                           'product_option_id': 95,
                           'type': 'Product list',
                           'value': '77'},
                          {'display_name': 'Trencher size',
                           'display_style': 'Pick list',
                           'display_value': 'Bachelor and Masters Trencher, '
                                            'Size L',
                           'id': 3,
                           'name': 'Trencher size',
                           'option_id': 20,
                           'order_product_id': 3,
                           'product_option_id': 97,
                           'type': 'Product list',
                           'value': '80'}],
      'quantity': 1,
      'sku': 'S-TIM-BAC-STD',
      'total_ex_tax': '99.0000',
      'total_inc_tax': '99.0000',
      'weight': '3.0000'},
     {'id': 4,
      'name': 'Cambridge-Style Bachelor Gown, Size L',
      'order_address_id': 2,
      'order_id': 614534,
      'parent_order_product_id': 3,
      'product_id': 80,
      'quantity': 1,
      'sku': 'G-CAM-BAC-L'},
     {'id': 5,
      'name': 'Bachelor and Masters Trencher, Size L',
      'order_address_id': 2,
      'order_id': 614534,
      'parent_order_product_id': 3,
      'product_id': 87,
      'quantity': 1,
      'sku': 'C-STD-B&M-L'}]
    

