There is data stored in JSON file with the following structure:goods_data = {year : {month : {day : {01 : {name : some item, price : 10.01, category : root | first | sub first | twice sub first, 02 : {name : another item, price : 10.99, category :  root | first | sub first} }}}}}The task is to fill a QTreeView in which categories must have tree structure (the first column). The second column must contain the total sum of purchases  and percentage of the year for current category. The other columns are suppose to contain the total sum for each month for the current category.The script I have written does what it is supposed to do but it takes too much time for the given data.I understand that I include too many iterations inside each iteration, but I don't know the other way to do it. I think there is no reason to put all data that is being used, so I just have included the simplified dictionary.How can I do it in more efficient and pythonic way?#!/usr/bin/env python -tt# -*- coding: utf-8 -*-#from PySide.QtGui import *from PyQt5.QtGui import *from PyQt5.QtCore import *from PyQt5.QtWidgets import *from PyQt5.QtCore import pyqtSlotimport timeimport typesimport jsonimport sysreload(sys)sys.setdefaultencoding('utf8')from datetime import datetimeimport collectionsimport operatorimport sipsip.setapi('QVariant', 2)goods_data = {2015 : {09 : {    01 : {        1 : {                name : Red apples, price : 10.01, category : Food and Grocery | Fruit and Vegetables | Fruit | Apples              },        2 : {                name : Green  apples, price : 10.99, category :  Food and Grocery | Fruit and Vegetables | Fruit | Apples              }            },    15 : {        1 :             {            name : Blue apples, price : 10.01, category : Food and Grocery | Fruit and Vegetables | Fruit | Apples            },        2 : {        name : Black  apples, price : 10.99, category :  Food and Grocery | Fruit and Vegetables | Fruit | Apples            }}            }, 10 : {    01 : {        1 : {                name : Oranges, price : 10, category : Food and Grocery | Fruit and Vegetables | Fruit | Oranges              },        2 : {                name : Oranges, price : 10, category :  Food and Grocery | Fruit and Vegetables | Fruit | Oranges              }            },    15 : {        1 :             {            name : Oranges, price : 10, category : Food and Grocery | Fruit and Vegetables | Fruit | Oranges            },        2 : {        name : Apples, price : 10, category :  Food and Grocery | Fruit and Vegetables | Fruit | Apples            }}            }                           }            }#gj = 'goods.json'#with open(gj) as f:    #goods_data = json.load(f)@pyqtSlot(float, float) def percentage(part,whole):    if whole>0:        return 100.00* float(part)/float(whole)    else:        return 0@pyqtSlot(float, float) def perc_diff(prev,curr):    return((float(curr)-prev)/abs(prev))*100.00@pyqtSlot(dict, dict)   def update(d, u):    '''    returns updated dictionary (initial dict, updating dict)    '''    for k, v in u.iteritems():        if isinstance(v, collections.Mapping):            r = update(d.get(k, {}), v)                 d[k] = r        else:            d[k] = u[k]    return dclass MainFrame(QWidget):    def __init__(self):        #super(MainFrame, self).__init__(parent)        QWidget.__init__(self)        self.statusBar = QStatusBar()        self.expences_cat_tree_stat = QTreeView()        self.expences_cat_tree_stat.setAutoExpandDelay(0)        self.expences_cat_tree_stat_model = QStandardItemModel()        self.expences_cat_tree_stat.setModel( self.expences_cat_tree_stat_model )        MainWindow = QVBoxLayout(self)        MainWindow.addWidget(self.expences_cat_tree_stat)        MainWindow.addWidget(self.statusBar)        self.statusBar.showMessage(READY)        self.setLayout(MainWindow)        self.stat_cat_tree = self.load_stat_cat_tree(goods_data, self.expences_cat_tree_stat_model)    @pyqtSlot(QObject, dict, QObject, dict, int,  QStandardItemModel)    def TreeCatStat(self, children, parent, cat_dict, year, model):        for child in sorted(children):            child_item = QStandardItem(child)            data_row = list([child_item])            #   adding items without data to the row            for i in range(model.columnCount()):                data_row.append(QStandardItem())            parent.appendRow(data_row)            #   creating the path            varforpath = child_item.parent()            list_for_path = list([child_item.text()])            while varforpath.parent() is not None:                list_for_path.append(varforpath.text())                varforpath = varforpath.parent()            cat_path_str = str()                cat_path_str = ' > '.join(reversed(list_for_path))            self.statusBar.showMessage(cat_path_str)            #   iterating categories            for sbct in sorted(cat_dict):                cat_tot_year = float()                #   if iterated category starts with path created from item                if sbct.startswith(cat_path_str):                    #   iterating months                     for i in range(2, model.columnCount()):                        cat_tot_month = float()                        #   inside each month iterating categories                        for kpd in sorted(cat_dict):                            # if iterated category inside the month starts with path from item                            if kpd.startswith(cat_path_str):                                 #   iterating month of categories that starts with path from item                                for mm in sorted(cat_dict[kpd][year]):                                    #   if name of the month from header matches name of the month from dictionary                                    if model.headerData(i, Qt.Horizontal) == QDate.fromString(mm, M).toString(MMMM):                                        #   iterating dates of month                                        for dd in sorted(cat_dict[kpd][year][mm]):                                            #   iterating numbers of purchases                                            for ii in sorted(cat_dict[kpd][year][mm][dd]):                                                #   iterating names of purchases                                                for itn in sorted(cat_dict[kpd][year][mm][dd][ii]):                                                    cat_tot_month += cat_dict[kpd][year][mm][dd][ii][itn]['sum']                                                    cat_tot_year += cat_dict[kpd][year][mm][dd][ii][itn]['sum']                                model.setData(model.indexFromItem(data_row[i]), cat_tot_month)                    model.setData(model.indexFromItem(data_row[1]), cat_tot_year)            whole = float(data_row[1].text())            #   adding percentage to months                         for val in range(2, model.columnCount()):                part = model.indexFromItem(data_row[val]).data()                str_out = str()                if part is not None:                    part = float(part)                    if part > 0:                        perc_str = format(percentage(part, whole), .2f)                         str_out = str(format(part, .2f)) +  ( + perc_str + %)                    model.setData(model.indexFromItem(data_row[val]), str_out )            if isinstance(children, types.DictType):                self.TreeCatStat(children[child], child_item, cat_dict, year, model)    @pyqtSlot(QObject, dict, QStandardItemModel)                def load_stat_cat_tree(self, data, model):        start=time.clock()        res = dict()        ch_y = 2015        tree = self.expences_cat_tree_stat        exp_cat_tree_header = list([Year / Category, Total])        month_perc_item_list = list()        overall_spent = float()        prev_month_perc =float()        one_month_sum_list  = list()        cat_dict = dict()        initial_dict_creation_time_start = time.clock()        for yy in sorted(data):            year_item = QStandardItem(yy)            one_year_sum = float()            for mm in sorted(data[yy]):                date =  QDate.fromString(mm, M)                chosenMonth = date.toString(MMMM)                month_item  = QStandardItem(chosenMonth)                one_month_sum = float()                for dd in sorted(data[yy][mm]):                    one_day_sum = float()                    day_str =  QDate.fromString(dd +  + mm +  + yy, dd MM yyyy)                    day_str_out = day_str.toString(d, dddd)                    day_item = QStandardItem(day_str_out)                    day_item.setToolTip(day_str.toString(d.MM.yy, dddd))                    for ii in sorted(data[yy][mm][dd]):                        item_pr = float(data[yy][mm][dd][ii][price])                        item_name=data[yy][mm][dd][ii][name]                        #item_descr=data[yy][mm][dd][ii][descr]                        item_cat = data[yy][mm][dd][ii][category].replace(|, >)                        path = item_cat                        if path not in cat_dict:                            cat_dict[path] = {yy: {mm: {dd: {ii: {item_name:  {sum: item_pr} }}}}}                        else:                            if yy not in cat_dict[path]:                                cat_dict[path][yy] = {}                            if mm not in cat_dict[path][yy]:                                cat_dict[path][yy][mm] = {}                            if dd not in cat_dict[path][yy][mm]:                                cat_dict[path][yy][mm][dd] = {}                            if ii not in cat_dict[path][yy][mm][dd]:                                cat_dict[path][yy][mm][dd][ii] = {}                            if item_name not in cat_dict[path][yy][mm][dd][ii]:                                cat_dict[path][yy][mm][dd][ii][item_name] = {sum: item_pr}                        if path:                            d = t = {} # t is my temporary dictionary                            for i in path.split( > ):                                t[i] = {}                                t = t[i]                                dct = update(res,d)                        else:                            dct = res                        one_month_sum += float(item_pr)                        one_year_sum += float(item_pr)                if one_month_sum > 0:                    exp_cat_tree_header.append(month_item.text())                    one_month_sum_list.append( float(one_month_sum))            initial_dict_creation_time_end = time.clock()            initial_dict_creation_time_diff = str(initial_dict_creation_time_end - initial_dict_creation_time_start)            print initial dict creation time for  + yy +  ->  + initial_dict_creation_time_diff            if one_year_sum > 0:                year_row_list = [year_item, QStandardItem(str(format(one_year_sum, .2f)))]                for tm in one_month_sum_list:                    year_row_list.append(QStandardItem(str(format(tm, .2f)) +  ( +  format(percentage (tm, one_year_sum), .2f) + %) ))                    self.expences_cat_tree_stat_model.appendRow(year_row_list)            model.setHorizontalHeaderLabels(exp_cat_tree_header)            self.TreeCatStat(res, year_item, cat_dict, yy, model)        tree.expandAll()        c = 0        while c < len(exp_cat_tree_header):            tree.resizeColumnToContents(c)            c=c+1        tree.collapseAll()        item_to_expand = year_item        tree.setExpanded(model.indexFromItem(item_to_expand), True)        tree.scrollTo(model.indexFromItem(item_to_expand))        end = time.clock()        #print %.2gs % (end - start)        print calculations:, str(end - start)        self.statusBar.showMessage(Calculations: + str(end - start))if __name__ == __main__:    app = QApplication(sys.argv)    main = MainFrame()    main.show()    main.move(app.desktop().screen().rect().center() -     main.rect().center())    sys.exit(app.exec_())
Calculating sum of values in each child of dictionary for QTreeView
python;performance;beginner;dictionary;pyqt
For various reasonnings Ill need slightly different goods_data than yours. As such, I used the following ones which includes 2 different years with duplicate months:goods_data = {    2015 : {        09 : {            01 : {                01 : {name : Red apples, price : 10.01, category : Food and Grocery | Fruit and Vegetables | Fruit | Apples, descr:},                 02 : {name : Green  apples, price : 10.99, category :  Food and Grocery | Fruit and Vegetables | Fruit | Apples, descr:},            },            15 : {                01 : {name : Blue apples, price : 10.01, category : Food and Grocery | Fruit and Vegetables | Fruit | Apples, descr:},                02 : {name : Black  apples, price : 10.99, category :  Food and Grocery | Fruit and Vegetables | Fruit | Apples, descr:},            },        },        11 : {            01 : {                01 : {name : Red apples, price : 10.01, category : Food and Grocery | Fruit and Vegetables | Fruit | Apples, descr:},                 02 : {name : Green  apples, price : 10.99, category :  Food and Grocery | Fruit and Vegetables | Fruit | Grappes, descr:},            },            15 : {                01 : {name : Blue apples, price : 10.01, category : Food and Grocery | Fruit and Vegetables | Fruit | Apples, descr:},                02 : {name : Black  apples, price : 10.99, category :  Food and Grocery | Fruit and Vegetables | Fruit | Grappes, descr:},            },        },    },    2016 : {        11 : {            01 : {                01 : {name : Red apples, price : 10.01, category : Food and Grocery | Fruit and Vegetables | Fruit | Apples, descr:},                 02 : {name : Green  apples, price : 10.99, category :  Food and Grocery | Fruit and Vegetables | Fruit | Grappes, descr:},            },            15 : {                01 : {name : Blue apples, price : 10.01, category : Food and Grocery | Fruit and Vegetables | Fruit | Apples, descr:},                02 : {name : Black  apples, price : 10.99, category :  Food and Grocery | Fruit and Vegetables | Fruit | Grappes, descr:},            },        },        12 : {            01 : {                01 : {name : Red apples, price : 10.01, category : Food and Grocery | Fruit and Vegetables | Fruit | Apples, descr:},                 02 : {name : Green  apples, price : 10.99, category :  Food and Grocery | Fruit and Vegetables | Fruit | Grappes, descr:},            },            15 : {                01 : {name : Blue apples, price : 10.01, category : Food and Grocery | Fruit and Vegetables | Fruit | Apples, descr:},                02 : {name : Black  apples, price : 10.99, category :  Food and Grocery | Fruit and Vegetables | Fruit | Grappes, descr:},            },        },    },}Youve got a bugEver wonder what happens when you use data that span on two years? Well things works strangely:initial dict creation time for 2015 -> 0.000836QStandardItem::insertRows: Ignoring duplicate insertion of item 0x557e570b4030QStandardItem::insertRows: Ignoring duplicate insertion of item 0x557e570dcd20QStandardItem::insertRows: Ignoring duplicate insertion of item 0x557e570e1bb0initial dict creation time for 2016 -> 0.006231QStandardItem::insertRows: Ignoring duplicate insertion of item 0x557e56d9fc30QStandardItem::insertRows: Ignoring duplicate insertion of item 0x557e570d7690QStandardItem::insertRows: Ignoring duplicate insertion of item 0x557e570dcd70QStandardItem::insertRows: Ignoring duplicate insertion of item 0x557e56d9fc30QStandardItem::insertRows: Ignoring duplicate insertion of item 0x557e570d7690QStandardItem::insertRows: Ignoring duplicate insertion of item 0x557e570dcd70QStandardItem::insertRows: Ignoring duplicate insertion of item 0x557e570d6280QStandardItem::insertRows: Ignoring duplicate insertion of item 0x557e56d9fc30QStandardItem::insertRows: Ignoring duplicate insertion of item 0x557e570d7690QStandardItem::insertRows: Ignoring duplicate insertion of item 0x557e570dcd70QStandardItem::insertRows: Ignoring duplicate insertion of item 0x557e570d6280QStandardItem::insertRows: Ignoring duplicate insertion of item 0x557e56e5a070calculations: 0.032324On top of that, the September column appears twice in the QTreeView and data are displayed strangely within it.This is probably due to model.setHorizontalHeaderLabels(exp_cat_tree_header) being called within for yy in sorted(data). Basically, you should first figure out the name of the columns, call setHorizontalHeaderLabels only once, and then process your data using your for loop.You also add your data_row to both the tree-view and the parent element, which lead to weird results.Due to other issues in your code, I won't develop further and well see latter how to improve it.ImportsImporting modules with from some_module import * is frowned upon in Python. It pollutes the global namespace. Specify the list of things you want to import from a module instead, explicit is better than implicit.I also have a tendancy to use the from .. import form when using modules for very specific features. Id do from time import clock for instance, to make it more clear that the time module is for benchmarking purposes and not really related to your computations.Also try to clean up unnecessary modules: you don't use operator or datetime for instance.Lastly, reload(sys);sys.setdefaultencoding(utf-8) is discouraged and sys.setdefaultencoding is a no-op in Python 3. It can be used in very specific cases but doesn't apply to yours. See this SO answer for more informations.pyqtSlotFirst of, you don't even need to from PyQt5.QtCore import pyqtSlot since you already have pyqtSlot in the global namespace after from PyQt5.QtCore import *. One more reason to avoid from .. import *.Second, slots in Qt are roughly callback functions for signals. You connect slots to specific signals and when one of these signals is emitted, the corresponding slots runs. Since you don't use signals, you don't need slots.Lastly, slots in pyQt are much less constrained than in C++. Most of the time you can connect a signal to a function without declaring it a slot first. Slots are mainly used to disambiguate overloading.Just don't use pyqtSlot, you should figure out when to use it the day youll need it.Avoid unnecessary stuff[Year / Category, Total] is a list, no need to list([Year / Category, Total])format already return a string, no need to str(format(...))methods of MainFrame can access attributes of self, no need to pass them as parametersyou make no use of the descripton of an item, don't parse it (well, this one was removed in rev3 of the question)you could remove the need of importing collections.Mapping by using EAFP approach:def update(d, u):    for k,v in u.iteritems():        r = d.get(k, {})        try:            update(r, v)        except AttributeError: # v.iteritems() does not exist            d[k] = v        else:            d[k] = rAlso removed return at the end of the function. It is weird for this function that modifies d in place to return something.replaceing '|' by '>'  just to split('>') is useless, better split('|') directly.since you explicitly pass your input data to load_stat_cat_tree, you'd better of also passing it to the __init__ of MainFrame. That way you can import the json module and read the input file only if you run your script from the command line.NamingWith a code this dense, using descriptive variable names for loop variables is necessary. yy might be convenient to type but very hard to follow when decyphering the code. On the other hand, you might want to shorten the names of you GUI componnents.Use better data structures to simplify the logic8 nested for loops + some ifs is really too much. It is both unreadable and probably sign of a bad design. Seeing, that you iterate over sorted(cat_dict) while iterating over sorted(cat_dict), I gave up trying to deeply understand what your code does.Instead of trying to parse your data and build your tree-view all at once, you should split your problem:Parse your data and build a structure where each sub-category of each category of each year hold expenses for the relevant months;Use this data structure to build a tree-view.The main problem you might have faced is that you need a data structure that can handle an arbitrary number of sub-sub--sub-categories. In short, you need nested dictionnaries. But the burden of checking wether a sub-category already exists and create it if not and then expand it can make the code more verbose. Two solutions exists to simplify it:the setdefault method of any dictionary. d.setdefault(key, {}) will create d[key] = {} if it doesn't already exists and then return d[key].collections.defaultdict which are dictionaries who uses a callable to create missing elements on the fly:>>> d = defaultdict(int)>>> d['test']0>>> ddefaultdict(<type: 'int'>, {'test': 0})Where defaultdict can come in handy is that it allows to define recursive dictionnaries; allowing to access any nested level without troubles:>>> def recursive_dict():...     return defaultdict(recursive_dict)...>>> d = recursive_dict()>>> d[1][2][3][4]defaultdict(<function recursive_dict at 0x7f00d215a7d0>, {})>>> ddefaultdict(<function recursive_dict at 0x7f00d215a7d0>, {1: defaultdict(<function recursive_dict at 0x7f00d215a7d0>, {2: defaultdict(<function recursive_dict at 0x7f00d215a7d0>, {3: defaultdict(<function recursive_dict at 0x7f00d215a7d0>, {4: defaultdict(<function recursive_dict at 0x7f00d215a7d0>, {})})})})})But defaultdict are not magic, they are just regular dictionaries which implements the __missing__ method. And in fact you could just build your own recursive_dict on top of dict pretty easily (not sure which one is faster, though):>>> class recursive_dict(dict):...     def __missing__(self, key):...         self[key] = value = recursive_dict()...         return value...>>> d = recursive_dict()>>> d[1][2][3][4]{}>>> d{1: {2: {3: {4: {}}}}}Using this recursive dictionary, we can parse the data structure and, for each item (of each day, of each month, of each year), split its category into hierarchical parts and acces each level of the hierarchy easily. We can then, at each level, for the desired month, store its price:set it if it is the first price for this month; we can tell the first price appart since any month for any level will return a recursive_dict on first access. Once we set the value to a float, it will stay a float.add it if a total already exists.We can do the same, at the same time, for the 'Total' column. And end up with a dictionnary that, for each year and each category, yields the expenses of each sub-category for each month and the total for the year.Pretty handy. We then just have to recursively iterate over this dictionary to add rows to our tree-view. And since totals and partials per months are available at each level of recursion, we can easily compute percentage while adding to the GUI.Total: Only two pass over any data. One when building up totals and one when populating the GUI.Proposed improvements#!/usr/bin/env python -tt# -*- coding: utf-8 -*-from time import clockfrom PyQt5.QtCore import QDatefrom PyQt5.QtGui import QStandardItemModel, QStandardItemfrom PyQt5.QtWidgets import (QApplication, QWidget, QVBoxLayout,                             QStatusBar, QTreeView)import sipsip.setapi('QVariant', 2)TOTAL = 'Total'class recursive_dict(dict):    def __missing__(self, key):        self[key] = value = recursive_dict()        return valuedef add_price(category, month, price):    for column in (month, TOTAL):        try:            category[column] += price        except TypeError:            # First time we hit this month in this            # category. Just store the first price.            category[column] = priceclass MainFrame(QWidget):    def __init__(self, goods_data):        QWidget.__init__(self)        self.statusBar = QStatusBar()        self.expences_cat_tree_stat = QTreeView()        self.expences_cat_tree_stat.setAutoExpandDelay(0)        self.expences_cat_tree_stat_model = QStandardItemModel()        self.expences_cat_tree_stat.setModel(self.expences_cat_tree_stat_model)        MainWindow = QVBoxLayout(self)        MainWindow.addWidget(self.expences_cat_tree_stat)        MainWindow.addWidget(self.statusBar)        self.statusBar.showMessage(READY)        self.setLayout(MainWindow)        # Benchmark        build_time = clock()        self.expenses = recursive_dict()        self._load_data(goods_data)        months = sorted(set(col for year in self.expenses                            for col, value in self.expenses[year].iteritems()                            if isinstance(value, float) and col != TOTAL))        # Debug: use every month of the year        #months = [{:02}.format(m) for m in xrange(1,13)]        month_names = [QDate.fromString(month, M).toString(MMMM)                       for month in months]        self.expences_cat_tree_stat_model.setHorizontalHeaderLabels(                ['Year / Category', TOTAL] + month_names)        for year in sorted(self.expenses):            item = QStandardItem(year)            row = self._build_category_row(self.expenses[year], item, months)            self.expences_cat_tree_stat_model.appendRow(row)        self.expences_cat_tree_stat.collapseAll()        # Recursively expand first category of last year        expanded_item = item        while expanded_item is not None:            expanded_index = self.expences_cat_tree_stat_model.indexFromItem(                    expanded_item)            self.expences_cat_tree_stat.setExpanded(expanded_index, True)            expanded_item = expanded_item.child(0)        self.expences_cat_tree_stat.scrollTo(                self.expences_cat_tree_stat_model.indexFromItem(item))        for i in xrange(len(month_names)+2):            self.expences_cat_tree_stat.resizeColumnToContents(i)        # Show benchmark        build_time = clock() - build_time        message = Building time: {:.6f}s.format(build_time)        print message        self.statusBar.showMessage(message)    def _load_data(self, goods_data):        for year in goods_data:            build_time = clock()            year_expenses = self.expenses[year]            for month in goods_data[year]:                for items in goods_data[year][month].itervalues():                    for good in items.itervalues():                        price = float(good['price'])                        current_category = year_expenses                        add_price(current_category, month, price)                        for category in good['category'].split('|'):                            category = category.strip()                            current_category = current_category[category]                            add_price(current_category, month, price)            build_time = clock() - build_time            message = Initial dict creation time for {}: {:.6f}s.format(                    year, build_time)            print message            self.statusBar.showMessage(message)    def _build_category_row(self, goods_values, parent, months):        total = goods_values[TOTAL]        data_row = [parent, QStandardItem({:.2f}.format(total))]        for month in months:            if month in goods_values:                partial = goods_values[month]                percent = 100 * partial / total                label = {:.2f} ({:.2f}%).format(partial, percent)                data_row.append(QStandardItem(label))            else:                data_row.append(QStandardItem())        for category in sorted(goods_values):            if category != TOTAL and category not in months:                row = self._build_category_row(                        goods_values[category],                        QStandardItem(category),                        months)                parent.appendRow(row)        return data_rowif __name__ == __main__:    import sys    import json    app = QApplication(sys.argv)    with open('goods.json') as f:        goods_data = json.load(f)    main = MainFrame(goods_data)    main.show()    main.move(app.desktop().screen().rect().center() - main.rect().center())    sys.exit(app.exec_())Which outputs:Initial dict creation time for 2015: 0.000545sInitial dict creation time for 2016: 0.000115sBuilding time: 0.022126s