          selection = self.tree.focus()
          values = self.tree.item(selection, "values")
          if values:
              logger.debug("Selected values: %s", values)
              self.selected_id.set(values[0])
              if region == "tree" and self.check_valid_data(values):
                  datapoints = int(values[self.columns.index("iterations")])
                  self.data_popup(datapoints)
  
      def check_valid_data(self, values):
          """ Check there is valid data available for popping up a graph """
          col_indices = [self.columns.index("batch"), self.columns.index("iterations")]
          for idx in col_indices:
              if (isinstance(values[idx], int) or values[idx].isdigit()) and int(values[idx]) == 0:
                  logger.warning("No data to graph for selected session")
                  return False
          return True
 
</s> remove     def select_item(self, event):
        """ Update the session summary info with
            the selected item or launch graph """
        region = self.tree.identify("region", event.x, event.y)
        selection = self.tree.focus()
        values = self.tree.item(selection, "values")
 </s> add     def _select_item(self, event):
        """ Update the session summary info with the selected item or launch graph.

        If the mouse is clicked on the graph icon, then the session summary pop-up graph is
        launched. Otherwise the selected ID is stored.

        Parameters
        ----------
        event: :class:`tkinter.Event`
            The tkinter mouse button release event
        """
        region = self._tree.identify("region", event.x, event.y)
        selection = self._tree.focus()
        values = self._tree.item(selection, "values") </s> remove     def data_popup(self, datapoints):
 </s> add     def _data_popup(self, data_points): </s> remove 

class SessionPopUp(tk.Toplevel):
    """ Pop up for detailed graph/stats for selected session """
    def __init__(self, model_dir, model_name, session_id, datapoints):
        logger.debug("Initializing: %s: (model_dir: %s, model_name: %s, session_id: %s, "
                     "datapoints: %s)", self.__class__.__name__, model_dir, model_name, session_id,
                     datapoints)
        super().__init__()
        self.thread = None  # Thread for loading data in a background task
        self.default_avg = 500
        self.default_view = "avg" if datapoints > self.default_avg * 2 else "smoothed"
        self.session_id = session_id
        self.session = Session(model_dir=model_dir, model_name=model_name)
        self.initialize_session()

        self.graph_frame = None
        self.graph = None
        self.display_data = None

        self.vars = {"status": tk.StringVar()}
        self.graph_initialised = False
        self.build()
        logger.debug("Initialized: %s", self.__class__.__name__)

    @property
    def is_totals(self):
        """ Return True if these are totals else False """
        return bool(self.session_id == "Total")

    def initialize_session(self):
        """ Initialize the session """
        logger.debug("Initializing session")
        kwargs = dict(is_training=False)
        if not self.is_totals:
            kwargs["session_id"] = int(self.session_id)
        logger.debug("Session kwargs: %s", kwargs)
        self.session.initialize_session(**kwargs)

    def build(self):
        """ Build the popup window """
        logger.debug("Building popup")
        optsframe = self.layout_frames()
        self.set_callback()
        self.opts_build(optsframe)
        self.compile_display_data()
        logger.debug("Built popup")

    def set_callback(self):
        """ Set a tkinter Boolean var to callback when graph is ready to build """
        logger.debug("Setting tk graph build variable")
        var = tk.BooleanVar()
        var.set(False)
        var.trace("w", self.graph_build)
        self.vars["buildgraph"] = var

    def layout_frames(self):
        """ Top level container frames """
        logger.debug("Layout frames")
        leftframe = ttk.Frame(self)
        leftframe.pack(side=tk.LEFT, expand=False, fill=tk.BOTH, pady=5)

        sep = ttk.Frame(self, width=2, relief=tk.RIDGE)
        sep.pack(fill=tk.Y, side=tk.LEFT)

        self.graph_frame = ttk.Frame(self)
        self.graph_frame.pack(side=tk.RIGHT, fill=tk.BOTH, pady=5, expand=True)
        logger.debug("Laid out frames")

        return leftframe

    def opts_build(self, frame):
        """ Build Options into the options frame """
        logger.debug("Building Options")
        self.opts_combobox(frame)
        self.opts_checkbuttons(frame)
        self.opts_loss_keys(frame)
        self.opts_slider(frame)
        self.opts_buttons(frame)
        sep = ttk.Frame(frame, height=2, relief=tk.RIDGE)
        sep.pack(fill=tk.X, pady=(5, 0), side=tk.BOTTOM)
        logger.debug("Built Options")

    def opts_combobox(self, frame):
        """ Add the options combo boxes """
        logger.debug("Building Combo boxes")
        choices = {"Display": ("Loss", "Rate"),
                   "Scale": ("Linear", "Log")}

        for item in ["Display", "Scale"]:
            var = tk.StringVar()

            cmbframe = ttk.Frame(frame)
            cmbframe.pack(fill=tk.X, pady=5, padx=5, side=tk.TOP)
            lblcmb = ttk.Label(cmbframe,
                               text="{}:".format(item),
                               width=7,
                               anchor=tk.W)
            lblcmb.pack(padx=(0, 2), side=tk.LEFT)

            cmb = ttk.Combobox(cmbframe, textvariable=var, width=10)
            cmb["values"] = choices[item]
            cmb.current(0)
            cmb.pack(fill=tk.X, side=tk.RIGHT)

            cmd = self.optbtn_reload if item == "Display" else self.graph_scale
            var.trace("w", cmd)
            self.vars[item.lower().strip()] = var

            hlp = self.set_help(item)
            Tooltip(cmbframe, text=hlp, wraplength=200)
        logger.debug("Built Combo boxes")

    @staticmethod
    def add_section(frame, title):
        """ Add a separator and section title """
        sep = ttk.Frame(frame, height=2, relief=tk.SOLID)
        sep.pack(fill=tk.X, pady=(5, 0), side=tk.TOP)
        lbl = ttk.Label(frame, text=title)
        lbl.pack(side=tk.TOP, padx=5, pady=0, anchor=tk.CENTER)

    def opts_checkbuttons(self, frame):
        """ Add the options check buttons """
        logger.debug("Building Check Buttons")

        self.add_section(frame, "Display")
        for item in ("raw", "trend", "avg", "smoothed", "outliers"):
            if item == "avg":
                text = "Show Rolling Average"
            elif item == "outliers":
                text = "Flatten Outliers"
            else:
                text = "Show {}".format(item.title())
            var = tk.BooleanVar()

            if item == self.default_view:
                var.set(True)

            self.vars[item] = var

            ctl = ttk.Checkbutton(frame, variable=var, text=text)
            ctl.pack(side=tk.TOP, padx=5, pady=5, anchor=tk.W)

            hlp = self.set_help(item)
            Tooltip(ctl, text=hlp, wraplength=200)
        logger.debug("Built Check Buttons")

    def opts_loss_keys(self, frame):
        """ Add loss key selections """
        logger.debug("Building Loss Key Check Buttons")
        loss_keys = self.session.loss_keys
        lk_vars = dict()
        section_added = False
        for loss_key in sorted(loss_keys):
            text = loss_key.replace("_", " ").title()
            helptext = "Display {}".format(text)
            var = tk.BooleanVar()
            if loss_key.startswith("total"):
                var.set(True)
            lk_vars[loss_key] = var

            if len(loss_keys) == 1:
                # Don't display if there's only one item
                var.set(True)
                break

            if not section_added:
                self.add_section(frame, "Keys")
                section_added = True

            ctl = ttk.Checkbutton(frame, variable=var, text=text)
            ctl.pack(side=tk.TOP, padx=5, pady=5, anchor=tk.W)
            Tooltip(ctl, text=helptext, wraplength=200)

        self.vars["loss_keys"] = lk_vars
        logger.debug("Built Loss Key Check Buttons")

    def opts_slider(self, frame):
        """ Add the options entry boxes """

        self.add_section(frame, "Parameters")
        logger.debug("Building Slider Controls")
        for item in ("avgiterations", "smoothamount"):
            if item == "avgiterations":
                dtype = int
                text = "Iterations to Average:"
                default = 500
                rounding = 25
                min_max = (25, 2500)
            elif item == "smoothamount":
                dtype = float
                text = "Smoothing Amount:"
                default = 0.90
                rounding = 2
                min_max = (0, 0.99)
            slider = ControlPanelOption(text,
                                        dtype,
                                        default=default,
                                        rounding=rounding,
                                        min_max=min_max,
                                        helptext=self.set_help(item))
            self.vars[item] = slider.tk_var
            ControlBuilder(frame, slider, 1, 19, None, True)
        logger.debug("Built Sliders")

    def opts_buttons(self, frame):
        """ Add the option buttons """
        logger.debug("Building Buttons")
        btnframe = ttk.Frame(frame)
        btnframe.pack(fill=tk.X, pady=5, padx=5, side=tk.BOTTOM)

        lblstatus = ttk.Label(btnframe,
                              width=40,
                              textvariable=self.vars["status"],
                              anchor=tk.W)
        lblstatus.pack(side=tk.LEFT, anchor=tk.W, fill=tk.X, expand=True)

        for btntype in ("reload", "save"):
            cmd = getattr(self, "optbtn_{}".format(btntype))
            btn = ttk.Button(btnframe,
                             image=get_images().icons[btntype],
                             command=cmd)
            btn.pack(padx=2, side=tk.RIGHT)
            hlp = self.set_help(btntype)
            Tooltip(btn, text=hlp, wraplength=200)
        logger.debug("Built Buttons")

    def optbtn_save(self):
        """ Action for save button press """
        logger.debug("Saving File")
        savefile = FileHandler("save", "csv").retfile
        if not savefile:
            logger.debug("Save Cancelled")
            return
        logger.debug("Saving to: %s", savefile)
        save_data = self.display_data.stats
        fieldnames = sorted(key for key in save_data.keys())

        with savefile as outfile:
            csvout = csv.writer(outfile, delimiter=",")
            csvout.writerow(fieldnames)
            csvout.writerows(zip(*[save_data[key] for key in fieldnames]))

    def optbtn_reload(self, *args):  # pylint: disable=unused-argument
        """ Action for reset button press and checkbox changes"""
        logger.debug("Refreshing Graph")
        if not self.graph_initialised:
            return
        valid = self.compile_display_data()
        if not valid:
            logger.debug("Invalid data")
            return
        self.graph.refresh(self.display_data,
                           self.vars["display"].get(),
                           self.vars["scale"].get())
        logger.debug("Refreshed Graph")

    def graph_scale(self, *args):  # pylint: disable=unused-argument
        """ Action for changing graph scale """
        if not self.graph_initialised:
            return
        self.graph.set_yscale_type(self.vars["scale"].get())

    @staticmethod
    def set_help(control):
        """ Set the help text for option buttons """
        hlp = ""
        control = control.lower()
        if control == "reload":
            hlp = "Refresh graph"
        elif control == "save":
            hlp = "Save display data to csv"
        elif control == "avgiterations":
            hlp = "Number of data points to sample for rolling average"
        elif control == "smoothamount":
            hlp = "Set the smoothing amount. 0 is no smoothing, 0.99 is maximum smoothing"
        elif control == "outliers":
            hlp = "Flatten data points that fall more than 1 standard " \
                  "deviation from the mean to the mean value."
        elif control == "avg":
            hlp = "Display rolling average of the data"
        elif control == "smoothed":
            hlp = "Smooth the data"
        elif control == "raw":
            hlp = "Display raw data"
        elif control == "trend":
            hlp = "Display polynormal data trend"
        elif control == "display":
            hlp = "Set the data to display"
        elif control == "scale":
            hlp = "Change y-axis scale"
        return hlp

    def compile_display_data(self):
        """ Compile the data to be displayed """
        if self.thread is None:
            logger.debug("Compiling Display Data in background thread")
            loss_keys = [key for key, val in self.vars["loss_keys"].items()
                         if val.get()]
            logger.debug("Selected loss_keys: %s", loss_keys)

            selections = self.selections_to_list()

            if not self.check_valid_selection(loss_keys, selections):
                logger.warning("No data to display. Not refreshing")
                return False
            self.vars["status"].set("Loading Data...")
            kwargs = dict(session=self.session,
                          display=self.vars["display"].get(),
                          loss_keys=loss_keys,
                          selections=selections,
                          avg_samples=self.vars["avgiterations"].get(),
                          smooth_amount=self.vars["smoothamount"].get(),
                          flatten_outliers=self.vars["outliers"].get(),
                          is_totals=self.is_totals)
            self.thread = LongRunningTask(target=self.get_display_data, kwargs=kwargs, widget=self)
            self.thread.start()
            self.after(1000, self.compile_display_data)
            return True
        if not self.thread.complete.is_set():
            logger.debug("Popup Data not yet available")
            self.after(1000, self.compile_display_data)
            return True

        logger.debug("Getting Popup from background Thread")
        self.display_data = self.thread.get_result()
        self.thread = None
        if not self.check_valid_data():
            logger.warning("No valid data to display. Not refreshing")
            self.vars["status"].set("")
            return False
        logger.debug("Compiled Display Data")
        self.vars["buildgraph"].set(True)
        return True

    @staticmethod
    def get_display_data(**kwargs):
        """ Get the display data in a LongRunningTask """
        return Calculations(**kwargs)

    def check_valid_selection(self, loss_keys, selections):
        """ Check that there will be data to display """
        display = self.vars["display"].get().lower()
        logger.debug("Validating selection. (loss_keys: %s, selections: %s, display: %s)",
                     loss_keys, selections, display)
        if not selections or (display == "loss" and not loss_keys):
            return False
        return True

    def check_valid_data(self):
        """ Check that the selections holds valid data to display
            NB: len-as-condition is used as data could be a list or a numpy array
        """
        logger.debug("Validating data. %s",
                     {key: len(val) for key, val in self.display_data.stats.items()})
        if any(len(val) == 0  # pylint:disable=len-as-condition
               for val in self.display_data.stats.values()):
            return False
        return True

    def selections_to_list(self):
        """ Compile checkbox selections to list """
        logger.debug("Compiling selections to list")
        selections = list()
        for key, val in self.vars.items():
            if (isinstance(val, tk.BooleanVar)
                    and key != "outliers"
                    and val.get()):
                selections.append(key)
        logger.debug("Compiling selections to list: %s", selections)
        return selections

    def graph_build(self, *args):  # pylint:disable=unused-argument
        """ Build the graph in the top right paned window """
        if not self.vars["buildgraph"].get():
            return
        self.vars["status"].set("Loading Data...")
        logger.debug("Building Graph")
        if self.graph is None:
            self.graph = SessionGraph(self.graph_frame,
                                      self.display_data,
                                      self.vars["display"].get(),
                                      self.vars["scale"].get())
            self.graph.pack(expand=True, fill=tk.BOTH)
            self.graph.build()
            self.graph_initialised = True
        else:
            self.graph.refresh(self.display_data,
                               self.vars["display"].get(),
                               self.vars["scale"].get())
        self.vars["status"].set("")
        self.vars["buildgraph"].set(False)
        logger.debug("Built Graph")
 </s> add  </s> remove         Returns
        -------
        dict
            The loss name as key with list of loss values as value
 </s> add         Parameters
        ----------
        amount: float
            The amount of smoothing to apply to smoothed graph
        """
        update = max(min(amount, 0.999), 0.001)
        logger.debug("Setting smooth amount to: %s (provided value: %s)", update, amount)
        self._args["smooth_amount"] = update

    def update_selections(self, selection, option):
        """ Update the type of selected data.

        Parameters
        ----------
        selection: str
            The selection to update (as can exist in :attr:`_selections`)
        option: bool
            ``True`` if the selection should be included, ``False`` if it should be removed
        """
        # TODO Somewhat hacky, to ensure values are inserted in the correct order. Fine for
        # now as this is only called from Live Graph and selections can only be "raw" and
        # smoothed.
        if option:
            if selection not in self._selections:
                if selection == "raw":
                    self._selections.insert(0, selection)
                else:
                    self._selections.append(selection)
        else:
            if selection in self._selections:
                self._selections.remove(selection)

    def set_iterations_limit(self, limit):
        """ Set the number of iterations to display in the calculations.

        If a value greater than 0 is passed, then the latest iterations up to the given
        limit will be calculated.

        Parameters
        ----------
        limit: int
            The number of iterations to calculate data for. `0` to calculate for all data </s> remove     @staticmethod
    def init_training_session(cliopt):
        """ Set the session stats for disable logging, model folder and model name """
        session = get_config().session
        if cliopt[0] == "-t":
            session.modelname = cliopt[1].lower().replace("-", "_")
            logger.debug("modelname: '%s'", session.modelname)
        if cliopt[0] == "-m":
            session.modeldir = cliopt[1]
            logger.debug("modeldir: '%s'", session.modeldir)
 </s> add     def _get_training_session_info(self, cli_option):
        """ Set the model folder and model name to :`attr:_training_session_location` so the global
        session picks them up for logging to the graph and analysis tab.

        Parameters
        ----------
        cli_option: list
            The command line option to be checked for model folder or name
        """
        if cli_option[0] == "-t":
            self._training_session_location["model_name"] = cli_option[1].lower().replace("-", "_")
            logger.debug("model_name: '%s'", self._training_session_location["model_name"])
        if cli_option[0] == "-m":
            self._training_session_location["model_folder"] = cli_option[1]
            logger.debug("model_folder: '%s'", self._training_session_location["model_folder"])
            self._selected_id.set(values[0])
            if region == "tree" and self._check_valid_data(values):
                data_points = int(values[self._columns.index("iterations")])
                self._data_popup(data_points)

    def _check_valid_data(self, values):
        """ Check there is valid data available for popping up a graph.

        Parameters
        ----------
        values: list
            The values that exist for a single session that are to be validated
        """
        col_indices = [self._columns.index("batch"), self._columns.index("iterations")]
          selection = self.tree.focus()
          values = self.tree.item(selection, "values")
          if values:
              logger.debug("Selected values: %s", values)
            self._selected_id.set(values[0])
            if region == "tree" and self._check_valid_data(values):
                data_points = int(values[self._columns.index("iterations")])
                self._data_popup(data_points)

    def _check_valid_data(self, values):
        """ Check there is valid data available for popping up a graph.

        Parameters
        ----------
        values: list
            The values that exist for a single session that are to be validated
        """
        col_indices = [self._columns.index("batch"), self._columns.index("iterations")]
            self._selected_id.set(values[0])
            if region == "tree" and self._check_valid_data(values):
                data_points = int(values[self._columns.index("iterations")])
                self._data_popup(data_points)

    def _check_valid_data(self, values):
        """ Check there is valid data available for popping up a graph.

        Parameters
        ----------
        values: list
            The values that exist for a single session that are to be validated
        """
        col_indices = [self._columns.index("batch"), self._columns.index("iterations")]
            self._selected_id.set(values[0])
            if region == "tree" and self._check_valid_data(values):
                data_points = int(values[self._columns.index("iterations")])
                self._data_popup(data_points)

    def _check_valid_data(self, values):
        """ Check there is valid data available for popping up a graph.

        Parameters
        ----------
        values: list
            The values that exist for a single session that are to be validated
        """
        col_indices = [self._columns.index("batch"), self._columns.index("iterations")]
            self._selected_id.set(values[0])
            if region == "tree" and self._check_valid_data(values):
                data_points = int(values[self._columns.index("iterations")])
                self._data_popup(data_points)

    def _check_valid_data(self, values):
        """ Check there is valid data available for popping up a graph.

        Parameters
        ----------
        values: list
            The values that exist for a single session that are to be validated
        """
        col_indices = [self._columns.index("batch"), self._columns.index("iterations")]
            self._selected_id.set(values[0])
            if region == "tree" and self._check_valid_data(values):
                data_points = int(values[self._columns.index("iterations")])
                self._data_popup(data_points)

    def _check_valid_data(self, values):
        """ Check there is valid data available for popping up a graph.

        Parameters
        ----------
        values: list
            The values that exist for a single session that are to be validated
        """
        col_indices = [self._columns.index("batch"), self._columns.index("iterations")]
            self._selected_id.set(values[0])
            if region == "tree" and self._check_valid_data(values):
                data_points = int(values[self._columns.index("iterations")])
                self._data_popup(data_points)

    def _check_valid_data(self, values):
        """ Check there is valid data available for popping up a graph.

        Parameters
        ----------
        values: list
            The values that exist for a single session that are to be validated
        """
        col_indices = [self._columns.index("batch"), self._columns.index("iterations")]
            self._selected_id.set(values[0])
            if region == "tree" and self._check_valid_data(values):
                data_points = int(values[self._columns.index("iterations")])
                self._data_popup(data_points)

    def _check_valid_data(self, values):
        """ Check there is valid data available for popping up a graph.

        Parameters
        ----------
        values: list
            The values that exist for a single session that are to be validated
        """
        col_indices = [self._columns.index("batch"), self._columns.index("iterations")]
            self._selected_id.set(values[0])
            if region == "tree" and self._check_valid_data(values):
                data_points = int(values[self._columns.index("iterations")])
                self._data_popup(data_points)

    def _check_valid_data(self, values):
        """ Check there is valid data available for popping up a graph.

        Parameters
        ----------
        values: list
            The values that exist for a single session that are to be validated
        """
        col_indices = [self._columns.index("batch"), self._columns.index("iterations")]
          for idx in col_indices:
              if (isinstance(values[idx], int) or values[idx].isdigit()) and int(values[idx]) == 0:
                  logger.warning("No data to graph for selected session")
                  return False
          return True
keep keep keep keep replace replace replace replace replace replace replace replace keep keep keep keep keep
 GUI - Stats optimization (#1067)

* Faster stat loading + caching

* Compress data in cache

* Optimize some calculations

* Vectorize smoothing

* stats.Calculations optimized

* Load latest training data from live iterator

* Add options to training graph 
lib/gui/display_analysis.py