              position = initial_position
          logger.debug("Returning poup boundaries: (initial_position: %s, position: %s)",
                       initial_position, position)
          return initial_position, position
  
  
  class SessionPopUp(tk.Toplevel):
      """ Pop up for detailed graph/stats for selected session """
      def __init__(self, model_dir, model_name, session_id, datapoints):
          logger.debug("Initializing: %s: (model_dir: %s, model_name: %s, session_id: %s, "
                       "datapoints: %s)", self.__class__.__name__, model_dir, model_name, session_id,
                       datapoints)
          super().__init__()
          self.thread = None  # Thread for loading data in a background task
          self.default_avg = 500
          self.default_view = "avg" if datapoints > self.default_avg * 2 else "smoothed"
          self.session_id = session_id
          self.session = Session(model_dir=model_dir, model_name=model_name)
          self.initialize_session()
  
          self.graph_frame = None
          self.graph = None
          self.display_data = None
  
          self.vars = {"status": tk.StringVar()}
          self.graph_initialised = False
          self.build()
          logger.debug("Initialized: %s", self.__class__.__name__)
  
      @property
      def is_totals(self):
          """ Return True if these are totals else False """
          return bool(self.session_id == "Total")
  
      def initialize_session(self):
          """ Initialize the session """
          logger.debug("Initializing session")
          kwargs = dict(is_training=False)
          if not self.is_totals:
              kwargs["session_id"] = int(self.session_id)
          logger.debug("Session kwargs: %s", kwargs)
          self.session.initialize_session(**kwargs)
  
      def build(self):
          """ Build the popup window """
          logger.debug("Building popup")
          optsframe = self.layout_frames()
          self.set_callback()
          self.opts_build(optsframe)
          self.compile_display_data()
          logger.debug("Built popup")
  
      def set_callback(self):
          """ Set a tkinter Boolean var to callback when graph is ready to build """
          logger.debug("Setting tk graph build variable")
          var = tk.BooleanVar()
          var.set(False)
          var.trace("w", self.graph_build)
          self.vars["buildgraph"] = var
  
      def layout_frames(self):
          """ Top level container frames """
          logger.debug("Layout frames")
          leftframe = ttk.Frame(self)
          leftframe.pack(side=tk.LEFT, expand=False, fill=tk.BOTH, pady=5)
  
          sep = ttk.Frame(self, width=2, relief=tk.RIDGE)
          sep.pack(fill=tk.Y, side=tk.LEFT)
  
          self.graph_frame = ttk.Frame(self)
          self.graph_frame.pack(side=tk.RIGHT, fill=tk.BOTH, pady=5, expand=True)
          logger.debug("Laid out frames")
  
          return leftframe
  
      def opts_build(self, frame):
          """ Build Options into the options frame """
          logger.debug("Building Options")
          self.opts_combobox(frame)
          self.opts_checkbuttons(frame)
          self.opts_loss_keys(frame)
          self.opts_slider(frame)
          self.opts_buttons(frame)
          sep = ttk.Frame(frame, height=2, relief=tk.RIDGE)
          sep.pack(fill=tk.X, pady=(5, 0), side=tk.BOTTOM)
          logger.debug("Built Options")
  
      def opts_combobox(self, frame):
          """ Add the options combo boxes """
          logger.debug("Building Combo boxes")
          choices = {"Display": ("Loss", "Rate"),
                     "Scale": ("Linear", "Log")}
  
          for item in ["Display", "Scale"]:
              var = tk.StringVar()
  
              cmbframe = ttk.Frame(frame)
              cmbframe.pack(fill=tk.X, pady=5, padx=5, side=tk.TOP)
              lblcmb = ttk.Label(cmbframe,
                                 text="{}:".format(item),
                                 width=7,
                                 anchor=tk.W)
              lblcmb.pack(padx=(0, 2), side=tk.LEFT)
  
              cmb = ttk.Combobox(cmbframe, textvariable=var, width=10)
              cmb["values"] = choices[item]
              cmb.current(0)
              cmb.pack(fill=tk.X, side=tk.RIGHT)
  
              cmd = self.optbtn_reload if item == "Display" else self.graph_scale
              var.trace("w", cmd)
              self.vars[item.lower().strip()] = var
  
              hlp = self.set_help(item)
              Tooltip(cmbframe, text=hlp, wraplength=200)
          logger.debug("Built Combo boxes")
  
      @staticmethod
      def add_section(frame, title):
          """ Add a separator and section title """
          sep = ttk.Frame(frame, height=2, relief=tk.SOLID)
          sep.pack(fill=tk.X, pady=(5, 0), side=tk.TOP)
          lbl = ttk.Label(frame, text=title)
          lbl.pack(side=tk.TOP, padx=5, pady=0, anchor=tk.CENTER)
  
      def opts_checkbuttons(self, frame):
          """ Add the options check buttons """
          logger.debug("Building Check Buttons")
  
          self.add_section(frame, "Display")
          for item in ("raw", "trend", "avg", "smoothed", "outliers"):
              if item == "avg":
                  text = "Show Rolling Average"
              elif item == "outliers":
                  text = "Flatten Outliers"
              else:
                  text = "Show {}".format(item.title())
              var = tk.BooleanVar()
  
              if item == self.default_view:
                  var.set(True)
  
              self.vars[item] = var
  
              ctl = ttk.Checkbutton(frame, variable=var, text=text)
              ctl.pack(side=tk.TOP, padx=5, pady=5, anchor=tk.W)
  
              hlp = self.set_help(item)
              Tooltip(ctl, text=hlp, wraplength=200)
          logger.debug("Built Check Buttons")
  
      def opts_loss_keys(self, frame):
          """ Add loss key selections """
          logger.debug("Building Loss Key Check Buttons")
          loss_keys = self.session.loss_keys
          lk_vars = dict()
          section_added = False
          for loss_key in sorted(loss_keys):
              text = loss_key.replace("_", " ").title()
              helptext = "Display {}".format(text)
              var = tk.BooleanVar()
              if loss_key.startswith("total"):
                  var.set(True)
              lk_vars[loss_key] = var
  
              if len(loss_keys) == 1:
                  # Don't display if there's only one item
                  var.set(True)
                  break
  
              if not section_added:
                  self.add_section(frame, "Keys")
                  section_added = True
  
              ctl = ttk.Checkbutton(frame, variable=var, text=text)
              ctl.pack(side=tk.TOP, padx=5, pady=5, anchor=tk.W)
              Tooltip(ctl, text=helptext, wraplength=200)
  
          self.vars["loss_keys"] = lk_vars
          logger.debug("Built Loss Key Check Buttons")
  
      def opts_slider(self, frame):
          """ Add the options entry boxes """
  
          self.add_section(frame, "Parameters")
          logger.debug("Building Slider Controls")
          for item in ("avgiterations", "smoothamount"):
              if item == "avgiterations":
                  dtype = int
                  text = "Iterations to Average:"
                  default = 500
                  rounding = 25
                  min_max = (25, 2500)
              elif item == "smoothamount":
                  dtype = float
                  text = "Smoothing Amount:"
                  default = 0.90
                  rounding = 2
                  min_max = (0, 0.99)
              slider = ControlPanelOption(text,
                                          dtype,
                                          default=default,
                                          rounding=rounding,
                                          min_max=min_max,
                                          helptext=self.set_help(item))
              self.vars[item] = slider.tk_var
              ControlBuilder(frame, slider, 1, 19, None, True)
          logger.debug("Built Sliders")
  
      def opts_buttons(self, frame):
          """ Add the option buttons """
          logger.debug("Building Buttons")
          btnframe = ttk.Frame(frame)
          btnframe.pack(fill=tk.X, pady=5, padx=5, side=tk.BOTTOM)
  
          lblstatus = ttk.Label(btnframe,
                                width=40,
                                textvariable=self.vars["status"],
                                anchor=tk.W)
          lblstatus.pack(side=tk.LEFT, anchor=tk.W, fill=tk.X, expand=True)
  
          for btntype in ("reload", "save"):
              cmd = getattr(self, "optbtn_{}".format(btntype))
              btn = ttk.Button(btnframe,
                               image=get_images().icons[btntype],
                               command=cmd)
              btn.pack(padx=2, side=tk.RIGHT)
              hlp = self.set_help(btntype)
              Tooltip(btn, text=hlp, wraplength=200)
          logger.debug("Built Buttons")
  
      def optbtn_save(self):
          """ Action for save button press """
          logger.debug("Saving File")
          savefile = FileHandler("save", "csv").retfile
          if not savefile:
              logger.debug("Save Cancelled")
              return
          logger.debug("Saving to: %s", savefile)
          save_data = self.display_data.stats
          fieldnames = sorted(key for key in save_data.keys())
  
          with savefile as outfile:
              csvout = csv.writer(outfile, delimiter=",")
              csvout.writerow(fieldnames)
              csvout.writerows(zip(*[save_data[key] for key in fieldnames]))
  
      def optbtn_reload(self, *args):  # pylint: disable=unused-argument
          """ Action for reset button press and checkbox changes"""
          logger.debug("Refreshing Graph")
          if not self.graph_initialised:
              return
          valid = self.compile_display_data()
          if not valid:
              logger.debug("Invalid data")
              return
          self.graph.refresh(self.display_data,
                             self.vars["display"].get(),
                             self.vars["scale"].get())
          logger.debug("Refreshed Graph")
  
      def graph_scale(self, *args):  # pylint: disable=unused-argument
          """ Action for changing graph scale """
          if not self.graph_initialised:
              return
          self.graph.set_yscale_type(self.vars["scale"].get())
  
      @staticmethod
      def set_help(control):
          """ Set the help text for option buttons """
          hlp = ""
          control = control.lower()
          if control == "reload":
              hlp = "Refresh graph"
          elif control == "save":
              hlp = "Save display data to csv"
          elif control == "avgiterations":
              hlp = "Number of data points to sample for rolling average"
          elif control == "smoothamount":
              hlp = "Set the smoothing amount. 0 is no smoothing, 0.99 is maximum smoothing"
          elif control == "outliers":
              hlp = "Flatten data points that fall more than 1 standard " \
                    "deviation from the mean to the mean value."
          elif control == "avg":
              hlp = "Display rolling average of the data"
          elif control == "smoothed":
              hlp = "Smooth the data"
          elif control == "raw":
              hlp = "Display raw data"
          elif control == "trend":
              hlp = "Display polynormal data trend"
          elif control == "display":
              hlp = "Set the data to display"
          elif control == "scale":
              hlp = "Change y-axis scale"
          return hlp
  
      def compile_display_data(self):
          """ Compile the data to be displayed """
          if self.thread is None:
              logger.debug("Compiling Display Data in background thread")
              loss_keys = [key for key, val in self.vars["loss_keys"].items()
                           if val.get()]
              logger.debug("Selected loss_keys: %s", loss_keys)
  
              selections = self.selections_to_list()
  
              if not self.check_valid_selection(loss_keys, selections):
                  logger.warning("No data to display. Not refreshing")
                  return False
              self.vars["status"].set("Loading Data...")
              kwargs = dict(session=self.session,
                            display=self.vars["display"].get(),
                            loss_keys=loss_keys,
                            selections=selections,
                            avg_samples=self.vars["avgiterations"].get(),
                            smooth_amount=self.vars["smoothamount"].get(),
                            flatten_outliers=self.vars["outliers"].get(),
                            is_totals=self.is_totals)
              self.thread = LongRunningTask(target=self.get_display_data, kwargs=kwargs, widget=self)
              self.thread.start()
              self.after(1000, self.compile_display_data)
              return True
          if not self.thread.complete.is_set():
              logger.debug("Popup Data not yet available")
              self.after(1000, self.compile_display_data)
              return True
  
          logger.debug("Getting Popup from background Thread")
          self.display_data = self.thread.get_result()
          self.thread = None
          if not self.check_valid_data():
              logger.warning("No valid data to display. Not refreshing")
              self.vars["status"].set("")
              return False
          logger.debug("Compiled Display Data")
          self.vars["buildgraph"].set(True)
          return True
  
      @staticmethod
      def get_display_data(**kwargs):
          """ Get the display data in a LongRunningTask """
          return Calculations(**kwargs)
  
      def check_valid_selection(self, loss_keys, selections):
          """ Check that there will be data to display """
          display = self.vars["display"].get().lower()
          logger.debug("Validating selection. (loss_keys: %s, selections: %s, display: %s)",
                       loss_keys, selections, display)
          if not selections or (display == "loss" and not loss_keys):
              return False
          return True
  
      def check_valid_data(self):
          """ Check that the selections holds valid data to display
              NB: len-as-condition is used as data could be a list or a numpy array
          """
          logger.debug("Validating data. %s",
                       {key: len(val) for key, val in self.display_data.stats.items()})
          if any(len(val) == 0  # pylint:disable=len-as-condition
                 for val in self.display_data.stats.values()):
              return False
          return True
  
      def selections_to_list(self):
          """ Compile checkbox selections to list """
          logger.debug("Compiling selections to list")
          selections = list()
          for key, val in self.vars.items():
              if (isinstance(val, tk.BooleanVar)
                      and key != "outliers"
                      and val.get()):
                  selections.append(key)
          logger.debug("Compiling selections to list: %s", selections)
          return selections
  
      def graph_build(self, *args):  # pylint:disable=unused-argument
          """ Build the graph in the top right paned window """
          if not self.vars["buildgraph"].get():
              return
          self.vars["status"].set("Loading Data...")
          logger.debug("Building Graph")
          if self.graph is None:
              self.graph = SessionGraph(self.graph_frame,
                                        self.display_data,
                                        self.vars["display"].get(),
                                        self.vars["scale"].get())
              self.graph.pack(expand=True, fill=tk.BOTH)
              self.graph.build()
              self.graph_initialised = True
          else:
              self.graph.refresh(self.display_data,
                                 self.vars["display"].get(),
                                 self.vars["scale"].get())
          self.vars["status"].set("")
          self.vars["buildgraph"].set(False)
          logger.debug("Built Graph")
 
</s> remove         if btntype == "reload":
 </s> add         if button_type == "reload": </s> remove     def _set_help(cls, btntype):
        """ Set the help text for option buttons """
 </s> add     def _set_help(cls, button_type):
        """ Set the help text for option buttons.

        Parameters
        ----------
        button_type: {"reload", "clear", "save", "load"}
            The type of button to set the help text for
        """ </s> remove             self.selected_id.set(values[0])
            if region == "tree" and self.check_valid_data(values):
                datapoints = int(values[self.columns.index("iterations")])
                self.data_popup(datapoints)

    def check_valid_data(self, values):
        """ Check there is valid data available for popping up a graph """
        col_indices = [self.columns.index("batch"), self.columns.index("iterations")]
 </s> add             self._selected_id.set(values[0])
            if region == "tree" and self._check_valid_data(values):
                data_points = int(values[self._columns.index("iterations")])
                self._data_popup(data_points)

    def _check_valid_data(self, values):
        """ Check there is valid data available for popping up a graph.

        Parameters
        ----------
        values: list
            The values that exist for a single session that are to be validated
        """
        col_indices = [self._columns.index("batch"), self._columns.index("iterations")] </s> remove def convert_time(timestamp):
    """ Convert time stamp to total hours, minutes and seconds """
    hrs = int(timestamp // 3600)
    if hrs < 10:
        hrs = "{0:02d}".format(hrs)
    mins = "{0:02d}".format((int(timestamp % 3600) // 60))
    secs = "{0:02d}".format((int(timestamp % 3600) % 60))
    return hrs, mins, secs
 </s> add class GlobalSession():
    """ Holds information about a loaded or current training session by accessing a model's state
    file and Tensorboard logs. This class should not be accessed directly, rather through
    :attr:`lib.stats.Session`
    """
    def __init__(self):
        logger.debug("Initializing %s", self.__class__.__name__)
        self._state = None
        self._model_dir = None
        self._model_name = None

        self._tb_logs = None
        self._summary = None

        self._is_training = False
        self._is_querying = Event()

        logger.debug("Initialized %s", self.__class__.__name__)

    @property
    def is_loaded(self):
        """ bool: ``True`` if session data is loaded otherwise ``False`` """
        return self._model_dir is not None

    @property
    def is_training(self):
        """ bool: ``True`` if the loaded session is the currently training model, otherwise
        ``False`` """
        return self._is_training

    @property
    def model_filename(self):
        """ str: The full model filename """
        return os.path.join(self._model_dir, self._model_name)

    @property
    def batch_sizes(self):
        """ dict: The batch sizes for each session_id for the model. """
        return {int(sess_id): sess["batchsize"]
                for sess_id, sess in self._state["sessions"].items()}

    @property
    def full_summary(self):
        """ list: List of dictionaries containing summary statistics for each session id. """
        return self._summary.get_summary_stats()

    @property
    def logging_disabled(self):
        """ bool: ``True`` if logging is enabled for the currently training session otherwise
        ``False``. """
        return self._state["sessions"][str(self.session_ids[-1])]["no_logs"]

    @property
    def session_ids(self):
        """ list: The sorted list of all existing session ids `int`s in the state file """
        return self._tb_logs.session_ids

    def _load_state_file(self):
        """ Load the current state file to :attr:`_state`. """
        state_file = os.path.join(self._model_dir, "{}_state.json".format(self._model_name))
        logger.debug("Loading State: '%s'", state_file)
        serializer = get_serializer("json")
        self._state = serializer.load(state_file)
        logger.debug("Loaded state: %s", self._state)

    def initialize_session(self, model_folder, model_name, is_training=False):
        """ Initialize a Session.

        Load's the model's state file, and sets the paths to any underlying Tensorboard logs, ready
        for access on request.

        Parameters
        ----------
        model_folder: str, optional
            If loading a session manually (e.g. for the analysis tab), then the path to the model
            folder must be provided. For training sessions, this should be left at ``None``
        model_name: str, optional
            If loading a session manually (e.g. for the analysis tab), then the model filename
            must be provided. For training sessions, this should be left at ``None``
        is_training: bool, optional
            ``True`` if the session is being initialized for a training session, otherwise
            ``False``. Default: ``False``
         """
        logger.debug("Initializing session: (is_training: %s)", is_training)

        if self._model_dir == model_folder and self._model_name == model_name:
            if is_training:
                self._tb_logs.refresh_log_filenames()
                self._load_state_file()
                self._is_training = True
            logger.debug("Requested session is already loaded. Not initializing: (model_folder: "
                         "%s, model_name: %s)", model_folder, model_name)
            return

        self._is_training = is_training
        self._model_dir = model_folder
        self._model_name = model_name
        self._load_state_file()
        self._tb_logs = TensorBoardLogs(os.path.join(self._model_dir,
                                                     "{}_logs".format(self._model_name)))

        self._summary = SessionsSummary(self)
        logger.debug("Initialized session. Session_IDS: %s", self.session_ids)

    def stop_training(self):
        """ Clears the internal training flag. To be called when training completes. """
        self._is_training = False

    def clear(self):
        """ Clear the currently loaded session. """
        self._state = None
        self._model_dir = None
        self._model_name = None

        del self._tb_logs
        self._tb_logs = None

        del self._summary
        self._summary = None

        self._is_training = False

    def get_loss(self, session_id):
        """ Obtain the loss values for the given session_id.

        Parameters
        ----------
        session_id: int or ``None``
            The session ID to return loss for. Pass ``None`` to return loss for all sessions.

        Returns
        -------
        dict
            Loss names as key, :class:`numpy.ndarray` as value. If No session ID was provided
            all session's losses are collated
        """
        self._wait_for_thread()

        if self._is_training:
            self._is_querying.set()

        loss_dict = self._tb_logs.get_loss(session_id=session_id, is_training=self._is_training)
        if session_id is None:
            retval = dict()
            for key in sorted(loss_dict):
                for loss_key, loss in loss_dict[key].items():
                    retval.setdefault(loss_key, []).extend(loss)
            retval = {key: np.array(val, dtype="float32") for key, val in retval.items()}
        else:
            retval = loss_dict[session_id]

        if self._is_training:
            self._is_querying.clear()
        return retval

    def get_timestamps(self, session_id):
        """ Obtain the time stamps keys for the given session_id.

        Parameters
        ----------
        session_id: int or ``None``
            The session ID to return the time stamps for. Pass ``None`` to return time stamps for
            all sessions.

        Returns
        -------
        dict or :class:`numpy.ndarray`
            If a session ID has been given then a single :class:`numpy.ndarray` will be returned
            with the session's time stamps. Otherwise a 'dict' will be returned with the session
            IDs as key with :class:`numpy.ndarray` of timestamps as values
        """
        self._wait_for_thread()

        if self._is_training:
            self._is_querying.set()

        retval = self._tb_logs.get_timestamps(session_id=session_id, is_training=self._is_training)
        if session_id is not None:
            retval = retval[session_id]

        if self._is_training:
            self._is_querying.clear()

        return retval

    def _wait_for_thread(self):
        """ If a thread is querying the log files for live data, then block until task clears. """
        while True:
            if self._is_training and self._is_querying.is_set():
                logger.debug("Waiting for available thread")
                time.sleep(1)
                continue
            break

    def get_loss_keys(self, session_id):
        """ Obtain the loss keys for the given session_id.

        Parameters
        ----------
        session_id: int or ``None``
            The session ID to return the loss keys for. Pass ``None`` to return loss keys for
            all sessions.

        Returns
        -------
        list
            The loss keys for the given session. If ``None`` is passed as session_id then a unique
            list of all loss keys for all sessions is returned
        """
        if session_id is None:
            retval = list(set(loss_key
                              for session in self._state["sessions"].values()
                              for loss_key in session["loss_names"]))
        else:
            retval = self._state["sessions"][str(session_id)]["loss_names"]
        return retval


Session = GlobalSession()
              position = initial_position
          logger.debug("Returning poup boundaries: (initial_position: %s, position: %s)",
                       initial_position, position)
          return initial_position, position









































































































































































































































































































































































































keep keep keep keep replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace
 GUI - Stats optimization (#1067)

* Faster stat loading + caching

* Compress data in cache

* Optimize some calculations

* Vectorize smoothing

* stats.Calculations optimized

* Load latest training data from live iterator

* Add options to training graph 
lib/gui/display_analysis.py