                          for session in self.state["sessions"].values()
                          for loss_key in session["loss_names"])
          return list(loss_keys)
  
      @property
      def total_timestamps(self):
          """ Return timestamps from logs separated per session for all sessions """
          return self.tb_logs.get_timestamps()
  
      def initialize_session(self, is_training=False, session_id=None):
          """ Initialize the training session """
          logger.debug("Initializing session: (is_training: %s, session_id: %s)",
                       is_training, session_id)
          self.load_state_file()
          self.tb_logs = TensorBoardLogs(os.path.join(self.modeldir,
                                                      "{}_logs".format(self.modelname)))
          if is_training:
              self.session_id = max(int(key) for key in self.state["sessions"].keys())
          else:
              self.session_id = session_id
          self.initialized = True
          logger.debug("Initialized session. Session_ID: %s", self.session_id)
  
      def load_state_file(self):
          """ Load the current state file """
          state_file = os.path.join(self.modeldir, "{}_state.json".format(self.modelname))
          logger.debug("Loading State: '%s'", state_file)
 
</s> remove     def load_state_file(self):
        """ Load the current state file """
        state_file = os.path.join(self.modeldir, "{}_state.json".format(self.modelname))
        logger.debug("Loading State: '%s'", state_file)
        self.state = self.serializer.load(state_file)
        logger.debug("Loaded state: %s", self.state)
 </s> add             self._time_stats[session_id] = dict(
                start_time=np.min(latest) if np.any(latest) else 0,
                end_time=np.max(latest) if np.any(latest) else 0,
                iterations=latest.shape[0] if np.any(latest) else 0) </s> remove     @property
    def _total_loss_keys(self):
        """ list: The loss keys for all sessions. """
        loss_keys = set(loss_key
                        for session in self.state["sessions"].values()
                        for loss_key in session["loss_names"])
        return list(loss_keys)
 </s> add         elif Session.is_training:
            logger.debug("Updating summary time stamps for training session") </s> remove     @property
    def loss_keys(self):
        """ list: The loss keys for the current session, or loss keys for all sessions. """
        if self.session_id is None:
            retval = self._total_loss_keys
        else:
            retval = self.session["loss_names"]
        return retval
 </s> add     Parameters
    ----------
    session: :class:`GlobalSession`
        The loaded or currently training session
    """
    def __init__(self, session):
        logger.debug("Initializing %s: (session: %s)", self.__class__.__name__, session)
        self._session = session
        self._state = session._state </s> remove def convert_time(timestamp):
    """ Convert time stamp to total hours, minutes and seconds """
    hrs = int(timestamp // 3600)
    if hrs < 10:
        hrs = "{0:02d}".format(hrs)
    mins = "{0:02d}".format((int(timestamp % 3600) // 60))
    secs = "{0:02d}".format((int(timestamp % 3600) % 60))
    return hrs, mins, secs
 </s> add class GlobalSession():
    """ Holds information about a loaded or current training session by accessing a model's state
    file and Tensorboard logs. This class should not be accessed directly, rather through
    :attr:`lib.stats.Session`
    """
    def __init__(self):
        logger.debug("Initializing %s", self.__class__.__name__)
        self._state = None
        self._model_dir = None
        self._model_name = None

        self._tb_logs = None
        self._summary = None

        self._is_training = False
        self._is_querying = Event()

        logger.debug("Initialized %s", self.__class__.__name__)

    @property
    def is_loaded(self):
        """ bool: ``True`` if session data is loaded otherwise ``False`` """
        return self._model_dir is not None

    @property
    def is_training(self):
        """ bool: ``True`` if the loaded session is the currently training model, otherwise
        ``False`` """
        return self._is_training

    @property
    def model_filename(self):
        """ str: The full model filename """
        return os.path.join(self._model_dir, self._model_name)

    @property
    def batch_sizes(self):
        """ dict: The batch sizes for each session_id for the model. """
        return {int(sess_id): sess["batchsize"]
                for sess_id, sess in self._state["sessions"].items()}

    @property
    def full_summary(self):
        """ list: List of dictionaries containing summary statistics for each session id. """
        return self._summary.get_summary_stats()

    @property
    def logging_disabled(self):
        """ bool: ``True`` if logging is enabled for the currently training session otherwise
        ``False``. """
        return self._state["sessions"][str(self.session_ids[-1])]["no_logs"]

    @property
    def session_ids(self):
        """ list: The sorted list of all existing session ids `int`s in the state file """
        return self._tb_logs.session_ids

    def _load_state_file(self):
        """ Load the current state file to :attr:`_state`. """
        state_file = os.path.join(self._model_dir, "{}_state.json".format(self._model_name))
        logger.debug("Loading State: '%s'", state_file)
        serializer = get_serializer("json")
        self._state = serializer.load(state_file)
        logger.debug("Loaded state: %s", self._state)

    def initialize_session(self, model_folder, model_name, is_training=False):
        """ Initialize a Session.

        Load's the model's state file, and sets the paths to any underlying Tensorboard logs, ready
        for access on request.

        Parameters
        ----------
        model_folder: str, optional
            If loading a session manually (e.g. for the analysis tab), then the path to the model
            folder must be provided. For training sessions, this should be left at ``None``
        model_name: str, optional
            If loading a session manually (e.g. for the analysis tab), then the model filename
            must be provided. For training sessions, this should be left at ``None``
        is_training: bool, optional
            ``True`` if the session is being initialized for a training session, otherwise
            ``False``. Default: ``False``
         """
        logger.debug("Initializing session: (is_training: %s)", is_training)

        if self._model_dir == model_folder and self._model_name == model_name:
            if is_training:
                self._tb_logs.refresh_log_filenames()
                self._load_state_file()
                self._is_training = True
            logger.debug("Requested session is already loaded. Not initializing: (model_folder: "
                         "%s, model_name: %s)", model_folder, model_name)
            return

        self._is_training = is_training
        self._model_dir = model_folder
        self._model_name = model_name
        self._load_state_file()
        self._tb_logs = TensorBoardLogs(os.path.join(self._model_dir,
                                                     "{}_logs".format(self._model_name)))

        self._summary = SessionsSummary(self)
        logger.debug("Initialized session. Session_IDS: %s", self.session_ids)

    def stop_training(self):
        """ Clears the internal training flag. To be called when training completes. """
        self._is_training = False

    def clear(self):
        """ Clear the currently loaded session. """
        self._state = None
        self._model_dir = None
        self._model_name = None

        del self._tb_logs
        self._tb_logs = None

        del self._summary
        self._summary = None

        self._is_training = False

    def get_loss(self, session_id):
        """ Obtain the loss values for the given session_id.

        Parameters
        ----------
        session_id: int or ``None``
            The session ID to return loss for. Pass ``None`` to return loss for all sessions.

        Returns
        -------
        dict
            Loss names as key, :class:`numpy.ndarray` as value. If No session ID was provided
            all session's losses are collated
        """
        self._wait_for_thread()

        if self._is_training:
            self._is_querying.set()

        loss_dict = self._tb_logs.get_loss(session_id=session_id, is_training=self._is_training)
        if session_id is None:
            retval = dict()
            for key in sorted(loss_dict):
                for loss_key, loss in loss_dict[key].items():
                    retval.setdefault(loss_key, []).extend(loss)
            retval = {key: np.array(val, dtype="float32") for key, val in retval.items()}
        else:
            retval = loss_dict[session_id]

        if self._is_training:
            self._is_querying.clear()
        return retval

    def get_timestamps(self, session_id):
        """ Obtain the time stamps keys for the given session_id.

        Parameters
        ----------
        session_id: int or ``None``
            The session ID to return the time stamps for. Pass ``None`` to return time stamps for
            all sessions.

        Returns
        -------
        dict or :class:`numpy.ndarray`
            If a session ID has been given then a single :class:`numpy.ndarray` will be returned
            with the session's time stamps. Otherwise a 'dict' will be returned with the session
            IDs as key with :class:`numpy.ndarray` of timestamps as values
        """
        self._wait_for_thread()

        if self._is_training:
            self._is_querying.set()

        retval = self._tb_logs.get_timestamps(session_id=session_id, is_training=self._is_training)
        if session_id is not None:
            retval = retval[session_id]

        if self._is_training:
            self._is_querying.clear()

        return retval

    def _wait_for_thread(self):
        """ If a thread is querying the log files for live data, then block until task clears. """
        while True:
            if self._is_training and self._is_querying.is_set():
                logger.debug("Waiting for available thread")
                time.sleep(1)
                continue
            break

    def get_loss_keys(self, session_id):
        """ Obtain the loss keys for the given session_id.

        Parameters
        ----------
        session_id: int or ``None``
            The session ID to return the loss keys for. Pass ``None`` to return loss keys for
            all sessions.

        Returns
        -------
        list
            The loss keys for the given session. If ``None`` is passed as session_id then a unique
            list of all loss keys for all sessions is returned
        """
        if session_id is None:
            retval = list(set(loss_key
                              for session in self._state["sessions"].values()
                              for loss_key in session["loss_names"]))
        else:
            retval = self._state["sessions"][str(session_id)]["loss_names"]
        return retval


Session = GlobalSession() </s> remove     @property
    def session_ids(self):
        """ Return sorted list of all existing session ids in the state file """
        return sorted([int(key) for key in self.state["sessions"].keys()])
 </s> add         Format the stats for display
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
                          for session in self.state["sessions"].values()
                          for loss_key in session["loss_names"])
          return list(loss_keys)
  
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
            session_id = Session.session_ids[-1]
            latest = self._session.get_timestamps(session_id)
  
      def load_state_file(self):
          """ Load the current state file """
          state_file = os.path.join(self.modeldir, "{}_state.json".format(self.modelname))
          logger.debug("Loading State: '%s'", state_file)
keep keep keep keep replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace keep keep keep keep keep
 GUI - Stats optimization (#1067)

* Faster stat loading + caching

* Compress data in cache

* Optimize some calculations

* Vectorize smoothing

* stats.Calculations optimized

* Load latest training data from live iterator

* Add options to training graph 
lib/gui/stats.py