          logger.debug("time_stats: %s", self._time_stats)
  
  
          If a training session is running, then updates the training sessions stats only.
          """
          if self._per_session_stats is None:
              logger.debug("Collating per session stats")
              compiled = list()
 
</s> remove class SessionsSummary():
    """ Calculations for analysis summary stats """
 </s> add         If a training session is running, then updates the training sessions stats only.
        """
        if self._per_session_stats is None:
            logger.debug("Collating per session stats")
            compiled = list()
            for session_id, ts_data in self._time_stats.items():
                logger.debug("Compiling session ID: %s", session_id)
                if self._state is None:
                    logger.debug("Session state dict doesn't exist. Most likely task has been "
                                 "terminated during compilation")
                    return
                compiled.append(self._collate_stats(session_id, ts_data))

            self._per_session_stats = list(sorted(compiled, key=lambda k: k["session"]))

        elif self._session.is_training:
            logger.debug("Collating per session stats for latest training data")
            session_id = self._session.session_ids[-1]
            ts_data = self._time_stats[session_id]

            if session_id > len(self._per_session_stats):
                self._per_session_stats.append(self._collate_stats(session_id, ts_data))

            stats = self._per_session_stats[-1]

            stats["start"] = ts_data["start_time"]
            stats["end"] = ts_data["end_time"]
            stats["elapsed"] = int(stats["end"] - stats["start"])
            stats["iterations"] = ts_data["iterations"]
            stats["rate"] = (((stats["batch"] * 2) * stats["iterations"])
                             / stats["elapsed"] if stats["elapsed"] != 0 else 0)
        logger.debug("per_session_stats: %s", self._per_session_stats)

    def _collate_stats(self, session_id, timestamps):
        """ Collate the session summary statistics for the given session ID. </s> remove     @property
    def total_iterations(self):
        """ Return session iterations """
        return self.state["iterations"]
 </s> add         If the main Session is currently training, then the training session ID is updated with the
        latest stats.
        """
        if self._time_stats is None:
            logger.debug("Collating summary time stamps") </s> remove     def get_iterations_for_session(self, session_id):
        """ Return the number of iterations for the given session id """
        session = self.state["sessions"].get(str(session_id), None)
        if session is None:
            logger.warning("No session data found for session id: %s", session_id)
            return 0
        return session["iterations"]
 </s> add         logger.debug("time_stats: %s", self._time_stats) </s> remove     @property
    def sessions_stats(self):
        """ Return compiled stats """
        compiled = list()
        for sess_idx, ts_data in self.time_stats.items():
            logger.debug("Compiling session ID: %s", sess_idx)
            if self.session.state is None:
                logger.debug("Session state dict doesn't exist. Most likely task has been "
                             "terminated during compilation")
                return None
            iterations = self.session.get_iterations_for_session(sess_idx)
            elapsed = ts_data["end_time"] - ts_data["start_time"]
            batchsize = self.session.total_batchsize.get(sess_idx, 0)
            compiled.append(
                {"session": sess_idx,
                 "start": ts_data["start_time"],
                 "end": ts_data["end_time"],
                 "elapsed": elapsed,
                 "rate": ((batchsize * 2) * iterations) / elapsed if elapsed != 0 else 0,
                 "batch": batchsize,
                 "iterations": iterations})
        compiled = sorted(compiled, key=lambda k: k["session"])
        return compiled

    def compile_stats(self):
        """ Compile sessions stats with totals, format and return """
        logger.debug("Compiling sessions summary data")
        compiled_stats = self.sessions_stats
        if not compiled_stats:
            return compiled_stats
        logger.debug("sessions_stats: %s", compiled_stats)
        total_stats = self.total_stats(compiled_stats)
        compiled_stats.append(total_stats)
        compiled_stats = self.format_stats(compiled_stats)
        logger.debug("Final stats: %s", compiled_stats)
        return compiled_stats

    @staticmethod
    def total_stats(sessions_stats):
        """ Return total stats """
 </s> add     def _total_stats(self):
        """ Compile the Totals stats.
        Totals are fully calculated each time as they will change on the basis of the training
        session.

        Returns
        -------
        dict:
            The Session name, start time, end time, elapsed time, rate, batch size and number of
            iterations for all session ids within the loaded data.
        """ </s> remove def convert_time(timestamp):
    """ Convert time stamp to total hours, minutes and seconds """
    hrs = int(timestamp // 3600)
    if hrs < 10:
        hrs = "{0:02d}".format(hrs)
    mins = "{0:02d}".format((int(timestamp % 3600) // 60))
    secs = "{0:02d}".format((int(timestamp % 3600) % 60))
    return hrs, mins, secs
 </s> add class GlobalSession():
    """ Holds information about a loaded or current training session by accessing a model's state
    file and Tensorboard logs. This class should not be accessed directly, rather through
    :attr:`lib.stats.Session`
    """
    def __init__(self):
        logger.debug("Initializing %s", self.__class__.__name__)
        self._state = None
        self._model_dir = None
        self._model_name = None

        self._tb_logs = None
        self._summary = None

        self._is_training = False
        self._is_querying = Event()

        logger.debug("Initialized %s", self.__class__.__name__)

    @property
    def is_loaded(self):
        """ bool: ``True`` if session data is loaded otherwise ``False`` """
        return self._model_dir is not None

    @property
    def is_training(self):
        """ bool: ``True`` if the loaded session is the currently training model, otherwise
        ``False`` """
        return self._is_training

    @property
    def model_filename(self):
        """ str: The full model filename """
        return os.path.join(self._model_dir, self._model_name)

    @property
    def batch_sizes(self):
        """ dict: The batch sizes for each session_id for the model. """
        return {int(sess_id): sess["batchsize"]
                for sess_id, sess in self._state["sessions"].items()}

    @property
    def full_summary(self):
        """ list: List of dictionaries containing summary statistics for each session id. """
        return self._summary.get_summary_stats()

    @property
    def logging_disabled(self):
        """ bool: ``True`` if logging is enabled for the currently training session otherwise
        ``False``. """
        return self._state["sessions"][str(self.session_ids[-1])]["no_logs"]

    @property
    def session_ids(self):
        """ list: The sorted list of all existing session ids `int`s in the state file """
        return self._tb_logs.session_ids

    def _load_state_file(self):
        """ Load the current state file to :attr:`_state`. """
        state_file = os.path.join(self._model_dir, "{}_state.json".format(self._model_name))
        logger.debug("Loading State: '%s'", state_file)
        serializer = get_serializer("json")
        self._state = serializer.load(state_file)
        logger.debug("Loaded state: %s", self._state)

    def initialize_session(self, model_folder, model_name, is_training=False):
        """ Initialize a Session.

        Load's the model's state file, and sets the paths to any underlying Tensorboard logs, ready
        for access on request.

        Parameters
        ----------
        model_folder: str, optional
            If loading a session manually (e.g. for the analysis tab), then the path to the model
            folder must be provided. For training sessions, this should be left at ``None``
        model_name: str, optional
            If loading a session manually (e.g. for the analysis tab), then the model filename
            must be provided. For training sessions, this should be left at ``None``
        is_training: bool, optional
            ``True`` if the session is being initialized for a training session, otherwise
            ``False``. Default: ``False``
         """
        logger.debug("Initializing session: (is_training: %s)", is_training)

        if self._model_dir == model_folder and self._model_name == model_name:
            if is_training:
                self._tb_logs.refresh_log_filenames()
                self._load_state_file()
                self._is_training = True
            logger.debug("Requested session is already loaded. Not initializing: (model_folder: "
                         "%s, model_name: %s)", model_folder, model_name)
            return

        self._is_training = is_training
        self._model_dir = model_folder
        self._model_name = model_name
        self._load_state_file()
        self._tb_logs = TensorBoardLogs(os.path.join(self._model_dir,
                                                     "{}_logs".format(self._model_name)))

        self._summary = SessionsSummary(self)
        logger.debug("Initialized session. Session_IDS: %s", self.session_ids)

    def stop_training(self):
        """ Clears the internal training flag. To be called when training completes. """
        self._is_training = False

    def clear(self):
        """ Clear the currently loaded session. """
        self._state = None
        self._model_dir = None
        self._model_name = None

        del self._tb_logs
        self._tb_logs = None

        del self._summary
        self._summary = None

        self._is_training = False

    def get_loss(self, session_id):
        """ Obtain the loss values for the given session_id.

        Parameters
        ----------
        session_id: int or ``None``
            The session ID to return loss for. Pass ``None`` to return loss for all sessions.

        Returns
        -------
        dict
            Loss names as key, :class:`numpy.ndarray` as value. If No session ID was provided
            all session's losses are collated
        """
        self._wait_for_thread()

        if self._is_training:
            self._is_querying.set()

        loss_dict = self._tb_logs.get_loss(session_id=session_id, is_training=self._is_training)
        if session_id is None:
            retval = dict()
            for key in sorted(loss_dict):
                for loss_key, loss in loss_dict[key].items():
                    retval.setdefault(loss_key, []).extend(loss)
            retval = {key: np.array(val, dtype="float32") for key, val in retval.items()}
        else:
            retval = loss_dict[session_id]

        if self._is_training:
            self._is_querying.clear()
        return retval

    def get_timestamps(self, session_id):
        """ Obtain the time stamps keys for the given session_id.

        Parameters
        ----------
        session_id: int or ``None``
            The session ID to return the time stamps for. Pass ``None`` to return time stamps for
            all sessions.

        Returns
        -------
        dict or :class:`numpy.ndarray`
            If a session ID has been given then a single :class:`numpy.ndarray` will be returned
            with the session's time stamps. Otherwise a 'dict' will be returned with the session
            IDs as key with :class:`numpy.ndarray` of timestamps as values
        """
        self._wait_for_thread()

        if self._is_training:
            self._is_querying.set()

        retval = self._tb_logs.get_timestamps(session_id=session_id, is_training=self._is_training)
        if session_id is not None:
            retval = retval[session_id]

        if self._is_training:
            self._is_querying.clear()

        return retval

    def _wait_for_thread(self):
        """ If a thread is querying the log files for live data, then block until task clears. """
        while True:
            if self._is_training and self._is_querying.is_set():
                logger.debug("Waiting for available thread")
                time.sleep(1)
                continue
            break

    def get_loss_keys(self, session_id):
        """ Obtain the loss keys for the given session_id.

        Parameters
        ----------
        session_id: int or ``None``
            The session ID to return the loss keys for. Pass ``None`` to return loss keys for
            all sessions.

        Returns
        -------
        list
            The loss keys for the given session. If ``None`` is passed as session_id then a unique
            list of all loss keys for all sessions is returned
        """
        if session_id is None:
            retval = list(set(loss_key
                              for session in self._state["sessions"].values()
                              for loss_key in session["loss_names"]))
        else:
            retval = self._state["sessions"][str(session_id)]["loss_names"]
        return retval


Session = GlobalSession()
    def _get_per_session_stats(self):
        """ Populate the attribute :attr:`_per_session_stats` with a sorted list by session ID
        of each ID in the training/loaded session. Stats contain the session ID, start, end and
        elapsed times, the training rate, batch size and number of iterations for each session.
          logger.debug("time_stats: %s", self._time_stats)
  
    def _get_per_session_stats(self):
        """ Populate the attribute :attr:`_per_session_stats` with a sorted list by session ID
        of each ID in the training/loaded session. Stats contain the session ID, start, end and
        elapsed times, the training rate, batch size and number of iterations for each session.
  
          If a training session is running, then updates the training sessions stats only.
          """
          if self._per_session_stats is None:
              logger.debug("Collating per session stats")
              compiled = list()
keep add keep keep keep keep keep keep
 GUI - Stats optimization (#1067)

* Faster stat loading + caching

* Compress data in cache

* Optimize some calculations

* Vectorize smoothing

* stats.Calculations optimized

* Load latest training data from live iterator

* Add options to training graph 
lib/gui/stats.py