          `batch_total` key.
  
          Parameters
          ----------
          session: int, optional
              The Session ID to return the timestamps for. Set to ``None`` to return all session
              timestamps. Default ``None``
  
          Returns
          -------
 
</s> remove         session: int, optional
 </s> add         session_id: int, optional </s> remove def convert_time(timestamp):
    """ Convert time stamp to total hours, minutes and seconds """
    hrs = int(timestamp // 3600)
    if hrs < 10:
        hrs = "{0:02d}".format(hrs)
    mins = "{0:02d}".format((int(timestamp % 3600) // 60))
    secs = "{0:02d}".format((int(timestamp % 3600) % 60))
    return hrs, mins, secs
 </s> add class GlobalSession():
    """ Holds information about a loaded or current training session by accessing a model's state
    file and Tensorboard logs. This class should not be accessed directly, rather through
    :attr:`lib.stats.Session`
    """
    def __init__(self):
        logger.debug("Initializing %s", self.__class__.__name__)
        self._state = None
        self._model_dir = None
        self._model_name = None

        self._tb_logs = None
        self._summary = None

        self._is_training = False
        self._is_querying = Event()

        logger.debug("Initialized %s", self.__class__.__name__)

    @property
    def is_loaded(self):
        """ bool: ``True`` if session data is loaded otherwise ``False`` """
        return self._model_dir is not None

    @property
    def is_training(self):
        """ bool: ``True`` if the loaded session is the currently training model, otherwise
        ``False`` """
        return self._is_training

    @property
    def model_filename(self):
        """ str: The full model filename """
        return os.path.join(self._model_dir, self._model_name)

    @property
    def batch_sizes(self):
        """ dict: The batch sizes for each session_id for the model. """
        return {int(sess_id): sess["batchsize"]
                for sess_id, sess in self._state["sessions"].items()}

    @property
    def full_summary(self):
        """ list: List of dictionaries containing summary statistics for each session id. """
        return self._summary.get_summary_stats()

    @property
    def logging_disabled(self):
        """ bool: ``True`` if logging is enabled for the currently training session otherwise
        ``False``. """
        return self._state["sessions"][str(self.session_ids[-1])]["no_logs"]

    @property
    def session_ids(self):
        """ list: The sorted list of all existing session ids `int`s in the state file """
        return self._tb_logs.session_ids

    def _load_state_file(self):
        """ Load the current state file to :attr:`_state`. """
        state_file = os.path.join(self._model_dir, "{}_state.json".format(self._model_name))
        logger.debug("Loading State: '%s'", state_file)
        serializer = get_serializer("json")
        self._state = serializer.load(state_file)
        logger.debug("Loaded state: %s", self._state)

    def initialize_session(self, model_folder, model_name, is_training=False):
        """ Initialize a Session.

        Load's the model's state file, and sets the paths to any underlying Tensorboard logs, ready
        for access on request.

        Parameters
        ----------
        model_folder: str, optional
            If loading a session manually (e.g. for the analysis tab), then the path to the model
            folder must be provided. For training sessions, this should be left at ``None``
        model_name: str, optional
            If loading a session manually (e.g. for the analysis tab), then the model filename
            must be provided. For training sessions, this should be left at ``None``
        is_training: bool, optional
            ``True`` if the session is being initialized for a training session, otherwise
            ``False``. Default: ``False``
         """
        logger.debug("Initializing session: (is_training: %s)", is_training)

        if self._model_dir == model_folder and self._model_name == model_name:
            if is_training:
                self._tb_logs.refresh_log_filenames()
                self._load_state_file()
                self._is_training = True
            logger.debug("Requested session is already loaded. Not initializing: (model_folder: "
                         "%s, model_name: %s)", model_folder, model_name)
            return

        self._is_training = is_training
        self._model_dir = model_folder
        self._model_name = model_name
        self._load_state_file()
        self._tb_logs = TensorBoardLogs(os.path.join(self._model_dir,
                                                     "{}_logs".format(self._model_name)))

        self._summary = SessionsSummary(self)
        logger.debug("Initialized session. Session_IDS: %s", self.session_ids)

    def stop_training(self):
        """ Clears the internal training flag. To be called when training completes. """
        self._is_training = False

    def clear(self):
        """ Clear the currently loaded session. """
        self._state = None
        self._model_dir = None
        self._model_name = None

        del self._tb_logs
        self._tb_logs = None

        del self._summary
        self._summary = None

        self._is_training = False

    def get_loss(self, session_id):
        """ Obtain the loss values for the given session_id.

        Parameters
        ----------
        session_id: int or ``None``
            The session ID to return loss for. Pass ``None`` to return loss for all sessions.

        Returns
        -------
        dict
            Loss names as key, :class:`numpy.ndarray` as value. If No session ID was provided
            all session's losses are collated
        """
        self._wait_for_thread()

        if self._is_training:
            self._is_querying.set()

        loss_dict = self._tb_logs.get_loss(session_id=session_id, is_training=self._is_training)
        if session_id is None:
            retval = dict()
            for key in sorted(loss_dict):
                for loss_key, loss in loss_dict[key].items():
                    retval.setdefault(loss_key, []).extend(loss)
            retval = {key: np.array(val, dtype="float32") for key, val in retval.items()}
        else:
            retval = loss_dict[session_id]

        if self._is_training:
            self._is_querying.clear()
        return retval

    def get_timestamps(self, session_id):
        """ Obtain the time stamps keys for the given session_id.

        Parameters
        ----------
        session_id: int or ``None``
            The session ID to return the time stamps for. Pass ``None`` to return time stamps for
            all sessions.

        Returns
        -------
        dict or :class:`numpy.ndarray`
            If a session ID has been given then a single :class:`numpy.ndarray` will be returned
            with the session's time stamps. Otherwise a 'dict' will be returned with the session
            IDs as key with :class:`numpy.ndarray` of timestamps as values
        """
        self._wait_for_thread()

        if self._is_training:
            self._is_querying.set()

        retval = self._tb_logs.get_timestamps(session_id=session_id, is_training=self._is_training)
        if session_id is not None:
            retval = retval[session_id]

        if self._is_training:
            self._is_querying.clear()

        return retval

    def _wait_for_thread(self):
        """ If a thread is querying the log files for live data, then block until task clears. """
        while True:
            if self._is_training and self._is_querying.is_set():
                logger.debug("Waiting for available thread")
                time.sleep(1)
                continue
            break

    def get_loss_keys(self, session_id):
        """ Obtain the loss keys for the given session_id.

        Parameters
        ----------
        session_id: int or ``None``
            The session ID to return the loss keys for. Pass ``None`` to return loss keys for
            all sessions.

        Returns
        -------
        list
            The loss keys for the given session. If ``None`` is passed as session_id then a unique
            list of all loss keys for all sessions is returned
        """
        if session_id is None:
            retval = list(set(loss_key
                              for session in self._state["sessions"].values()
                              for loss_key in session["loss_names"]))
        else:
            retval = self._state["sessions"][str(session_id)]["loss_names"]
        return retval


Session = GlobalSession() </s> remove     def get_loss(self, session=None):
 </s> add     def _cache_data(self, session_id, is_training=False):
        """ Cache TensorBoard logs for the given session ID on first access.

        Populates :attr:`_cache` with timestamps and loss data.

        If this is a training session and the data is being queried for the training session ID
        then get the latest available data and append to the cache

        Parameters
        -------
        session_id: int
            The session ID to cache the data for
        is_training: bool, optional
            ``True`` if a current training session is running otherwise ``False``.
            Default: ``False``
        """
        labels = []
        step = []
        loss = []
        timestamps = []
        last_step = -1
        live_data = is_training and session_id == max(self._log_filenames)

        if live_data:
            iterator = self._get_latest_live()
        else:
            iterator = tf.compat.v1.io.tf_record_iterator(self._log_filenames[session_id])

        try:
            for record in iterator:
                event = event_pb2.Event.FromString(record)
                if not event.summary.value or not event.summary.value[0].tag.startswith("batch_"):
                    continue

                if last_step == -1:
                    last_step = event.step if live_data else 0

                if event.step != last_step:
                    loss.append(step)
                    step = []
                    last_step = event.step

                summary = event.summary.value[0]
                tag = summary.tag

                if tag == "batch_total":
                    timestamps.append(event.wall_time)
                    continue

                lbl = tag.replace("batch_", "")
                if lbl not in labels:
                    labels.append(lbl)

                step.append(summary.simple_value)

        except tf_errors.DataLossError as err:
            logger.warning("The logs for Session %s are corrupted and cannot be displayed. "
                           "The totals do not include this session. Original error message: "
                           "'%s'", session_id, str(err))

        if step:
            loss.append(step)

        loss = np.array(loss, dtype="float32")
        timestamps = np.array(timestamps, dtype="float64")

        logger.debug("Caching session id: %s, labels: %s, loss: %s, timestamps: %s",
                     session_id, labels, loss.shape, timestamps.shape)

        if live_data and session_id in self._cache:
            self._add_latest_data_to_cache(session_id, loss, timestamps)
        else:
            self._cache[session_id] = dict(labels=labels,
                                           loss=zlib.compress(loss),
                                           loss_shape=loss.shape,
                                           timestamps=zlib.compress(timestamps),
                                           timestamps_shape=timestamps.shape)

    def _get_latest_live(self):
        """ Obtain the latest event logs for live training data and add to the cache """
        if self._training_iterator is None:
            training_session_id = self.session_ids[-1]
            filename = self._log_filenames[training_session_id]
            self._training_iterator = tf.compat.v1.io.tf_record_iterator(filename)
            logger.debug("Set live training iterator %s for session_id: %s",
                         self._training_iterator, training_session_id)

        i = 0
        while True:
            try:
                yield next(self._training_iterator)
                i += 1
            except StopIteration:
                logger.debug("End of data reached")
                break
            except tf.errors.DataLossError as err:
                # Truncated records are ignored. The iterator holds the offset, so the record will
                # be completed at the next call.
                logger.debug("Truncated record. Original Error: %s", err)
                break
        logger.debug("Collected %s records from live log file", i)

    def _add_latest_data_to_cache(self, session_id, loss, timestamps):
        """ Append the latest received live training data to the cached data.

        Parameters
        ----------
        session_id: int
            The training session ID to update the cache for
        loss: :class:`numpy.ndarray`
            The latest loss values returned from the iterator
        timestamps: :class:`numpy.ndarray`
            The latest time stamps returned from the iterator
        """
        if not np.any(loss) and not np.any(timestamps):
            logger.debug("No new live data to cache.")
            return

        logger.debug("Adding live data to cache: (loss: %s, timestamps: %s)",
                     loss.shape, timestamps.shape)

        cache = self._cache[session_id]

        past_loss = np.frombuffer(zlib.decompress(cache["loss"]),
                                  dtype="float32").reshape(cache["loss_shape"])
        new_loss = np.concatenate((past_loss, loss))
        cache["loss_shape"] = new_loss.shape
        cache["loss"] = zlib.compress(new_loss)
        del past_loss

        past_timestamps = np.frombuffer(zlib.decompress(cache["timestamps"]),
                                        dtype="float64").reshape(cache["timestamps_shape"])
        new_timestamps = np.concatenate((past_timestamps, timestamps))
        cache["timestamps_shape"] = new_timestamps.shape
        cache["timestamps"] = zlib.compress(new_timestamps)
        del past_timestamps

    def _from_cache(self, session_id=None, is_training=False):
        """ Get the session data from the cache.

        If the request data does not exist in the cache, then populate it.

        Parameters
        ----------
        session_id: int, optional
            The Session ID to return the data for. Set to ``None`` to return all session
            data. Default ``None`
        is_training: bool, optional
            ``True`` if a current training session is running otherwise ``False``.
            Default: ``False``

        Returns
        -------
        dict
            The session id(s) as key, with the event data as value
        """
        if session_id is not None and session_id not in self._cache:
            self._cache_data(session_id)
        elif is_training and session_id == self.session_ids[-1]:
            self._cache_data(session_id, is_training=is_training)
        elif session_id is None and not all(idx in self._cache for idx in self._log_filenames):
            for sess in self._log_filenames:
                if sess not in self._cache:
                    self._cache_data(sess)

        if session_id is None:
            return self._cache
        return {session_id: self._cache[session_id]}

    def get_loss(self, session_id=None, is_training=False): </s> remove     """ Class to pull raw data for given session(s) and perform calculations """
    def __init__(self, session, display="loss", loss_keys=["loss"], selections=["raw"],
                 avg_samples=500, smooth_amount=0.90, flatten_outliers=False, is_totals=False):
        logger.debug("Initializing %s: (session: %s, display: %s, loss_keys: %s, selections: %s, "
                     "avg_samples: %s, smooth_amount: %s, flatten_outliers: %s, is_totals: %s",
                     self.__class__.__name__, session, display, loss_keys, selections, avg_samples,
                     smooth_amount, flatten_outliers, is_totals)
 </s> add     """ Class that performs calculations on the :class:`GlobalSession` raw data for the given
    session id.

    Parameters
    ----------
    session_id: int or ``None``
        The session id number for the selected session from the Analysis tab. Should be ``None``
        if all sessions are being calculated
    display: {"loss", "rate"}, optional
        Whether to display a graph for loss or training rate. Default: `"loss"`
    loss_keys: list, optional
        The list of loss keys to display on the graph. Default: `["loss"]`
    selections: list, optional
        The selected annotations to display. Default: `["raw"]`
    avg_samples: int, optional
        The number of samples to use for performing moving average calculation. Default: `500`.
    smooth_amount: float, optional
        The amount of smoothing to apply for performing smoothing calculation. Default: `0.9`.
    flatten_outliers: bool, optional
        ``True`` if values significantly away from the average should be excluded, otherwise
        ``False``. Default: ``False``
    """
    def __init__(self, session_id,
                 display="loss",
                 loss_keys="loss",
                 selections="raw",
                 avg_samples=500,
                 smooth_amount=0.90,
                 flatten_outliers=False):
        logger.debug("Initializing %s: (session_id: %s, display: %s, loss_keys: %s, "
                     "selections: %s, avg_samples: %s, smooth_amount: %s, flatten_outliers: %s)",
                     self.__class__.__name__, session_id, display, loss_keys, selections,
                     avg_samples, smooth_amount, flatten_outliers)
        session_id: int, optional
          `batch_total` key.
  
          Parameters
          ----------
        session_id: int, optional
              The Session ID to return the timestamps for. Set to ``None`` to return all session
              timestamps. Default ``None``
  
          Returns
          -------
keep keep keep keep replace keep keep keep keep keep
 GUI - Stats optimization (#1067)

* Faster stat loading + caching

* Compress data in cache

* Optimize some calculations

* Vectorize smoothing

* stats.Calculations optimized

* Load latest training data from live iterator

* Add options to training graph 
lib/gui/stats.py