          -------
          dict
              A list of loss values for each step for the requested session
          """
          logger.debug("Getting loss: (session: %s)", session)
          all_loss = dict()
          for sess, logfile in self.log_filenames.items():
              if session is not None and sess != session:
                  logger.debug("Skipping session: %s", sess)
                  continue
              loss = dict()
              events = [event_pb2.Event.FromString(record.numpy())
                        for record in tf.data.TFRecordDataset(logfile)]
              for event in events:
                  if not event.summary.value or not event.summary.value[0].tag.startswith("batch_"):
                      continue
                  summary = event.summary.value[0]
                  tag = summary.tag.replace("batch_", "")
                  loss.setdefault(tag, []).append(summary.simple_value)
              all_loss[sess] = loss
          logger.debug(all_loss)
          return all_loss
  
      def get_timestamps(self, session=None):
          """ Read the timestamps from the TensorBoard logs.
  
          As loss timestamps are slightly different for each loss, we collect the timestamp from the
 
</s> remove     def get_timestamps(self, session=None):
 </s> add     def get_timestamps(self, session_id=None, is_training=False): </s> remove             A list of loss values for each step for the requested session
 </s> add             The session id(s) as key, with a further dictionary as value containing the loss name
            and list of loss values for each step </s> remove     def get_loss(self, session=None):
 </s> add     def _cache_data(self, session_id, is_training=False):
        """ Cache TensorBoard logs for the given session ID on first access.

        Populates :attr:`_cache` with timestamps and loss data.

        If this is a training session and the data is being queried for the training session ID
        then get the latest available data and append to the cache

        Parameters
        -------
        session_id: int
            The session ID to cache the data for
        is_training: bool, optional
            ``True`` if a current training session is running otherwise ``False``.
            Default: ``False``
        """
        labels = []
        step = []
        loss = []
        timestamps = []
        last_step = -1
        live_data = is_training and session_id == max(self._log_filenames)

        if live_data:
            iterator = self._get_latest_live()
        else:
            iterator = tf.compat.v1.io.tf_record_iterator(self._log_filenames[session_id])

        try:
            for record in iterator:
                event = event_pb2.Event.FromString(record)
                if not event.summary.value or not event.summary.value[0].tag.startswith("batch_"):
                    continue

                if last_step == -1:
                    last_step = event.step if live_data else 0

                if event.step != last_step:
                    loss.append(step)
                    step = []
                    last_step = event.step

                summary = event.summary.value[0]
                tag = summary.tag

                if tag == "batch_total":
                    timestamps.append(event.wall_time)
                    continue

                lbl = tag.replace("batch_", "")
                if lbl not in labels:
                    labels.append(lbl)

                step.append(summary.simple_value)

        except tf_errors.DataLossError as err:
            logger.warning("The logs for Session %s are corrupted and cannot be displayed. "
                           "The totals do not include this session. Original error message: "
                           "'%s'", session_id, str(err))

        if step:
            loss.append(step)

        loss = np.array(loss, dtype="float32")
        timestamps = np.array(timestamps, dtype="float64")

        logger.debug("Caching session id: %s, labels: %s, loss: %s, timestamps: %s",
                     session_id, labels, loss.shape, timestamps.shape)

        if live_data and session_id in self._cache:
            self._add_latest_data_to_cache(session_id, loss, timestamps)
        else:
            self._cache[session_id] = dict(labels=labels,
                                           loss=zlib.compress(loss),
                                           loss_shape=loss.shape,
                                           timestamps=zlib.compress(timestamps),
                                           timestamps_shape=timestamps.shape)

    def _get_latest_live(self):
        """ Obtain the latest event logs for live training data and add to the cache """
        if self._training_iterator is None:
            training_session_id = self.session_ids[-1]
            filename = self._log_filenames[training_session_id]
            self._training_iterator = tf.compat.v1.io.tf_record_iterator(filename)
            logger.debug("Set live training iterator %s for session_id: %s",
                         self._training_iterator, training_session_id)

        i = 0
        while True:
            try:
                yield next(self._training_iterator)
                i += 1
            except StopIteration:
                logger.debug("End of data reached")
                break
            except tf.errors.DataLossError as err:
                # Truncated records are ignored. The iterator holds the offset, so the record will
                # be completed at the next call.
                logger.debug("Truncated record. Original Error: %s", err)
                break
        logger.debug("Collected %s records from live log file", i)

    def _add_latest_data_to_cache(self, session_id, loss, timestamps):
        """ Append the latest received live training data to the cached data.

        Parameters
        ----------
        session_id: int
            The training session ID to update the cache for
        loss: :class:`numpy.ndarray`
            The latest loss values returned from the iterator
        timestamps: :class:`numpy.ndarray`
            The latest time stamps returned from the iterator
        """
        if not np.any(loss) and not np.any(timestamps):
            logger.debug("No new live data to cache.")
            return

        logger.debug("Adding live data to cache: (loss: %s, timestamps: %s)",
                     loss.shape, timestamps.shape)

        cache = self._cache[session_id]

        past_loss = np.frombuffer(zlib.decompress(cache["loss"]),
                                  dtype="float32").reshape(cache["loss_shape"])
        new_loss = np.concatenate((past_loss, loss))
        cache["loss_shape"] = new_loss.shape
        cache["loss"] = zlib.compress(new_loss)
        del past_loss

        past_timestamps = np.frombuffer(zlib.decompress(cache["timestamps"]),
                                        dtype="float64").reshape(cache["timestamps_shape"])
        new_timestamps = np.concatenate((past_timestamps, timestamps))
        cache["timestamps_shape"] = new_timestamps.shape
        cache["timestamps"] = zlib.compress(new_timestamps)
        del past_timestamps

    def _from_cache(self, session_id=None, is_training=False):
        """ Get the session data from the cache.

        If the request data does not exist in the cache, then populate it.

        Parameters
        ----------
        session_id: int, optional
            The Session ID to return the data for. Set to ``None`` to return all session
            data. Default ``None`
        is_training: bool, optional
            ``True`` if a current training session is running otherwise ``False``.
            Default: ``False``

        Returns
        -------
        dict
            The session id(s) as key, with the event data as value
        """
        if session_id is not None and session_id not in self._cache:
            self._cache_data(session_id)
        elif is_training and session_id == self.session_ids[-1]:
            self._cache_data(session_id, is_training=is_training)
        elif session_id is None and not all(idx in self._cache for idx in self._log_filenames):
            for sess in self._log_filenames:
                if sess not in self._cache:
                    self._cache_data(sess)

        if session_id is None:
            return self._cache
        return {session_id: self._cache[session_id]}

    def get_loss(self, session_id=None, is_training=False): </s> remove     @property
    def loss_keys(self):
        """ list: The loss keys for the current session, or loss keys for all sessions. """
        if self.session_id is None:
            retval = self._total_loss_keys
        else:
            retval = self.session["loss_names"]
        return retval
 </s> add     Parameters
    ----------
    session: :class:`GlobalSession`
        The loaded or currently training session
    """
    def __init__(self, session):
        logger.debug("Initializing %s: (session: %s)", self.__class__.__name__, session)
        self._session = session
        self._state = session._state </s> remove def convert_time(timestamp):
    """ Convert time stamp to total hours, minutes and seconds """
    hrs = int(timestamp // 3600)
    if hrs < 10:
        hrs = "{0:02d}".format(hrs)
    mins = "{0:02d}".format((int(timestamp % 3600) // 60))
    secs = "{0:02d}".format((int(timestamp % 3600) % 60))
    return hrs, mins, secs
 </s> add class GlobalSession():
    """ Holds information about a loaded or current training session by accessing a model's state
    file and Tensorboard logs. This class should not be accessed directly, rather through
    :attr:`lib.stats.Session`
    """
    def __init__(self):
        logger.debug("Initializing %s", self.__class__.__name__)
        self._state = None
        self._model_dir = None
        self._model_name = None

        self._tb_logs = None
        self._summary = None

        self._is_training = False
        self._is_querying = Event()

        logger.debug("Initialized %s", self.__class__.__name__)

    @property
    def is_loaded(self):
        """ bool: ``True`` if session data is loaded otherwise ``False`` """
        return self._model_dir is not None

    @property
    def is_training(self):
        """ bool: ``True`` if the loaded session is the currently training model, otherwise
        ``False`` """
        return self._is_training

    @property
    def model_filename(self):
        """ str: The full model filename """
        return os.path.join(self._model_dir, self._model_name)

    @property
    def batch_sizes(self):
        """ dict: The batch sizes for each session_id for the model. """
        return {int(sess_id): sess["batchsize"]
                for sess_id, sess in self._state["sessions"].items()}

    @property
    def full_summary(self):
        """ list: List of dictionaries containing summary statistics for each session id. """
        return self._summary.get_summary_stats()

    @property
    def logging_disabled(self):
        """ bool: ``True`` if logging is enabled for the currently training session otherwise
        ``False``. """
        return self._state["sessions"][str(self.session_ids[-1])]["no_logs"]

    @property
    def session_ids(self):
        """ list: The sorted list of all existing session ids `int`s in the state file """
        return self._tb_logs.session_ids

    def _load_state_file(self):
        """ Load the current state file to :attr:`_state`. """
        state_file = os.path.join(self._model_dir, "{}_state.json".format(self._model_name))
        logger.debug("Loading State: '%s'", state_file)
        serializer = get_serializer("json")
        self._state = serializer.load(state_file)
        logger.debug("Loaded state: %s", self._state)

    def initialize_session(self, model_folder, model_name, is_training=False):
        """ Initialize a Session.

        Load's the model's state file, and sets the paths to any underlying Tensorboard logs, ready
        for access on request.

        Parameters
        ----------
        model_folder: str, optional
            If loading a session manually (e.g. for the analysis tab), then the path to the model
            folder must be provided. For training sessions, this should be left at ``None``
        model_name: str, optional
            If loading a session manually (e.g. for the analysis tab), then the model filename
            must be provided. For training sessions, this should be left at ``None``
        is_training: bool, optional
            ``True`` if the session is being initialized for a training session, otherwise
            ``False``. Default: ``False``
         """
        logger.debug("Initializing session: (is_training: %s)", is_training)

        if self._model_dir == model_folder and self._model_name == model_name:
            if is_training:
                self._tb_logs.refresh_log_filenames()
                self._load_state_file()
                self._is_training = True
            logger.debug("Requested session is already loaded. Not initializing: (model_folder: "
                         "%s, model_name: %s)", model_folder, model_name)
            return

        self._is_training = is_training
        self._model_dir = model_folder
        self._model_name = model_name
        self._load_state_file()
        self._tb_logs = TensorBoardLogs(os.path.join(self._model_dir,
                                                     "{}_logs".format(self._model_name)))

        self._summary = SessionsSummary(self)
        logger.debug("Initialized session. Session_IDS: %s", self.session_ids)

    def stop_training(self):
        """ Clears the internal training flag. To be called when training completes. """
        self._is_training = False

    def clear(self):
        """ Clear the currently loaded session. """
        self._state = None
        self._model_dir = None
        self._model_name = None

        del self._tb_logs
        self._tb_logs = None

        del self._summary
        self._summary = None

        self._is_training = False

    def get_loss(self, session_id):
        """ Obtain the loss values for the given session_id.

        Parameters
        ----------
        session_id: int or ``None``
            The session ID to return loss for. Pass ``None`` to return loss for all sessions.

        Returns
        -------
        dict
            Loss names as key, :class:`numpy.ndarray` as value. If No session ID was provided
            all session's losses are collated
        """
        self._wait_for_thread()

        if self._is_training:
            self._is_querying.set()

        loss_dict = self._tb_logs.get_loss(session_id=session_id, is_training=self._is_training)
        if session_id is None:
            retval = dict()
            for key in sorted(loss_dict):
                for loss_key, loss in loss_dict[key].items():
                    retval.setdefault(loss_key, []).extend(loss)
            retval = {key: np.array(val, dtype="float32") for key, val in retval.items()}
        else:
            retval = loss_dict[session_id]

        if self._is_training:
            self._is_querying.clear()
        return retval

    def get_timestamps(self, session_id):
        """ Obtain the time stamps keys for the given session_id.

        Parameters
        ----------
        session_id: int or ``None``
            The session ID to return the time stamps for. Pass ``None`` to return time stamps for
            all sessions.

        Returns
        -------
        dict or :class:`numpy.ndarray`
            If a session ID has been given then a single :class:`numpy.ndarray` will be returned
            with the session's time stamps. Otherwise a 'dict' will be returned with the session
            IDs as key with :class:`numpy.ndarray` of timestamps as values
        """
        self._wait_for_thread()

        if self._is_training:
            self._is_querying.set()

        retval = self._tb_logs.get_timestamps(session_id=session_id, is_training=self._is_training)
        if session_id is not None:
            retval = retval[session_id]

        if self._is_training:
            self._is_querying.clear()

        return retval

    def _wait_for_thread(self):
        """ If a thread is querying the log files for live data, then block until task clears. """
        while True:
            if self._is_training and self._is_querying.is_set():
                logger.debug("Waiting for available thread")
                time.sleep(1)
                continue
            break

    def get_loss_keys(self, session_id):
        """ Obtain the loss keys for the given session_id.

        Parameters
        ----------
        session_id: int or ``None``
            The session ID to return the loss keys for. Pass ``None`` to return loss keys for
            all sessions.

        Returns
        -------
        list
            The loss keys for the given session. If ``None`` is passed as session_id then a unique
            list of all loss keys for all sessions is returned
        """
        if session_id is None:
            retval = list(set(loss_key
                              for session in self._state["sessions"].values()
                              for loss_key in session["loss_names"]))
        else:
            retval = self._state["sessions"][str(session_id)]["loss_names"]
        return retval


Session = GlobalSession()
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
          -------
          dict
              A list of loss values for each step for the requested session
          """
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
        logger.debug("Getting loss: (session_id: %s)", session_id)
        retval = dict()
        for sess, info in self._from_cache(session_id, is_training).items():
            arr = np.frombuffer(zlib.decompress(info["loss"]),
                                dtype="float32").reshape(info["loss_shape"])
            for idx, title in enumerate(info["labels"]):
                retval.setdefault(sess, dict())[title] = arr[:, idx]
        logger.debug({key: {k: v.shape for k, v in val.items()}
                      for key, val in retval.items()})
        return retval
  
      def get_timestamps(self, session=None):
          """ Read the timestamps from the TensorBoard logs.
  
          As loss timestamps are slightly different for each loss, we collect the timestamp from the
keep keep keep keep replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace keep keep keep keep keep
 GUI - Stats optimization (#1067)

* Faster stat loading + caching

* Compress data in cache

* Optimize some calculations

* Vectorize smoothing

* stats.Calculations optimized

* Load latest training data from live iterator

* Add options to training graph 
lib/gui/stats.py