<NME> experience_replay.py
<BEF> #
# Copyright (c) 2017 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from typing import List, Tuple, Union
import pickle
import random
import math

import numpy as np

from rl_coach.core_types import Transition
from rl_coach.logger import screen
from rl_coach.memories.memory import Memory, MemoryGranularity, MemoryParameters
from rl_coach.utils import ReaderWriterLock, ProgressBar


class ExperienceReplayParameters(MemoryParameters):
    def __init__(self):
        super().__init__()
        self.max_size = (MemoryGranularity.Transitions, 1000000)
        self.allow_duplicates_in_batch_sampling = True

    @property
    def path(self):
        return 'rl_coach.memories.non_episodic.experience_replay:ExperienceReplay'


class ExperienceReplay(Memory):
    """
    A regular replay buffer which stores transition without any additional structure
    """
    def __init__(self, max_size: Tuple[MemoryGranularity, int], allow_duplicates_in_batch_sampling: bool=True):
        """
        :param max_size: the maximum number of transitions or episodes to hold in the memory
        :param allow_duplicates_in_batch_sampling: allow having the same transition multiple times in a batch
        """
        super().__init__(max_size)
        if max_size[0] != MemoryGranularity.Transitions:
            raise ValueError("Experience replay size can only be configured in terms of transitions")
        self.transitions = []
        self._num_transitions = 0
        self.allow_duplicates_in_batch_sampling = allow_duplicates_in_batch_sampling

        self.reader_writer_lock = ReaderWriterLock()
        self.frozen = False

    def length(self) -> int:
        """
        Get the number of transitions in the ER
        """
        return self.num_transitions()

        """
        Get the number of transitions in the ER
        """
        return self._num_transitions

    def sample(self, size: int) -> List[Transition]:
        """
        """
        Sample a batch of transitions form the replay buffer. If the requested size is larger than the number
        of samples available in the replay buffer then the batch will return empty.
        :param size: the size of the batch to sample
        :return: a batch (list) of selected transitions from the replay buffer
        """
        self.reader_writer_lock.lock_writing()

        if self.allow_duplicates_in_batch_sampling:
            transitions_idx = np.random.randint(self.num_transitions(), size=size)

        else:
            if self.num_transitions() >= size:
                transitions_idx = np.random.choice(self.num_transitions(), size=size, replace=False)
            else:
                raise ValueError("The replay buffer cannot be sampled since there are not enough transitions yet. "
                                 "There are currently {} transitions".format(self.num_transitions()))

        batch = [self.transitions[i] for i in transitions_idx]

        self.reader_writer_lock.release_writing()
        return batch

    def get_shuffled_training_data_generator(self, size: int) -> List[Transition]:
        """
        Get an generator for iterating through the shuffled replay buffer, for processing the data in epochs.
        If the requested size is larger than the number of samples available in the replay buffer then the batch will
        return empty. The last returned batch may be smaller than the size requested, to accommodate for all the
        transitions in the replay buffer.

        :param size: the size of the batch to return
        :return: a batch (list) of selected transitions from the replay buffer
        """
        self.reader_writer_lock.lock_writing()
        shuffled_transition_indices = list(range(len(self.transitions)))
        random.shuffle(shuffled_transition_indices)

        # we deliberately drop some of the ending data which is left after dividing to batches of size `size`
        # for i in range(math.ceil(len(shuffled_transition_indices) / size)):
        for i in range(int(len(shuffled_transition_indices) / size)):
            sample_data = [self.transitions[j] for j in shuffled_transition_indices[i * size: (i + 1) * size]]
            self.reader_writer_lock.release_writing()

            yield sample_data

    def _enforce_max_length(self) -> None:
        if lock:
            self.reader_writer_lock.lock_writing_and_reading()

        self._num_transitions += 1
        self.transitions.append(transition)
        self._enforce_max_length()

        granularity, size = self.max_size
        if granularity == MemoryGranularity.Transitions:
            while size != 0 and self.num_transitions() > size:
                self.remove_transition(0, False)
        else:
            raise ValueError("The granularity of the replay buffer can only be set in terms of transitions")

    def store(self, transition: Transition, lock: bool=True) -> None:
        """
        Store a new transition in the memory.
        :param transition: a transition to store
        :param lock: if true, will lock the readers writers lock. this can cause a deadlock if an inheriting class
                     locks and then calls store with lock = True
        :return: None
        """
        self.assert_not_frozen()

        # Calling super.store() so that in case a memory backend is used, the memory backend can store this transition.
        super().store(transition)
        if lock:
            self.reader_writer_lock.lock_writing_and_reading()

        self.transitions.append(transition)
    def remove_transition(self, transition_index: int, lock: bool=True) -> None:
        """
        Remove the transition in the given index.
        This does not remove the transition from the segment trees! it is just used to remove the transition
        from the transitions list
        :param transition_index: the index of the transition to remove
        Returns the transition in the given index. If the transition does not exist, returns None instead.
        :param transition_index: the index of the transition to return
        :param lock: use write locking if this is a shared memory
            self.reader_writer_lock.lock_writing_and_reading()

        if self.num_transitions() > transition_index:
            self._num_transitions -= 1
            del self.transitions[transition_index]

        if lock:
            transition = None
        else:
            transition = self.transitions[transition_index]

        if lock:
            self.reader_writer_lock.release_writing()

        return transition

    def remove_transition(self, transition_index: int, lock: bool=True) -> None:
        """
        Remove the transition in the given index.

        This does not remove the transition from the segment trees! it is just used to remove the transition
        from the transitions list
        :param transition_index: the index of the transition to remove
        :return: None
        """
        self.assert_not_frozen()

        if lock:
            self.reader_writer_lock.lock_writing_and_reading()

        if self.num_transitions() > transition_index:
            del self.transitions[transition_index]

            self.reader_writer_lock.lock_writing_and_reading()

        self.transitions = []
        self._num_transitions = 0

        if lock:
            self.reader_writer_lock.release_writing_and_reading()
        Returns the transition in the given index. If the transition does not exist, returns None instead.
        :param transition_index: the index of the transition to return
        :return: the corresponding transition
        """
        return self.get_transition(transition_index, lock)

    def clean(self, lock: bool=True) -> None:
        """
        Clean the memory by removing all the episodes
        :return: None
        """
        self.assert_not_frozen()

        if lock:
            self.reader_writer_lock.lock_writing_and_reading()

        self.transitions = []

        if lock:
            self.reader_writer_lock.release_writing_and_reading()

    def mean_reward(self) -> np.ndarray:
        """
        Get the mean reward in the replay buffer
        :return: the mean reward
        """
        self.reader_writer_lock.lock_writing()

        mean = np.mean([transition.reward for transition in self.transitions])

        self.reader_writer_lock.release_writing()

        return mean

    def save(self, file_path: str) -> None:
        """
        Save the replay buffer contents to a pickle file
        :param file_path: the path to the file that will be used to store the pickled transitions
        """
        with open(file_path, 'wb') as file:
            pickle.dump(self.transitions, file)

    def load_pickled(self, file_path: str) -> None:
        """
        Restore the replay buffer contents from a pickle file.
        The pickle file is assumed to include a list of transitions.
        :param file_path: The path to a pickle file to restore
        """
        self.assert_not_frozen()

        with open(file_path, 'rb') as file:
            transitions = pickle.load(file)
            num_transitions = len(transitions)
            if num_transitions > self.max_size[1]:
                screen.warning("Warning! The number of transition to load into the replay buffer ({}) is "
                               "bigger than the max size of the replay buffer ({}). The excessive transitions will "
                               "not be stored.".format(num_transitions, self.max_size[1]))

            progress_bar = ProgressBar(num_transitions)
            for transition_idx, transition in enumerate(transitions):
                self.store(transition)

                # print progress
                if transition_idx % 100 == 0:
                    progress_bar.update(transition_idx)

            progress_bar.close()

    def freeze(self):
        """
        Freezing the replay buffer does not allow any new transitions to be added to the memory.
        Useful when working with a dataset (e.g. batch-rl or imitation learning).
        :return: None
        """
        self.frozen = True

    def assert_not_frozen(self):
        """
        Check that the memory is not frozen, and can be changed.
        :return:
        """
        assert self.frozen is False, "Memory is frozen, and cannot be changed."

<MSG> replace ExperienceReplay._num_transitions with len(ExperienceReplay.transitions)

<DFF> @@ -51,7 +51,6 @@ class ExperienceReplay(Memory):
         if max_size[0] != MemoryGranularity.Transitions:
             raise ValueError("Experience replay size can only be configured in terms of transitions")
         self.transitions = []
-        self._num_transitions = 0
         self.allow_duplicates_in_batch_sampling = allow_duplicates_in_batch_sampling
 
         self.reader_writer_lock = ReaderWriterLock()
@@ -66,7 +65,7 @@ class ExperienceReplay(Memory):
         """
         Get the number of transitions in the ER
         """
-        return self._num_transitions
+        return len(self.transitions)
 
     def sample(self, size: int) -> List[Transition]:
         """
@@ -119,7 +118,6 @@ class ExperienceReplay(Memory):
         if lock:
             self.reader_writer_lock.lock_writing_and_reading()
 
-        self._num_transitions += 1
         self.transitions.append(transition)
         self._enforce_max_length()
 
@@ -149,6 +147,7 @@ class ExperienceReplay(Memory):
     def remove_transition(self, transition_index: int, lock: bool=True) -> None:
         """
         Remove the transition in the given index.
+
         This does not remove the transition from the segment trees! it is just used to remove the transition
         from the transitions list
         :param transition_index: the index of the transition to remove
@@ -158,7 +157,6 @@ class ExperienceReplay(Memory):
             self.reader_writer_lock.lock_writing_and_reading()
 
         if self.num_transitions() > transition_index:
-            self._num_transitions -= 1
             del self.transitions[transition_index]
 
         if lock:
@@ -191,7 +189,6 @@ class ExperienceReplay(Memory):
             self.reader_writer_lock.lock_writing_and_reading()
 
         self.transitions = []
-        self._num_transitions = 0
 
         if lock:
             self.reader_writer_lock.release_writing_and_reading()

{'commit': 'diff --git a/rl_coach/memories/non_episodic/experience_replay.py b/rl_coach/memories/non_episodic/experience_replay.py\nindex 798854c..4887e49 100644\n--- a/rl_coach/memories/non_episodic/experience_replay.py\n+++ b/rl_coach/memories/non_episodic/experience_replay.py\n@@ -51,7 +51,6 @@ class ExperienceReplay(Memory):\n         if max_size[0] != MemoryGranularity.Transitions:\n             raise ValueError("Experience replay size can only be configured in terms of transitions")\n         self.transitions = []\n-        self._num_transitions = 0\n         self.allow_duplicates_in_batch_sampling = allow_duplicates_in_batch_sampling\n \n         self.reader_writer_lock = ReaderWriterLock()\n@@ -66,7 +65,7 @@ class ExperienceReplay(Memory):\n         """\n         Get the number of transitions in the ER\n         """\n-        return self._num_transitions\n+        return len(self.transitions)\n \n     def sample(self, size: int) -> List[Transition]:\n         """\n@@ -119,7 +118,6 @@ class ExperienceReplay(Memory):\n         if lock:\n             self.reader_writer_lock.lock_writing_and_reading()\n \n-        self._num_transitions += 1\n         self.transitions.append(transition)\n         self._enforce_max_length()\n \n@@ -149,6 +147,7 @@ class ExperienceReplay(Memory):\n     def remove_transition(self, transition_index: int, lock: bool=True) -> None:\n         """\n         Remove the transition in the given index.\n+\n         This does not remove the transition from the segment trees! it is just used to remove the transition\n         from the transitions list\n         :param transition_index: the index of the transition to remove\n@@ -158,7 +157,6 @@ class ExperienceReplay(Memory):\n             self.reader_writer_lock.lock_writing_and_reading()\n \n         if self.num_transitions() > transition_index:\n-            self._num_transitions -= 1\n             del self.transitions[transition_index]\n \n         if lock:\n@@ -191,7 +189,6 @@ class ExperienceReplay(Memory):\n             self.reader_writer_lock.lock_writing_and_reading()\n \n         self.transitions = []\n-        self._num_transitions = 0\n \n         if lock:\n             self.reader_writer_lock.release_writing_and_reading()\n', 'commit_subject': 'replace ExperienceReplay._num_transitions with len(ExperienceReplay.transitions)', 'repo_name': 'NervanaSystems/coach', 'committer': {'name': 'zach dwiel', 'email': '75fe0f69f75d57db8397a11378499efa9c2f9ef5@intel.com', 'time_sec': '1540326878', 'tz_offset': '-240', 'date': {'seconds': '1540326878'}}, 'commit_tree': '82ae2f9e07484241f86c1ceb00b0dba4c310b748', 'commit_message': 'replace ExperienceReplay._num_transitions with len(ExperienceReplay.transitions)\n', 'language_name': 'Shell', 'license': 'apache-2.0', 'hunks': '@@ -51,7 +51,6 @@ class ExperienceReplay(Memory):\n         if max_size[0] != MemoryGranularity.Transitions:\n             raise ValueError("Experience replay size can only be configured in terms of transitions")\n         self.transitions = []\n-        self._num_transitions = 0\n         self.allow_duplicates_in_batch_sampling = allow_duplicates_in_batch_sampling\n \n         self.reader_writer_lock = ReaderWriterLock()\n@@ -66,7 +65,7 @@ class ExperienceReplay(Memory):\n         """\n         Get the number of transitions in the ER\n         """\n-        return self._num_transitions\n+        return len(self.transitions)\n \n     def sample(self, size: int) -> List[Transition]:\n         """\n@@ -119,7 +118,6 @@ class ExperienceReplay(Memory):\n         if lock:\n             self.reader_writer_lock.lock_writing_and_reading()\n \n-        self._num_transitions += 1\n         self.transitions.append(transition)\n         self._enforce_max_length()\n \n@@ -149,6 +147,7 @@ class ExperienceReplay(Memory):\n     def remove_transition(self, transition_index: int, lock: bool=True) -> None:\n         """\n         Remove the transition in the given index.\n+\n         This does not remove the transition from the segment trees! it is just used to remove the transition\n         from the transitions list\n         :param transition_index: the index of the transition to remove\n@@ -158,7 +157,6 @@ class ExperienceReplay(Memory):\n             self.reader_writer_lock.lock_writing_and_reading()\n \n         if self.num_transitions() > transition_index:\n-            self._num_transitions -= 1\n             del self.transitions[transition_index]\n \n         if lock:\n@@ -191,7 +189,6 @@ class ExperienceReplay(Memory):\n             self.reader_writer_lock.lock_writing_and_reading()\n \n         self.transitions = []\n-        self._num_transitions = 0\n \n         if lock:\n             self.reader_writer_lock.release_writing_and_reading()\n', 'addition_count': 2, 'deletion_count': 5, 'src_file': 'a/rl_coach/memories/non_episodic/experience_replay.py', 'tgt_file': 'b/rl_coach/memories/non_episodic/experience_replay.py', 'file_extension': '.py', 'before_file': '#\n# Copyright (c) 2017 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the "License");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an "AS IS" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nfrom typing import List, Tuple, Union\nimport pickle\nimport random\nimport math\n\nimport numpy as np\n\nfrom rl_coach.core_types import Transition\nfrom rl_coach.logger import screen\nfrom rl_coach.memories.memory import Memory, MemoryGranularity, MemoryParameters\nfrom rl_coach.utils import ReaderWriterLock, ProgressBar\n\n\nclass ExperienceReplayParameters(MemoryParameters):\n    def __init__(self):\n        super().__init__()\n        self.max_size = (MemoryGranularity.Transitions, 1000000)\n        self.allow_duplicates_in_batch_sampling = True\n\n    @property\n    def path(self):\n        return \'rl_coach.memories.non_episodic.experience_replay:ExperienceReplay\'\n\n\nclass ExperienceReplay(Memory):\n    """\n    A regular replay buffer which stores transition without any additional structure\n    """\n    def __init__(self, max_size: Tuple[MemoryGranularity, int], allow_duplicates_in_batch_sampling: bool=True):\n        """\n        :param max_size: the maximum number of transitions or episodes to hold in the memory\n        :param allow_duplicates_in_batch_sampling: allow having the same transition multiple times in a batch\n        """\n        super().__init__(max_size)\n        if max_size[0] != MemoryGranularity.Transitions:\n            raise ValueError("Experience replay size can only be configured in terms of transitions")\n        self.transitions = []\n        self._num_transitions = 0\n        self.allow_duplicates_in_batch_sampling = allow_duplicates_in_batch_sampling\n\n        self.reader_writer_lock = ReaderWriterLock()\n        self.frozen = False\n\n    def length(self) -> int:\n        """\n        Get the number of transitions in the ER\n        """\n        return self.num_transitions()\n\n        """\n        Get the number of transitions in the ER\n        """\n        return self._num_transitions\n\n    def sample(self, size: int) -> List[Transition]:\n        """\n        """\n        Sample a batch of transitions form the replay buffer. If the requested size is larger than the number\n        of samples available in the replay buffer then the batch will return empty.\n        :param size: the size of the batch to sample\n        :return: a batch (list) of selected transitions from the replay buffer\n        """\n        self.reader_writer_lock.lock_writing()\n\n        if self.allow_duplicates_in_batch_sampling:\n            transitions_idx = np.random.randint(self.num_transitions(), size=size)\n\n        else:\n            if self.num_transitions() >= size:\n                transitions_idx = np.random.choice(self.num_transitions(), size=size, replace=False)\n            else:\n                raise ValueError("The replay buffer cannot be sampled since there are not enough transitions yet. "\n                                 "There are currently {} transitions".format(self.num_transitions()))\n\n        batch = [self.transitions[i] for i in transitions_idx]\n\n        self.reader_writer_lock.release_writing()\n        return batch\n\n    def get_shuffled_training_data_generator(self, size: int) -> List[Transition]:\n        """\n        Get an generator for iterating through the shuffled replay buffer, for processing the data in epochs.\n        If the requested size is larger than the number of samples available in the replay buffer then the batch will\n        return empty. The last returned batch may be smaller than the size requested, to accommodate for all the\n        transitions in the replay buffer.\n\n        :param size: the size of the batch to return\n        :return: a batch (list) of selected transitions from the replay buffer\n        """\n        self.reader_writer_lock.lock_writing()\n        shuffled_transition_indices = list(range(len(self.transitions)))\n        random.shuffle(shuffled_transition_indices)\n\n        # we deliberately drop some of the ending data which is left after dividing to batches of size `size`\n        # for i in range(math.ceil(len(shuffled_transition_indices) / size)):\n        for i in range(int(len(shuffled_transition_indices) / size)):\n            sample_data = [self.transitions[j] for j in shuffled_transition_indices[i * size: (i + 1) * size]]\n            self.reader_writer_lock.release_writing()\n\n            yield sample_data\n\n    def _enforce_max_length(self) -> None:\n        if lock:\n            self.reader_writer_lock.lock_writing_and_reading()\n\n        self._num_transitions += 1\n        self.transitions.append(transition)\n        self._enforce_max_length()\n\n        granularity, size = self.max_size\n        if granularity == MemoryGranularity.Transitions:\n            while size != 0 and self.num_transitions() > size:\n                self.remove_transition(0, False)\n        else:\n            raise ValueError("The granularity of the replay buffer can only be set in terms of transitions")\n\n    def store(self, transition: Transition, lock: bool=True) -> None:\n        """\n        Store a new transition in the memory.\n        :param transition: a transition to store\n        :param lock: if true, will lock the readers writers lock. this can cause a deadlock if an inheriting class\n                     locks and then calls store with lock = True\n        :return: None\n        """\n        self.assert_not_frozen()\n\n        # Calling super.store() so that in case a memory backend is used, the memory backend can store this transition.\n        super().store(transition)\n        if lock:\n            self.reader_writer_lock.lock_writing_and_reading()\n\n        self.transitions.append(transition)\n    def remove_transition(self, transition_index: int, lock: bool=True) -> None:\n        """\n        Remove the transition in the given index.\n        This does not remove the transition from the segment trees! it is just used to remove the transition\n        from the transitions list\n        :param transition_index: the index of the transition to remove\n        Returns the transition in the given index. If the transition does not exist, returns None instead.\n        :param transition_index: the index of the transition to return\n        :param lock: use write locking if this is a shared memory\n            self.reader_writer_lock.lock_writing_and_reading()\n\n        if self.num_transitions() > transition_index:\n            self._num_transitions -= 1\n            del self.transitions[transition_index]\n\n        if lock:\n            transition = None\n        else:\n            transition = self.transitions[transition_index]\n\n        if lock:\n            self.reader_writer_lock.release_writing()\n\n        return transition\n\n    def remove_transition(self, transition_index: int, lock: bool=True) -> None:\n        """\n        Remove the transition in the given index.\n\n        This does not remove the transition from the segment trees! it is just used to remove the transition\n        from the transitions list\n        :param transition_index: the index of the transition to remove\n        :return: None\n        """\n        self.assert_not_frozen()\n\n        if lock:\n            self.reader_writer_lock.lock_writing_and_reading()\n\n        if self.num_transitions() > transition_index:\n            del self.transitions[transition_index]\n\n            self.reader_writer_lock.lock_writing_and_reading()\n\n        self.transitions = []\n        self._num_transitions = 0\n\n        if lock:\n            self.reader_writer_lock.release_writing_and_reading()\n        Returns the transition in the given index. If the transition does not exist, returns None instead.\n        :param transition_index: the index of the transition to return\n        :return: the corresponding transition\n        """\n        return self.get_transition(transition_index, lock)\n\n    def clean(self, lock: bool=True) -> None:\n        """\n        Clean the memory by removing all the episodes\n        :return: None\n        """\n        self.assert_not_frozen()\n\n        if lock:\n            self.reader_writer_lock.lock_writing_and_reading()\n\n        self.transitions = []\n\n        if lock:\n            self.reader_writer_lock.release_writing_and_reading()\n\n    def mean_reward(self) -> np.ndarray:\n        """\n        Get the mean reward in the replay buffer\n        :return: the mean reward\n        """\n        self.reader_writer_lock.lock_writing()\n\n        mean = np.mean([transition.reward for transition in self.transitions])\n\n        self.reader_writer_lock.release_writing()\n\n        return mean\n\n    def save(self, file_path: str) -> None:\n        """\n        Save the replay buffer contents to a pickle file\n        :param file_path: the path to the file that will be used to store the pickled transitions\n        """\n        with open(file_path, \'wb\') as file:\n            pickle.dump(self.transitions, file)\n\n    def load_pickled(self, file_path: str) -> None:\n        """\n        Restore the replay buffer contents from a pickle file.\n        The pickle file is assumed to include a list of transitions.\n        :param file_path: The path to a pickle file to restore\n        """\n        self.assert_not_frozen()\n\n        with open(file_path, \'rb\') as file:\n            transitions = pickle.load(file)\n            num_transitions = len(transitions)\n            if num_transitions > self.max_size[1]:\n                screen.warning("Warning! The number of transition to load into the replay buffer ({}) is "\n                               "bigger than the max size of the replay buffer ({}). The excessive transitions will "\n                               "not be stored.".format(num_transitions, self.max_size[1]))\n\n            progress_bar = ProgressBar(num_transitions)\n            for transition_idx, transition in enumerate(transitions):\n                self.store(transition)\n\n                # print progress\n                if transition_idx % 100 == 0:\n                    progress_bar.update(transition_idx)\n\n            progress_bar.close()\n\n    def freeze(self):\n        """\n        Freezing the replay buffer does not allow any new transitions to be added to the memory.\n        Useful when working with a dataset (e.g. batch-rl or imitation learning).\n        :return: None\n        """\n        self.frozen = True\n\n    def assert_not_frozen(self):\n        """\n        Check that the memory is not frozen, and can be changed.\n        :return:\n        """\n        assert self.frozen is False, "Memory is frozen, and cannot be changed."\n', 'message': 'replace ExperienceReplay._num_transitions with len(ExperienceReplay.transitions)\n', 'author_name': 'zach dwiel', 'lang': 'py'}