          do_snapshot = (snapshot_interval != 0 and
                         self.model.iterations >= snapshot_interval and
                         self.model.iterations % snapshot_interval == 0)
          loss = dict()
          for side, batcher in self.batchers.items():
              if self.pingpong.active and side != self.pingpong.side:
                  continue
              loss[side] = batcher.train_one_batch(do_preview)
              if not do_preview and not do_timelapse:
                  continue
              if do_preview:
                  self.samples.images[side] = batcher.compile_sample(None)
              if do_timelapse:
                  self.timelapse.get_sample(side, timelapse_kwargs)
  
          self.model.state.increment_iterations()
  
          for side, side_loss in loss.items():
              self.store_history(side, side_loss)
              self.log_tensorboard(side, side_loss)
  
          if not self.pingpong.active:
              self.print_loss(loss)
          else:
              for key, val in loss.items():
                  self.pingpong.loss[key] = val
              self.print_loss(self.pingpong.loss)
  
          if do_preview:
              samples = self.samples.show_sample()
              if samples is not None:
                  viewer(samples, "Training - 'S': Save Now. 'ENTER': Save and Quit")
 
</s> remove         if do_timelapse:
            self.timelapse.output_timelapse()
 </s> add             if do_timelapse:
                self.timelapse.output_timelapse() </s> remove         if do_preview:
            samples = self.samples.show_sample()
            if samples is not None:
                viewer(samples, "Training - 'S': Save Now. 'ENTER': Save and Quit")
 </s> add             if do_preview:
                samples = self.samples.show_sample()
                if samples is not None:
                    viewer(samples, "Training - 'S': Save Now. 'ENTER': Save and Quit")
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
          do_snapshot = (snapshot_interval != 0 and
                         self.model.iterations >= snapshot_interval and
                         self.model.iterations % snapshot_interval == 0)
          loss = dict()
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
        try:
            for side, batcher in self.batchers.items():
                if self.pingpong.active and side != self.pingpong.side:
                    continue
                loss[side] = batcher.train_one_batch(do_preview)
                if not do_preview and not do_timelapse:
                    continue
                if do_preview:
                    self.samples.images[side] = batcher.compile_sample(None)
                if do_timelapse:
                    self.timelapse.get_sample(side, timelapse_kwargs)

            self.model.state.increment_iterations()

            for side, side_loss in loss.items():
                self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)

            if not self.pingpong.active:
                self.print_loss(loss)
            else:
                for key, val in loss.items():
                    self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
  
          if do_preview:
              samples = self.samples.show_sample()
              if samples is not None:
                  viewer(samples, "Training - 'S': Save Now. 'ENTER': Save and Quit")
keep keep keep keep replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace keep keep keep keep keep
 Training Fixups

Only show model folder deprecation warning for existing models
Gracefully exit FixedProducerDispatcher on error 
plugins/train/trainer/_base.py