src/transformers/trainer_utils.py
def stop(self, stage):
        """ stop tracking for the passed stage """

        # deal with nested calls of eval during train - simply ignore those
        if self.cur_stage is not None and self.cur_stage != stage:
            return

        if self.torch is not None:
            self.torch.cuda.empty_cache()

        gc.collect()

        # gpu
        if self.torch is not None:
            mem_cur = self.torch.cuda.memory_allocated()
            # this is the difference between the start and the end allocated memory
            self.gpu[self.cur_stage]["alloc"] = mem_cur - self.gpu[self.cur_stage]["alloc"]  # can be negative
            # this is the difference if any between the start and the peak
            self.gpu[self.cur_stage]["peaked"] = max(0, self.torch.cuda.max_memory_allocated() - mem_cur)

        # cpu
        cpu_mem_used_delta, cpu_mem_used_peak = tracemalloc.get_traced_memory()
        tracemalloc.stop()  # reset accounting
        self.cpu[self.cur_stage]["alloc"] = cpu_mem_used_delta  # can be negative
        self.cpu[self.cur_stage]["peaked"] = max(0, cpu_mem_used_peak - cpu_mem_used_delta)

        # reset - cycle finished
        self.cur_stage = None
 stop tracking for the passed stage 
stop tracking for the passed stage
stop tracking for the passed stage
def stop(self, stage):
        if self.cur_stage is not None and self.cur_stage != stage:
            return
        if self.torch is not None:
            self.torch.cuda.empty_cache()
        gc.collect()
        if self.torch is not None:
            mem_cur = self.torch.cuda.memory_allocated()
            self.gpu[self.cur_stage]["alloc"] = mem_cur - self.gpu[self.cur_stage]["alloc"]  
            self.gpu[self.cur_stage]["peaked"] = max(0, self.torch.cuda.max_memory_allocated() - mem_cur)
        cpu_mem_used_delta, cpu_mem_used_peak = tracemalloc.get_traced_memory()
        tracemalloc.stop()  
        self.cpu[self.cur_stage]["alloc"] = cpu_mem_used_delta  
        self.cpu[self.cur_stage]["peaked"] = max(0, cpu_mem_used_peak - cpu_mem_used_delta)
        self.cur_stage = None
def stop ( self , stage ) : if self . cur_stage is not None and self . cur_stage != stage : return if self . torch is not None : self . torch . cuda . empty_cache ( ) gc . collect ( ) if self . torch is not None : mem_cur = self . torch . cuda . memory_allocated ( ) self . gpu [ self . cur_stage ] [ "alloc" ] = mem_cur - self . gpu [ self . cur_stage ] [ "alloc" ] self . gpu [ self . cur_stage ] [ "peaked" ] = max ( 0 , self . torch . cuda . max_memory_allocated ( ) - mem_cur ) cpu_mem_used_delta , cpu_mem_used_peak = tracemalloc . get_traced_memory ( ) tracemalloc . stop ( ) self . cpu [ self . cur_stage ] [ "alloc" ] = cpu_mem_used_delta self . cpu [ self . cur_stage ] [ "peaked" ] = max ( 0 , cpu_mem_used_peak - cpu_mem_used_delta ) self . cur_stage = None
stop tracking for the passed stage
stop tracking for the passed stage
""" stop tracking for the passed stage """ # deal with nested calls of eval during train - simply ignore those # gpu # this is the difference between the start and the end allocated memory # can be negative # this is the difference if any between the start and the peak # cpu # reset accounting # can be negative # reset - cycle finished
self
stage