pyanomaly/core/hook/functions/ocae_hooks.py
def evaluate(self, current_step):
        '''
        Evaluate the results of the model
        !!! Will change, e.g. accuracy, mAP.....
        !!! Or can call other methods written by the official
        '''
        # self.trainer.set_requires_grad(self.trainer.A, False)
        # self.trainer.set_requires_grad(self.trainer.B, False)
        # self.trainer.set_requires_grad(self.trainer.C, False)
        # self.trainer.set_requires_grad(self.trainer.Detector, False)
        # self.trainer.A.eval()
        # self.trainer.B.eval()
        # self.trainer.C.eval()
        # self.trainer.Detector.eval()
        self.engine.set_all(False)
        frame_num = self.engine.config.DATASET.test_clip_length
        tb_writer = self.engine.kwargs['writer_dict']['writer']
        global_steps = self.engine.kwargs['writer_dict']['global_steps_{}'.format(self.engine.kwargs['model_type'])]
        score_records = []
        # psnr_records = []
        total = 0
        random_video_sn = torch.randint(0, len(self.engine.test_dataset_keys), (1,))
        # random_video_sn = 0
        for sn, video_name in enumerate(self.engine.test_dataset_keys):
            # _temp_test_folder = os.path.join(self.testing_data_folder, dir)
            # need to improve
            # dataset = AvenueTestOld(_temp_test_folder, clip_length=frame_num)
            dataset = self.engine.test_dataset_dict[video_name]
            len_dataset = dataset.pics_len
            test_iters = len_dataset - frame_num + 1
            test_counter = 0
            # feature_record = []
        
            data_loader = DataLoader(dataset=dataset, batch_size=1, shuffle=False, num_workers=1)
            # import ipdb; ipdb.set_trace()
            scores = np.empty(shape=(len_dataset,),dtype=np.float32)
            # for test_input, _ in data_loader:
            random_frame_sn = torch.randint(0, len_dataset,(1,))
            for frame_sn, (test_input, anno, meta) in enumerate(data_loader):
                feature_record_object = []
                future = test_input[:, :, 2, :, :].cuda()
                current = test_input[:, :, 1, :, :].cuda()
                past = test_input[:, :, 0, :, :].cuda()
                bboxs = get_batch_dets(self.engine.Detector, current)
                for index, bbox in enumerate(bboxs):
                    # import ipdb; ipdb.set_trace()
                    if bbox.numel() == 0:
                        bbox = bbox.new_zeros([1,4])
                        # print('NO objects')
                        # continue
                    current_object, _ = multi_obj_grid_crop(current[index], bbox)
                    future_object, _ = multi_obj_grid_crop(future[index], bbox)
                    future2current = torch.stack([future_object, current_object], dim=1)

                    past_object, _ = multi_obj_grid_crop(past[index], bbox)
                    current2past = torch.stack([current_object, past_object], dim=1)

                    _, _, A_input = frame_gradient(future2current)
                    A_input = A_input.sum(1)
                    _, _, C_input = frame_gradient(current2past)
                    C_input = C_input.sum(1)
                    A_feature, temp_a, _ = self.engine.A(A_input)
                    B_feature, temp_b, _ = self.engine.B(current_object)
                    C_feature, temp_c, _ = self.engine.C(C_input)

                    # import ipdb; ipdb.set_trace()
                    if sn == random_video_sn and frame_sn == random_frame_sn:
                        vis_objects = OrderedDict({
                            'eval_oc_input_a': A_input.detach(),
                            'eval_oc_output_a': temp_a.detach(),
                            'eval_oc_input_b': current_object.detach(),
                            'eval_oc_output_b':  temp_b.detach(),
                            'eval_oc_input_c': C_input.detach(),
                            'eval_oc_output_c': temp_c.detach(),
                        })
                        tensorboard_vis_images(vis_objects, tb_writer, global_steps, normalize=self.engine.normalize.param['val'])

                    A_flatten_feature = A_feature.flatten(start_dim=1)
                    B_flatten_feature = B_feature.flatten(start_dim=1)
                    C_flatten_feature = C_feature.flatten(start_dim=1)
                    ABC_feature = torch.cat([A_flatten_feature, B_flatten_feature, C_flatten_feature], dim=1).detach()
                    ABC_feature_s = torch.chunk(ABC_feature, ABC_feature.size(0), dim=0)

                    for abc_f in ABC_feature_s:
                        temp = abc_f.squeeze(0).cpu().numpy()
                        feature_record_object.append(temp)
                
                predict_input = np.array(feature_record_object)
                self.engine.ovr_model = joblib.load(self.engine.ovr_model_path)
                g_i = self.engine.ovr_model.decision_function(predict_input) # the svm score of each object in one frame
                frame_score = oc_score(g_i)

                # test_psnr = psnr_error(g_output, test_target)
                # test_psnr = test_psnr.tolist()
                scores[test_counter+frame_num-1] = frame_score

                test_counter += 1
                total+=1
                if test_counter >= test_iters:
                    scores[:frame_num-1]=scores[frame_num-1]
                    score_records.append(scores)
                    print(f'finish test video set {video_name}')
                    break
        
        self.engine.pkl_path = save_score_results(self.engine.config, self.engine.logger, verbose=self.engine.verbose, config_name=self.engine.config_name, current_step=current_step, time_stamp=self.engine.kwargs["time_stamp"],score=score_records)
        results = self.engine.evaluate_function(self.engine.pkl_path, self.engine.logger, self.engine.config)
        self.engine.logger.info(results)
        tb_writer.add_text('AUC of ROC curve', f'AUC is {results.auc:.5f}',global_steps)
        return results.auc

        Evaluate the results of the model
        !!! Will change, e.g. accuracy, mAP.....
        !!! Or can call other methods written by the official
        
Evaluate the results of the model
Will change, e.g. accuracy, mAP
Or can call other methods written by the official
Evaluate the results of the model Will change e . g . accuracy mAP Or can call other methods written by the official
def evaluate(self, current_step):
        self.engine.set_all(False)
        frame_num = self.engine.config.DATASET.test_clip_length
        tb_writer = self.engine.kwargs['writer_dict']['writer']
        global_steps = self.engine.kwargs['writer_dict']['global_steps_{}'.format(self.engine.kwargs['model_type'])]
        score_records = []
        total = 0
        random_video_sn = torch.randint(0, len(self.engine.test_dataset_keys), (1,))
        for sn, video_name in enumerate(self.engine.test_dataset_keys):
            dataset = self.engine.test_dataset_dict[video_name]
            len_dataset = dataset.pics_len
            test_iters = len_dataset - frame_num + 1
            test_counter = 0
            data_loader = DataLoader(dataset=dataset, batch_size=1, shuffle=False, num_workers=1)
            scores = np.empty(shape=(len_dataset,),dtype=np.float32)
            random_frame_sn = torch.randint(0, len_dataset,(1,))
            for frame_sn, (test_input, anno, meta) in enumerate(data_loader):
                feature_record_object = []
                future = test_input[:, :, 2, :, :].cuda()
                current = test_input[:, :, 1, :, :].cuda()
                past = test_input[:, :, 0, :, :].cuda()
                bboxs = get_batch_dets(self.engine.Detector, current)
                for index, bbox in enumerate(bboxs):
                    if bbox.numel() == 0:
                        bbox = bbox.new_zeros([1,4])
                    current_object, _ = multi_obj_grid_crop(current[index], bbox)
                    future_object, _ = multi_obj_grid_crop(future[index], bbox)
                    future2current = torch.stack([future_object, current_object], dim=1)
                    past_object, _ = multi_obj_grid_crop(past[index], bbox)
                    current2past = torch.stack([current_object, past_object], dim=1)
                    _, _, A_input = frame_gradient(future2current)
                    A_input = A_input.sum(1)
                    _, _, C_input = frame_gradient(current2past)
                    C_input = C_input.sum(1)
                    A_feature, temp_a, _ = self.engine.A(A_input)
                    B_feature, temp_b, _ = self.engine.B(current_object)
                    C_feature, temp_c, _ = self.engine.C(C_input)
                    if sn == random_video_sn and frame_sn == random_frame_sn:
                        vis_objects = OrderedDict({
                            'eval_oc_input_a': A_input.detach(),
                            'eval_oc_output_a': temp_a.detach(),
                            'eval_oc_input_b': current_object.detach(),
                            'eval_oc_output_b':  temp_b.detach(),
                            'eval_oc_input_c': C_input.detach(),
                            'eval_oc_output_c': temp_c.detach(),
                        })
                        tensorboard_vis_images(vis_objects, tb_writer, global_steps, normalize=self.engine.normalize.param['val'])
                    A_flatten_feature = A_feature.flatten(start_dim=1)
                    B_flatten_feature = B_feature.flatten(start_dim=1)
                    C_flatten_feature = C_feature.flatten(start_dim=1)
                    ABC_feature = torch.cat([A_flatten_feature, B_flatten_feature, C_flatten_feature], dim=1).detach()
                    ABC_feature_s = torch.chunk(ABC_feature, ABC_feature.size(0), dim=0)
                    for abc_f in ABC_feature_s:
                        temp = abc_f.squeeze(0).cpu().numpy()
                        feature_record_object.append(temp)
                predict_input = np.array(feature_record_object)
                self.engine.ovr_model = joblib.load(self.engine.ovr_model_path)
                g_i = self.engine.ovr_model.decision_function(predict_input) 
                frame_score = oc_score(g_i)
                scores[test_counter+frame_num-1] = frame_score
                test_counter += 1
                total+=1
                if test_counter >= test_iters:
                    scores[:frame_num-1]=scores[frame_num-1]
                    score_records.append(scores)
                    print(f'finish test video set {video_name}')
                    break
        self.engine.pkl_path = save_score_results(self.engine.config, self.engine.logger, verbose=self.engine.verbose, config_name=self.engine.config_name, current_step=current_step, time_stamp=self.engine.kwargs["time_stamp"],score=score_records)
        results = self.engine.evaluate_function(self.engine.pkl_path, self.engine.logger, self.engine.config)
        self.engine.logger.info(results)
        tb_writer.add_text('AUC of ROC curve', f'AUC is {results.auc:.5f}',global_steps)
        return results.auc
def evaluate ( self , current_step ) : self . engine . set_all ( False ) frame_num = self . engine . config . DATASET . test_clip_length tb_writer = self . engine . kwargs [ 'writer_dict' ] [ 'writer' ] global_steps = self . engine . kwargs [ 'writer_dict' ] [ 'global_steps_{}' . format ( self . engine . kwargs [ 'model_type' ] ) ] score_records = [ ] total = 0 random_video_sn = torch . randint ( 0 , len ( self . engine . test_dataset_keys ) , ( 1 , ) ) for sn , video_name in enumerate ( self . engine . test_dataset_keys ) : dataset = self . engine . test_dataset_dict [ video_name ] len_dataset = dataset . pics_len test_iters = len_dataset - frame_num + 1 test_counter = 0 data_loader = DataLoader ( dataset = dataset , batch_size = 1 , shuffle = False , num_workers = 1 ) scores = np . empty ( shape = ( len_dataset , ) , dtype = np . float32 ) random_frame_sn = torch . randint ( 0 , len_dataset , ( 1 , ) ) for frame_sn , ( test_input , anno , meta ) in enumerate ( data_loader ) : feature_record_object = [ ] future = test_input [ : , : , 2 , : , : ] . cuda ( ) current = test_input [ : , : , 1 , : , : ] . cuda ( ) past = test_input [ : , : , 0 , : , : ] . cuda ( ) bboxs = get_batch_dets ( self . engine . Detector , current ) for index , bbox in enumerate ( bboxs ) : if bbox . numel ( ) == 0 : bbox = bbox . new_zeros ( [ 1 , 4 ] ) current_object , _ = multi_obj_grid_crop ( current [ index ] , bbox ) future_object , _ = multi_obj_grid_crop ( future [ index ] , bbox ) future2current = torch . stack ( [ future_object , current_object ] , dim = 1 ) past_object , _ = multi_obj_grid_crop ( past [ index ] , bbox ) current2past = torch . stack ( [ current_object , past_object ] , dim = 1 ) _ , _ , A_input = frame_gradient ( future2current ) A_input = A_input . sum ( 1 ) _ , _ , C_input = frame_gradient ( current2past ) C_input = C_input . sum ( 1 ) A_feature , temp_a , _ = self . engine . A ( A_input ) B_feature , temp_b , _ = self . engine . B ( current_object ) C_feature , temp_c , _ = self . engine . C ( C_input ) if sn == random_video_sn and frame_sn == random_frame_sn : vis_objects = OrderedDict ( { 'eval_oc_input_a' : A_input . detach ( ) , 'eval_oc_output_a' : temp_a . detach ( ) , 'eval_oc_input_b' : current_object . detach ( ) , 'eval_oc_output_b' : temp_b . detach ( ) , 'eval_oc_input_c' : C_input . detach ( ) , 'eval_oc_output_c' : temp_c . detach ( ) , } ) tensorboard_vis_images ( vis_objects , tb_writer , global_steps , normalize = self . engine . normalize . param [ 'val' ] ) A_flatten_feature = A_feature . flatten ( start_dim = 1 ) B_flatten_feature = B_feature . flatten ( start_dim = 1 ) C_flatten_feature = C_feature . flatten ( start_dim = 1 ) ABC_feature = torch . cat ( [ A_flatten_feature , B_flatten_feature , C_flatten_feature ] , dim = 1 ) . detach ( ) ABC_feature_s = torch . chunk ( ABC_feature , ABC_feature . size ( 0 ) , dim = 0 ) for abc_f in ABC_feature_s : temp = abc_f . squeeze ( 0 ) . cpu ( ) . numpy ( ) feature_record_object . append ( temp ) predict_input = np . array ( feature_record_object ) self . engine . ovr_model = joblib . load ( self . engine . ovr_model_path ) g_i = self . engine . ovr_model . decision_function ( predict_input ) frame_score = oc_score ( g_i ) scores [ test_counter + frame_num - 1 ] = frame_score test_counter += 1 total += 1 if test_counter >= test_iters : scores [ : frame_num - 1 ] = scores [ frame_num - 1 ] score_records . append ( scores ) print ( f'finish test video set {video_name}' ) break self . engine . pkl_path = save_score_results ( self . engine . config , self . engine . logger , verbose = self . engine . verbose , config_name = self . engine . config_name , current_step = current_step , time_stamp = self . engine . kwargs [ "time_stamp" ] , score = score_records ) results = self . engine . evaluate_function ( self . engine . pkl_path , self . engine . logger , self . engine . config ) self . engine . logger . info ( results ) tb_writer . add_text ( 'AUC of ROC curve' , f'AUC is {results.auc:.5f}' , global_steps ) return results . auc
Evaluate the results of the model
 Will change, e.g.
Evaluate the results of the model Will change e . g .
'''
        Evaluate the results of the model
        !!! Will change, e.g. accuracy, mAP.....
        !!! Or can call other methods written by the official
        ''' # self.trainer.set_requires_grad(self.trainer.A, False)
 # self.trainer.set_requires_grad(self.trainer.B, False)
 # self.trainer.set_requires_grad(self.trainer.C, False)
 # self.trainer.set_requires_grad(self.trainer.Detector, False)
 # self.trainer.A.eval()
 # self.trainer.B.eval()
 # self.trainer.C.eval()
 # self.trainer.Detector.eval()
 # psnr_records = []
 # random_video_sn = 0
 # _temp_test_folder = os.path.join(self.testing_data_folder, dir)
 # need to improve
 # dataset = AvenueTestOld(_temp_test_folder, clip_length=frame_num)
 # feature_record = []
 # import ipdb; ipdb.set_trace()
 # for test_input, _ in data_loader:
 # import ipdb; ipdb.set_trace()
 # print('NO objects')
 # continue
 # import ipdb; ipdb.set_trace()
 # the svm score of each object in one frame
 # test_psnr = psnr_error(g_output, test_target)
 # test_psnr = test_psnr.tolist()

self
current_step