import gradio as gr
from load_model import extract_sel_mean_std_bias_assignemnt
from pathlib import Path
from architectures.model_mapping import get_model
from configs.dataset_params import dataset_constants
import torch
import torchvision.transforms as transforms
import pandas as pd
import cv2
import numpy as np
from PIL import Image
from get_data import get_augmentation
from configs.dataset_params import normalize_params
import random
from evaluation.diversity import MultiKCrossChannelMaxPooledSum
def overlapping_features_on_input(model,output, feature_maps, input, target):
W=model.linear.layer.weight
feature_maps=feature_maps.detach().cpu().numpy().squeeze()
print("feature_maps",feature_maps.shape)
if target !=None:
label=target-1
else:
output=output.detach().cpu().numpy()
label=np.argmax(output)
Interpretable_Selection= W[label,:]
print("W",Interpretable_Selection)
input_np=np.array(input)
h,w= input.shape[:2]
print("h,w:",h,w)
Interpretable_Features=[]
input_np=cv2.resize(input_np,(448,448))
Feature_image_list=[input_np]
# color_id=0 #set each feature to singel color
# COLOR=['R','G','B','Y','P','C']
for S in range(len(Interpretable_Selection)):
if Interpretable_Selection[S] != 0:
Interpretable_Features.append(feature_maps[S])
Feature_image=cv2.resize(feature_maps[S],(448,448))
Feature_image=np.uint((Feature_image-np.min(Feature_image))/(np.max(Feature_image)-np.min(Feature_image)) * 255)
Feature_image=Feature_image.astype(np.uint8)
#set each feature to singel color
# if color_id>len(COLOR)-1:
# color_id=color_id%len(COLOR)
# color=COLOR[color_id]
# if color == 'R':
# Feature_image_color=np.zeros_like(input_np)
# Feature_image_color[:,:,0]=Feature_image
# Feature_image=Feature_image_color
# if color == 'G':
# Feature_image_color=np.zeros_like(input_np)
# Feature_image_color[:,:,1]=Feature_image
# Feature_image=Feature_image_color
# if color == 'B':
# Feature_image_color=np.zeros_like(input_np)
# Feature_image_color[:,:,2]=Feature_image
# Feature_image=Feature_image_color
# if color == 'Y':
# Feature_image_color=np.zeros_like(input_np)
# Feature_image_color[:,:,0]=Feature_image
# Feature_image_color[:,:,1]=Feature_image
# Feature_image=Feature_image_color
# if color == 'P':
# Feature_image_color=np.zeros_like(input_np)
# Feature_image_color[:,:,0]=Feature_image
# Feature_image_color[:,:,2]=Feature_image
# Feature_image=Feature_image_color
# if color == 'C':
# Feature_image_color=np.zeros_like(input_np)
# Feature_image_color[:,:,1]=Feature_image
# Feature_image_color[:,:,2]=Feature_image
# Feature_image=Feature_image_color
# color_id+=1
# use Gamma correction
# Feature_image=np.power(Feature_image,1.5)
# use Gamma correction
#set each feature to singel color
Feature_image=cv2.applyColorMap(Feature_image,cv2.COLORMAP_JET)
Feature_image=0.3*Feature_image+0.7*input_np
Feature_image=np.uint((Feature_image-np.min(Feature_image))/(np.max(Feature_image)-np.min(Feature_image)) * 255)
Feature_image=Feature_image.astype(np.uint8)
# path_to_featureimage=f"/home/qixuan/tmp/FeatureImage/FI{S}.jpg"
# cv2.imwrite(path_to_featureimage,Feature_image)
Feature_image = cv2.cvtColor(Feature_image, cv2.COLOR_RGB2BGR)
Feature_image_list.append(Feature_image)
print("len of Features:",len(Interpretable_Features))
return Feature_image_list
def genreate_intepriable_output(input,dataset="CUB2011", arch="resnet50",seed=123456, model_type="qsenn", n_features = 50, n_per_class=5, img_size=448, reduced_strides=False, folder = None, with_featuremaps=True):
n_classes = dataset_constants[dataset]["num_classes"]
# image_re=np.array(input)
input=Image.fromarray(input)
print("input shape",input.size)
model = get_model(arch, n_classes, reduced_strides)
tr=transform_input_img(input,img_size)
# tr=transforms.Compose([
# transforms.Resize(500),
# transforms.CenterCrop(img_size),
# transforms.ToTensor(),
# ])
#TR=get_augmentation(0.1, img_size, False, False, True, True, normalize_params["CUB2011"])
device = torch.device("cpu")
if folder is None:
folder = Path(f"tmp/{arch}/{dataset}/{seed}/")
model.load_state_dict(torch.load(folder / "Trained_DenseModel.pth",map_location=torch.device('cpu')))
state_dict = torch.load(folder / f"{model_type}_{n_features}_{n_per_class}_FinetunedModel.pth",map_location=torch.device('cpu'))
selection= torch.load(folder / f"SlDD_Selection_50.pt",map_location=torch.device('cpu'))
state_dict['linear.selection']=selection
feature_sel, sparse_layer, current_mean, current_std, bias_sparse = extract_sel_mean_std_bias_assignemnt(state_dict)
model.set_model_sldd(feature_sel, sparse_layer, current_mean, current_std, bias_sparse)
model.load_state_dict(state_dict)
input = tr(input)
# path_to_input="/home/qixuan/tmp/FeatureImage/croped.jpg"
# path_to_input_re="/home/qixuan/tmp/FeatureImage/re.jpg"
# path_to_input_concat="/home/qixuan/tmp/FeatureImage/concate.jpg"
# image_re=cv2.cvtColor(image_re, cv2.COLOR_RGB2BGR)
# image_re=cv2.resize(image_re,(448,448))
# image_np = (input * 255).clamp(0, 255).byte()
# image_np = image_np.permute(1, 2, 0).numpy()
# image_np = cv2.cvtColor(image_np, cv2.COLOR_RGB2BGR)
# print("????",input.shape)
# concat=np.vstack((image_re, image_np))
# cv2.imwrite(path_to_input,image_np)
# cv2.imwrite(path_to_input_re,image_re)
# cv2.imwrite(path_to_input_concat,concat)
input= input.unsqueeze(0)
input= input.to(device)
model = model.to(device)
model.eval()
with torch.no_grad():
output, feature_maps, final_features = model(input, with_feature_maps=True, with_final_features=True)
print("featuresmap size:",feature_maps.size())
output_np=output.detach().cpu().numpy()
output_np= np.argmax(output_np)+1
if with_featuremaps:
return output_np,model,feature_maps
else:
return output_np, model
def get_options_from_trainingset(output, model, TR, device,with_other_class):
print("outputclass:",output)
data_dir=Path("tmp/Datasets/CUB200/CUB_200_2011/")
labels = pd.read_csv("image_class_labels.txt", sep=' ', names=['img_id', 'target'])
namelist=pd.read_csv(data_dir/"images.txt",sep=' ',names=['img_id','file_name'])
classlist=pd.read_csv(data_dir/"classes.txt",sep=' ',names=['cl_id','class_name'])
options_output=labels[labels['target']==output]
print(options_output)
print(labels)
options=options_output.sample(4)
#mode 2
if with_other_class:
other_targets=random.sample([i for i in range(1,200)if i != output],3)
all_targets=[output]+other_targets
for tg in other_targets:
others=labels[labels['target']==tg]
options_others=others.sample(4)
options = pd.concat([options, options_others], ignore_index=True)
else:
all_targets=[output]
#shuffled_options = options.sample(frac=1).reset_index(drop=True)
print("shuffled:",options)
op=[]
# resample_img_id_list=[]#resample filter
W=model.linear.layer.weight# intergrate negative features
model.eval()
with torch.no_grad():
for t in all_targets:
# intergrate negative features
W_class=W[t-1,:]
features_id=[ f for f in W_class if f !=0 ]
features_id_neg= [i+1 for i, x in enumerate(features_id) if x < 0]
# intergrate negative features
image = cv2.imread(f"options_heatmap/{t}.jpg")
concatenate_class = np.array(image)
concatenate_class = cv2.cvtColor(concatenate_class, cv2.COLOR_RGB2BGR)
op.append((concatenate_class,features_id_neg))# intergrate negative features
return op
def transform_input_img(input,img_size):
h,w=input.size
rate=h/w
if h >= w:
w_new=img_size
h_new=int(w_new*rate)
else:
h_new=img_size
w_new=int(h_new/rate)
return transforms.Compose([
transforms.Resize((w_new,h_new)),
transforms.CenterCrop(img_size),
transforms.ToTensor(),
])
def post_next_image(OPT: str,key:str):
if OPT==key:
return ("Congradulations! you can simulate the prediction of Model this time",gr.update(interactive=False),gr.update(interactive=False),gr.update(interactive=False),gr.update(interactive=False))
else:
return (f"sorry, what the model predicted is {key}",gr.update(interactive=False),gr.update(interactive=False),gr.update(interactive=False),gr.update(interactive=False))
def get_features_on_interface(input):
img_size=448
output,model=genreate_intepriable_output(input,dataset="CUB2011",
arch="resnet50",seed=123456,
model_type="qsenn", n_features = 50,n_per_class=5,
img_size=448, reduced_strides=False, folder = None,with_featuremaps=False)
TR=get_augmentation(0.1, img_size, False, False, True, True, normalize_params["CUB2011"])
device = torch.device("cpu")
op= get_options_from_trainingset(output, model, TR, device,with_other_class=True)
key=op[0][0]# intergrate negative features
random.shuffle(op)
option=[(op[0][0],"A"),
(op[1][0],"B"),
(op[2][0],"C"),
(op[3][0],"D")]
for value,char in option:
if np.array_equal(value,key):
key_op=char
print("key",key_op)
# if op[0][1]!=[]:
# option[0][1]=f"A,features{', '.join(map(str, op[0][1]))} are negative."
# if op[1][1]!=[]:
# option[1][1]=f"B,features{', '.join(map(str, op[0][1]))} are negative."
# if op[2][1]!=[]:
# option[2][1]=f"C,features{', '.join(map(str, op[0][1]))} are negative."
# if op[3][1]!=[]:
# option[3][1]=f"D,features{', '.join(map(str, op[0][1]))} are negative."
return option, key_op," These are some class explanations from our model for different classes,which of these classes has our model predicted?",gr.update(interactive=False)
def direct_inference(input):
img_size=448
output, model,feature_maps=genreate_intepriable_output(input,dataset="CUB2011",
arch="resnet50",seed=123456,
model_type="qsenn", n_features = 50,n_per_class=5,
img_size=448, reduced_strides=False, folder = None,with_featuremaps=True)
# image_list=overlapping_features_on_input(model,output,feature_maps,input,target=None)
# image_arrays = [np.array(img) for img in image_list]
# concatenated_image = np.concatenate(image_arrays, axis=0)
TR=get_augmentation(0.1, img_size, False, False, True, True, normalize_params["CUB2011"])
device = torch.device("cpu")
concatenated_image=get_options_from_trainingset(output, model, TR, device, with_other_class=False)
#original
Input=Image.fromarray(input)
tr=transform_input_img(Input,img_size)
Input=tr(Input)
image_np = (Input * 255).clamp(0, 255).byte()
image_np = image_np.permute(1, 2, 0).numpy()
# image_np = cv2.cvtColor(image_np, cv2.COLOR_RGB2BGR)
ORI= overlapping_features_on_input(model,output, feature_maps, image_np,output)#input image_np
ORI_arrays = [np.array(img) for img in ORI]
concatenated_ORI = np.concatenate(ORI_arrays, axis=0)
print(concatenated_ORI.shape,concatenated_image[0][0].shape)
concatenated_image_final_array=np.concatenate((concatenated_ORI,concatenated_image[0][0]),axis=1)
print(concatenated_image_final_array.shape)
#original
data_dir=Path("tmp/Datasets/CUB200/CUB_200_2011/")
classlist=pd.read_csv(data_dir/"classes.txt",sep=' ',names=['cl_id','class_name'])
output_name=classlist.loc[classlist['cl_id']==output,'class_name'].values[0]
if concatenated_image[0][1]!=[]:
output_name_and_features=f"{output_name}, features{', '.join(map(str, concatenated_image[0][1]))} are negative."
else:
output_name_and_features=f"{output_name}, all features are positive."
return concatenated_image_final_array, output_name_and_features
def filter_with_diversity(featuremaps,output,weight):
localizer = MultiKCrossChannelMaxPooledSum(range(1, 6), weight, None)
localizer(output.to("cpu"),featuremaps.to("cpu"))
locality, exlusive_locality = localizer.get_result()
diversity = locality[4]
diversity=diversity.item()
return diversity