text: from __future__ import print_function
  import tensorflow as tf
  from tensorflow.keras.datasets import cifar10
  from tensorflow.keras.preprocessing.image import ImageDataGenerator
  from tensorflow.keras.models import Sequential
  from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten
  from tensorflow.keras.layers import Conv2D, MaxPooling2D, BatchNormalization
  from tensorflow.keras import optimizers
  import numpy as np
  from tensorflow.keras.layers import Lambda
  from tensorflow.keras import backend as K
  from tensorflow.keras import regularizers
  from tensorflow.keras.models import Model
  from numpy.linalg import norm
  import matplotlib.pyplot as plt
  import os
  from scipy import io
  
  # parameters for computing the similarity
  layer_num = 15
  layer_name = 'activation_'
  # end parameters
  
  class cifar10vgg:
      def __init__(self, train=True, model_path=None):
          self.num_classes = 10
          self.weight_decay = 0.0005
          self.x_shape = [32,32,3]
  
          self.model = self.build_model()
          if train:
              self.model = self.train(self.model)
          else:
              self.model.load_weights(model_path)
  
  
      def build_model(self):
          # Build the network of vgg for 10 classes with massive dropout and weight decay as described in the paper.
  
          model = Sequential()
          weight_decay = self.weight_decay
  
          model.add(Conv2D(64, (3, 3), padding='same',
                           input_shape=self.x_shape,kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
          model.add(Dropout(0.3))
  
          model.add(Conv2D(64, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
  
          model.add(MaxPooling2D(pool_size=(2, 2)))
  
          model.add(Conv2D(128, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
          model.add(Dropout(0.4))
  
          model.add(Conv2D(128, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
  
          model.add(MaxPooling2D(pool_size=(2, 2)))
  
          model.add(Conv2D(256, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
          model.add(Dropout(0.4))
  
          model.add(Conv2D(256, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
          model.add(Dropout(0.4))
  
          model.add(Conv2D(256, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
  
          model.add(MaxPooling2D(pool_size=(2, 2)))
  
  
          model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
          model.add(Dropout(0.4))
  
          model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
          model.add(Dropout(0.4))
  
          model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
  
          model.add(MaxPooling2D(pool_size=(2, 2)))
  
  
          model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
          model.add(Dropout(0.4))
  
          model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
          model.add(Dropout(0.4))
  
          model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
  
          model.add(MaxPooling2D(pool_size=(2, 2)))
          model.add(Dropout(0.5))
  
          model.add(Flatten())
          model.add(Dense(512,kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
  
          model.add(Dropout(0.5))
          model.add(Dense(self.num_classes))
          model.add(Activation('softmax'))
          return model
  
  
      def normalize(self,X_train,X_test):
          #this function normalize inputs for zero mean and unit variance
          # it is used when training a model.
          # Input: training set and test set
          # Output: normalized training set and test set according to the trianing set statistics.
          mean = np.mean(X_train,axis=(0,1,2,3))
          std = np.std(X_train, axis=(0, 1, 2, 3))
          X_train = (X_train-mean)/(std+1e-7)
          X_test = (X_test-mean)/(std+1e-7)
          return X_train, X_test
  
      def normalize_production(self,x):
          #this function is used to normalize instances in production according to saved training set statistics
          # Input: X - a training set
          # Output X - a normalized training set according to normalization constants.
  
          #these values produced during first training and are general for the standard cifar10 training set normalization
          mean = 120.707
          std = 64.15
          return (x-mean)/(std+1e-7)
  
      def predict(self,x,normalize=True,batch_size=50):
          if normalize:
              x = self.normalize_production(x)
          return self.model.predict(x,batch_size)
  
      def train(self,model):
  
          #training parameters
          batch_size = 128
          maxepoches = 250
          learning_rate = 0.1
          lr_decay = 1e-6
          lr_drop = 20
          # The data, shuffled and split between train and test sets:
          (x_train, y_train), (x_test, y_test) = cifar10.load_data()
          x_train = x_train.astype('float32')
          x_test = x_test.astype('float32')
          x_train, x_test = self.normalize(x_train, x_test)
  
          y_train = tf.keras.utils.to_categorical(y_train, self.num_classes)
          y_test = tf.keras.utils.to_categorical(y_test, self.num_classes)
  
          def lr_scheduler(epoch):
              return learning_rate * (0.5 ** (epoch // lr_drop))
          reduce_lr = tf.keras.callbacks.LearningRateScheduler(lr_scheduler)
  
          #data augmentation
          datagen = ImageDataGenerator(
              featurewise_center=False,  # set input mean to 0 over the dataset
              samplewise_center=False,  # set each sample mean to 0
              featurewise_std_normalization=False,  # divide inputs by std of the dataset
              samplewise_std_normalization=False,  # divide each input by its std
              zca_whitening=False,  # apply ZCA whitening
              rotation_range=15,  # randomly rotate images in the range (degrees, 0 to 180)
              width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)
              height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)
              horizontal_flip=True,  # randomly flip images
              vertical_flip=False)  # randomly flip images
          # (std, mean, and principal components if ZCA whitening is applied).
          datagen.fit(x_train)
  
  
  
          #optimization details
          sgd = optimizers.SGD(lr=learning_rate, decay=lr_decay, momentum=0.9, nesterov=True)
          model.compile(loss='categorical_crossentropy', optimizer=sgd,metrics=['accuracy'])
  
  
          # training process in a for loop with learning rate drop every 25 epoches.
  
          historytemp = model.fit_generator(datagen.flow(x_train, y_train,
                                           batch_size=batch_size),
                              steps_per_epoch=x_train.shape[0] // batch_size,
                              epochs=maxepoches,
                              validation_data=(x_test, y_test),callbacks=[reduce_lr],verbose=2)
          model.save_weights('cifar10vgg.h5')
          return model
  
  
  
  class cifar100vgg:
      def __init__(self,train=True, model_path=None):
          self.num_classes = 100
          self.weight_decay = 0.0005
          self.x_shape = [32,32,3]
  
          self.model = self.build_model()
          if train:
              self.model = self.train(self.model)
          else:
              self.model.load_weights(model_path)
  
  
      def build_model(self):
          # Build the network of vgg for 10 classes with massive dropout and weight decay as described in the paper.
  
          model = Sequential()
          weight_decay = self.weight_decay
  
          model.add(Conv2D(64, (3, 3), padding='same',
                           input_shape=self.x_shape,kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
          model.add(Dropout(0.3))
  
          model.add(Conv2D(64, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
  
          model.add(MaxPooling2D(pool_size=(2, 2)))
  
          model.add(Conv2D(128, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
          model.add(Dropout(0.4))
  
          model.add(Conv2D(128, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
  
          model.add(MaxPooling2D(pool_size=(2, 2)))
  
          model.add(Conv2D(256, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
          model.add(Dropout(0.4))
  
          model.add(Conv2D(256, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
          model.add(Dropout(0.4))
  
          model.add(Conv2D(256, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
  
          model.add(MaxPooling2D(pool_size=(2, 2)))
  
  
          model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
          model.add(Dropout(0.4))
  
          model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
          model.add(Dropout(0.4))
  
          model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
  
          model.add(MaxPooling2D(pool_size=(2, 2)))
  
  
          model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
          model.add(Dropout(0.4))
  
          model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
          model.add(Dropout(0.4))
  
          model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
  
          model.add(MaxPooling2D(pool_size=(2, 2)))
          model.add(Dropout(0.5))
  
          model.add(Flatten())
          model.add(Dense(512,kernel_regularizer=regularizers.l2(weight_decay)))
          model.add(Activation('relu'))
          model.add(BatchNormalization())
  
          model.add(Dropout(0.5))
          model.add(Dense(self.num_classes))
          model.add(Activation('softmax'))
          return model
          
      def normalize(self,X_train,X_test):
          #this function normalize inputs for zero mean and unit variance
          # it is used when training a model.
          # Input: training set and test set
          # Output: normalized training set and test set according to the trianing set statistics.
          mean = np.mean(X_train,axis=(0,1,2,3))
          std = np.std(X_train, axis=(0, 1, 2, 3))
          X_train = (X_train-mean)/(std+1e-7)
          X_test = (X_test-mean)/(std+1e-7)
          return X_train, X_test
  
      def normalize_production(self,x):
          #this function is used to normalize instances in production according to saved training set statistics
          # Input: X - a training set
          # Output X - a normalized training set according to normalization constants.
  
          #these values produced during first training and are general for the standard cifar10 training set normalization
          mean = 121.936
          std = 68.389
          return (x-mean)/(std+1e-7)
  
      def predict(self,x,normalize=True,batch_size=50):
          if normalize:
              x = self.normalize_production(x)
          return self.model.predict(x,batch_size)
          
          
          
  
  (x_train, y_train), (x_test, y_test) = cifar10.load_data()
  x_train = x_train.astype('float32')
  x_test = x_test.astype('float32')
  
  mean = np.mean(x_train,axis=(0,1,2,3))
  std = np.std(x_train, axis=(0, 1, 2, 3))
  #X_train = (X_train-mean)/(std+1e-7)
  x_test = (x_test-mean)/(std+1e-7)
  
  y_train = tf.keras.utils.to_categorical(y_train, 10)
  y_test = tf.keras.utils.to_categorical(y_test, 10)
  
  model1 = cifar10vgg(train=False, model_path='cifar10vgg_0.h5').model
  model1.summary()
  model2 = cifar100vgg(train=False, model_path='cifar100vgg.h5').model
  model2.summary()
  
  #compute each layer's output
  list_conv = []
  all = 6
  print(layer_num)
  for i in range(layer_num-all,layer_num):
      if i!=0:
          s = layer_name + str(i)
      else:
  	    s = layer_name[0:-1]
  
      temp = Model(inputs=model1.input, outputs=model1.get_layer(s).output).predict(x_test).reshape((10000, -1))
      temp_mean = np.sum(temp, axis=0)/10000
      temp = temp - temp_mean
      temp = temp.transpose()
      list_conv.append(temp)
  
  for i in range(layer_num-all,layer_num):
      s = layer_name + str(i+layer_num)
      temp = Model(inputs=model2.input, outputs=model2.get_layer(s).output).predict(x_test).reshape((10000,-1))
      temp_mean = np.sum(temp,axis=0)/10000
      temp = temp - temp_mean
      temp = temp.transpose()
      list_conv.append(temp)
  
  #the linear CKA
  def CKA(x,y):
      a = norm(y.dot(x.transpose()))
      b = norm(x.dot(x.transpose()))
      c = norm(y.dot(y.transpose()))
      return (a*a) / (b*c)
  
  #matlab
  def OMMD(y, z):
      io.savemat(r"Y.mat", {'data': y})
      io.savemat(r"Z.mat", {'data': z})
      os.system("matlab -nodesktop -nosplash -r test")
      d = io.loadmat(r"DIST.mat")
      print('DIST calculated')
      print(d['DIST'][0][0])
      return d['DIST'][0][0]
  
  #compute the similarity
  list_sim = []
  for i in range(all):
      print("compute:", i)
      for j in range(all):
          list_sim.append(CKA(list_conv[i], list_conv[j+all]))
          
  
  #visualize
  list_sim = np.array(list_sim).reshape(all,all)
  print(list_sim)
  np.save("sim_vgg16_all.npy",list_sim)
  plt.imshow(list_sim, cmap='hot')
  plt.colorbar(shrink=.92)
  
  plt.xticks(np.arange(0,all))
  plt.yticks(np.arange(0,all))
  plt.show()
  