text: 
  
  # %% [code]
  bool_random_flip_left_right=1
  bool_random_flip_up_down=1
  bool_random_brightness=0
  bool_random_contrast=0
  bool_random_hue=0
  bool_random_saturation=0
  bool_random_crop = 0
  
  bool_rotation_transform=1
  cutmix_rate=0.
  mixup_rate= 0.
  gridmask_rate = 0.5
  
  pre_trained='noisy-student' # None,'imagenet','noisy-student'
  dense_activation='softmax' #'softmax','sigmoid'
  bool_lr_scheduler=1
  
  # 交叉验证和tta目前最多只允许使用一个
  tta_times=0 #15 #当tta_times=i>0时，使用i+1倍测试集
  cross_validation_folds=5 #当tta_times=i>1时，使用i折交叉验证
  
  
  #focal_loss和label_smoothing最多同时使用一个
  bool_focal_loss = 1
  label_smoothing_rate=0.
  
  #伪标签
  bool_pseudo=0
  
  # 增加auc记录
  special_monitor='auc'
  
  # %% [code]
  #针对前面opts的一些中间处理
  
  #tta只有在使用了data_aug时才允许启用
  bool_tta =  tta_times and max(  bool_random_flip_left_right,
                                  bool_random_flip_up_down,
                                  bool_random_brightness,
                                  bool_random_contrast,
                                  bool_random_hue,
                                  bool_random_saturation)
  
  print(bool_tta)
  
  assert (bool_focal_loss and label_smoothing_rate) == 0 , 'focal_loss和label_smoothing最多同时使用一个'
  assert (tta_times and cross_validation_folds) == 0 , 'focal_loss和label_smoothing最多同时使用一个'
  
  
  # %% [code]
  #安装包
  !pip install -U efficientnet
  !pip install tensorflow_addons
  
  # %% [code]
  #导入包
  import numpy as np
  import pandas as pd
  import matplotlib.pyplot as plt 
  import os
  import tensorflow as tf
  import random, re, math
  from tensorflow.keras.layers import Dense
  from tensorflow.keras.models import Model
  from tensorflow.keras import optimizers
  import tensorflow_addons as tfa
  
  from kaggle_datasets import KaggleDatasets
  import efficientnet.tfkeras as efn
  from sklearn.model_selection import KFold
  from tensorflow.keras.callbacks import ModelCheckpoint
  
  print(tf.__version__)
  print(tf.keras.__version__)
  
  # %% [code]
  #针对不同硬件产生不同配置
  AUTO = tf.data.experimental.AUTOTUNE
  # Detect hardware, return appropriate distribution strategy
  try:
      tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.
      print('Running on TPU ', tpu.master())
  except ValueError:
      tpu = None
  
  if tpu:
      tf.config.experimental_connect_to_cluster(tpu)
      tf.tpu.experimental.initialize_tpu_system(tpu)
      strategy = tf.distribute.experimental.TPUStrategy(tpu)
      GCS_DS_PATH = KaggleDatasets().get_gcs_path()
      print(GCS_DS_PATH)
  else:
      strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.
  print("REPLICAS: ", strategy.num_replicas_in_sync)
  
  # %% [code]
  #path、label都作为全局变量
  
  #超参数，根据数据和策略调参
  BATCH_SIZE = 1 * strategy.num_replicas_in_sync
  img_size = 768
  EPOCHS = 25
  lr_if_without_scheduler = 0.0003
  nb_classes = 3
  print('BATCH_SIZE是：',BATCH_SIZE)
  
  my_metrics = ['accuracy']
  
  
  
  import glob
  train_paths = []
  train_labels = []
  test_paths = []
  test_labels = []
  
  
  # path='../input/small-big-culture/Candida'
  # img=glob.glob(path+'/*')
  # for i in range(len(img)):
  #     train_paths.append(img[i])
  #     train_labels.append([1,0,0,0])
  
      
  # path='../input/small-big-culture/Microsporum'
  # img=glob.glob(path+'/*')
  # for i in range(len(img)):
  #     train_paths.append(img[i])
  #     train_labels.append([0,1,0,0])
  
      
  # path='../input/small-big-culture/Trichophyton'
  # img=glob.glob(path+'/*')
  # for i in range(len(img)):
  #     train_paths.append(img[i])
  #     train_labels.append([0,0,1,0])
  
      
  # path='../input/small-big-culture/Trichosporon'
  # img=glob.glob(path+'/*')
  # for i in range(len(img)):
  #     train_paths.append(img[i])
  #     train_labels.append([0,0,0,1])
      
  
      
  paths = os.listdir('../input/slide-culture')
  for i in range(3):
      img_dir = '../input/slide-culture/'+paths[i]
      img = os.listdir(img_dir)
      for ele in img:
          train_paths.append(img_dir+'/'+ele)     
          train_labels.append(np.eye(3)[i])
      
      
  
      
  train_labels=np.array(train_labels)
  
  
  
      
  
  # print((train_paths))
  # print((train_labels))    
      
  print(len(train_paths))
  print(len(train_labels))
  print(len(test_paths))
  
  print('\n',train_labels[0])
  
  
  from matplotlib import pyplot as plt
  #随便看一张
  img = plt.imread(train_paths[0])
  print('\n',img.shape)
  plt.imshow(img)
  
  # %% [code]
  #lr_scheduler
  #数值按实际情况设置
  
  LR_START = 0.00001
  LR_MAX = 0.0001 * strategy.num_replicas_in_sync
  LR_MIN = 0.00001
  LR_RAMPUP_EPOCHS = 3
  LR_SUSTAIN_EPOCHS = 5
  LR_EXP_DECAY = .8
  
  def lrfn(epoch):
      if epoch < LR_RAMPUP_EPOCHS:
          lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START
      elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
          lr = LR_MAX
      else:
          lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN
      return lr
      
  lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
  
  rng = [i for i in range(EPOCHS)]
  y = [lrfn(x) for x in rng]
  if bool_lr_scheduler:
      plt.plot(rng, y)
      print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
  
  # %% [code]
  def decode_image(filename, label=None,image_size=(img_size, img_size)):
      bits = tf.io.read_file(filename)
      image = tf.image.decode_jpeg(bits, channels=3)
      image = tf.cast(image, tf.float32) / 255.0
      image = tf.image.resize(image, image_size)
      
      if label is None:
          return image
      else:
          return image, label
  
  
  
  
  
  # 只能写入test data 也能用的 aug   
  def data_aug(image, label=None, image_size=[img_size, img_size]):
      if bool_random_flip_left_right:
          image = tf.image.random_flip_left_right(image)
      if bool_random_flip_up_down:    
          image = tf.image.random_flip_up_down(image)
      if bool_random_brightness:
          image = tf.image.random_brightness(image,0.2)
      if bool_random_contrast:
          image = tf.image.random_contrast(image,0.6,1.4)
      if bool_random_hue:
          image = tf.image.random_hue(image,0.07)
      if bool_random_saturation:
          image = tf.image.random_saturation(image,0.5,1.5)
      if bool_random_crop:
          image = tf.image.resize(image, [int(1.2*ele) for ele in image_size])
          image = tf.image.random_crop(image,image_size+[3])
      
  
      
      if label is None:
          return image
      else:
          return image, label
  
  # %% [code]
  import tensorflow as tf, tensorflow.keras.backend as K
  def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
      # returns 3x3 transformmatrix which transforms indicies
          
      # CONVERT DEGREES TO RADIANS
      rotation = math.pi * rotation / 180.
      shear = math.pi * shear / 180.
      
      # ROTATION MATRIX
      c1 = tf.math.cos(rotation)
      s1 = tf.math.sin(rotation)
      one = tf.constant([1],dtype='float32')
      zero = tf.constant([0],dtype='float32')
      rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )
          
      # SHEAR MATRIX
      c2 = tf.math.cos(shear)
      s2 = tf.math.sin(shear)
      shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    
      
      # ZOOM MATRIX
      zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )
      
      # SHIFT MATRIX
      shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )
      
      return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
  
  
  
  def rotation_transform(image,label):
      # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]
      # output - image randomly rotated, sheared, zoomed, and shifted
      DIM = img_size
      XDIM = DIM%2 #fix for size 331
      
      rot = 15. * tf.random.normal([1],dtype='float32')
      shr = 5. * tf.random.normal([1],dtype='float32') 
      h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
      w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
      h_shift = 16. * tf.random.normal([1],dtype='float32') 
      w_shift = 16. * tf.random.normal([1],dtype='float32') 
    
      # GET TRANSFORMATION MATRIX
      m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) 
  
      # LIST DESTINATION PIXEL INDICES
      x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )
      y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )
      z = tf.ones([DIM*DIM],dtype='int32')
      idx = tf.stack( [x,y,z] )
      
      # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS
      idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
      idx2 = K.cast(idx2,dtype='int32')
      idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
      
      # FIND ORIGIN PIXEL VALUES           
      idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )
      d = tf.gather_nd(image,tf.transpose(idx3))
          
      return tf.reshape(d,[DIM,DIM,3]),label
  
  # %% [code]
  # 在batch内部互相随机取图
  def cutmix(image, label, PROBABILITY = cutmix_rate):
      # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]
      # output - a batch of images with cutmix applied
      label=tf.cast(label,tf.float32)
      
      DIM = img_size    
      imgs = []; labs = []
      
      for j in range(BATCH_SIZE):
          
          #random_uniform( shape, minval=0, maxval=None)        
          # DO CUTMIX WITH PROBABILITY DEFINED ABOVE
          P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)
          
          # CHOOSE RANDOM IMAGE TO CUTMIX WITH
          k = tf.cast(tf.random.uniform([], 0, BATCH_SIZE), tf.int32)
          
          # CHOOSE RANDOM LOCATION
          #选一个随机的中心点
          x = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)
          y = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)
          
          # Beta(1, 1)等于均匀分布
          b = tf.random.uniform([], 0, 1) # this is beta dist with alpha=1.0
          
          #P只随机出0或1，就是裁剪或是不裁剪
          WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32) * P
          ya = tf.math.maximum(0,y-WIDTH//2)
          yb = tf.math.minimum(DIM,y+WIDTH//2)
          xa = tf.math.maximum(0,x-WIDTH//2)
          xb = tf.math.minimum(DIM,x+WIDTH//2)
          
          # MAKE CUTMIX IMAGE
          one = image[j,ya:yb,0:xa,:]
          two = image[k,ya:yb,xa:xb,:]
          three = image[j,ya:yb,xb:DIM,:]        
          #得出了ya:yb区间内的输出图像
          middle = tf.concat([one,two,three],axis=1)
          #得到了完整输出图像
          img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)
          imgs.append(img)
          
          # MAKE CUTMIX LABEL
          #按面积来加权的
          a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)
          lab1 = label[j,]
          lab2 = label[k,]
          labs.append((1-a)*lab1 + a*lab2)
  
      image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))
      label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE, nb_classes))
      return image2,label2
  
  # %% [code]
  def mixup(image, label, PROBABILITY = mixup_rate):
      # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]
      # output - a batch of images with mixup applied
      DIM = img_size
      
      imgs = []; labs = []
      for j in range(BATCH_SIZE):
          
          # CHOOSE RANDOM
          k = tf.cast( tf.random.uniform([],0,BATCH_SIZE),tf.int32)
          a = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0
  
          #根据概率抽取执不执行mixup
          P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)
          if P==1:
              a=0.
          
          # MAKE MIXUP IMAGE
          img1 = image[j,]
          img2 = image[k,]
          imgs.append((1-a)*img1 + a*img2)
          
          # MAKE CUTMIX LABEL
          lab1 = label[j,]
          lab2 = label[k,]
          labs.append((1-a)*lab1 + a*lab2)
              
      # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)
      image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))
      label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE,nb_classes))
      return image2,label2
  
  # %% [code]
  # gridmask
  def transform(image, inv_mat, image_shape):
      h, w, c = image_shape
      cx, cy = w//2, h//2
      new_xs = tf.repeat( tf.range(-cx, cx, 1), h)
      new_ys = tf.tile( tf.range(-cy, cy, 1), [w])
      new_zs = tf.ones([h*w], dtype=tf.int32)
      old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))
      old_coords_x, old_coords_y = tf.round(old_coords[0, :] + w//2), tf.round(old_coords[1, :] + h//2)
      clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)
      clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)
      clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)
      old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))
      old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))
      new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))
      new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))
      old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)
      new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)
      rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))
      rotated_image_channel = list()
      for i in range(c):
          vals = rotated_image_values[:,i]
          sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])
          rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))
      return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])
  
  def random_rotate(image, angle, image_shape):
      def get_rotation_mat_inv(angle):
          # transform to radian
          angle = math.pi * angle / 180
          cos_val = tf.math.cos(angle)
          sin_val = tf.math.sin(angle)
          one = tf.constant([1], tf.float32)
          zero = tf.constant([0], tf.float32)
          rot_mat_inv = tf.concat([cos_val, sin_val, zero, -sin_val, cos_val, zero, zero, zero, one], axis=0)
          rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])
          return rot_mat_inv
      angle = float(angle) * tf.random.normal([1],dtype='float32')
      rot_mat_inv = get_rotation_mat_inv(angle)
      return transform(image, rot_mat_inv, image_shape)
  
  
  def GridMask(image_height, image_width, d1, d2, rotate_angle=1, ratio=0.5):
      h, w = image_height, image_width
      hh = int(np.ceil(np.sqrt(h*h+w*w)))
      hh = hh+1 if hh%2==1 else hh
      d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)
      l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)
  
      st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
      st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
  
      y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)
      x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)
  
      for i in range(0, hh//d+1):
          s1 = i * d + st_h
          s2 = i * d + st_w
          y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)
          x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)
  
      x_clip_mask = tf.logical_or(x_ranges < 0 , x_ranges > hh-1)
      y_clip_mask = tf.logical_or(y_ranges < 0 , y_ranges > hh-1)
      clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)
  
      x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))
      y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))
  
      hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)), tf.int32)])
      x_ranges = tf.repeat(x_ranges, hh)
      y_ranges = tf.repeat(y_ranges, hh)
  
      y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))
      x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))
  
      y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64),  tf.zeros_like(y_ranges), [hh, hh])
      y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)
  
      x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])
      x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)
  
      mask = tf.expand_dims( tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)
  
      mask = random_rotate(mask, rotate_angle, [hh, hh, 1])
      mask = tf.image.crop_to_bounding_box(mask, (hh-h)//2, (hh-w)//2, image_height, image_width)
  
      return mask
  
  def apply_grid_mask(image, image_shape, PROBABILITY = gridmask_rate):
      AugParams = {
          'd1' : 100,
          'd2': 160,
          'rotate' : 45,
          'ratio' : 0.3
      }
      
          
      mask = GridMask(image_shape[0], image_shape[1], AugParams['d1'], AugParams['d2'], AugParams['rotate'], AugParams['ratio'])
      if image_shape[-1] == 3:
          mask = tf.concat([mask, mask, mask], axis=-1)
          mask = tf.cast(mask,tf.float32)
          #print(mask.shape) # (299,299,3)
  
  # 会报错，放弃
  #     imgs = []
  #     BATCH_SIZE=len(image)
  #     for j in range(BATCH_SIZE):
  #         P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)
  #         if P==1:
  #             imgs.append(image[j,]*mask)
  #         else:
  #             imgs.append(image[j,])
  #     return tf.cast(imgs,tf.float32)
  
          
      # 整个batch启用或者不启用
      P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)
      if P==1:
          return image*mask
      else:
          return image
  
  def gridmask(img_batch, label_batch):
      return apply_grid_mask(img_batch, (img_size,img_size, 3)), label_batch
  
  # %% [code]
  def get_train_dataset(train_paths,train_labels=None):
  
      # num_parallel_calls并发处理数据的并发数
      train_dataset = tf.data.Dataset.from_tensor_slices((train_paths, train_labels.astype(np.float32))).map(decode_image, num_parallel_calls=AUTO)
  
      #train_dataset = train_dataset.cache().map(data_aug, num_parallel_calls=AUTO).repeat()
      train_dataset = train_dataset.map(data_aug, num_parallel_calls=AUTO).repeat()
      
      if bool_rotation_transform:
          train_dataset =train_dataset.map(rotation_transform)
                       
      train_dataset = train_dataset.shuffle(512).batch(BATCH_SIZE,drop_remainder=True)
  
  
      if cutmix_rate:  
          print('启用cutmix')
          train_dataset =train_dataset.map(cutmix, num_parallel_calls=AUTO)
      if mixup_rate:  
          print('启用mixup')
          train_dataset =train_dataset.map(mixup, num_parallel_calls=AUTO)
      if gridmask_rate:
          print('启用gridmask')
          train_dataset =train_dataset.map(gridmask, num_parallel_calls=AUTO)
      if (cutmix_rate or mixup_rate):
          train_dataset =train_dataset.unbatch().shuffle(512).batch(BATCH_SIZE)
  
  
      # repeat()代表无限制复制原始数据，这里可以用count指明复制份数，但要注意要比fit中的epochs大才可
      # 直接调用repeat()的话，生成的序列就会无限重复下去
      # prefetch: prefetch next batch while training (autotune prefetch buffer size)
      train_dataset = train_dataset.prefetch(AUTO)
  
      return train_dataset
  
  # %% [code]
  # #暂时取消这一段，减少爆内存问题
  # try:
  #     view_train_dataset=get_train_dataset(train_paths,train_labels)
  # except:
  #     view_train_dataset=get_train_dataset(train_paths)
      
  # it = view_train_dataset.__iter__()
  
  # #看看train_dataset 是否正常显示
  # show_x, show_y = it.next()#TPU下File system scheme '[local]' not implemented
  # print(show_x.shape,'\n\n',show_y[0])
  
  # plt.figure(figsize=(12, 6))
  # for i in range(1):
  #     plt.subplot(2, 4, i+1)
  #     plt.imshow(show_x[i])
  
  # %% [code]
  def get_validation_dataset(valid_paths,valid_labels=None):
      dataset = tf.data.Dataset.from_tensor_slices((valid_paths, valid_labels))
      dataset = dataset.map(decode_image, num_parallel_calls=AUTO)
      
  
      
      
  
      #dataset = dataset.cache()
      dataset = dataset.batch(BATCH_SIZE)
      dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)
      return dataset
  
  # %% [code]
  #生成测试集
  def re_produce_test_dataset(test_paths):
      test_dataset = tf.data.Dataset.from_tensor_slices(test_paths)
      test_dataset = test_dataset.map(decode_image, num_parallel_calls=AUTO)
      
  
      
  
      if bool_tta:
          test_dataset = test_dataset.cache().map(data_aug, num_parallel_calls=AUTO)
  
      test_dataset = test_dataset.batch(BATCH_SIZE)
      return test_dataset
  
  # %% [code]
  # #tta时可重复运行这块观察是否多次运行时生成了不同的测试数据
  # view_dataset = re_produce_test_dataset(test_paths[:8])
  # it = view_dataset.__iter__()
  # show_x= it.next()
  # try:
  #     print(show_x.shape)
  # except:
  #     print(show_x[0].shape)
  #     show_x=show_x[0]
      
  # plt.figure(figsize=(12, 6))
  # for i in range(1):
  #     plt.subplot(2, 4, i+1)
  #     plt.imshow(show_x[i])
  
  # %% [code]
  if special_monitor=='auc':
      my_metrics.append(tf.keras.metrics.AUC(name='auc'))
  
  
  #创建模型
  def get_model():
      with strategy.scope():
          base_model =  efn.EfficientNetB7(weights=pre_trained, include_top=False, pooling='avg', input_shape=(img_size, img_size, 3))
          x = base_model.output
          predictions = Dense(nb_classes, activation=dense_activation)(x)
          model = Model(inputs=base_model.input, outputs=predictions)
  
      if label_smoothing_rate:
          print('启用label_smoothing')
          my_loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=label_smoothing_rate)
      elif bool_focal_loss:
  
          my_loss = tfa.losses.SigmoidFocalCrossEntropy(reduction=tf.keras.losses.Reduction.AUTO)
  
      else:
          my_loss='categorical_crossentropy'
  
      model.compile(optimizer=tf.keras.optimizers.Adam(lr_if_without_scheduler), 
                    loss=my_loss,
                    metrics=my_metrics
                   )
  
      return model
  
  # %% [code]
  callbacks=[]
  if bool_lr_scheduler:
      callbacks.append(lr_callback)
  
  # %% [code]
  def training():
      probabilities =[]
      
      if cross_validation_folds:
          
          histories = []
  
          # early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 3)
          kfold = KFold(cross_validation_folds , shuffle = True)
  
          i=1
  
  
  
  
          for trn_ind, val_ind in kfold.split(train_paths,train_labels):
              print(); print('#'*25)
              print('### FOLD',i)
              print('#'*25)
              
              # print(trn_ind)
              print(val_ind)
  
              # 暂停checkpoint，防止爆内存
              # 每轮都应该重置 ModelCheckpoint
              # ch_p1 = ModelCheckpoint(filepath="temp_best.h5", monitor='val_accuracy', save_weights_only=True,verbose=1,save_best_only=True)
  
  
              if special_monitor=='auc': 
                  ch_p1 = ModelCheckpoint(filepath="temp_best.h5", monitor='val_auc', mode='max',save_weights_only=True,verbose=1,save_best_only=True)
  
              temp_callbacks=callbacks.copy()
              temp_callbacks.append(ch_p1)
  
              trn_paths = np.array(train_paths)[trn_ind]
              val_paths=np.array(train_paths)[val_ind]
              
              
  
              trn_labels = train_labels[trn_ind]
              val_labels=train_labels[val_ind]
              test_paths = val_paths
  
              model = get_model()
              history = model.fit(
                  get_train_dataset(trn_paths,trn_labels), 
                  steps_per_epoch = trn_labels.shape[0]//BATCH_SIZE,
  
              
  
                  epochs = EPOCHS,
                  callbacks = temp_callbacks,
                  validation_data = (get_validation_dataset(val_paths,val_labels)),
              )
              
              
  
              i+=1
              histories.append(history)
  
              #用val_loss最小的权重来预测
              model.load_weights("temp_best.h5")
              prob = model.predict(re_produce_test_dataset(test_paths), verbose=1)       
              
              
              probabilities.append(prob)
              probabilities.append(val_labels)
              break
              
              
  
      #if not cross_validation_folds:
      else:
  
          model = get_model()
  
  
  
          histories = model.fit(
              get_train_dataset(train_paths,train_labels), 
              steps_per_epoch=train_labels.shape[0] // BATCH_SIZE,
  
  
  
  
              callbacks=callbacks,
              epochs=EPOCHS
          )
          
          
      return model,histories,probabilities
  
  # %% [code]
  #画history咯
  def display_training_curves(training, title, subplot, validation=None):
      if subplot%10==1: # set up the subplots on the first call
          plt.subplots(figsize=(10,10), facecolor='#F0F0F0')
          plt.tight_layout()
      ax = plt.subplot(subplot)
      ax.set_facecolor('#F8F8F8')
      ax.plot(training)
      if validation is not None:
          ax.plot(validation)
      ax.set_title('model '+ title)
      ax.set_ylabel(title)
      #ax.set_ylim(0.28,1.05)
      ax.set_xlabel('epoch')
      if validation is not None:
          ax.legend(['train', 'valid.'])
      else:
          ax.legend(['train'])
  
  # %% [code]
  
  
  # %% [code]
  def draw_training_curves(histories):
      # 写开，防止交叉验证时out of memory导致什么都没保存
      if cross_validation_folds:
  
  
      #然后画画- -
          for h in range(len(histories)):
              display_training_curves(histories[h].history['loss'], 'loss', 211, histories[h].history['val_loss'])
              display_training_curves(histories[h].history['accuracy'], 'accuracy', 212, histories[h].history['val_accuracy'])
  
      #if not cross_validation_folds:
      else:
          display_training_curves(histories.history['loss'], 'loss', 211)
          display_training_curves(histories.history['accuracy'], 'accuracy', 212)
  
  # %% [code]
  def predict(model,probabilities):
      if cross_validation_folds:
          y_pred = np.mean(probabilities,axis =0)
      #if not cross_validation_folds:
      else:
          if bool_tta:
              probabilities = []
              for i in range(tta_times+1):
                  print('TTA Number: ',i,'\n')
                  test_dataset = re_produce_test_dataset(test_paths)
                  probabilities.append(model.predict(test_dataset))
                  y_pred = np.mean(probabilities,axis =0)
  
  
          else:
              test_dataset = re_produce_test_dataset(test_paths)       
              y_pred = model.predict(test_dataset)
              
      return y_pred
  
  # %% [code]
  model,histories,probabilities=training()
  draw_training_curves(histories)
  y_pred=predict(model,probabilities)
  
  # %% [code]
  %matplotlib inline
  
  
  
  from sklearn.metrics import confusion_matrix
  import matplotlib.pyplot as plt
  import numpy as np
  
  def plot_confusion_matrix(cm, savename, title='Confusion Matrix',classes = paths):
  
      plt.figure(figsize=(12, 8), dpi=100)
      np.set_printoptions(precision=2)
  
      # 在混淆矩阵中每格的概率值
      ind_array = np.arange(len(classes))
      x, y = np.meshgrid(ind_array, ind_array)
      for x_val, y_val in zip(x.flatten(), y.flatten()):
          c = cm[y_val][x_val]
          if c > 0.001:
              plt.text(x_val, y_val, "%0.2f" % (c,), color='red', fontsize=15, va='center', ha='center')
      
      plt.imshow(cm, interpolation='nearest', cmap=plt.cm.binary)
      plt.title(title)
      plt.colorbar()
      xlocations = np.array(range(len(classes)))
      plt.xticks(xlocations, classes, rotation=90)
      plt.yticks(xlocations, classes)
      plt.ylabel('Actual label')
      plt.xlabel('Predict label')
      
      # offset the tick
      tick_marks = np.array(range(len(classes))) + 0.5
      plt.gca().set_xticks(tick_marks, minor=True)
      plt.gca().set_yticks(tick_marks, minor=True)
      plt.gca().xaxis.set_ticks_position('none')
      plt.gca().yaxis.set_ticks_position('none')
      plt.grid(True, which='minor', linestyle='-')
      plt.gcf().subplots_adjust(bottom=0.15)
      
      # show confusion matrix
      plt.savefig(savename, format='png')
      plt.show()
  
  # %% [code]
  # val_labels = [train_labels[i] for i in [7,9,13,15,38,40,56,57,58,61,67,70,72,74,76,88,92,95
  # ,100,107,111,116,124,126,130,134,138,139,140,148,149,150,151,154,155,160
  # ,164,167,171,172,193,197,201,207,209,216,226,232,234,238,241,244,245,249
  # ,250,254,259,262,264,269,270,273,275,284,300,306,307,308,313,318,320,321
  # ,322,334,337,339,344,349,354,356,359,364,371,379,387,391,394,413,418,423
  # ,424,426,459,462,464,465,466,468,470,471,485,487,502,507,508,513,516,518
  # ,529,536,544,547,557,566,567,568,570,572,584,589,599,604,605,610,611,614
  # ,615,619,621,632,637,638,639,640,654,669,671,675,682,684,685,692,693,694
  # ,702,707,710,727,734,741,743,762,767,770,774,778,781,788]]
  # val_labels = np.array(val_labels)
  # test_dataset = re_produce_test_dataset([train_paths[i] for i in [7,9,13,15,38,40,56,57,58,61,67,70,72,74,76,88,92,95
  # ,100,107,111,116,124,126,130,134,138,139,140,148,149,150,151,154,155,160
  # ,164,167,171,172,193,197,201,207,209,216,226,232,234,238,241,244,245,249
  # ,250,254,259,262,264,269,270,273,275,284,300,306,307,308,313,318,320,321
  # ,322,334,337,339,344,349,354,356,359,364,371,379,387,391,394,413,418,423
  # ,424,426,459,462,464,465,466,468,470,471,485,487,502,507,508,513,516,518
  # ,529,536,544,547,557,566,567,568,570,572,584,589,599,604,605,610,611,614
  # ,615,619,621,632,637,638,639,640,654,669,671,675,682,684,685,692,693,694
  # ,702,707,710,727,734,741,743,762,767,770,774,778,781,788]])
  
  # %% [code]
  # pred = model.predict(test_dataset)
  
  # %% [code]
  pred = (np.array([np.argmax(ele) for ele in probabilities[0]])).astype(int)
  val_labels = (np.array([np.argmax(ele) for ele in probabilities[1]])).astype(int)
  
  # %% [code]
  print(pred)
  
  # %% [code]
  cm = confusion_matrix((val_labels), np.array(pred))
  plot_confusion_matrix(cm, 'confusion_matrix.png', title='confusion matrix')
  
  # %% [code]
  #############画图部分
  fpr, tpr, threshold = metrics.roc_curve(val_labels, pred)
  roc_auc = metrics.auc(fpr, tpr)
  plt.figure(figsize=(6,6))
  plt.title('Validation ROC')
  plt.plot(fpr, tpr, 'b', label = 'Val AUC = %0.3f' % roc_auc)
  plt.legend(loc = 'lower right')
  plt.plot([0, 1], [0, 1],'r--')
  plt.xlim([0, 1])
  plt.ylim([0, 1])
  plt.ylabel('True Positive Rate')
  plt.xlabel('False Positive Rate')
  
  # %% [code]
  def sub(name):
      ret=[]
      for i in y_pred:
          if i[0]>=i[1]:
              ret.append('AD')
          elif i[0]<i[1]:
              ret.append('CN')
  
      sub = pd.read_csv('../input/petpet/pet.csv')
      sub.loc[:, 'label'] = ret
  
      print(len(sub))
  
      sub.to_csv(name, index=False)
      sub.head(30)
  
  # %% [code]
  sub('submission.csv')
  
  # %% [code]
  #保留副本，防止错误操作
  train_paths_copy=train_paths.copy()
  train_labels_copy=train_labels.copy()
  
  # %% [code]
  if bool_pseudo:
      print("启用伪标签")
      
      train_paths=train_paths_copy.copy()
      train_labels=train_labels_copy.copy()
      
      
      print(len(train_paths),len(train_labels))
      for i in range(len(y_pred)):
          print(y_pred[i])
  
  
          if y_pred[i][0]>=0.9:
              train_paths=np.append(train_paths,test_paths[i])
              train_labels=np.append(train_labels,[[1,0]],axis = 0)
          elif y_pred[i][0]<=0.1:
              train_paths=np.append(train_paths,test_paths[i])
              train_labels=np.append(train_labels,[[0,1]],axis = 0)
      print(len(train_paths),len(train_labels))
      
  else:
      for i in range(len(y_pred)):
          print(y_pred[i])
  
  # %% [code]
  if bool_pseudo:
      model,histories,probabilities=training()
      draw_training_curves(histories)
      y_pred=predict(model,probabilities)
      sub('submission_pseudo.csv')
  