!cp ../input/gdcm-conda-install/gdcm.tar .
!tar -xvzf gdcm.tar
!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2
import pandas as pd 
df = pd.read_csv('../input/siim-covid19-detection/sample_submission.csv')
if df.shape[0] == 2477:
    fast_sub = True
    df.to_csv('submission.csv', index=False)
else:
    fast_sub = False
import os

from PIL import Image
import pandas as pd
from tqdm.auto import tqdm
import numpy as np
import pydicom
from pydicom.pixel_data_handlers.util import apply_voi_lut

def read_xray(path, voi_lut = True, fix_monochrome = True):
    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way
    dicom = pydicom.read_file(path)
    
    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to 
    # "human-friendly" view
    if voi_lut:
        data = apply_voi_lut(dicom.pixel_array, dicom)
    else:
        data = dicom.pixel_array
               
    # depending on this value, X-ray may look inverted - fix that:
    if fix_monochrome and dicom.PhotometricInterpretation == "MONOCHROME1":
        data = np.amax(data) - data
        
    data = data - np.min(data)
    data = data / np.max(data)
    data = (data * 255).astype(np.uint8)
        
    return data

def resize(array, size, keep_ratio=False, resample=Image.LANCZOS):
    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image
    im = Image.fromarray(array)
    
    if keep_ratio:
        im.thumbnail((size, size), resample)
    else:
        im = im.resize((size, size), resample)
    
    return im

image_id = []
study_id = []
dim0 = []
dim1 = []
split = 'test'
save_dir = f'/kaggle/tmp/{split}/image/'
os.makedirs(save_dir, exist_ok=True)

for dirname, _, filenames in tqdm(os.walk(f'../input/siim-covid19-detection/{split}')):
    for file in filenames:
        # set keep_ratio=True to have original aspect ratio
        xray = read_xray(os.path.join(dirname, file))
        im = resize(xray, size=512)  
        im.save(os.path.join(save_dir, file.replace('.dcm', '.png')))
        image_id.append(file.replace('.dcm', ''))
        study_id.append(dirname.split('/')[-2])
        dim0.append(xray.shape[0])
        dim1.append(xray.shape[1])
        
        if len(dim0) >10 and fast_sub:
            break
    if len(dim0) >10 and fast_sub:
            break
meta = pd.DataFrame.from_dict({'image_id': image_id, 'dim0': dim0, 'dim1': dim1, 'study_id': study_id})
meta.to_csv('test_meta.csv', index=False)
!mkdir det_txt
!mkdir det
! pip install ../input/siim-libs/addict-2.4.0-py3-none-any.whl >> /dev/null
! pip install ../input/siim-libs/timm-0.4.12-py3-none-any.whl >> /dev/null
! pip install ../input/siim-libs/ensemble_boxes-1.0.6-py3-none-any.whl >> /dev/null
! pip install ../input/siim-libs/loguru-0.5.3-py3-none-any.whl >> /dev/null
! pip install ../input/siim-libs/thop-0.0.31.post2005241907-py3-none-any.whl >> /dev/null
! pip install ../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl >> /dev/null
! pip install ../input/omegaconf/omegaconf-2.0.5-py3-none-any.whl >> /dev/null
!cp -r ../input/siim-v5-mh/* .
!python inference.py --is_val 0 --output_path det_txt/ --input_path '/kaggle/tmp/test/image/*png' --weight_path '/kaggle/input/siim-v5-weights/*/*/*/best.pt'
!cp -r ../input/yolox-inference/* .
!python inference.py -f ../input/yolox-weights/yolox_weights/yolox_siim_d_f0.py -c ss --path '/kaggle/tmp/test/image/*png' \
    --wei_dir ../input/yolox-weights/yolox_weights/ --conf 0.0001 --nms 0.5 --tsize 384 --save_result --device gpu
!cp -r ../input/siimeffdet/* .
!python predict_oof_det.py --is_val 0 --test_path '/kaggle/tmp/test/image/*png' --model_dir ../input/siim-det-models/
! cp det/* det_txt
! python ensemble1.py --input_path  'det_txt/*txt' --image_path '/kaggle/tmp/test/image/*png' --thr 0.001
#mask_dir = f'/kaggle/tmp/{split}/mask/'
#os.makedirs(mask_dir, exist_ok=True)
#!cp -r ../input/siimsegs/* .
#!python train_seg.py --image_path '/kaggle/tmp/test/image/*png' --weight_path best_loss.pth
!cp -r ../input/siim-cls-code/* .
!python main.py -C n_cf11_6 -M test -W ../input/siim-cls-weights/n_cf11_6_f3/
!python main.py -C n_cf11 -M test -W ../input/siim-cls-weights/n_cf11_l1/
!python main.py -C n_cf11_7 -M test -W ../input/siim-cls-weights/n_cf11_7/
#!python main.py -C n_cf11_8 -M test -W ../input/siim-cls-weights/n_cf11_8/
!python main.py -C n_cf11_9 -M test -W ../input/siim-cls-weights/n_cf11_9/
!python main.py -C n_cf11_10 -M test -W ../input/siim-cls-weights/n_cf11_10/
!python main.py -C n_cf11_1 -M test -W ../input/siim-cls-weights/n_cf_11_1/
!python main.py -C n_cf11_rot1 -M test -W ../input/siim-cls-weights/n_cf11_rot1/
!pip install -q ../input/landmark-additional-packages/EfficientNet-PyTorch/EfficientNet-PyTorch-master 
!pip install -U ../input/landmark-additional-packages/timm-0.4.12-py3-none-any.whl # to fix 
!pip install ../input/siim-libs/segmentation_models_pytorch-0.1.3-py3-none-any.whl --no-deps  
"""
## Model difinition
"""
import sys
sys.path.append('../usr/lib/siim_cov_model_v0/')
from siim_cov_model_v0 import *
"""
## tools and difinition
"""
import numpy as np
import pandas as pd
import tqdm
import cv2
import glob
import math
import csv
import torch
import operator
import sys
import os
from path import Path
from skimage.io import imread
from scipy.ndimage.interpolation import zoom
import matplotlib.pyplot as plt
from skimage.transform import resize
import torchvision
from scipy.stats import rankdata

from torchvision.transforms import (
    ToTensor, Normalize, Compose, Resize, CenterCrop, RandomCrop,
    RandomHorizontalFlip, RandomAffine, RandomVerticalFlip, RandomChoice, ColorJitter, RandomRotation)

sys.path.append('../input/siim-covid-aggron-eecf6c/covid19-aggron')

# from utils import parse_args, prepare_for_result
# from torch.utils.data import DataLoader, Dataset, WeightedRandomSampler
# from models import get_model
from losses import get_loss, get_class_balanced_weighted
# from dataloaders import get_dataloader
# from utils import load_matched_state
from configs import Config
# import seaborn as sns
# from dataloaders.transform_loader import get_tfms

from sklearn.metrics import f1_score, roc_auc_score, average_precision_score
"""
## Uncompress test data
"""
test_path = '/kaggle/tmp/test/image/'
class COVIDDataset(torch.utils.data.Dataset):
    def __init__(self, df, cfg=None, tfms=None, path='.'):
        self.df = df
        self.cfg = cfg
        self.tfms = tfms
        self.tensor_tfms = torchvision.transforms.Compose([
            torchvision.transforms.ToTensor(),
            torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
            # torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
        ])
        self.path = './'
        # self.path = Path('/home/sheep/kaggle/siim')
        self.studys = self.df['StudyInstanceUID'].unique()
        print(len(self.studys))
        self.cols = ['Negative for Pneumonia', 'Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance']
        self.cols2index = {x: i for i, x in enumerate(self.cols)}
        self.path = path

    def __len__(self):
        return len(self.studys)

    def __getitem__(self, idx):
        study_id = self.studys[idx]
        sub_df = self.df[self.df.StudyInstanceUID == study_id].copy()
        images = []
        masks = []
        study = [idx for _ in range(sub_df.shape[0])]
        image_as_study = []
        bbox = []
        label_study = 0
        has_masks = []
        iids = []
        for i, row in sub_df.iterrows():
            img = cv2.imread(str(self.path + f'/{row.ImageUID}.png'))
            sz = self.cfg.transform.size
            mask = np.zeros((sz, sz))
            has_mask = 1
            has_masks.append(has_mask)
            label = 0
            if self.tfms:
                tf = self.tfms(image=img, mask=mask)
                img = tf['image']
                mask = tf['mask']
            if not img.shape[0] == self.cfg.transform.size:
                img = cv2.resize(img, (self.cfg.transform.size, self.cfg.transform.size))
            # resize to aux
            if self.cfg.transform.size == 512:
                msksz = 32
            elif self.cfg.transform.size == 384:
                msksz = 24
            elif self.cfg.transform.size == 640:
                msksz = 40
            elif self.cfg.transform.size == 700:
                msksz = 44
            elif self.cfg.transform.size == 720:
                msksz = 45
            elif self.cfg.transform.size == 768:
                msksz = 48
            else:
                msksz = 32
            mask = cv2.resize(mask, (msksz, msksz))
            masks.append(torch.FloatTensor(mask).view(1, mask.shape[0], mask.shape[1]))
            img = self.tensor_tfms(img)
            images.append(img)
            image_as_study.append(label)
            iids.append(row.ImageUID)
        images = torch.stack(images)
        masks = torch.stack(masks)
        return images, study, label_study, image_as_study, masks, iids
    
def idoit_collect_func(batch):
    img, study, lbl, image_as_study, bbox, has_masks = [], [], [], [], [], []
    for im, st, lb, ias, bb, has_m in batch:
        img.extend(im)
        study.extend(st)
        lbl.append(lb)
        image_as_study.extend(ias)
        bbox.extend(bb)
        has_masks.extend(has_m)
    return torch.stack(img), study, torch.tensor(lbl), torch.tensor(image_as_study), torch.stack(bbox), has_masks
def load_model(cfg):
    if cfg.model.name == 'v2m_aux':
        #     def __init__(self, name, dropout=0, pool='AdaptiveAvgPool2d'):
        drop = cfg.model.param.get('dropout', 0)
        pool = cfg.model.param.get('last_pool', 'AdaptiveAvgPool2d')
        return AUXNet(name='tf_efficientnetv2_m', dropout=drop, pool=pool)
    elif cfg.model.name == 'v2m_aux_v2':
        drop = cfg.model.param.get('dropout', 0)
        pool = cfg.model.param.get('last_pool', 'AdaptiveAvgPool2d')
        return AUXNetV2(name='tf_efficientnetv2_m', dropout=drop, pool=pool)
    elif cfg.model.name == 'b5_aux':
        drop = cfg.model.param.get('dropout', 0)
        pool = cfg.model.param.get('last_pool', 'AdaptiveAvgPool2d')
        return AUXNetb5(name='tf_efficientnetv2_m', dropout=drop)
    elif cfg.model.name == 'v2l_aux':
        drop = cfg.model.param.get('dropout', 0)
        pool = cfg.model.param.get('last_pool', 'AdaptiveAvgPool2d')
        return AUXNetL(name='tf_efficientnetv2_l', dropout=drop)
def predict_run(RUN, test, is_none = False):
    df = pd.read_csv(f'{RUN}/train.log', sep='\t')
    fold2epochs = {}
    for i in range(0, 5):
        eph = df[df.Fold == i].sort_values('F1@0.3', ascending=False).iloc[0].Epochs
        fold2epochs[i] = int(eph)

    print(fold2epochs.values())

    predicted = []
    # load the model
    models = []
    for f in range(5):
        cfg = Config.load_json(f'{RUN}/config.json')
        cfg.experiment.run_fold = f
        model = load_model(cfg).cuda()
        load_matched_state(model, torch.load(
            glob.glob(f'{RUN}/checkpoints/f{f}*-{fold2epochs[f]}*')[0]))
        model.eval()
        models.append(model)
        
    # inference
    test_ds = COVIDDataset(test, cfg=cfg, path=test_path)
    test_dl = torch.utils.data.DataLoader(test_ds, num_workers=2, batch_size=32, collate_fn=idoit_collect_func)
    with torch.no_grad():
        results = []
        predicted, image_ids = [], []
        for i, (img, study_index, lbl_study, label_image, mask_t, iids) in tqdm.tqdm(enumerate(test_dl)):
            img = img.cuda()
            sz = img.size()[0]
            img = torch.stack([img,img.flip(-1)],0) # hflip
            img = img.view(-1, 3, img.shape[-1], img.shape[-1])
            preds = []
            for m in models:
                with torch.cuda.amp.autocast():
                    logits, mask = m(img)
                logits = logits.float()
                if cfg.loss.name == 'bce':
                    logits = torch.sigmoid(logits)
                else:
                    logits = torch.softmax(logits, 1)
                cls = (logits[:sz] + logits[sz:]) / 2
                preds.append(cls)
            predicted.append(torch.stack(preds).mean(0).cpu())
            image_ids.extend(iids)
    if is_none:
        return pd.DataFrame(torch.cat(predicted).numpy(), index=image_ids,
             columns=['Negative for Pneumonia', 'Typical Appearance'])
    else:
        return pd.DataFrame(torch.cat(predicted).numpy(), index=image_ids,
             columns=['Negative for Pneumonia', 'Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance'])
"""
## Test assets
"""
test = pd.read_csv('test_meta.csv')[["image_id", "study_id"]]
test = test.rename(columns={"image_id": "ImageUID", "study_id": "StudyInstanceUID"})
for e in ['Negative for Pneumonia', 'Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance']:
    test[e] = 0
test.head()
"""
## BCE loss with pl
"""
r9 = predict_run(
    '../input/siim-covid-aux-bce-agg-exp-rot-30-20-v2l-pl/aux_bce_agg_exp_rot_30_20_v2l_pl.upload', test)
r8 = predict_run(
    '../input/aux-bce-agg-exp-rot-30-20-b5-pl/aux_bce_agg_exp_rot_30_20_b5_pl.upload', test)
r = predict_run(
    '../input/siim-covid-aux-aug-v2m-lm-aggron-40-clean-cut1/aux_aug_v2m_lm_aggron_40_clean_cut1.yaml_upload', test)
r3 = predict_run(
    '../input/siim-covid-aux-bce-agg-exp-rot-30-20-pl/aux_bce_agg_exp_rot_30_20_pl.upload', test)
"""
## CE model
"""
r2 = predict_run(
    '../input/siim-covid-aux-bce-v2m-lm-aggron-40-clean-cut1-pl/aux_bce_v2m_lm_aggron_40_clean_cut1_bce_pl.upload', test)
r4 = predict_run(
    '../input/siim-cov-dddddd-dbg-1-aux-2/dddddd_dbg_1_aux_2_upload', test)
r5 = predict_run(
    '../input/siim-cov-clean-oof-clean-agree-upload/clean_oof_clean_agree_upload', test)
r6 = predict_run(
    '../input/siim-cov-aux-aug-agg-exp-rot-30/aux_aug_agg_exp_rot_30.yaml_upload', test)
r7 = predict_run(
    '../input/siim-cov-model-modelv2upload/model_modelV2.upload', test)
image_result = (r + r2 + r3 + 0.5 * r4 + r5 + r6 + 0.75 * r7 + r8 + r9) / 8.25
image_result = image_result.reset_index()
image_result.to_csv('sheep_df.csv')
"""
**2 class**
"""
class AUXNet(nn.Module):
    def __init__(self, name, dropout=0, pool='AdaptiveAvgPool2d'):
        super(AUXNet, self).__init__()

        print('[ AUX model ] dropout: {}, pool: {}'.format(dropout, pool))
        e = timm.models.__dict__[name](pretrained=False, drop_rate=0.3, drop_path_rate=0.2)
        self.model = e
        self.b0 = nn.Sequential(
            e.conv_stem,
            e.bn1,
            e.act1,
        )
        self.b1 = e.blocks[0]
        self.b2 = e.blocks[1]
        self.b3 = e.blocks[2]
        self.b4 = e.blocks[3]
        self.b5 = e.blocks[4]
        self.b6 = e.blocks[5]
        self.b7 = e.blocks[6]
        self.b8 = nn.Sequential(
            e.conv_head, #384, 1536
            e.bn2,
            e.act2,
        )

        self.logit = nn.Linear(1280,2)
        self.mask = nn.Sequential(
            nn.Conv2d(176, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            nn.Conv2d(128, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            nn.Conv2d(128, 1, kernel_size=1, padding=0),
        )

        self.dropout = nn.Dropout(p=dropout)

        if pool == 'AdaptiveAvgPool2d':
            self.pooling = nn.AdaptiveAvgPool2d(1)
        elif pool == 'gem':
            self.pooling = GeM()

    # @torch.cuda.amp.autocast()
    def forward(self, image):
        batch_size = len(image)
        # x = 2*image-1     # ; print('input ',   x.shape)
        x = image

        x = self.b0(x) #; print (x.shape)  # torch.Size([2, 40, 256, 256])
        x = self.b1(x) #; print (x.shape)  # torch.Size([2, 24, 256, 256])
        x = self.b2(x) #; print (x.shape)  # torch.Size([2, 32, 128, 128])
        x = self.b3(x) #; print (x.shape)  # torch.Size([2, 48, 64, 64])
        x = self.b4(x) #; print (x.shape)  # torch.Size([2, 96, 32, 32])
        x = self.b5(x) #; print (x.shape)  # torch.Size([2, 136, 32, 32])
        #------------
        mask = self.mask(x)
        #-------------
        x = self.b6(x) #; print (x.shape)  # torch.Size([2, 232, 16, 16])
        x = self.b7(x) #; print (x.shape)  # torch.Size([2, 384, 16, 16])
        x = self.b8(x) #; print (x.shape)  # torch.Size([2, 1536, 16, 16])
        # x = F.adaptive_avg_pool2d(x,1).reshape(batch_size,-1)
        x = nn.Flatten()(self.pooling(x))
        x = self.dropout(x)
        logit = self.logit(x)
        return logit, mask
none_1 = predict_run(
    '../input/two-class-bce-fix-valid-bbox-two-classes-12-pl-2x/two_class_bce_fix_valid_bbox_two_classes_12_pl_2x.upload', test, True)
none_2 = predict_run(
    '../input/two-class-bce-fix-valid-bbox-two-classes-12upload/two_class_bce_fix_valid_bbox_two_classes_12.upload', test, True)
none_result = (none_1 + none_2) / 2
none_result.to_csv('public_test_sheep_predict_none.csv')
! python tocsv.py --input_path  'test_v5neg_2a.txt' --meta_path 'test_meta.csv'
def prob2str(row):
    return f'negative {row.pred_cls1:.6f} 0 0 1 1 typical {row.pred_cls2:.6f} 0 0 1 1 indeterminate {row.pred_cls3:.6f} 0 0 1 1 atypical {row.pred_cls4:.6f} 0 0 1 1'
    #return f''
def combine_image(row):
     return f'none {row.pred_cls5:.6f} 0 0 1 1 {row.PredictionString}'
        #return f''
def tosub(df1):
    df1_image = df1[['image_id', 'pred_cls1', 'pred_cls5', 'PredictionString']].copy()
    df1_image.rename(columns={"image_id": "id"}, inplace=True)
    df1_image['id'] = df1_image['id'].apply(lambda x: f'{x}_image')
    df1_image['PredictionString'] = df1_image.apply(lambda r: combine_image(r), axis=1) 
    df1_image = df1_image[['id', 'PredictionString']]
    
    df1_study = df1[['study_id', 'pred_cls1', 'pred_cls2', 'pred_cls3', 'pred_cls4']].copy()
    df1_study = df1_study.groupby('study_id').agg('mean').reset_index()
    df1_study.rename(columns={"study_id": "id"}, inplace=True)
    df1_study['id'] = df1_study['id'].apply(lambda x: f'{x}_study')
    df1_study["PredictionString"] = df1_study.apply(lambda r: prob2str(r), axis=1) 
    df1_study = df1_study[['id', 'PredictionString']]

    df1_sub = pd.concat([df1_study, df1_image])

    return df1_sub
df1 = pd.read_csv('n_cf11_9.csv') 
df2 = pd.read_csv('n_cf11.csv')#.head(10)
df3 = pd.read_csv('n_cf11_6.csv')
df4 = pd.read_csv('n_cf11_7.csv')
df5 = pd.read_csv('n_cf11_10.csv')
df6 = pd.read_csv('n_cf11_rot1.csv')
df7 = pd.read_csv('n_cf11_1.csv')
#df8 = pd.read_csv('n_cf11_8.csv')

df2 = df1[["image_id"]].merge(df2, on=["image_id"])
df3 = df1[["image_id"]].merge(df3, on=["image_id"])
df4 = df1[["image_id"]].merge(df4, on=["image_id"])
df5 = df1[["image_id"]].merge(df5, on=["image_id"])
df6 = df1[["image_id"]].merge(df6, on=["image_id"])
df7 = df1[["image_id"]].merge(df7, on=["image_id"])
#df8 = df1[["image_id"]].merge(df8, on=["image_id"])

sheep_df = pd.read_csv('sheep_df.csv')
sheep_df = sheep_df.rename(columns={"index": "image_id", "Negative for Pneumonia": "pred_cls1", 
                         "Typical Appearance": "pred_cls2", "Indeterminate Appearance": "pred_cls3",
                        "Atypical Appearance": "pred_cls4"})
sheep_df = df2[["image_id"]].merge(sheep_df, on=["image_id"])

for col in ['pred_cls1', 'pred_cls2', 'pred_cls3', 'pred_cls4', 'pred_cls5']:
    df1[col] = (df1[col] + df2[col] + df3[col] + df4[col] + df5[col] + df6[col] + df7[col])/7
    
for col in ['pred_cls1', 'pred_cls2', 'pred_cls3', 'pred_cls4']:
    df1[col] = (1*df1[col] + sheep_df[col])/2
    
sheep_none_df = pd.read_csv('public_test_sheep_predict_none.csv')
sheep_none_df["image_id"] = sheep_none_df['Unnamed: 0']
sheep_none_df = df1[["image_id"]].merge(sheep_none_df, on=["image_id"])

#df1['pred_cls5'] = sheep_none_df['Negative for Pneumonia']
df1['pred_cls5'] = 2*(1-df1['pred_cls5']) + 1*sheep_df['pred_cls1'] + 1*sheep_none_df['Negative for Pneumonia']

image_sub = pd.read_csv('v5_50.csv')
image_sub = df1[["image_id"]].merge(image_sub, on=["image_id"])
df1['PredictionString'] = image_sub['PredictionString']
df_sub  = tosub(df1)
df_sub.tail()
df_sub.head()
!rm -r ./*
df_sub.to_csv('submission.csv', index=False)