# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/pranith7867/PyTorch-Computer-Vision-Cookbook/blob/master/Dogs-vs-Cata.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="NkDV_hsUnloj"
# #Data processing
#

# + id="bWMn18jbAXzc"
# !pip install opendatasets --upgrade --quiet


# + colab={"base_uri": "https://localhost:8080/"} id="rem6AL0lfwHq" outputId="81f3bb4f-322c-4670-8843-2312ba633915"
import opendatasets as od
dataset_url = "https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition/data"
od.download(dataset_url)
# 18053a5c5a08203aeb457de2f5efac0f

# + id="rJt-b74Ffypi"
# !unzip "/content/Dogs-vs-Cats/data/train.zip" -d "/content/Dogs-vs-Cats/data" &> /dev/null

# + id="Yy8me8u1g1fT"
# !unzip "/content/Dogs-vs-Cats/data/test.zip" -d "/content/Dogs-vs-Cats/data" &> /dev/null

# + id="fX21MYP3hU9O"
import os

# + id="xyciOIMVBczo"
os.mkdir(os.path.join("/content/Dogs-vs-Cats/data/test","test"))

# + id="FBJlxctghlIW"
os.mkdir(os.path.join("/content/Dogs-vs-Cats/data/train","cats"))
os.mkdir(os.path.join("/content/Dogs-vs-Cats/data/train","dogs"))

# + id="aE39x2CohvhD"
os.mkdir(os.path.join("/content/Dogs-vs-Cats/data","valid"))
os.mkdir(os.path.join("/content/Dogs-vs-Cats/data/valid","cats"))
os.mkdir(os.path.join("/content/Dogs-vs-Cats/data/valid","dogs"))

# + id="dIMar4U3h5iH"
train_dir = "/content/Dogs-vs-Cats/data/train"

# + id="IifRaClOiY97"
import os
import pandas as pd
import numpy as np
import seaborn as sns
import random
import torch
import torchvision
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torchvision.datasets.utils import download_url
from torch.utils.data import random_split
from tqdm.notebook import tqdm
from torchvision import transforms, utils, datasets
from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler
from sklearn.metrics import classification_report, confusion_matrix
import matplotlib.pyplot as plt
# %matplotlib inline

# + colab={"base_uri": "https://localhost:8080/"} id="kPtmXfmgibZA" outputId="bf098698-e388-4e89-b6a5-db33d220f5d3"
import os
# train_dir = "./data/train"
train_dogs_dir = f'{train_dir}/dogs'
train_cats_dir = f'{train_dir}/cats'
val_dir = "/content/Dogs-vs-Cats/data/valid"
val_dogs_dir = f'{val_dir}/dogs'
val_cats_dir = f'{val_dir}/cats'
print("Printing data dir")
print(os.listdir("/content/Dogs-vs-Cats/data"))
print("Printing train dir")
# !ls {train_dir} | head -n 5
print("Printing train dog dir")
# !ls {train_dogs_dir} | head -n 5
print("Printing train cat dir")
# !ls {train_cats_dir} | head -n 5
print("Printing val dir")
# !ls {val_dir} | head -n 5
print("Printing val dog dir")
# !ls {val_dogs_dir} | head -n 5
print("Printing val cat dir")
# !ls {val_cats_dir} | head -n 5

# + id="E1BpUV63igBI"
import shutil
import re

files = os.listdir(train_dir)

# Move all train cat images to cats folder, dog images to dogs folder
for f in files:
    catSearchObj = re.search("cat", f)
    dogSearchObj = re.search("dog", f)
    if catSearchObj:
        shutil.move(f'{train_dir}/{f}', train_cats_dir)
    elif dogSearchObj:
        shutil.move(f'{train_dir}/{f}', train_dogs_dir)

# + colab={"base_uri": "https://localhost:8080/"} id="Go2k924vjNem" outputId="3b34404b-91a8-4184-af0e-a142ec0500b3"
print("Printing training cats dir")
# !ls {train_cats_dir} | head -n 5

# + colab={"base_uri": "https://localhost:8080/"} id="7qd9Z0FQjIod" outputId="bda7d7df-fe97-4f38-9883-0374994c7718"
print("Printing train dog dir")
# !ls {train_dogs_dir} | head -n 5

# + id="2uclwKniiwiF"
files = os.listdir(train_dogs_dir)

for f in files:
    validationDogsSearchObj = re.search("5\d\d\d", f)
    if validationDogsSearchObj:
        shutil.move(f'{train_dogs_dir}/{f}', val_dogs_dir)

# + colab={"base_uri": "https://localhost:8080/"} id="af079yh0i1u1" outputId="b60b844c-b35f-4e8f-806e-a873b5f05ad0"
print("Printing val dog dir")
# !ls {val_dogs_dir} | head -n 5

# + id="oOdWwfYOjGQ5"
files = os.listdir(train_cats_dir)

for f in files:
    validationCatsSearchObj = re.search("5\d\d\d", f)
    if validationCatsSearchObj:
        shutil.move(f'{train_cats_dir}/{f}', val_cats_dir)

# + colab={"base_uri": "https://localhost:8080/"} id="qu3V6Kgqi6Be" outputId="187696e7-d38f-4ccc-976a-2e25bc141183"
print("Printing val cats dir")
# !ls {val_cats_dir} | head -n 5

# + id="4oPTpGNJj5wL"
test_dir = "./Dogs-vs-Cats/data/test/test"
files = os.listdir("./Dogs-vs-Cats/data/test")

for f in files:
  shutil.move(f'{"./Dogs-vs-Cats/data/test"}/{f}', test_dir)

# + colab={"base_uri": "https://localhost:8080/"} id="Fi8CkSH3i-Wo" outputId="45ec863a-acc9-4466-fa59-0f079cb64cf2"
test_dir = "./Dogs-vs-Cats/data/test/test"
print("Printing from test dir")
# !ls {test_dir} | head -n 5

# + [markdown] id="kqFpY1ttnfCs"
# #Load Data
#

# + id="-MKyCRGmjsj9"

import os
import random
import collections
import shutil
import time
import glob
import csv
import numpy as np

import torch
import torch.backends.cudnn as cudnn
import torch.nn as nn
import torch.optim as optim
import torch.utils.data as data
import torchvision.datasets as datasets
import torchvision.models as models
import torchvision.transforms as transforms

from PIL import Image

ROOT_DIR = os.getcwd()
DATA_HOME_DIR = './Dogs-vs-Cats/data'

# + colab={"base_uri": "https://localhost:8080/", "height": 36} id="CleQsrhFofN_" outputId="2076c6c2-1213-4663-dcf0-39b28da7cd0a"
DATA_HOME_DIR

# + id="tzk-ZfMYkq0W"
# paths
data_path = DATA_HOME_DIR 
split_train_path = data_path + '/train'
# full_train_path = data_path + '/train_full/'
valid_path = data_path + '/valid'
test_path = DATA_HOME_DIR + '/test/test'
saved_model_path =  './Dogs-vs-Cats/models'
submission_path =  './Dogs-vs-Cats/submissions'

# data
batch_size = 8

# model
nb_runs = 1
nb_aug = 3
epochs = 5
lr = 1e-4
clip = 0.001
archs = ["resnet152"]

model_names = sorted(name for name in models.__dict__ if name.islower() and not name.startswith("__"))
best_prec1 = 0

# + colab={"base_uri": "https://localhost:8080/"} id="mpHbK1okk7cw" outputId="a35f6dec-76a5-4496-d896-53404c59fb65"
model_names


# + id="9MPLQKi2lJRb"
def train(train_loader, model, criterion, optimizer, epoch):
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    acc = AverageMeter()
    end = time.time()
    
    # switch to train mode
    model.train()
    
    for i, (images, target) in enumerate(train_loader):
        # measure data loading time
        data_time.update(time.time() - end)

        target = target.cuda()
        image_var = torch.autograd.Variable(images)
        label_var = torch.autograd.Variable(target)

        # compute y_pred
        y_pred = model(image_var)
        loss = criterion(y_pred, label_var)

        # measure accuracy and record loss
        prec1, prec1 = accuracy(y_pred.data, target, topk=(1, 1))
        losses.update(loss.data, images.size(0))
        acc.update(prec1, images.size(0))

        # compute gradient and do SGD step
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()


# + id="NpRG-C1XlNGm"
def validate(val_loader, model, criterion, epoch):
    batch_time = AverageMeter()
    losses = AverageMeter()
    acc = AverageMeter()

    # switch to evaluate mode
    model.eval()

    end = time.time()
    for i, (images, labels) in enumerate(val_loader):
        labels = labels.cuda()
        image_var = torch.autograd.Variable(images)
        label_var = torch.autograd.Variable(labels)

        # compute y_pred
        y_pred = model(image_var)
        loss = criterion(y_pred, label_var)

        # measure accuracy and record loss
        prec1, temp_var = accuracy(y_pred.data, labels, topk=(1, 1))
        losses.update(loss.data, images.size(0))
        acc.update(prec1, images.size(0))

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

    print('   * EPOCH {epoch} | Accuracy: {acc.avg:.3f} | Loss: {losses.avg:.3f}'.format(epoch=epoch,
                                                                                         acc=acc,
                                                                                         losses=losses))

    return acc.avg


# + id="NXN9CycSlSwi"
def test(test_loader, model):
    csv_map = collections.defaultdict(float)
    
    # switch to evaluate mode
    model.eval()
    
    for aug in range(nb_aug):
        print("   * Predicting on test augmentation {}".format(aug + 1))
        
        for i, (images, filepath) in enumerate(test_loader):
            # pop extension, treat as id to map
            filepath = os.path.splitext(os.path.basename(filepath[0]))[0]
            filepath = int(filepath)

            image_var = torch.autograd.Variable(images)
            y_pred = model(image_var)
            # get the index of the max log-probability
            smax = nn.Softmax()
            smax_out = smax(y_pred)[0]
            cat_prob = smax_out.data[0]
            dog_prob = smax_out.data[1]
            prob = dog_prob
            if cat_prob > dog_prob:
                prob = 1 - cat_prob
            prob = np.around(prob.cpu(), decimals=4)
            prob = np.clip(prob, clip, 1-clip)
            csv_map[filepath] += (prob / nb_aug)

    sub_fn = submission_path + '{0}epoch_{1}clip_{2}runs'.format(epochs, clip, nb_runs)
    
    for arch in archs:
        sub_fn += "_{}".format(arch)
        
    print("Writing Predictions to CSV...")
    with open(sub_fn + '.csv', 'w') as csvfile:
        fieldnames = ['id', 'label']
        csv_w = csv.writer(csvfile)
        csv_w.writerow(('id', 'label'))
        for row in sorted(csv_map.items()):
            csv_w.writerow(row)
    print("Done.")
# In [7]:
# def save_checkpoint(


# + id="06fqrCgylYE9"
def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'):
    torch.save(state, filename)
    if is_best:
        shutil.copyfile(filename, 'model_best.pth.tar')


# + id="zUPe-955la_R"
class AverageMeter(object):
    """Computes and stores the average and current value"""

    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count


# + id="JxBW_an1ld1o"

def adjust_learning_rate(optimizer, epoch):
    """Sets the learning rate to the initial LR decayed by 10 every 30 epochs"""
    global lr
    lr = lr * (0.1**(epoch // 30))
    for param_group in optimizer.state_dict()['param_groups']:
        param_group['lr'] = lr


def accuracy(y_pred, y_actual, topk=(1, )):
    """Computes the precision@k for the specified values of k"""
    maxk = max(topk)
    batch_size = y_actual.size(0)

    _, pred = y_pred.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(y_actual.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0)
        res.append(correct_k.mul_(100.0 / batch_size))

    return res


# + id="9Z22_ExhlgF5"
class TestImageFolder(data.Dataset):
    def __init__(self, root, transform=None):
        images = []
        for filename in sorted(glob.glob(test_path + "*.jpg")):
            images.append('{}'.format(filename))

        self.root = root
        self.imgs = images
        self.transform = transform

    def __getitem__(self, index):
        filename = self.imgs[index]
        img = Image.open(os.path.join(self.root, filename))
        if self.transform is not None:
            img = self.transform(img)
        return img, filename

    def __len__(self):
        return len(self.imgs)


# + id="BNQUlIAhljF7"
def shear(img):
    width, height = img.size
    m = random.uniform(-0.05, 0.05)
    xshift = abs(m) * width
    new_width = width + int(round(xshift))
    img = img.transform((new_width, height), Image.AFFINE,
                        (1, m, -xshift if m > 0 else 0, 0, 1, 0),
                        Image.BICUBIC)
    return img


# + id="4u_AVeVLllFz"
def main(mode="train", resume=False):
    
    global best_prec1
    
    for arch in archs:

        # create model
        print("=> Starting {0} on '{1}' model".format(mode, arch))
        model = models.__dict__[arch](pretrained=True)
        # Don't update non-classifier learned features in the pretrained networks
        for param in model.parameters():
            param.requires_grad = False
        # Replace the last fully-connected layer
        # Parameters of newly constructed modules have requires_grad=True by default
        # Final dense layer needs to replaced with the previous out chans, and number of classes
        # in this case -- resnet 101 - it's 2048 with two classes (cats and dogs)
        model.fc = nn.Linear(2048, 2)

        if arch.startswith('alexnet') or arch.startswith('vgg'):
            model.features = torch.nn.DataParallel(model.features)
            model.cuda()
        else:
            model = torch.nn.DataParallel(model).cuda()
            
        # optionally resume from a checkpoint
        if resume:
            if os.path.isfile(resume):
                print("=> Loading checkpoint '{}'".format(resume))
                checkpoint = torch.load(resume)
                start_epoch = checkpoint['epoch']
                best_prec1 = checkpoint['best_prec1']
                model.load_state_dict(checkpoint['state_dict'])
                print("=> Loaded checkpoint (epoch {})".format(checkpoint['epoch']))
            else:
                print("=> No checkpoint found at '{}'".format(args.resume))

        cudnn.benchmark = True

        # Data loading code
        traindir = split_train_path
        valdir = valid_path
        testdir = test_path

        normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])

        train_loader = data.DataLoader(
            datasets.ImageFolder(traindir,
                                 transforms.Compose([
                                     # transforms.Lambda(shear),
                                     transforms.RandomResizedCrop(224),
                                     transforms.RandomHorizontalFlip(),
                                     transforms.ToTensor(),
                                     normalize,
                                 ])),
            batch_size=batch_size,
            shuffle=True,
            num_workers=4,
            pin_memory=True)

        val_loader = data.DataLoader(
            datasets.ImageFolder(valdir,
                                 transforms.Compose([
                                     transforms.Resize(256),
                                     transforms.CenterCrop(224),
                                     transforms.ToTensor(),
                                     normalize,
                                 ])),
            batch_size=batch_size,
            shuffle=True,
            num_workers=4,
            pin_memory=True)

        test_loader = data.DataLoader(
            TestImageFolder(testdir,
                            transforms.Compose([
                                # transforms.Lambda(shear),
                                transforms.Resize(256),
                                transforms.CenterCrop(224),
                                transforms.RandomHorizontalFlip(),
                                transforms.ToTensor(),
                                normalize,
                            ])),
            batch_size=1,
            shuffle=False,
            num_workers=1,
            pin_memory=False)
        
        
        if mode == "test":
            test(test_loader, model)
            return
        
        # define loss function (criterion) and pptimizer
        criterion = nn.CrossEntropyLoss().cuda()
        
        if mode == "validate":
            validate(val_loader, model, criterion, 0)
            return

        optimizer = optim.Adam(model.module.fc.parameters(), lr, weight_decay=1e-4)

        for epoch in range(epochs):
            adjust_learning_rate(optimizer, epoch)

            # train for one epoch
            train(train_loader, model, criterion, optimizer, epoch)

            # evaluate on validation set
            prec1 = validate(val_loader, model, criterion, epoch)

            # remember best Accuracy and save checkpoint
            is_best = prec1 > best_prec1
            best_prec1 = max(prec1, best_prec1)
            save_checkpoint({
                'epoch': epoch + 1,
                'arch': arch,
                'state_dict': model.state_dict(),
                'best_prec1': best_prec1,
            }, is_best)

# + colab={"base_uri": "https://localhost:8080/"} id="bMEqN1QCATci" outputId="3a55cd8c-0ede-4f17-c285-820b0009abd8"
main(mode="train")

# + colab={"base_uri": "https://localhost:8080/"} id="Ki983OYtDQPK" outputId="ced0d049-be3e-44ba-8399-6f5efc531921"
main(mode="validate", resume='model_best.pth.tar')

# + id="6L6PKWUAAS0l"


# + colab={"base_uri": "https://localhost:8080/"} id="gzf_-fJAOjax" outputId="16a40740-70cc-4fe3-aa57-598e9b17aa11"
main(mode="test", resume='model_best.pth.tar')

# + id="r1G4FMcSO6GV"


StarcoderdataJupyterScriptsDedupFiltered