# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Environment (conda_tensorflow_p36)
#     language: python
#     name: conda_tensorflow_p36
# ---

# +
import warnings
warnings.simplefilter('ignore')

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import sys, os, time, gc, random, pickle
from sklearn.neighbors import NearestNeighbors
import requests, shutil
import tensorflow as tf
import keras
from skimage import io
from skimage.transform import resize
from PIL import Image

from keras import backend as K
from keras.preprocessing.image import load_img, img_to_array
from keras.models import load_model

# %matplotlib inline
# -

# # Load the data set

# +
# Load the data
train = pd.read_csv('./data/all/train.csv')
test = pd.read_csv('./data/all/test.csv')

print('Train:\t\t', train.shape)
print('Test:\t\t', test.shape)
print('Landmarks:\t', len(train['landmark_id'].unique()))
# -

train.head()


# # Extract Valid Image IDs

def valid(path):
    """ function to determine whether or not the given image is valid """
    try:
        img = Image.open(path)
        if img.width < 256 or img.height < 256 or img.format != 'JPEG':
            return False
        _ = img.resize((256, 256))
    except:
        return False
    
    return True


if not os.path.exists('./result/train_imgs_id.npy'):
    # Process the training set
    train_imgs_id = []
    train_landmark = []
    t0 = time.time()

    for i in range(len(train)):
        # Helpful information
        if i % 100000 == 0:
            t = time.time() - t0
            print('\nProcess: {:9d}'.format(i), '   Used time: {} s'.format(np.round(t, 0)))
            t0 = time.time()
        if i % 1250 == 0:
            print('=', end='')

        path = './data/all/train_images/' + str(i) + '.jpg'
        if valid(path):
            train_imgs_id.append(i)
            train_landmark.append(train.loc[i, 'landmark_id'])

    np.save('./result/train_imgs_id.npy', train_imgs_id)
    np.save('./result/train_landmark.npy', train_landmark)

if not os.path.exists('./result/test_imgs_id.npy'):
    # Process the test set
    test_imgs_id = []
    t0 = time.time()

    for i in range(len(test)):
        # Helpful information
        if i % 50000 == 0:
            t = time.time() - t0
            print('\nProcess: {:9d}'.format(i), '   Used time: {} s'.format(np.round(t, 0)))
            t0 = time.time()
        if i % 625 == 0:
            print('=', end='')

        path = './data/all/test_images/' + str(i) + '.jpg'
        if valid(path):
            test_imgs_id.append(i)

    np.save('./result/test_imgs_id.npy', test_imgs_id)

# # Load fine-tuned InceptionV3 Model

# Load trained model
model = load_model('./models/inception-base-0.1-model.h5')
model.summary()

# # Extract training and test image features

# +
# Process all the training images
train_imgs_id = np.load('./result/train_imgs_id.npy')
train_landmark = np.load('./result/train_landmark.npy')

test_imgs_id = np.load('./result/test_imgs_id.npy')
train_features = np.zeros(shape=(len(train_imgs_id), 2048), dtype=np.float32)

print('Train:\t', len(train_imgs_id), len(train_landmark), len(np.unique(train_landmark)), train_features.shape)
print('Test:\t', len(test_imgs_id))

# +
steps = 20000
t0 = time.time()

for i in range(0, len(train_imgs_id), steps):
    # Helpful information
    if i % steps == 0:
        t = time.time() - t0
        print('\nProcess: {:9d}'.format(i), '   Used time: {} s'.format(np.round(t, 0)))
        t0 = time.time()
        
    tmp_imgs = []
    start = i
    end = min(len(train_imgs_id), i + steps)
    for idx in range(start, end):
        if idx % 250 == 0:
            print('=', end='')
            
        img_id = train_imgs_id[idx]
        path = './data/all/train_images/' + str(img_id) + '.jpg'
        img = load_img(path, target_size=(224, 224))
        img = img_to_array(img)
        tmp_imgs.append(img)
        
    tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0
    tmp_feature = model.predict(tmp_imgs)
    train_features[start:end, ] = tmp_feature
    _ = gc.collect()

# Save to disk
np.save('./data/all/train_features.npy', train_features)
np.save('./data/all/train_id.npy', train_landmark)

# +
# Process all the test images
test_imgs_id = np.load('./result/test_imgs_id.npy')
test_features = np.zeros(shape=(len(test_imgs_id), 2048), dtype=np.float32)

steps = 20000
t0 = time.time()

for i in range(0, len(test_imgs_id), steps):
    # Helpful information
    if i % steps == 0:
        t = time.time() - t0
        print('\nProcess: {:9d}'.format(i), '   Used time: {} s'.format(np.round(t, 0)))
        t0 = time.time()
        
    tmp_imgs = []
    start = i
    end = min(len(test_imgs_id), i + steps)
    for idx in range(start, end):
        if idx % 250 == 0:
            print('=', end='')
            
        img_id = test_imgs_id[idx]
        path = './data/all/test_images/' + str(img_id) + '.jpg'
        img = load_img(path, target_size=(224, 224))
        img = img_to_array(img)
        tmp_imgs.append(img)
        
    tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0
    tmp_feature = model.predict(tmp_imgs)
    test_features[start:end, ] = tmp_feature
    _ = gc.collect()

# Save to disk
np.save('./data/all/test_features.npy', test_features)
# -



StarcoderdataJupyterScriptsDedupFiltered