# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/aupe/apai/blob/master/dl_cv_pet_detection.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="SxjD9fZHC02P" colab_type="text"
# # Oxford Pets

# + [markdown] id="c7B0PG8WC02R" colab_type="text"
# ## Setup

# + id="CadPzcshC02S" colab_type="code" colab={}
SETUP = True

# + id="lpgOZwfaGXBm" colab_type="code" colab={}
# a = []
# while(1):
#     a.append('1')

# + id="E4zT3_bjC02W" colab_type="code" outputId="d438d432-8ac5-41ca-f379-125f6efd0a00" colab={"base_uri": "https://localhost:8080/", "height": 34}
import os

try:
    # %tensorflow_version 2.x
    from google.colab import drive
    drive.mount('/content/drive', force_remount=True)
    # %matplotlib inline
except:
    # %load_ext lab_black
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"] = "0"

# + id="h9LAWcbmC02a" colab_type="code" colab={}
# if SETUP:
# #     !bash ./setup.sh

# + id="XZBxWqUAZium" colab_type="code" outputId="cba38561-e642-4836-8bb9-4c061a177447" colab={"base_uri": "https://localhost:8080/", "height": 247}
# !pip install -q -U toai

# + id="2R1UMsioC02i" colab_type="code" outputId="750681e7-8a5e-4e54-e80f-94689cc1e6fb" colab={"base_uri": "https://localhost:8080/", "height": 52}
print(__import__("toai").__version__)
print(__import__("tensorflow").__version__)

# + id="xFAJCHMqC02m" colab_type="code" outputId="c8df37a0-7d48-4db8-de82-5a218a4e1288" colab={"base_uri": "https://localhost:8080/", "height": 72}
from toai.imports import *
from toai.image import ImageAugmentor, ImageParser, ImageResizer, ImageDataContainer
from toai.data.utils import split_df
from toai.encode import CategoricalEncoder
from typing import *
import tensorflow as tf
from tensorflow import keras
import tensorflow_datasets as tfds
import tensorflow_hub as hub
from glob import glob
from matplotlib.patches import Rectangle

# + [markdown] id="HlSPx5C4C02r" colab_type="text"
# ## Loading the data

# + id="Pxc2VWsQC02s" colab_type="code" colab={}
DATA_DIR = Path("data/pets")
DATA_DIR.mkdir(parents=True, exist_ok=True)


# + id="Z-Eo-D62EIvq" colab_type="code" colab={}
TEMP_DIR = Path('./drive/My Drive/Projects/16_Advanced_CV/temp/pets')

# + id="BbATjroeC02v" colab_type="code" colab={}
# if SETUP:
#     shutil.rmtree(str(DATA_DIR))
#     DATA_DIR.mkdir(parents=True, exist_ok=True)
#     TEMP_DIR.mkdir(parents=True, exist_ok=True)
#     kaggle.api.authenticate()
#     kaggle.api.dataset_download_files(
#         dataset="jutrera/stanford-car-dataset-by-classes-folder",
#         path=DATA_DIR,
#         unzip=True,
#     )


# + id="VYREx9dzaC_j" colab_type="code" colab={}
def setup_kaggle():
    # !rm -rf /root/.kaggle
    # x = !ls kaggle.json
    assert x == ['kaggle.json'], 'Upload kaggle.json'
    # !mkdir /root/.kaggle
    # !mv kaggle.json /root/.kaggle
    # !chmod 600 /root/.kaggle/kaggle.json
    
setup_kaggle()

# + id="veTITsyYaISD" colab_type="code" colab={}
DOWNLOAD_DATA = True

if DOWNLOAD_DATA:
    # !kaggle datasets download -q -d devdgohil/the-oxfordiiit-pet-dataset --unzip -p {str(DATA_DIR)}

# + id="lMpAiijqKvsC" colab_type="code" colab={}
import xml.etree.ElementTree as ET

# + id="-98gMK0PK1_l" colab_type="code" colab={}
tree = ET.parse(DATA_DIR/'annotations/annotations/xmls/Abyssinian_1.xml')
root = tree.getroot()

# + id="nCQYPAujLf82" colab_type="code" outputId="1500ae6c-c1cb-4441-ee2a-4e1cd9f56994" colab={"base_uri": "https://localhost:8080/", "height": 34}
root.tag

# + id="2pwfip5TLjLG" colab_type="code" outputId="f9f15e67-7be0-4ef1-b4d5-c39a154d2dfb" colab={"base_uri": "https://localhost:8080/", "height": 34}
root.attrib

# + id="GRLSe5C_LpEx" colab_type="code" outputId="f29ee90c-4513-4bca-ec2f-76b262a2c0c8" colab={"base_uri": "https://localhost:8080/", "height": 123}
for child in root:
    print(child.tag, child.attrib)

# + id="hubSDdzELtvD" colab_type="code" outputId="bca193a7-879b-439b-d834-06afd23067c0" colab={"base_uri": "https://localhost:8080/", "height": 425}
[elem.tag for elem in root.iter()]

# + id="w1wGI6PIOVZM" colab_type="code" outputId="9bcae2a3-3b64-48b6-f872-6e5c35a91da2" colab={"base_uri": "https://localhost:8080/", "height": 54}
ET.tostring(root, encoding='utf8').decode('utf8')

# + id="0U9b6TJPK2CB" colab_type="code" colab={}
origin_img1 = Image.open(str(DATA_DIR/'images/images/Abyssinian_1.jpg'))

# + id="SX-ICy5uRC9B" colab_type="code" colab={}
from PIL import Image, ImageDraw

# + id="C-fmWBCIUlWK" colab_type="code" colab={}
# %matplotlib inline

# + id="Bs9rfZsuUroq" colab_type="code" colab={}
origin_img

# + id="MmoSI8LRUbRo" colab_type="code" outputId="b40fd697-9745-489a-8f31-7c596d62ff82" colab={"base_uri": "https://localhost:8080/", "height": 286}
plt.imshow(plt.imread(str(DATA_DIR/'images/images/Abyssinian_1.jpg')))
plt.plot([xmin, xmax, xmax, xmin, xmin], [ymin, ymin, ymax, ymax, ymin])


# + id="amtE1k_vRqoQ" colab_type="code" colab={}
def save_cropped_img(path, annotation):
    tree = ET.parse(annotation)
    xmin = int(tree.getroot().findall('.//xmin')[0].text)
    xmax = int(tree.getroot().findall('.//xmax')[0].text)
    ymin = int(tree.getroot().findall('.//ymin')[0].text)
    ymax = int(tree.getroot().findall('.//ymax')[0].text)



# + id="hucaBhxESI0X" colab_type="code" colab={}
xmin = int(tree.getroot().findall('.//xmin')[0].text)
xmax = int(tree.getroot().findall('.//xmax')[0].text)
ymin = int(tree.getroot().findall('.//ymin')[0].text)
ymax = int(tree.getroot().findall('.//ymax')[0].text)
filename = tree.getroot().findall('.//filename')[0].text
width = tree.getroot().findall('.//width')[0].text
height= tree.getroot().findall('.//height')[0].text
depth= tree.getroot().findall('.//depth')[0].text
name= tree.getroot().findall('.//name')[0].text

# + id="ha_PWzgjMgY4" colab_type="code" outputId="20e6d3a0-2caa-456e-a689-36c3a5cc98b8" colab={"base_uri": "https://localhost:8080/", "height": 34}
filename, xmin, xmax,ymin, ymax, width, height, depth, name

# + id="6S8j6TUCSSpv" colab_type="code" outputId="640a5e61-6992-47d6-e588-8a972fd7a60e" colab={"base_uri": "https://localhost:8080/", "height": 34}
# df['V'] = df['V'].str.split('-').str[0]

# + id="dc3gbLs6UB9K" colab_type="code" colab={}
df_cols = ['filename', 'xmin', 'xmax','ymin', 'ymax',  'width', 'height', 'depth', 'name']

# + id="qtnUgmzyVkAp" colab_type="code" colab={}
annotations = sorted(os.listdir(DATA_DIR/'annotations/annotations/xmls'))

# + id="Cp_3pytJVsUN" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="bff9ec46-638d-4314-bc40-d583344c1ff0"
len(annotations)

# + id="UYc7mJ5PYJTx" colab_type="code" colab={}
df = pd.DataFrame(columns = df_cols)

# + id="RHGwmssQYYBj" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 47} outputId="fcb3fbc8-3644-4509-d56b-445b5ad27b0c"
df

# + id="gAz2E-HWWW_8" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 36} outputId="6319fd40-8af6-4601-80dc-f20440346485"
for xml_file in progress_bar(annotations):
    tree = ET.parse(str(DATA_DIR/'annotations/annotations/xmls/') + '/' + xml_file)

    for node in xroot:
        filename = tree.getroot().findall('.//filename')[0].text
        xmin = int(tree.getroot().findall('.//xmin')[0].text)
        xmax = int(tree.getroot().findall('.//xmax')[0].text)
        ymin = int(tree.getroot().findall('.//ymin')[0].text)
        ymax = int(tree.getroot().findall('.//ymax')[0].text)
        width = int(tree.getroot().findall('.//width')[0].text)
        height= int(tree.getroot().findall('.//height')[0].text)
        depth= int(tree.getroot().findall('.//depth')[0].text)
        name= tree.getroot().findall('.//name')[0].text
    df = df.append(pd.Series([filename, xmin, xmax,ymin, ymax, width, height, depth, name],index = df_cols), ignore_index = True)

# + id="Txrc_dCFht2z" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 137} outputId="4526e209-7268-4395-84b4-30f6f6015dcf"
df.head(3)

# + id="tZvVrgrwkXGp" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 137} outputId="b7e195a3-5630-469e-a055-62f6fab845c2"
df.tail(3)

# + id="9RwvMej-kf0U" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 137} outputId="15b720d5-ec15-4f94-fda1-ab34e2a466ff"
df.sample(3)

# + id="ph_ua5GDiwIU" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 167} outputId="8043ae22-1f24-49e8-f715-98d0c9945eb8"
df.describe()

# + id="MZz6goOJik1R" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 265} outputId="2b8104ea-ea54-4424-93b3-cdba50ac62f8"
df.info()

# + id="TtcCuuVpnBWM" colab_type="code" colab={}
df["x0"] = df["xmin"] / df["width"]
df["x1"] = df["xmax"] / df["width"]
df["y0"] = df["ymin"] / df["height"]
df["y1"] = df["ymax"] / df["height"]

# + id="UIDBtYO9nsej" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 137} outputId="d7d4062c-3f81-42bc-b1b4-201d1c0a79e7"
df.head(3)

# + id="8s3aqwC6oTtM" colab_type="code" colab={}
df["object_width"] = df["x1"] - df["x0"]
df["object_height"] = df["y1"] - df["y0"]

# + id="qAFW0JoToYXm" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 157} outputId="99aeb211-7771-46b6-bbaf-e92ff087fa82"
df.head(3)

# + id="lAF-vENNpRdv" colab_type="code" colab={}
if SETUP:
    for filename in glob(str(DATA_DIR / "**/*.jpg"), recursive=True):
        try:
            shutil.move(filename, DATA_DIR)
        except:
            pass
    shutil.rmtree(DATA_DIR / "images")

# + id="vdl1hPvlo3cT" colab_type="code" colab={}
df["file_path"] = f"{DATA_DIR}/" + df["filename"]

# + id="Cmn0PWLwp81L" colab_type="code" colab={}
df = df[df["file_path"].apply(lambda x: Path(x).is_file())]


# + id="8htlksfwnt1J" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 157} outputId="f4e4fff1-966a-4694-e7e6-25ec34351b0f"
df.head(3)

# + id="YyEhAw_osQWd" colab_type="code" colab={}
df['breed'] = df["filename"].apply(lambda x: x.rsplit('_', 1)[0])

# + id="3bSQngmsud5A" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 194} outputId="4e4ed65e-32f6-43ff-b227-78717195d7f0"
df.breed.unique()

# + id="F4J5s1Fgupk-" colab_type="code" colab={}
from sklearn.preprocessing import LabelEncoder

# + id="w1kfrtAAwFn3" colab_type="code" colab={}
le = LabelEncoder()

# + id="alzPJS5gxXsf" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="a9fd9515-6986-4c30-bcd3-80ce039c9ec5"
le.fit(df['breed'])

# + id="uZ4MZq7VxfmO" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="2903db6c-8b5f-4460-c0df-712e6b613ed4"
joblib.dump(le, TEMP_DIR / "le.pickle")

# + id="tdCPOedRxr6r" colab_type="code" colab={}
le = joblib.load( TEMP_DIR / "le.pickle")

# + id="hY0xe674xfp2" colab_type="code" colab={}
df['breed_encoded'] = le.transform(df['breed'])

# + id="s6oJQaLWx9YU" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 157} outputId="a16e6351-a3ac-4181-e7ad-9bb3c7609e79"
df.sample(3)

# + id="koQufY1ax9bZ" colab_type="code" colab={}
le_name = LabelEncoder()

# + colab_type="code" outputId="2c7247ad-4068-4b95-80fa-335b870b63cf" id="AlAo8gQuyROX" colab={"base_uri": "https://localhost:8080/", "height": 34}
le_name.fit(df['name'])

# + colab_type="code" outputId="bdab4f46-b9e7-476d-f36e-3018453dc1b3" id="Vfxv5stQyROj" colab={"base_uri": "https://localhost:8080/", "height": 34}
joblib.dump(le_name, TEMP_DIR / "le_name.pickle")

# + colab_type="code" id="67UT5mepyROo" colab={}
le_name = joblib.load( TEMP_DIR / "le_name.pickle")

# + colab_type="code" id="T5xYTSuqyROr" colab={}
df['name_encoded'] = le_name.transform(df['name'])

# + colab_type="code" outputId="afe9a607-b43d-451e-f636-91ce107b62d7" id="8Z5wwexMyROu" colab={"base_uri": "https://localhost:8080/", "height": 157}
df.sample(3)

# + id="oRRY4x4jkmo2" colab_type="code" colab={}
df.to_csv(TEMP_DIR/'pets_df.csv', index=False)

# + id="I6nSSjXdkxNx" colab_type="code" colab={}
df = pd.read_csv(TEMP_DIR/'pets_df.csv')

# + [markdown] id="f9nBGsFd1TSO" colab_type="text"
# ## Preparing the dataset
#

# + id="YF9oPP4U1Uld" colab_type="code" colab={}
BATCH_SIZE = 32
SHUFFLE_SIZE = 1024
IMG_DIMS = (299, 299, 3)
N_IMAGES = df.shape[0]

# + id="ZwBaWv-v1X1d" colab_type="code" colab={}
train_data, validation_data, test_data = split_df(df, 0.2, "breed_encoded")

# + id="a6Eh7fay1o1q" colab_type="code" colab={}
train_set = (
    tf.data.Dataset.from_tensor_slices(
        (
            train_data["file_path"],
            tuple(
                [
                    train_data["breed_encoded"].values,
                    train_data[["x0", "y0", "object_width", "object_height"]].values,
                ]
            ),
        )
    )
    .map(ImageParser())
    .map(ImageResizer(IMG_DIMS, "stretch"))
    .map(ImageAugmentor(level=3, rotate=False))
    .repeat()
    .shuffle(SHUFFLE_SIZE)
    .batch(BATCH_SIZE)
    .prefetch(tf.data.experimental.AUTOTUNE)
)

# + id="FVlR0yIR1o5Q" colab_type="code" colab={}
validation_set = (
    tf.data.Dataset.from_tensor_slices(
        (
            validation_data["file_path"],
            tuple(
                [
                    validation_data["breed_encoded"].values,
                    validation_data[
                        ["x0", "y0", "object_width", "object_height"]
                    ].values,
                ]
            ),
        )
    )
    .map(ImageParser())
    .map(ImageResizer(IMG_DIMS, "stretch"))
    .batch(BATCH_SIZE)
    .prefetch(tf.data.experimental.AUTOTUNE)
)

# + id="MstUqfJZ1o8J" colab_type="code" colab={}
test_set = (
    tf.data.Dataset.from_tensor_slices(
        (
            test_data["file_path"],
            tuple(
                [
                    test_data["breed_encoded"].values,
                    test_data[["x0", "y0", "object_width", "object_height"]].values,
                ]
            ),
        )
    )
    .map(ImageParser())
    .map(ImageResizer(IMG_DIMS, "stretch"))
    .batch(BATCH_SIZE)
    .prefetch(tf.data.experimental.AUTOTUNE)
)


# + id="KLe0ML932AXc" colab_type="code" colab={}
def make_rectangle(x0, y0, width, height, img_dims, color, linewidth=2):
    return Rectangle(
        (x0 * img_dims[1], y0 * img_dims[0]),
        width * img_dims[1],
        height * img_dims[0],
        linewidth=linewidth,
        edgecolor=color,
        facecolor="none",
    )


# + id="Lq2sjmRs2Ag1" colab_type="code" colab={}
def get_category_name(transformer, category) -> str:
    category_title = transformer.inverse_transform([category])
    category_title = ' '.join([str(elem) for elem in category_title])
    return category_title


# + id="AMIIZk2L2Fgb" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 268} outputId="84826b68-14a3-42d9-8461-666b6d77d1f7"
for x, y in train_set.take(1):
    n_image = 0
    categories, bounding_boxes = y
    category = categories[n_image].numpy()
    bounding_box = bounding_boxes[n_image].numpy()
    fig, ax = plt.subplots(1)
    ax.imshow(x[n_image].numpy())
    plt.text(
        bounding_box[0] * IMG_DIMS[0],
        bounding_box[1] * IMG_DIMS[1],
        get_category_name(le, category),
        backgroundcolor="yellow",
        fontsize=12,
    )
    y_rect = make_rectangle(*bounding_box, IMG_DIMS, "y")
    ax.add_patch(y_rect)
    plt.show()

# + id="CV6QHeo05x4y" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="e89b5486-7d1e-4e3d-8fe4-b314fbdfbc67"
n_categories = train_data["breed_encoded"].nunique()
n_categories


# + [markdown] id="sxdyax9J5-id" colab_type="text"
# ## Creating the model

# + id="DnexhzHr5_uo" colab_type="code" colab={}
class GlobalConcatPooling2D(keras.layers.Layer):
    def __call__(self, layer: keras.layers.Layer) -> keras.layers.Layer:
        return keras.layers.concatenate(
            [
                keras.layers.GlobalAvgPool2D()(layer),
                keras.layers.GlobalMaxPool2D()(layer),
            ]
        )


# + id="1RocpgBQ6EYH" colab_type="code" colab={}
def make_model():
    base = tf.keras.applications.Xception(
        input_shape=IMG_DIMS, include_top=False, weights="imagenet"
    )
    x = GlobalConcatPooling2D()(base.output)
    x = keras.layers.Dropout(0.4)(x)
    category = keras.layers.Dense(n_categories, activation=keras.activations.softmax)(x)
    bounding_box = keras.layers.Dense(4, activation=keras.activations.sigmoid)(x)
    model = keras.Model(inputs=base.input, outputs=[category, bounding_box])
    model.compile(
        loss=[
            keras.losses.SparseCategoricalCrossentropy(),
            keras.losses.BinaryCrossentropy(),
        ],
        loss_weights=[0.5, 2.0],
        optimizer=keras.optimizers.Adam(3e-4),
    )
    return model


# + id="giVCHGW46Ebs" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 72} outputId="7abf47c3-ec09-4408-d1d1-d8053ad3056e"
model = make_model()


# + id="uQ3-A-s86JRG" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 1000} outputId="cbd09c41-c218-401a-c3ac-275d587cdc6d"
model.summary()


# + id="JwdIPB4d6NVt" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 250} outputId="be1a9b14-58df-4af0-9bf6-56c772e08577"
model.fit(
    x=train_set,
    validation_data=validation_set,
    steps_per_epoch=math.ceil(train_data.shape[0] / BATCH_SIZE),
    epochs=5,
    callbacks=[
        keras.callbacks.ModelCheckpoint(
            str(TEMP_DIR / "model1.h5"), save_best_only=True, save_weights_only=True
        ),
        keras.callbacks.ReduceLROnPlateau(factor=0.3, patience=1),
        keras.callbacks.EarlyStopping(patience=3, restore_best_weights=True),
    ],
)


# + id="P08AZWZ46S1w" colab_type="code" colab={}
def predict(model, dataset, n_image=0):
    for x, y in dataset.take(1):
        fig, ax = plt.subplots(1)
        ax.imshow(x[n_image].numpy())

        p_categories, p_bounding_boxes = model.predict(x)
        p_category = p_categories[n_image].argmax()
        p_bounding_box = p_bounding_boxes[n_image]
        plt.text(
            (p_bounding_box[0] + p_bounding_box[2]) * IMG_DIMS[0] - 25,
            p_bounding_box[1] * IMG_DIMS[1],
            get_category_name(le, p_category),
            color="white",
            backgroundcolor="blue",
            fontsize=16,
        )
        p_rect = make_rectangle(*p_bounding_box, IMG_DIMS, "b", 3)
        ax.add_patch(p_rect)

        y_categories, y_bounding_boxes = y
        y_category = y_categories[n_image].numpy()
        y_bounding_box = y_bounding_boxes[n_image].numpy()
        plt.text(
            y_bounding_box[0] * IMG_DIMS[0] + 5,
            y_bounding_box[1] * IMG_DIMS[1],
            f"true:{get_category_name(le, y_category)}",
            backgroundcolor="red",
            fontsize=16,
        )
        y_rect = make_rectangle(*y_bounding_box, IMG_DIMS, "r")
        ax.add_patch(y_rect)

        plt.show()


# + id="jRVWkf0p7rnu" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 268} outputId="f1729cf1-91af-42f3-bc66-6247253fd4e4"
predict(model, validation_set, 1)


# + id="dUQUCfyl8Ijj" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 268} outputId="098bcb60-e729-4a04-a0b0-6943b10ed903"
predict(model, validation_set, 10)


# + id="39RGUDQ99zlN" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 275} outputId="0cc88e70-0b50-49f0-e53d-0ae6b8be20e8"
predict(model, test_set, 10)


# + id="DzJWIRLy85DN" colab_type="code" colab={}
def make_report(model, dataset):
    print(
        classification_report(
            [category.numpy() for _, (category, _) in dataset.unbatch()],
            model.predict(dataset)[0].argmax(axis=1),
            target_names=list(category_map.values()),
        )
    )


# + id="87il2cHe9g5c" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 292} outputId="c9aa1678-cd6e-4f70-c07d-cd815452c0e0"
make_report(model, validation_set)


# + id="jtg6uZtj9g9c" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered