first commit

.gitignore

@@ -0,0 +1,86 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# DotEnv configuration
+.env
+
+# Database
+*.db
+*.rdb
+
+# Pycharm
+.idea
+
+# VS Code
+.vscode/
+
+# Spyder
+.spyproject/
+
+# Jupyter NB Checkpoints
+.ipynb_checkpoints/
+
+# Mac OS-specific storage files
+.DS_Store
+
+# vim
+*.swp
+*.swo
+
+# Mypy cache
+.mypy_cache/

Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.9
+COPY requirements.txt /app/requirements.txt
+WORKDIR /app
+RUN pip install seaborn
+RUN pip install gradio
+RUN pip install datetime
+RUN pip install numpy
+RUN pip install opencv-python
+RUN apt-get update
+RUN apt-get install ffmpeg libsm6 libxext6 -y
+RUN pip install -r requirements.txt
+COPY . /app
+# EXPOSE 7860
+CMD python app.py
+# hello world

README.md

@@ -0,0 +1,40 @@
+# Welcome to the project inno-satellite-images-segmentation-gan
+![](docs/assets/banner.png)
+
+- **Project name**: inno-satellite-images-segmentation-gan
+- **Library name**: library
+- **Authors**: Ekimetrics
+- **Description**: Segmenting satellite images in a large scale is challenging because grondtruth labels are spurious for medium resolution images (Sentinel 2). We want to improve our algorithm either with data augmentation from a GAN, or to correct or adjust Corine labels. 
+
+
+
+## Project Structure
+```
+- library/ # Your python library
+- data/
+    - raw/
+    - processed/
+- docs/
+- tests/                            # Where goes each unitary test in your folder
+- scripts/                          # Where each automation script will go
+- requirements.txt                  # Where you should put the libraries version used in your library
+```
+
+
+## Branch strategy
+TBD
+
+
+## Ethics checklist
+TBD
+
+
+
+## Starter package
+This project has been created using the Ekimetrics Python Starter Package to enforce best coding practices, reusability and industrialization. <br>
+If you have any questions please reach out to the inno team and [Théo Alves Da Costa](mailto:[email protected])
+
+
+
+
+

biomap/.gitignore

@@ -0,0 +1,6 @@
+#wsl
+*.Zone.Identifier
+
+#python
+*__pycache__
+

biomap/.private-key.json

@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "cvimg-377115",
+  "private_key_id": "a162152bd26f4bcc287c44b130109892b5517875",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDCr/zwOTwyVVdF\nk1cKcdju9jeDWceVJ/zj73b5IS8IbQJbFGWSjaL6Ft1HfJ3rSdGbj+Xy26jY9OFJ\n4rIhpY0M0cCWpYo+gqS9p4JAL6lHqZvSnkRTglpx9QYOT8o9ibCWhMVVAPH71QZ/\n4BEfGC6s2+zdEn+cbCkGIqLLhZTq655kDOaGSycwV/bk+TOLI/An4gjMoEIimhsD\nS6TRmqZnQGoI6m6aj3xPZGVMkid3I37h+BOC64YjKeXnpAhTQ4LbpQz1BxvIKDt6\ncJ1FBOmdSvUC+dLEGMN3yijpJZ74nXUnSVbwYYt3K8Kz2PtmgDYvswER53NlrMoW\n1AF9ImDFAgMBAAECggEAASO7IXWBo90d6CiEdhQF9uFy2S5d9ol9N6EBl+JHOuwB\nwne4tSZJ3jT/Rus7wvX67tXI9dgf3fIgjsv92NLnwEn1Wq/xQppMm8iyK1DKU3vH\n8xrvf8iG048ojGqQXqf0ZEUoWd+/YDGZ2qNuZjmVgwZKwF2h2pcnQ25uIvWdYHrb\n3XhYLDAROVTTtyscYcl8UKmAZ35moVVBQxdakGunYg6o/s6rESRbc+gCyqHR5v+r\nCl3Z4XEKDdukIVI72Ybk0F8eZpQztN97uzK/zm9jl4NmAPXrnWLEwuJdwdm1cWUF\n/LTTuNPmRzCm7IGUpkx0AKEs6s0BRbJbwlZaj4QVJwKBgQDjb2rSO6kRLRHyv+4w\ny/OLmqOrMY7fpSCj0mH41GhiZUaoZqhDznmuhqEjo1kVipfuW94Fn5NsGmWpbmEC\nJlObUEg1umX/ceOJrtRdY3AQMSQXR6u7oc2mTgj3Opd0V1L1Lopj4Ijj43ARg/fU\nu4RnrCGHcXXzT2LCchY0ZhLg3wKBgQDbI6bzt/RNW8+IGKCvLLi41bxM/9r83GNO\nQI4a6yTT09N11okjP9h00JKYBgU3fYivA1aBloFB4kOYaBzomfWSZHEyyFWCr9y0\ndGyIDbfUaI/jFx2CaKomLnPDF5LA3IWHAsTRZ/c1JGhiOUseEq/TR0cJAo69kgf0\nkVmoGjo+2wKBgQCo7crkGJg9P8LDEbgz2mktWlETCR5cE2SpCczna62U2DChSI7W\nvng3H5x0whGbJHQxAV9pwdtYQksci/XWCO20wO7BqY+1KrydOZRXQVKtVDLAb+Wo\n2kfLrM6QA58XNP1TS5xTDyXeTsKg3+qmwhlYf8vvtGCttltenirMBL0k9QKBgFpL\nanNqDOQDPJQbcbo8dzDSAPDJS/Z86P5JY0R8N4SA99TKPV+k4w/fEUhK0sN2mmdi\nvLZQyZnYHXojDCZbqfBUKsB+A54B0LMadc3puSFwpDkyQRqG/fUVluWARRvqwapL\n3cVbTWU8RzaR3P3bPU+VQxPXVfGOxnBjo8m8ZNuZAoGBANTC20T9rZ9Won9FCbi3\nSMkGY59smx19CdytQ2rjGFOEeAVMVotP5viXFuKfv5g2E/kvyJUjuOoulA3dxddN\nQzXnOIT3dlyBmvXkHJHUIKiidyuX4JqQFdPTAmkt6KaTceRNb7VN1OqIk1AJ1SDb\nkGxerLg4WuGfSqOIV0Wk4cLI\n-----END PRIVATE KEY-----\n",
+  "client_email": "[email protected]",
+  "client_id": "115144831673857322488",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/cvimg-355%40cvimg-377115.iam.gserviceaccount.com"
+}

biomap/app.py

@@ -0,0 +1,110 @@
+from plot_functions import *
+import hydra
+
+import torch
+from model import LitUnsupervisedSegmenter
+from helper import inference_on_location_and_month, inference_on_location
+from plot_functions import segment_region
+
+from functools import partial
+import gradio as gr
+import logging
+
+import geopandas as gpd
+mapbox_access_token = "pk.eyJ1IjoiamVyZW15LWVraW1ldHJpY3MiLCJhIjoiY2xrNjBwNGU2MDRhMjNqbWw0YTJrbnpvNCJ9.poVyIzhJuJmD6ffrL9lm2w"
+geo_df = gpd.read_file(gpd.datasets.get_path('naturalearth_cities'))
+
+def get_geomap(long, lat ):
+    
+
+    fig = go.Figure(go.Scattermapbox(
+            lat=geo_df.geometry.y,
+            lon=geo_df.geometry.x,
+            mode='markers',
+            marker=go.scattermapbox.Marker(
+                size=14
+            ),
+            text=geo_df.name,
+        ))
+    
+    fig.add_trace(go.Scattermapbox(lat=[lat],
+        lon=[long],
+        mode='markers',
+        marker=go.scattermapbox.Marker(
+            size=14
+        ),
+        marker_color="green",
+        text=['Actual position']))
+
+    fig.update_layout(
+        showlegend=False,
+        hovermode='closest',
+        mapbox=dict(
+            accesstoken=mapbox_access_token,
+            center=go.layout.mapbox.Center(
+                lat=lat,
+                lon=long
+            ),
+            zoom=3
+        )
+    )
+    
+    return fig
+
+
+if __name__ == "__main__":
+    
+    
+    logging.basicConfig(filename='example.log', encoding='utf-8', level=logging.INFO)
+    # Initialize hydra with configs
+    #hydra.initialize(config_path="configs", job_name="corine")
+    cfg = hydra.compose(config_name="my_train_config.yml")
+    logging.info(f"config : {cfg}")
+    # Load the model
+
+    nbclasses = cfg.dir_dataset_n_classes
+    model = LitUnsupervisedSegmenter(nbclasses, cfg)
+    logging.info(f"Model Initialiazed")
+    
+    model_path = "checkpoint/model/model.pt"
+    saved_state_dict = torch.load(model_path, map_location=torch.device("cpu"))
+    logging.info(f"Model weights Loaded")
+    model.load_state_dict(saved_state_dict)
+    logging.info(f"Model Loaded")
+# css=".VIDEO video{height: 100%;width:50%;margin:auto};.VIDEO{height: 50%;};.svelte-1vnmhm4{height:auto}"
+    with gr.Blocks() as demo:
+        gr.Markdown("Estimate Biodiversity in the world.")
+        with gr.Tab("Single Image"):
+            with gr.Row():
+                input_map = gr.Plot().style()  
+                with gr.Column():
+                    input_latitude = gr.Number(label="lattitude", value=2.98)
+                    input_longitude = gr.Number(label="longitude", value=48.81)
+                    input_date = gr.Textbox(label="start_date", value="2020-03-20")
+
+            single_button = gr.Button("Predict")
+            with gr.Row():
+                raw_image = gr.Image(label = "Localisation visualization")
+                output_image = gr.Image(label = "Labeled visualisation")
+                score_biodiv = gr.Number(label = "Biodiversity score")
+
+        with gr.Tab("TimeLapse"):
+            with gr.Row():
+                input_map_2 = gr.Plot().style() 
+            with gr.Row():
+                timelapse_input_latitude = gr.Number(value=2.98, label="Latitude")
+                timelapse_input_longitude = gr.Number(value=48.81, label="Longitude")
+                timelapse_start_date = gr.Textbox(value='2020-05-01', label="Start Date")
+                timelapse_end_date = gr.Textbox(value='2020-06-30', label="End Date")
+            segmentation = gr.CheckboxGroup(choices=['month', 'year', '2months'], value=['month'], label="Select Segmentation Level:")
+            timelapse_button = gr.Button(value="Predict")     
+            map = gr.Plot().style()
+        
+        demo.load(get_geomap, [input_latitude, input_longitude], input_map)
+        single_button.click(get_geomap, [input_latitude, input_longitude], input_map)
+        single_button.click(partial(inference_on_location_and_month, model), inputs=[input_latitude, input_longitude, input_date], outputs=[raw_image, output_image,score_biodiv])
+
+        demo.load(get_geomap, [timelapse_input_latitude, timelapse_input_longitude], input_map_2)
+        timelapse_button.click(get_geomap, [timelapse_input_latitude, timelapse_input_longitude], input_map_2)
+        timelapse_button.click(segment_region, inputs=[timelapse_input_latitude, timelapse_input_longitude, timelapse_start_date, timelapse_end_date,segmentation], outputs=[map])
+    demo.launch(share=True)

biomap/configs/my_train_config.yml

@@ -0,0 +1,197 @@
+output_root: '../'
+pytorch_data_dir: '/home/duong_nguyen/pytorch-data'
+experiment_name: "unet_7classes"
+log_dir: "france"
+# experiment_name: "unet"
+# log_dir: "potsdam"
+azureml_logging: False
+submitting_to_aml: False
+full_name: ~
+
+# Loader params
+num_workers: 24
+max_steps: 80000
+batch_size: 16
+
+num_neighbors: 7
+dataset_name: "directory"
+# dataset_name: "potsdam"
+
+# Used if dataset_name is "directory"
+dir_dataset_name: "corine"
+dir_dataset_n_classes: 7
+
+has_labels: False
+# crop_type: "five"
+crop_type: ~
+crop_ratio: .5
+res: 224
+loader_crop_type: "center"
+
+# Model Params
+extra_clusters: 0
+use_true_labels: False
+use_recalibrator: False
+model_type: "vit_small"
+arch: "dino"
+use_fit_model: False
+dino_feat_type: "feat"
+projection_type: "nonlinear"
+#projection_type: linear
+dino_patch_size: 8
+granularity: 1
+continuous: True
+dim: 70
+dropout: True
+zero_clamp: True
+
+lr: 5e-4
+pretrained_weights: ~
+use_salience: False
+stabalize: False
+stop_at_zero: True
+
+# Feature Contrastive params
+pointwise: True
+feature_samples: 11
+neg_samples: 5
+aug_alignment_weight: 0.0
+
+correspondence_weight: 1.0
+
+
+# # Corine vit small 24/11/22
+neg_inter_weight: 0.63
+pos_inter_weight: 0.25
+pos_intra_weight: 0.67
+neg_inter_shift: 0.46
+pos_inter_shift: 0.02
+pos_intra_shift: 0.08
+
+# # Corine vit small 11/09/22
+# neg_inter_weight: 0.63
+# pos_inter_weight: 0.25
+# pos_intra_weight: 0.67
+# neg_inter_shift: 0.46
+# pos_inter_shift: 0.24
+# pos_intra_shift: 0.36
+
+# # IAROA vit small 1/31/22
+# neg_inter_weight: 0.63
+# pos_inter_weight: 0.25
+# pos_intra_weight: 0.67
+# neg_inter_shift: 0.46
+# pos_inter_shift: 0.12
+# pos_intra_shift: 0.18
+
+# Potsdam vit small 1/31/22
+# neg_inter_weight: 0.63
+# pos_inter_weight: 0.25
+# pos_intra_weight: 0.67
+# neg_inter_shift: 0.46
+# pos_inter_shift: 0.02
+# pos_intra_shift: 0.08
+
+# Cocostuff27 vit small 1/31/22
+#neg_inter_weight: 0.63
+#pos_inter_weight: 0.25
+#pos_intra_weight: 0.67
+#neg_inter_shift: 0.66
+#pos_inter_shift: 0.02
+#pos_intra_shift: 0.08
+
+
+## Cocostuff27 10/3 vit_base
+
+#neg_inter_weight: 0.1538476246415498
+#pos_inter_weight: 1
+#pos_intra_weight: 0.1
+#
+#neg_inter_shift: 1
+#pos_inter_shift: 0.2
+#pos_intra_shift: 0.12
+
+
+## Cocostuff27 10/3 vit_small
+#neg_inter_weight: .63
+#pos_inter_weight: .25
+#pos_intra_weight: .67
+#
+#neg_inter_shift: .16
+#pos_inter_shift: .02
+#pos_intra_shift: .08
+
+
+
+## Cocostuff27 10/3 moco
+#neg_inter_weight: .63
+#pos_inter_weight: .25
+#pos_intra_weight: .67
+#
+#neg_inter_shift: .26
+#pos_inter_shift: .36
+#pos_intra_shift: .32
+
+#pos_inter_shift: .12
+#pos_intra_shift: .18
+
+## Cocostuff27
+#neg_inter_weight: .72
+#pos_inter_weight: .80
+#pos_intra_weight: .29
+#
+#neg_inter_shift: .86
+#pos_inter_shift: .04
+#pos_intra_shift: .34
+
+# Cityscapes 10/3
+
+# neg_inter_weight: 0.9058762625226623
+# pos_inter_weight: 0.577453483136995
+# pos_intra_weight: 1
+
+# neg_inter_shift: 0.31361241889448443
+# pos_inter_shift: 0.1754346515479633
+# pos_intra_shift: 0.45828472207
+
+
+# Cityscapes
+#neg_inter_weight: .72
+#pos_inter_weight: .18
+#pos_intra_weight: .46
+#
+#neg_inter_shift: .25
+#pos_inter_shift: .20
+#pos_intra_shift: .25
+
+
+rec_weight: 0.0
+repulsion_weight: 0.0
+
+# CRF Params
+crf_weight: 0.0
+alpha: .5
+beta: .15
+gamma: .05
+w1: 10.0
+w2: 3.0
+shift: 0.00
+crf_samples: 1000
+color_space: "rgb"
+
+reset_probe_steps: ~
+
+# Logging params
+n_images: 5
+scalar_log_freq: 10
+checkpoint_freq: 50
+val_freq: 100
+hist_freq: 100
+
+
+hydra:
+  run:
+    dir: "."
+  output_subdir: ~
+  #job_logging: "disabled"
+  #hydra_logging: "disabled"

biomap/data.py

@@ -0,0 +1,584 @@
+import os
+import random
+from os.path import join
+
+import numpy as np
+import torch.multiprocessing
+from PIL import Image
+from scipy.io import loadmat
+from torch.utils.data import DataLoader
+from torch.utils.data import Dataset
+from torchvision.datasets.cityscapes import Cityscapes
+from torchvision.transforms.functional import to_pil_image
+from tqdm import tqdm
+
+
+def bit_get(val, idx):
+    """Gets the bit value.
+    Args:
+      val: Input value, int or numpy int array.
+      idx: Which bit of the input val.
+    Returns:
+      The "idx"-th bit of input val.
+    """
+    return (val >> idx) & 1
+
+
+def create_pascal_label_colormap():
+    """Creates a label colormap used in PASCAL VOC segmentation benchmark.
+    Returns:
+      A colormap for visualizing segmentation results.
+    """
+    colormap = np.zeros((512, 3), dtype=int)
+    ind = np.arange(512, dtype=int)
+
+    for shift in reversed(list(range(8))):
+        for channel in range(3):
+            colormap[:, channel] |= bit_get(ind, channel) << shift
+        ind >>= 3
+
+    return colormap
+
+
+def create_cityscapes_colormap():
+    colors = [(128, 64, 128),
+              (244, 35, 232),
+              (250, 170, 160),
+              (230, 150, 140),
+              (70, 70, 70),
+              (102, 102, 156),
+              (190, 153, 153),
+              (180, 165, 180),
+              (150, 100, 100),
+              (150, 120, 90),
+              (153, 153, 153),
+              (153, 153, 153),
+              (250, 170, 30),
+              (220, 220, 0),
+              (107, 142, 35),
+              (152, 251, 152),
+              (70, 130, 180),
+              (220, 20, 60),
+              (255, 0, 0),
+              (0, 0, 142),
+              (0, 0, 70),
+              (0, 60, 100),
+              (0, 0, 90),
+              (0, 0, 110),
+              (0, 80, 100),
+              (0, 0, 230),
+              (119, 11, 32),
+              (0, 0, 0)]
+    return np.array(colors)
+
+
+class DirectoryDataset(Dataset):
+    def __init__(self, root, path, image_set, transform, target_transform):
+        super(DirectoryDataset, self).__init__()
+        self.split = image_set
+        self.dir = join(root, path)
+        self.img_dir = join(self.dir, "imgs", self.split)
+        self.label_dir = join(self.dir, "labels", self.split)
+
+        self.transform = transform
+        self.target_transform = target_transform
+
+        self.img_files = np.array(sorted(os.listdir(self.img_dir)))
+        assert len(self.img_files) > 0
+        if os.path.exists(join(self.dir, "labels")):
+            self.label_files = np.array(sorted(os.listdir(self.label_dir)))
+            assert len(self.img_files) == len(self.label_files)
+        else:
+            self.label_files = None
+        self.fine_to_coarse = {0: 0,  
+                               1: 1,  
+                               2: 2, 
+                               3: 3,
+                               4: 4,
+                               5: 5,
+                               6: 6,
+                               7: -1,
+                               }
+            
+    def __getitem__(self, index):
+        image_fn = self.img_files[index]
+        img = Image.open(join(self.img_dir, image_fn))
+
+        if self.label_files is not None:
+            label_fn = self.label_files[index]
+            label = Image.open(join(self.label_dir, label_fn))
+
+        seed = np.random.randint(2147483647)
+        random.seed(seed)
+        torch.manual_seed(seed)
+        img = self.transform(img)
+
+        if self.label_files is not None:
+            random.seed(seed)
+            torch.manual_seed(seed)
+            label = self.target_transform(label)
+            new_label_map = torch.zeros_like(label)
+            for fine, coarse in self.fine_to_coarse.items():
+                new_label_map[label == fine] = coarse
+            label = new_label_map
+        else:
+            label = torch.zeros(img.shape[1], img.shape[2], dtype=torch.int64) - 1
+
+        mask = (label > 0).to(torch.float32)
+        return img, label, mask
+    
+    
+    def __len__(self):
+        return len(self.img_files)
+
+
+class Potsdam(Dataset):
+    def __init__(self, root, image_set, transform, target_transform, coarse_labels):
+        super(Potsdam, self).__init__()
+        self.split = image_set
+        self.root = os.path.join(root, "potsdam")
+        self.transform = transform
+        self.target_transform = target_transform
+        split_files = {
+            "train": ["labelled_train.txt"],
+            "unlabelled_train": ["unlabelled_train.txt"],
+            # "train": ["unlabelled_train.txt"],
+            "val": ["labelled_test.txt"],
+            "train+val": ["labelled_train.txt", "labelled_test.txt"],
+            "all": ["all.txt"]
+        }
+        assert self.split in split_files.keys()
+
+        self.files = []
+        for split_file in split_files[self.split]:
+            with open(join(self.root, split_file), "r") as f:
+                self.files.extend(fn.rstrip() for fn in f.readlines())
+
+        self.coarse_labels = coarse_labels
+        self.fine_to_coarse = {0: 0, 4: 0,  # roads and cars
+                               1: 1, 5: 1,  # buildings and clutter
+                               2: 2, 3: 2,  # vegetation and trees
+                               255: -1
+                               }
+
+    def __getitem__(self, index):
+        image_id = self.files[index]
+        img = loadmat(join(self.root, "imgs", image_id + ".mat"))["img"]
+        img = to_pil_image(torch.from_numpy(img).permute(2, 0, 1)[:3])  # TODO add ir channel back
+        try:
+            label = loadmat(join(self.root, "gt", image_id + ".mat"))["gt"]
+            label = to_pil_image(torch.from_numpy(label).unsqueeze(-1).permute(2, 0, 1))
+        except FileNotFoundError:
+            label = to_pil_image(torch.ones(1, img.height, img.width))
+
+        seed = np.random.randint(2147483647)
+        random.seed(seed)
+        torch.manual_seed(seed)
+        img = self.transform(img)
+
+        random.seed(seed)
+        torch.manual_seed(seed)
+        label = self.target_transform(label).squeeze(0)
+        if self.coarse_labels:
+            new_label_map = torch.zeros_like(label)
+            for fine, coarse in self.fine_to_coarse.items():
+                new_label_map[label == fine] = coarse
+            label = new_label_map
+
+        mask = (label > 0).to(torch.float32)
+        return img, label, mask
+
+    def __len__(self):
+        return len(self.files)
+
+
+class PotsdamRaw(Dataset):
+    def __init__(self, root, image_set, transform, target_transform, coarse_labels):
+        super(PotsdamRaw, self).__init__()
+        self.split = image_set
+        self.root = os.path.join(root, "potsdamraw", "processed")
+        self.transform = transform
+        self.target_transform = target_transform
+        self.files = []
+        for im_num in range(38):
+            for i_h in range(15):
+                for i_w in range(15):
+                    self.files.append("{}_{}_{}.mat".format(im_num, i_h, i_w))
+
+        self.coarse_labels = coarse_labels
+        self.fine_to_coarse = {0: 0, 4: 0,  # roads and cars
+                               1: 1, 5: 1,  # buildings and clutter
+                               2: 2, 3: 2,  # vegetation and trees
+                               255: -1
+                               }
+
+    def __getitem__(self, index):
+        image_id = self.files[index]
+        img = loadmat(join(self.root, "imgs", image_id))["img"]
+        img = to_pil_image(torch.from_numpy(img).permute(2, 0, 1)[:3])  # TODO add ir channel back
+        try:
+            label = loadmat(join(self.root, "gt", image_id))["gt"]
+            label = to_pil_image(torch.from_numpy(label).unsqueeze(-1).permute(2, 0, 1))
+        except FileNotFoundError:
+            label = to_pil_image(torch.ones(1, img.height, img.width))
+
+        seed = np.random.randint(2147483647)
+        random.seed(seed)
+        torch.manual_seed(seed)
+        img = self.transform(img)
+
+        random.seed(seed)
+        torch.manual_seed(seed)
+        label = self.target_transform(label).squeeze(0)
+        if self.coarse_labels:
+            new_label_map = torch.zeros_like(label)
+            for fine, coarse in self.fine_to_coarse.items():
+                new_label_map[label == fine] = coarse
+            label = new_label_map
+
+        mask = (label > 0).to(torch.float32)
+        return img, label, mask
+
+    def __len__(self):
+        return len(self.files)
+
+
+class Coco(Dataset):
+    def __init__(self, root, image_set, transform, target_transform,
+                 coarse_labels, exclude_things, subset=None):
+        super(Coco, self).__init__()
+        self.split = image_set
+        self.root = join(root, "cocostuff")
+        self.coarse_labels = coarse_labels
+        self.transform = transform
+        self.label_transform = target_transform
+        self.subset = subset
+        self.exclude_things = exclude_things
+
+        if self.subset is None:
+            self.image_list = "Coco164kFull_Stuff_Coarse.txt"
+        elif self.subset == 6:  # IIC Coarse
+            self.image_list = "Coco164kFew_Stuff_6.txt"
+        elif self.subset == 7:  # IIC Fine
+            self.image_list = "Coco164kFull_Stuff_Coarse_7.txt"
+
+        assert self.split in ["train", "val", "train+val"]
+        split_dirs = {
+            "train": ["train2017"],
+            "val": ["val2017"],
+            "train+val": ["train2017", "val2017"]
+        }
+
+        self.image_files = []
+        self.label_files = []
+        for split_dir in split_dirs[self.split]:
+            with open(join(self.root, "curated", split_dir, self.image_list), "r") as f:
+                img_ids = [fn.rstrip() for fn in f.readlines()]
+                for img_id in img_ids:
+                    self.image_files.append(join(self.root, "images", split_dir, img_id + ".jpg"))
+                    self.label_files.append(join(self.root, "annotations", split_dir, img_id + ".png"))
+
+        self.fine_to_coarse = {0: 9, 1: 11, 2: 11, 3: 11, 4: 11, 5: 11, 6: 11, 7: 11, 8: 11, 9: 8, 10: 8, 11: 8, 12: 8,
+                               13: 8, 14: 8, 15: 7, 16: 7, 17: 7, 18: 7, 19: 7, 20: 7, 21: 7, 22: 7, 23: 7, 24: 7,
+                               25: 6, 26: 6, 27: 6, 28: 6, 29: 6, 30: 6, 31: 6, 32: 6, 33: 10, 34: 10, 35: 10, 36: 10,
+                               37: 10, 38: 10, 39: 10, 40: 10, 41: 10, 42: 10, 43: 5, 44: 5, 45: 5, 46: 5, 47: 5, 48: 5,
+                               49: 5, 50: 5, 51: 2, 52: 2, 53: 2, 54: 2, 55: 2, 56: 2, 57: 2, 58: 2, 59: 2, 60: 2,
+                               61: 3, 62: 3, 63: 3, 64: 3, 65: 3, 66: 3, 67: 3, 68: 3, 69: 3, 70: 3, 71: 0, 72: 0,
+                               73: 0, 74: 0, 75: 0, 76: 0, 77: 1, 78: 1, 79: 1, 80: 1, 81: 1, 82: 1, 83: 4, 84: 4,
+                               85: 4, 86: 4, 87: 4, 88: 4, 89: 4, 90: 4, 91: 17, 92: 17, 93: 22, 94: 20, 95: 20, 96: 22,
+                               97: 15, 98: 25, 99: 16, 100: 13, 101: 12, 102: 12, 103: 17, 104: 17, 105: 23, 106: 15,
+                               107: 15, 108: 17, 109: 15, 110: 21, 111: 15, 112: 25, 113: 13, 114: 13, 115: 13, 116: 13,
+                               117: 13, 118: 22, 119: 26, 120: 14, 121: 14, 122: 15, 123: 22, 124: 21, 125: 21, 126: 24,
+                               127: 20, 128: 22, 129: 15, 130: 17, 131: 16, 132: 15, 133: 22, 134: 24, 135: 21, 136: 17,
+                               137: 25, 138: 16, 139: 21, 140: 17, 141: 22, 142: 16, 143: 21, 144: 21, 145: 25, 146: 21,
+                               147: 26, 148: 21, 149: 24, 150: 20, 151: 17, 152: 14, 153: 21, 154: 26, 155: 15, 156: 23,
+                               157: 20, 158: 21, 159: 24, 160: 15, 161: 24, 162: 22, 163: 25, 164: 15, 165: 20, 166: 17,
+                               167: 17, 168: 22, 169: 14, 170: 18, 171: 18, 172: 18, 173: 18, 174: 18, 175: 18, 176: 18,
+                               177: 26, 178: 26, 179: 19, 180: 19, 181: 24}
+
+        self._label_names = [
+            "ground-stuff",
+            "plant-stuff",
+            "sky-stuff",
+        ]
+        self.cocostuff3_coarse_classes = [23, 22, 21]
+        self.first_stuff_index = 12
+
+    def __getitem__(self, index):
+        image_path = self.image_files[index]
+        label_path = self.label_files[index]
+        seed = np.random.randint(2147483647)
+        random.seed(seed)
+        torch.manual_seed(seed)
+        img = self.transform(Image.open(image_path).convert("RGB"))
+
+        random.seed(seed)
+        torch.manual_seed(seed)
+        label = self.label_transform(Image.open(label_path)).squeeze(0)
+        label[label == 255] = -1  # to be consistent with 10k
+        coarse_label = torch.zeros_like(label)
+        for fine, coarse in self.fine_to_coarse.items():
+            coarse_label[label == fine] = coarse
+        coarse_label[label == -1] = -1
+
+        if self.coarse_labels:
+            coarser_labels = -torch.ones_like(label)
+            for i, c in enumerate(self.cocostuff3_coarse_classes):
+                coarser_labels[coarse_label == c] = i
+            return img, coarser_labels, coarser_labels >= 0
+        else:
+            if self.exclude_things:
+                return img, coarse_label - self.first_stuff_index, (coarse_label >= self.first_stuff_index)
+            else:
+                return img, coarse_label, coarse_label >= 0
+
+    def __len__(self):
+        return len(self.image_files)
+
+
+class CityscapesSeg(Dataset):
+    def __init__(self, root, image_set, transform, target_transform):
+        super(CityscapesSeg, self).__init__()
+        self.split = image_set
+        self.root = join(root, "cityscapes")
+        if image_set == "train":
+            # our_image_set = "train_extra"
+            # mode = "coarse"
+            our_image_set = "train"
+            mode = "fine"
+        else:
+            our_image_set = image_set
+            mode = "fine"
+        self.inner_loader = Cityscapes(self.root, our_image_set,
+                                       mode=mode,
+                                       target_type="semantic",
+                                       transform=None,
+                                       target_transform=None)
+        self.transform = transform
+        self.target_transform = target_transform
+        self.first_nonvoid = 7
+
+    def __getitem__(self, index):
+        if self.transform is not None:
+            image, target = self.inner_loader[index]
+
+            seed = np.random.randint(2147483647)
+            random.seed(seed)
+            torch.manual_seed(seed)
+            image = self.transform(image)
+            random.seed(seed)
+            torch.manual_seed(seed)
+            target = self.target_transform(target)
+
+            target = target - self.first_nonvoid
+            target[target < 0] = -1
+            mask = target == -1
+            return image, target.squeeze(0), mask
+        else:
+            return self.inner_loader[index]
+
+    def __len__(self):
+        return len(self.inner_loader)
+
+
+class CroppedDataset(Dataset):
+    def __init__(self, root, dataset_name, crop_type, crop_ratio, image_set, transform, target_transform):
+        super(CroppedDataset, self).__init__()
+        self.dataset_name = dataset_name
+        self.split = image_set
+        self.root = join(root, "cropped", "{}_{}_crop_{}".format(dataset_name, crop_type, crop_ratio))
+        self.transform = transform
+        self.target_transform = target_transform
+        self.img_dir = join(self.root, "img", self.split)
+        self.label_dir = join(self.root, "label", self.split)
+        self.num_images = len(os.listdir(self.img_dir))
+        assert self.num_images == len(os.listdir(self.label_dir))
+
+    def __getitem__(self, index):
+        image = Image.open(join(self.img_dir, "{}.jpg".format(index))).convert('RGB')
+        target = Image.open(join(self.label_dir, "{}.png".format(index)))
+
+        seed = np.random.randint(2147483647)
+        random.seed(seed)
+        torch.manual_seed(seed)
+        image = self.transform(image)
+        random.seed(seed)
+        torch.manual_seed(seed)
+        target = self.target_transform(target)
+
+        target = target - 1
+        mask = target == -1
+        return image, target.squeeze(0), mask
+
+    def __len__(self):
+        return self.num_images
+
+
+class MaterializedDataset(Dataset):
+
+    def __init__(self, ds):
+        self.ds = ds
+        self.materialized = []
+        loader = DataLoader(ds, num_workers=12, collate_fn=lambda l: l[0])
+        for batch in tqdm(loader):
+            self.materialized.append(batch)
+
+    def __len__(self):
+        return len(self.ds)
+
+    def __getitem__(self, ind):
+        return self.materialized[ind]
+
+
+class ContrastiveSegDataset(Dataset):
+    def __init__(self,
+                 pytorch_data_dir,
+                 dataset_name,
+                 crop_type,
+                 image_set,
+                 transform,
+                 target_transform,
+                 cfg,
+                 aug_geometric_transform=None,
+                 aug_photometric_transform=None,
+                 num_neighbors=5,
+                 compute_knns=False,
+                 mask=False,
+                 pos_labels=False,
+                 pos_images=False,
+                 extra_transform=None,
+                 model_type_override=None
+                 ):
+        super(ContrastiveSegDataset).__init__()
+        self.num_neighbors = num_neighbors
+        self.image_set = image_set
+        self.dataset_name = dataset_name
+        self.mask = mask
+        self.pos_labels = pos_labels
+        self.pos_images = pos_images
+        self.extra_transform = extra_transform
+
+        if dataset_name == "potsdam":
+            self.n_classes = 3
+            dataset_class = Potsdam
+            extra_args = dict(coarse_labels=True)
+        elif dataset_name == "potsdamraw":
+            self.n_classes = 3
+            dataset_class = PotsdamRaw
+            extra_args = dict(coarse_labels=True)
+        elif dataset_name == "directory":
+            self.n_classes = cfg.dir_dataset_n_classes
+            dataset_class = DirectoryDataset
+            extra_args = dict(path=cfg.dir_dataset_name)
+        elif dataset_name == "cityscapes" and crop_type is None:
+            self.n_classes = 27
+            dataset_class = CityscapesSeg
+            extra_args = dict()
+        elif dataset_name == "cityscapes" and crop_type is not None:
+            self.n_classes = 27
+            dataset_class = CroppedDataset
+            extra_args = dict(dataset_name="cityscapes", crop_type=crop_type, crop_ratio=cfg.crop_ratio)
+        elif dataset_name == "cocostuff3":
+            self.n_classes = 3
+            dataset_class = Coco
+            extra_args = dict(coarse_labels=True, subset=6, exclude_things=True)
+        elif dataset_name == "cocostuff15":
+            self.n_classes = 15
+            dataset_class = Coco
+            extra_args = dict(coarse_labels=False, subset=7, exclude_things=True)
+        elif dataset_name == "cocostuff27" and crop_type is not None:
+            self.n_classes = 27
+            dataset_class = CroppedDataset
+            extra_args = dict(dataset_name="cocostuff27", crop_type=cfg.crop_type, crop_ratio=cfg.crop_ratio)
+        elif dataset_name == "cocostuff27" and crop_type is None:
+            self.n_classes = 27
+            dataset_class = Coco
+            extra_args = dict(coarse_labels=False, subset=None, exclude_things=False)
+            if image_set == "val":
+                extra_args["subset"] = 7
+        else:
+            raise ValueError("Unknown dataset: {}".format(dataset_name))
+
+        self.aug_geometric_transform = aug_geometric_transform
+        self.aug_photometric_transform = aug_photometric_transform
+
+        self.dataset = dataset_class(
+            root=pytorch_data_dir,
+            image_set=self.image_set,
+            transform=transform,
+            target_transform=target_transform, **extra_args)
+
+        if model_type_override is not None:
+            model_type = model_type_override
+        else:
+            model_type = cfg.model_type
+
+        nice_dataset_name = cfg.dir_dataset_name if dataset_name == "directory" else dataset_name
+        feature_cache_file = join(pytorch_data_dir, "nns", "nns_{}_{}_{}_{}_{}.npz".format(
+            model_type, nice_dataset_name, image_set, crop_type, cfg.res))
+        if pos_labels or pos_images:
+            if not os.path.exists(feature_cache_file) or compute_knns:
+                raise ValueError("could not find nn file {} please run precompute_knns".format(feature_cache_file))
+            else:
+                loaded = np.load(feature_cache_file)
+                self.nns = loaded["nns"]
+            assert len(self.dataset) == self.nns.shape[0]
+
+    def __len__(self):
+        return len(self.dataset)
+
+    def _set_seed(self, seed):
+        random.seed(seed)  # apply this seed to img tranfsorms
+        torch.manual_seed(seed)  # needed for torchvision 0.7
+
+    def __getitem__(self, ind):
+        pack = self.dataset[ind]
+
+        if self.pos_images or self.pos_labels:
+            ind_pos = self.nns[ind][torch.randint(low=1, high=self.num_neighbors + 1, size=[]).item()]
+            pack_pos = self.dataset[ind_pos]
+
+        seed = np.random.randint(2147483647)  # make a seed with numpy generator
+
+        self._set_seed(seed)
+        coord_entries = torch.meshgrid([torch.linspace(-1, 1, pack[0].shape[1]),
+                                        torch.linspace(-1, 1, pack[0].shape[2])])
+        coord = torch.cat([t.unsqueeze(0) for t in coord_entries], 0)
+
+        if self.extra_transform is not None:
+            extra_trans = self.extra_transform
+        else:
+            extra_trans = lambda i, x: x
+
+        def squeeze_tuple(label_raw):
+            if type(label_raw) == tuple:
+                return tuple(x.squeeze() for x in label_raw)
+            else:
+                return label_raw.squeeze()
+        ret = {
+            "ind": ind,
+            "img": extra_trans(ind, pack[0]),
+            "label": squeeze_tuple(extra_trans(ind, pack[1]))
+        }
+
+        if self.pos_images:
+            ret["img_pos"] = extra_trans(ind, pack_pos[0])
+            ret["ind_pos"] = ind_pos
+
+        if self.mask:
+            ret["mask"] = pack[2]
+
+        if self.pos_labels:
+            ret["label_pos"] = squeeze_tuple(extra_trans(ind, pack_pos[1]))
+            ret["mask_pos"] = pack_pos[2]
+
+        if self.aug_photometric_transform is not None:
+            img_aug = self.aug_photometric_transform(self.aug_geometric_transform(pack[0]))
+
+            self._set_seed(seed)
+            coord_aug = self.aug_geometric_transform(coord)
+
+            ret["img_aug"] = img_aug
+            ret["coord_aug"] = coord_aug.permute(1, 2, 0)
+
+        return ret

biomap/dataset_generator/__init__.py

@@ -0,0 +1,6 @@
+from .data_loader import DataLoader
+
+
+__all__ = [
+    'DataLoader',
+]

biomap/dataset_generator/data_loader.py

@@ -0,0 +1,356 @@
+from datetime import datetime
+import ee
+from func_timeout import func_set_timeout
+import pandas as pd
+from PIL import Image
+import requests
+import tempfile
+import io
+from tqdm import tqdm
+import functools
+import re  # Used in an eval statement
+from typing import List
+from typing import Union
+from typing import Any
+
+
+class DataLoader:
+    """
+    Main class for loading and exploring data from satellite images.
+    The goal is to load an ImageCollection and to filter that collection according to needs, with methods like
+    filter, filterDate, filterBounds, select. These will work just like earth engine's methods with the same names.
+
+    This class, just like earth engine, works with lazy loading and compute. This means that running filterBounds
+    will not actually filter the image collection until required, e.g. when counting the images by accessing .count
+    property.
+    However, it will only load once the information it needs, unless additional filtering is made.
+
+    This works thanks to the signal_change decorator. If you develop a new filtering method for this class,
+    you will need to decorate your method with @signal_change.
+    In addition, if you develop a new method that will require to run getInfo to actually load data from
+    Google Earth Engine, you will need to use _get_timeout_info(your object before getInfo). This will run
+    getInfo with a timeout (currently set to 10 seconds).
+    It is important to use a timeout to avoid unexpected run times.
+
+    Usage:
+    >>> dl = DataLoader(satellite_name="COPERNICUS/S2_SR",  \
+        start_date='2021-01-01',                            \
+        end_date='2021-01-15',                              \
+        bands=["TCI_R", "TCI_G", "TCI_B"],                  \
+        geographic_bounds=ee.Geometry.Point(*[5.238728194366604, 44.474864056855935]).buffer(500) \
+               )
+
+    Get a pandas dataframe with all pixel values as a timeseries:
+    >>> dl.getRegion(dl.bounds, 500)
+    >>> dl.region.head(2)
+    [Out]
+            id	longitude	latitude	time	B1	B2	B3	B4	B5	B6	...	WVP	SCL	TCI_R	TCI_G	TCI_B	MSK_CLDPRB	MSK_SNWPRB	QA10	QA20	QA60
+    0	20210102T104441_20210102T104435_T31TFK	5.234932	44.473344	2021-01-02 10:48:36.299	6297	5955	5768	5773	5965	5883	...	393	8	255	255	255	0	95	0	0	1024
+    1	20210104T103329_20210104T103331_T31TFK	5.234932	44.473344	2021-01-04 10:38:38.304	5547	5355	5184	5090	5254	5229	...	314	9	255	255	255	29	9	0	0	1024
+
+    >>> dl.date_range
+    [Out]
+    {'max': datetime.datetime(2021, 1, 14, 11, 38, 39, 208000),
+     'min': datetime.datetime(2021, 1, 2, 11, 48, 36, 299000)}
+
+    >>> dl.count
+    [Out]
+    6
+
+    >>> dl.collection_info  # constains a html description of the dataset in "description"
+
+    >>> dl.image_ids
+    [Out]
+    ['COPERNICUS/S2_SR/20210102T104441_20210102T104435_T31TFK',
+     'COPERNICUS/S2_SR/20210104T103329_20210104T103331_T31TFK',
+     'COPERNICUS/S2_SR/20210107T104329_20210107T104328_T31TFK',
+     'COPERNICUS/S2_SR/20210109T103421_20210109T103431_T31TFK',
+     'COPERNICUS/S2_SR/20210112T104411_20210112T104438_T31TFK',
+     'COPERNICUS/S2_SR/20210114T103309_20210114T103305_T31TFK']
+
+    # Download the image
+    >>> img = dl.download_image(dl.image_ids[3])
+
+    # Download all images as a list
+    >>> imgs = dl.download_all_images(scale=1)
+
+    """
+    def __init__(self,
+                 satellite_name: str,
+                 bands: Union[List, str] = None,
+                 start_date: str = None,
+                 end_date: str = None,
+                 geographic_bounds: ee.geometry = None,
+                 scale: int = 10,
+                 crs: str = "EPSG:32630"
+                 ):
+        """
+
+        Args:
+            satellite_name: satellite to use. Examples: COPERNICUS/S2_SR, COPERNICUS/CORINE/V20/100m.
+                See https://developers.google.com/earth-engine/datasets for the full list.
+            bands: list of bands to load.
+            start_date: lowest possible date. Might be lower than the actual date of the first picture.
+            end_date: Latest possible date.
+            geographic_bounds: Region of interest.
+        """
+        self.satellite_name = satellite_name
+        if isinstance(bands, str):
+            bands = [bands]
+        self.bands = bands if bands is not None else list()
+        if start_date is None or end_date is None:
+            assert (start_date is not None) and (end_date is not None), "start_date and end_date must both be provided"
+        self.start_date = start_date
+        self.end_date = end_date
+        self.bounds = geographic_bounds
+
+        # Lazy computed
+        self._available_images = None
+
+        # Start getting info from google cloud
+        if satellite_name:
+            self.image_collection = ee.ImageCollection(self.satellite_name)
+        if self.bounds:
+            self.filterBounds(self.bounds)
+        if self.start_date is not None:
+            self.filterDate(self.start_date, self.end_date)
+        self.scale = scale
+        self.crs = crs
+        self.image_list = None
+        self._df_image_list = None
+        self.image_collection_info = None
+        self._date_range = None
+        self.date_filter_change = False
+        self._count = None
+
+        # Bool for caching
+        self.filter_change = True
+        self._describe = None
+
+    def signal_change(func):
+        """Signals that additional filtering was performed. To be used
+        as a decorator."""
+        @functools.wraps(func)
+        def wrap(self, *args, **kwargs):
+            self.filter_change = True
+            self.date_filter_change = True
+            return func(self, *args, **kwargs)
+        return wrap
+
+    @staticmethod
+    @func_set_timeout(10)
+    def _get_timeout_info(instance: Any):
+        """Runs getInfo on anything that is passed, with a timeout."""
+        return instance.getInfo()
+
+    @staticmethod
+    def _authenticate_gee():
+        """Authenticates earth engine if needed, and initializes."""
+        try:
+            ee.Initialize()
+        except Exception as e:
+            # Trigger the authentication flow.
+            ee.Authenticate()
+            # Initialize the library.
+            ee.Initialize()
+
+    def filter(self, ee_filter: ee.Filter):
+        """Applies a filter to the image_collection attribute. This can be useful for example
+        to filter out clouds
+
+        Args:
+            ee_filter: Filter to apply, must be an instance of ee.Filter.
+
+        Returns: self, for operation chaining as possible with the earth engine API.
+
+        """
+        self.image_collection = self.image_collection.filter(ee_filter)
+
+        return self
+
+    @property
+    def count(self):
+        """Number of images in the ImageCollection"""
+        if self.filter_change or self._count is None:
+            self._count = self._get_timeout_info(self.image_collection.size())
+            self.filter_change = False
+        return self._count
+
+    @property
+    def available_images(self):
+        """Gets the ImageCollection info"""
+        if self.filter_change or self._available_images is None:
+            self._available_images = self._get_timeout_info(self.image_collection)
+        return self._available_images
+
+    @signal_change
+    def filterDate(self, *args, **kwargs):
+        """Wrapper for the filterDate method in earth engine on the ImageCollection"""
+        self.image_collection = self.image_collection.filterDate(*args, **kwargs)
+        return self
+
+    @signal_change
+    def getRegion(self, *args, **kwargs):
+        """Wrapper for the getRegion method in earth engine on the ImageCollection.
+        Caveat! getRegion does not return an image collection, so the image_list attribute gets
+        updated instead of the image_collection attribute. However, the instance of the DataLoader class
+        is still returned, so this could be chained with another method on ImageCollection, which wouldn't be
+        possible using earth engine.
+        """
+        self.image_list = self.image_collection.getRegion(*args, **kwargs)
+        return self
+
+    @signal_change
+    def filterBounds(self, geometry, *args, **kwargs):
+        """Wrapper for the filterBounds method in earth engine on the ImageCollection"""
+        self.image_collection = self.image_collection.filterBounds(geometry, *args, **kwargs)
+        self.bounds = geometry
+        return self
+
+    @signal_change
+    def select(self, *bands, **kwargs):
+        """Wrapper for the select method in earth engine on the ImageCollection"""
+        self.image_collection = self.image_collection.select(*bands, **kwargs)
+        self.bands = list(set(self.bands) | set(bands))  # Unique bands
+        return self
+
+    @property
+    def date_range(self):
+        """Gets the actual date range of the images in the image collection."""
+        if self.date_filter_change or self._date_range is None:
+            date_range = self.image_collection.reduceColumns(ee.Reducer.minMax(), ["system:time_start"]).getInfo()
+            self._date_range = {key: datetime.fromtimestamp(value/1e3) for key, value in date_range.items()}
+            self.date_filter_change = False
+        return self._date_range
+
+    @property
+    def region(self):
+        """Gets a time series as a pandas DataFrame of the band values for the specified region."""
+        if self.filter_change:
+            if self.image_list is None:
+                self.getRegion()
+            res_list = self._get_timeout_info(self.image_list)
+            df = pd.DataFrame(res_list[1:], columns=res_list[0])
+            df.loc[:, "time"] = pd.to_datetime(df.loc[:, "time"], unit="ms")
+            self._df_image_list = df
+            self.filter_change = False
+        return self._df_image_list
+
+    @property
+    def collection_info(self):
+        """Runs getInfo on the image collection (the first time the next time the previously
+        populated attribute will be returned)."""
+        if self.count > 5000:
+            raise Exception("Too many images to load. Try filtering more")
+        if self.filter_change or self.image_collection_info is None:
+            self.image_collection_info = self._get_timeout_info(self.image_collection)
+        return self.image_collection_info
+
+    @property
+    def image_ids(self):
+        """list of names of available images in the image collection"""
+        return [i["id"] for i in self.collection_info["features"]]
+
+    def __repr__(self):
+        try:
+            return f"""
+Size: {self.count}
+
+Dataset date ranges:
+From: {self.date_range["min"]}
+To: {self.date_range["max"]}
+
+Selected bands:
+{self.bands}
+            
+            """
+        except Exception as e:
+            raise Exception("Impossible to represent the dataset. Try filtering more. Error handling to do.")
+
+    def reproject(self, image, **kwargs):
+        def resolve(name: str):
+            # Resolve crs
+            if name in kwargs:
+                item = kwargs[name]
+            elif getattr(self, name):
+                item = getattr(self, name)
+            else:
+                item = None
+            return item
+        crs = resolve("crs")
+        scale = resolve("scale")
+        if crs is not None or scale is not None:
+            image = image.reproject(crs, None, scale)
+        return image
+
+    def download_image(self, image_id: str, **kwargs):
+        """Downloads an image based on its id / name. The additional arguments are passed
+        to getThumbUrl, and could be scale, max, min...
+        """
+        img = ee.Image(image_id).select(*self.bands)
+        img = self.reproject(img, **kwargs)
+        input_args = {'region': self.bounds}
+        input_args.update(**kwargs)
+        all_bands = self.collection_info["features"][0]["bands"]
+        selected_bands = [band for i, band in enumerate(all_bands) if all_bands[i]["id"] in self.bands]
+        if "min" not in input_args:
+            input_args.update({"min": selected_bands[0]["data_type"]["min"]})
+        if "max" not in input_args:
+            input_args.update({"max": selected_bands[0]["data_type"]["max"]})
+        url = img.getThumbUrl(input_args)
+        buffer = tempfile.SpooledTemporaryFile(max_size=1e9)
+        r = requests.get(url, stream=True)
+        if r.status_code == 200:
+            downloaded = 0
+            # filesize = int(r.headers['content-length'])
+            for chunk in r.iter_content(chunk_size=1024):
+                downloaded += len(chunk)
+                buffer.write(chunk)
+            buffer.seek(0)
+            img = Image.open(io.BytesIO(buffer.read()))
+        buffer.close()
+        return img
+
+    @staticmethod
+    def _regex(regex: str, im_id_list: List[str], include: bool) -> list:
+        """
+        Filters the im_id_list based on a regular expression. This is useful before downloading
+        a collection of images. For example, using (.*)TXT with include=True will only download images
+        that end with TXT, wich for Nantes means filtering out empty or half empty images.
+        Args:
+            regex: python regex as a strng
+            im_id_list: list, image id list
+            include: whether to include or exclude elements that match the regex.
+
+        Returns: filtered list.
+
+        """
+        expression = "re.match('{regex}', '{im_id}') is not None"
+        if not include:
+            expression = "not " + expression
+        filtered_list = list()
+        for im_id in im_id_list:
+            if eval(expression.format(regex=regex, im_id=im_id)):
+                filtered_list.append(im_id)
+        return filtered_list
+
+    def download_all_images(self, regex_exclude: str = None, regex_include: str = None, **kwargs):
+        """
+        Runs download_image in a for loop around the available images.
+        Makes it possible to filter images to download based on a regex.
+        Args:
+            regex_exclude: any image that matches this regex will be excluded.
+            regex_include: any image that matches this regex will be included
+            **kwargs: arguments to be passed to getThumbUrl
+
+        Returns: list of PIL images
+        """
+        images = list()
+        image_ids = self.image_ids
+        if regex_exclude is not None:
+            image_ids = self._regex(regex_exclude, image_ids, include=False)
+        if regex_include is not None:
+            image_ids = self._regex(regex_include, image_ids, include=True)
+        for i in tqdm(range(len(image_ids))):
+            images.append(self.download_image(image_ids[i], **kwargs))
+        return images

biomap/dino/utils.py

@@ -0,0 +1,619 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Misc functions.
+
+Mostly copy-paste from torchvision references or other public repos like DETR:
+https://github.com/facebookresearch/detr/blob/master/util/misc.py
+"""
+import os
+import sys
+import time
+import math
+import random
+import datetime
+import subprocess
+from collections import defaultdict, deque
+
+import numpy as np
+import torch
+from torch import nn
+import torch.distributed as dist
+from PIL import ImageFilter, ImageOps
+
+
+class GaussianBlur(object):
+    """
+    Apply Gaussian Blur to the PIL image.
+    """
+    def __init__(self, p=0.5, radius_min=0.1, radius_max=2.):
+        self.prob = p
+        self.radius_min = radius_min
+        self.radius_max = radius_max
+
+    def __call__(self, img):
+        do_it = random.random() <= self.prob
+        if not do_it:
+            return img
+
+        return img.filter(
+            ImageFilter.GaussianBlur(
+                radius=random.uniform(self.radius_min, self.radius_max)
+            )
+        )
+
+
+class Solarization(object):
+    """
+    Apply Solarization to the PIL image.
+    """
+    def __init__(self, p):
+        self.p = p
+
+    def __call__(self, img):
+        if random.random() < self.p:
+            return ImageOps.solarize(img)
+        else:
+            return img
+
+
+def load_pretrained_weights(model, pretrained_weights, checkpoint_key, model_name, patch_size):
+    if os.path.isfile(pretrained_weights):
+        state_dict = torch.load(pretrained_weights, map_location="cpu")
+        if checkpoint_key is not None and checkpoint_key in state_dict:
+            print(f"Take key {checkpoint_key} in provided checkpoint dict")
+            state_dict = state_dict[checkpoint_key]
+        # remove `module.` prefix
+        state_dict = {k.replace("module.", ""): v for k, v in state_dict.items()}
+        # remove `backbone.` prefix induced by multicrop wrapper
+        state_dict = {k.replace("backbone.", ""): v for k, v in state_dict.items()}
+        msg = model.load_state_dict(state_dict, strict=False)
+        print('Pretrained weights found at {} and loaded with msg: {}'.format(pretrained_weights, msg))
+    else:
+        print("Please use the `--pretrained_weights` argument to indicate the path of the checkpoint to evaluate.")
+        url = None
+        if model_name == "vit_small" and patch_size == 16:
+            url = "dino_deitsmall16_pretrain/dino_deitsmall16_pretrain.pth"
+        elif model_name == "vit_small" and patch_size == 8:
+            url = "dino_deitsmall8_pretrain/dino_deitsmall8_pretrain.pth"
+        elif model_name == "vit_base" and patch_size == 16:
+            url = "dino_vitbase16_pretrain/dino_vitbase16_pretrain.pth"
+        elif model_name == "vit_base" and patch_size == 8:
+            url = "dino_vitbase8_pretrain/dino_vitbase8_pretrain.pth"
+        if url is not None:
+            print("Since no pretrained weights have been provided, we load the reference pretrained DINO weights.")
+            state_dict = torch.hub.load_state_dict_from_url(url="https://dl.fbaipublicfiles.com/dino/" + url)
+            model.load_state_dict(state_dict, strict=True)
+        else:
+            print("There is no reference weights available for this model => We use random weights.")
+
+
+def clip_gradients(model, clip):
+    norms = []
+    for name, p in model.named_parameters():
+        if p.grad is not None:
+            param_norm = p.grad.data.norm(2)
+            norms.append(param_norm.item())
+            clip_coef = clip / (param_norm + 1e-6)
+            if clip_coef < 1:
+                p.grad.data.mul_(clip_coef)
+    return norms
+
+
+def cancel_gradients_last_layer(epoch, model, freeze_last_layer):
+    if epoch >= freeze_last_layer:
+        return
+    for n, p in model.named_parameters():
+        if "last_layer" in n:
+            p.grad = None
+
+
+def restart_from_checkpoint(ckp_path, run_variables=None, **kwargs):
+    """
+    Re-start from checkpoint
+    """
+    if not os.path.isfile(ckp_path):
+        return
+    print("Found checkpoint at {}".format(ckp_path))
+
+    # open checkpoint file
+    checkpoint = torch.load(ckp_path, map_location="cpu")
+
+    # key is what to look for in the checkpoint file
+    # value is the object to load
+    # example: {'state_dict': model}
+    for key, value in kwargs.items():
+        if key in checkpoint and value is not None:
+            try:
+                msg = value.load_state_dict(checkpoint[key], strict=False)
+                print("=> loaded {} from checkpoint '{}' with msg {}".format(key, ckp_path, msg))
+            except TypeError:
+                try:
+                    msg = value.load_state_dict(checkpoint[key])
+                    print("=> loaded {} from checkpoint '{}'".format(key, ckp_path))
+                except ValueError:
+                    print("=> failed to load {} from checkpoint '{}'".format(key, ckp_path))
+        else:
+            print("=> failed to load {} from checkpoint '{}'".format(key, ckp_path))
+
+    # re load variable important for the run
+    if run_variables is not None:
+        for var_name in run_variables:
+            if var_name in checkpoint:
+                run_variables[var_name] = checkpoint[var_name]
+
+
+def cosine_scheduler(base_value, final_value, epochs, niter_per_ep, warmup_epochs=0, start_warmup_value=0):
+    warmup_schedule = np.array([])
+    warmup_iters = warmup_epochs * niter_per_ep
+    if warmup_epochs > 0:
+        warmup_schedule = np.linspace(start_warmup_value, base_value, warmup_iters)
+
+    iters = np.arange(epochs * niter_per_ep - warmup_iters)
+    schedule = final_value + 0.5 * (base_value - final_value) * (1 + np.cos(np.pi * iters / len(iters)))
+
+    schedule = np.concatenate((warmup_schedule, schedule))
+    assert len(schedule) == epochs * niter_per_ep
+    return schedule
+
+
+def bool_flag(s):
+    """
+    Parse boolean arguments from the command line.
+    """
+    FALSY_STRINGS = {"off", "false", "0"}
+    TRUTHY_STRINGS = {"on", "true", "1"}
+    if s.lower() in FALSY_STRINGS:
+        return False
+    elif s.lower() in TRUTHY_STRINGS:
+        return True
+    else:
+        raise argparse.ArgumentTypeError("invalid value for a boolean flag")
+
+
+def fix_random_seeds(seed=31):
+    """
+    Fix random seeds.
+    """
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    np.random.seed(seed)
+
+
+class SmoothedValue(object):
+    """Track a series of values and provide access to smoothed values over a
+    window or the global series average.
+    """
+
+    def __init__(self, window_size=20, fmt=None):
+        if fmt is None:
+            fmt = "{median:.6f} ({global_avg:.6f})"
+        self.deque = deque(maxlen=window_size)
+        self.total = 0.0
+        self.count = 0
+        self.fmt = fmt
+
+    def update(self, value, n=1):
+        self.deque.append(value)
+        self.count += n
+        self.total += value * n
+
+    def synchronize_between_processes(self):
+        """
+        Warning: does not synchronize the deque!
+        """
+        if not is_dist_avail_and_initialized():
+            return
+        t = torch.tensor([self.count, self.total], dtype=torch.float64, device='cuda')
+        dist.barrier()
+        dist.all_reduce(t)
+        t = t.tolist()
+        self.count = int(t[0])
+        self.total = t[1]
+
+    @property
+    def median(self):
+        d = torch.tensor(list(self.deque))
+        return d.median().item()
+
+    @property
+    def avg(self):
+        d = torch.tensor(list(self.deque), dtype=torch.float32)
+        return d.mean().item()
+
+    @property
+    def global_avg(self):
+        return self.total / self.count
+
+    @property
+    def max(self):
+        return max(self.deque)
+
+    @property
+    def value(self):
+        return self.deque[-1]
+
+    def __str__(self):
+        return self.fmt.format(
+            median=self.median,
+            avg=self.avg,
+            global_avg=self.global_avg,
+            max=self.max,
+            value=self.value)
+
+
+def reduce_dict(input_dict, average=True):
+    """
+    Args:
+        input_dict (dict): all the values will be reduced
+        average (bool): whether to do average or sum
+    Reduce the values in the dictionary from all processes so that all processes
+    have the averaged results. Returns a dict with the same fields as
+    input_dict, after reduction.
+    """
+    world_size = get_world_size()
+    if world_size < 2:
+        return input_dict
+    with torch.no_grad():
+        names = []
+        values = []
+        # sort the keys so that they are consistent across processes
+        for k in sorted(input_dict.keys()):
+            names.append(k)
+            values.append(input_dict[k])
+        values = torch.stack(values, dim=0)
+        dist.all_reduce(values)
+        if average:
+            values /= world_size
+        reduced_dict = {k: v for k, v in zip(names, values)}
+    return reduced_dict
+
+
+class MetricLogger(object):
+    def __init__(self, delimiter="\t"):
+        self.meters = defaultdict(SmoothedValue)
+        self.delimiter = delimiter
+
+    def update(self, **kwargs):
+        for k, v in kwargs.items():
+            if isinstance(v, torch.Tensor):
+                v = v.item()
+            assert isinstance(v, (float, int))
+            self.meters[k].update(v)
+
+    def __getattr__(self, attr):
+        if attr in self.meters:
+            return self.meters[attr]
+        if attr in self.__dict__:
+            return self.__dict__[attr]
+        raise AttributeError("'{}' object has no attribute '{}'".format(
+            type(self).__name__, attr))
+
+    def __str__(self):
+        loss_str = []
+        for name, meter in self.meters.items():
+            loss_str.append(
+                "{}: {}".format(name, str(meter))
+            )
+        return self.delimiter.join(loss_str)
+
+    def synchronize_between_processes(self):
+        for meter in self.meters.values():
+            meter.synchronize_between_processes()
+
+    def add_meter(self, name, meter):
+        self.meters[name] = meter
+
+    def log_every(self, iterable, print_freq, header=None):
+        i = 0
+        if not header:
+            header = ''
+        start_time = time.time()
+        end = time.time()
+        iter_time = SmoothedValue(fmt='{avg:.6f}')
+        data_time = SmoothedValue(fmt='{avg:.6f}')
+        space_fmt = ':' + str(len(str(len(iterable)))) + 'd'
+        if torch.cuda.is_available():
+            log_msg = self.delimiter.join([
+                header,
+                '[{0' + space_fmt + '}/{1}]',
+                'eta: {eta}',
+                '{meters}',
+                'time: {time}',
+                'data: {data}',
+                'max mem: {memory:.0f}'
+            ])
+        else:
+            log_msg = self.delimiter.join([
+                header,
+                '[{0' + space_fmt + '}/{1}]',
+                'eta: {eta}',
+                '{meters}',
+                'time: {time}',
+                'data: {data}'
+            ])
+        MB = 1024.0 * 1024.0
+        for obj in iterable:
+            data_time.update(time.time() - end)
+            yield obj
+            iter_time.update(time.time() - end)
+            if i % print_freq == 0 or i == len(iterable) - 1:
+                eta_seconds = iter_time.global_avg * (len(iterable) - i)
+                eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
+                if torch.cuda.is_available():
+                    print(log_msg.format(
+                        i, len(iterable), eta=eta_string,
+                        meters=str(self),
+                        time=str(iter_time), data=str(data_time),
+                        memory=torch.cuda.max_memory_allocated() / MB))
+                else:
+                    print(log_msg.format(
+                        i, len(iterable), eta=eta_string,
+                        meters=str(self),
+                        time=str(iter_time), data=str(data_time)))
+            i += 1
+            end = time.time()
+        total_time = time.time() - start_time
+        total_time_str = str(datetime.timedelta(seconds=int(total_time)))
+        print('{} Total time: {} ({:.6f} s / it)'.format(
+            header, total_time_str, total_time / len(iterable)))
+
+
+def get_sha():
+    cwd = os.path.dirname(os.path.abspath(__file__))
+
+    def _run(command):
+        return subprocess.check_output(command, cwd=cwd).decode('ascii').strip()
+    sha = 'N/A'
+    diff = "clean"
+    branch = 'N/A'
+    try:
+        sha = _run(['git', 'rev-parse', 'HEAD'])
+        subprocess.check_output(['git', 'diff'], cwd=cwd)
+        diff = _run(['git', 'diff-index', 'HEAD'])
+        diff = "has uncommited changes" if diff else "clean"
+        branch = _run(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
+    except Exception:
+        pass
+    message = f"sha: {sha}, status: {diff}, branch: {branch}"
+    return message
+
+
+def is_dist_avail_and_initialized():
+    if not dist.is_available():
+        return False
+    if not dist.is_initialized():
+        return False
+    return True
+
+
+def get_world_size():
+    if not is_dist_avail_and_initialized():
+        return 1
+    return dist.get_world_size()
+
+
+def get_rank():
+    if not is_dist_avail_and_initialized():
+        return 0
+    return dist.get_rank()
+
+
+def is_main_process():
+    return get_rank() == 0
+
+
+def save_on_master(*args, **kwargs):
+    if is_main_process():
+        torch.save(*args, **kwargs)
+
+
+def setup_for_distributed(is_master):
+    """
+    This function disables printing when not in master process
+    """
+    import builtins as __builtin__
+    builtin_print = __builtin__.print
+
+    def print(*args, **kwargs):
+        force = kwargs.pop('force', False)
+        if is_master or force:
+            builtin_print(*args, **kwargs)
+
+    __builtin__.print = print
+
+
+def init_distributed_mode(args):
+    # launched with torch.distributed.launch
+    if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ:
+        args.rank = int(os.environ["RANK"])
+        args.world_size = int(os.environ['WORLD_SIZE'])
+        args.gpu = int(os.environ['LOCAL_RANK'])
+    # launched with submitit on a slurm cluster
+    elif 'SLURM_PROCID' in os.environ:
+        args.rank = int(os.environ['SLURM_PROCID'])
+        args.gpu = args.rank % torch.cuda.device_count()
+    # launched naively with `python main_dino.py`
+    # we manually add MASTER_ADDR and MASTER_PORT to env variables
+    elif torch.cuda.is_available():
+        print('Will run the code on one GPU.')
+        args.rank, args.gpu, args.world_size = 0, 0, 1
+        os.environ['MASTER_ADDR'] = '127.0.0.1'
+        os.environ['MASTER_PORT'] = '29500'
+    else:
+        print('Does not support training without GPU.')
+        sys.exit(1)
+
+    dist.init_process_group(
+        backend="nccl",
+        init_method=args.dist_url,
+        world_size=args.world_size,
+        rank=args.rank,
+    )
+
+    torch.cuda.set_device(args.gpu)
+    print('| distributed init (rank {}): {}'.format(
+        args.rank, args.dist_url), flush=True)
+    dist.barrier()
+    setup_for_distributed(args.rank == 0)
+
+
+def accuracy(output, target, topk=(1,)):
+    """Computes the accuracy over the k top predictions for the specified values of k"""
+    maxk = max(topk)
+    batch_size = target.size(0)
+    _, pred = output.topk(maxk, 1, True, True)
+    pred = pred.t()
+    correct = pred.eq(target.reshape(1, -1).expand_as(pred))
+    return [correct[:k].reshape(-1).float().sum(0) * 100. / batch_size for k in topk]
+
+
+def _no_grad_trunc_normal_(tensor, mean, std, a, b):
+    # Cut & paste from PyTorch official master until it's in a few official releases - RW
+    # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
+    def norm_cdf(x):
+        # Computes standard normal cumulative distribution function
+        return (1. + math.erf(x / math.sqrt(2.))) / 2.
+
+    if (mean < a - 2 * std) or (mean > b + 2 * std):
+        warnings.warn("mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
+                      "The distribution of values may be incorrect.",
+                      stacklevel=2)
+
+    with torch.no_grad():
+        # Values are generated by using a truncated uniform distribution and
+        # then using the inverse CDF for the normal distribution.
+        # Get upper and lower cdf values
+        l = norm_cdf((a - mean) / std)
+        u = norm_cdf((b - mean) / std)
+
+        # Uniformly fill tensor with values from [l, u], then translate to
+        # [2l-1, 2u-1].
+        tensor.uniform_(2 * l - 1, 2 * u - 1)
+
+        # Use inverse cdf transform for normal distribution to get truncated
+        # standard normal
+        tensor.erfinv_()
+
+        # Transform to proper mean, std
+        tensor.mul_(std * math.sqrt(2.))
+        tensor.add_(mean)
+
+        # Clamp to ensure it's in the proper range
+        tensor.clamp_(min=a, max=b)
+        return tensor
+
+
+def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.):
+    # type: (Tensor, float, float, float, float) -> Tensor
+    return _no_grad_trunc_normal_(tensor, mean, std, a, b)
+
+
+class LARS(torch.optim.Optimizer):
+    """
+    Almost copy-paste from https://github.com/facebookresearch/barlowtwins/blob/main/main.py
+    """
+    def __init__(self, params, lr=0, weight_decay=0, momentum=0.9, eta=0.001,
+                 weight_decay_filter=None, lars_adaptation_filter=None):
+        defaults = dict(lr=lr, weight_decay=weight_decay, momentum=momentum,
+                        eta=eta, weight_decay_filter=weight_decay_filter,
+                        lars_adaptation_filter=lars_adaptation_filter)
+        super().__init__(params, defaults)
+
+    @torch.no_grad()
+    def step(self):
+        for g in self.param_groups:
+            for p in g['params']:
+                dp = p.grad
+
+                if dp is None:
+                    continue
+
+                if p.ndim != 1:
+                    dp = dp.add(p, alpha=g['weight_decay'])
+
+                if p.ndim != 1:
+                    param_norm = torch.norm(p)
+                    update_norm = torch.norm(dp)
+                    one = torch.ones_like(param_norm)
+                    q = torch.where(param_norm > 0.,
+                                    torch.where(update_norm > 0,
+                                                (g['eta'] * param_norm / update_norm), one), one)
+                    dp = dp.mul(q)
+
+                param_state = self.state[p]
+                if 'mu' not in param_state:
+                    param_state['mu'] = torch.zeros_like(p)
+                mu = param_state['mu']
+                mu.mul_(g['momentum']).add_(dp)
+
+                p.add_(mu, alpha=-g['lr'])
+
+
+class MultiCropWrapper(nn.Module):
+    """
+    Perform forward pass separately on each resolution input.
+    The inputs corresponding to a single resolution are clubbed and single
+    forward is run on the same resolution inputs. Hence we do several
+    forward passes = number of different resolutions used. We then
+    concatenate all the output features and run the head forward on these
+    concatenated features.
+    """
+    def __init__(self, backbone, head):
+        super(MultiCropWrapper, self).__init__()
+        # disable layers dedicated to ImageNet labels classification
+        backbone.fc, backbone.head = nn.Identity(), nn.Identity()
+        self.backbone = backbone
+        self.head = head
+
+    def forward(self, x):
+        # convert to list
+        if not isinstance(x, list):
+            x = [x]
+        idx_crops = torch.cumsum(torch.unique_consecutive(
+            torch.tensor([inp.shape[-1] for inp in x]),
+            return_counts=True,
+        )[1], 0)
+        start_idx = 0
+        for end_idx in idx_crops:
+            _out = self.backbone(torch.cat(x[start_idx: end_idx]))
+            if start_idx == 0:
+                output = _out
+            else:
+                output = torch.cat((output, _out))
+            start_idx = end_idx
+        # Run the head forward on the concatenated features.
+        return self.head(output)
+
+
+def get_params_groups(model):
+    regularized = []
+    not_regularized = []
+    for name, param in model.named_parameters():
+        if not param.requires_grad:
+            continue
+        # we do not regularize biases nor Norm parameters
+        if name.endswith(".bias") or len(param.shape) == 1:
+            not_regularized.append(param)
+        else:
+            regularized.append(param)
+    return [{'params': regularized}, {'params': not_regularized, 'weight_decay': 0.}]
+
+
+def has_batchnorms(model):
+    bn_types = (nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d, nn.SyncBatchNorm)
+    for name, module in model.named_modules():
+        if isinstance(module, bn_types):
+            return True
+    return False

biomap/dino/vision_transformer.py

@@ -0,0 +1,314 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Mostly copy-paste from timm library.
+https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py
+"""
+import math
+from functools import partial
+
+import torch
+import torch.nn as nn
+from dino.utils import trunc_normal_
+
+def drop_path(x, drop_prob: float = 0., training: bool = False):
+    if drop_prob == 0. or not training:
+        return x
+    keep_prob = 1 - drop_prob
+    shape = (x.shape[0],) + (1,) * (x.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
+    random_tensor = keep_prob + torch.rand(shape, dtype=x.dtype, device=x.device)
+    random_tensor.floor_()  # binarize
+    output = x.div(keep_prob) * random_tensor
+    return output
+
+
+class DropPath(nn.Module):
+    """Drop paths (Stochastic Depth) per sample  (when applied in main path of residual blocks).
+    """
+    def __init__(self, drop_prob=None):
+        super(DropPath, self).__init__()
+        self.drop_prob = drop_prob
+
+    def forward(self, x):
+        return drop_path(x, self.drop_prob, self.training)
+
+
+class Mlp(nn.Module):
+    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+
+class Attention(nn.Module):
+    def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.):
+        super().__init__()
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = qk_scale or head_dim ** -0.5
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+    def forward(self, x, return_qkv=False):
+        B, N, C = x.shape
+        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv[0], qkv[1], qkv[2]
+
+        attn = (q @ k.transpose(-2, -1)) * self.scale
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+
+        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x,attn, qkv
+        
+
+
+class Block(nn.Module):
+    def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,
+                 drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm):
+        super().__init__()
+        self.norm1 = norm_layer(dim)
+        self.attn = Attention(
+            dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
+
+    def forward(self, x, return_attention=False, return_qkv = False):
+        y, attn, qkv = self.attn(self.norm1(x))
+        if return_attention:
+            return attn
+        x = x + self.drop_path(y)
+        x = x + self.drop_path(self.mlp(self.norm2(x)))
+        if return_qkv:
+            return x,attn, qkv
+        return x
+
+
+class PatchEmbed(nn.Module):
+    """ Image to Patch Embedding
+    """
+    def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768):
+        super().__init__()
+        num_patches = (img_size // patch_size) * (img_size // patch_size)
+        self.img_size = img_size
+        self.patch_size = patch_size
+        self.num_patches = num_patches
+
+        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
+
+    def forward(self, x):
+        B, C, H, W = x.shape
+        x = self.proj(x).flatten(2).transpose(1, 2)
+        return x
+
+
+class VisionTransformer(nn.Module):
+    """ Vision Transformer """
+    def __init__(self, img_size=[224], patch_size=16, in_chans=3, num_classes=0, embed_dim=768, depth=12,
+                 num_heads=12, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0.,
+                 drop_path_rate=0., norm_layer=nn.LayerNorm, **kwargs):
+        super().__init__()
+
+        self.num_features = self.embed_dim = embed_dim
+
+        self.patch_embed = PatchEmbed(
+            img_size=img_size[0], patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
+        num_patches = self.patch_embed.num_patches
+
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
+        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))
+        self.pos_drop = nn.Dropout(p=drop_rate)
+
+        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
+        self.blocks = nn.ModuleList([
+            Block(
+                dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale,
+                drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer)
+            for i in range(depth)])
+        self.norm = norm_layer(embed_dim)
+
+        # Classifier head
+        self.head = nn.Linear(embed_dim, num_classes) if num_classes > 0 else nn.Identity()
+
+        trunc_normal_(self.pos_embed, std=.02)
+        trunc_normal_(self.cls_token, std=.02)
+        self.apply(self._init_weights)
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+
+    def interpolate_pos_encoding(self, x, w, h):
+        npatch = x.shape[1] - 1
+        N = self.pos_embed.shape[1] - 1
+        if npatch == N and w == h:
+            return self.pos_embed
+        class_pos_embed = self.pos_embed[:, 0]
+        patch_pos_embed = self.pos_embed[:, 1:]
+        dim = x.shape[-1]
+        w0 = w // self.patch_embed.patch_size
+        h0 = h // self.patch_embed.patch_size
+        # we add a small number to avoid floating point error in the interpolation
+        # see discussion at https://github.com/facebookresearch/dino/issues/8
+        w0, h0 = w0 + 0.1, h0 + 0.1
+        patch_pos_embed = nn.functional.interpolate(
+            patch_pos_embed.reshape(1, int(math.sqrt(N)), int(math.sqrt(N)), dim).permute(0, 3, 1, 2),
+            scale_factor=(w0 / math.sqrt(N), h0 / math.sqrt(N)),
+            mode='bicubic',
+        )
+        assert int(w0) == patch_pos_embed.shape[-2] and int(h0) == patch_pos_embed.shape[-1]
+        patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
+        return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1)
+
+    def prepare_tokens(self, x):
+        B, nc, w, h = x.shape
+        x = self.patch_embed(x)  # patch linear embedding
+
+        # add the [CLS] token to the embed patch tokens
+        cls_tokens = self.cls_token.expand(B, -1, -1)
+        x = torch.cat((cls_tokens, x), dim=1)
+
+        # add positional encoding to each token
+        x = x + self.interpolate_pos_encoding(x, w, h)
+
+        return self.pos_drop(x)
+
+    def forward(self, x):
+        x = self.prepare_tokens(x)
+        for blk in self.blocks:
+            x = blk(x)
+        x = self.norm(x)
+        return x[:, 0]
+
+    def forward_feats(self, x):
+        x = self.prepare_tokens(x)
+        for blk in self.blocks:
+            x = blk(x)
+        x = self.norm(x)
+        return x
+
+    def get_intermediate_feat(self, x, n=1):
+        x = self.prepare_tokens(x)
+        # we return the output tokens from the `n` last blocks
+        feat = []
+        attns = []
+        qkvs = []
+        for i, blk in enumerate(self.blocks):
+            x,attn,qkv = blk(x, return_qkv=True)
+            if len(self.blocks) - i <= n:
+                feat.append(self.norm(x))
+                qkvs.append(qkv)
+                attns.append(attn)
+        return feat, attns, qkvs
+
+    def get_last_selfattention(self, x):
+        x = self.prepare_tokens(x)
+        for i, blk in enumerate(self.blocks):
+            if i < len(self.blocks) - 1:
+                x = blk(x)
+            else:
+                # return attention of the last block
+                return blk(x, return_attention=True)
+
+    def get_intermediate_layers(self, x, n=1):
+        x = self.prepare_tokens(x)
+        # we return the output tokens from the `n` last blocks
+        output = []
+        for i, blk in enumerate(self.blocks):
+            x = blk(x)
+            if len(self.blocks) - i <= n:
+                output.append(self.norm(x))
+        return output
+
+
+def vit_tiny(patch_size=16, **kwargs):
+    model = VisionTransformer(
+        patch_size=patch_size, embed_dim=192, depth=12, num_heads=3, mlp_ratio=4,
+        qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)
+    return model
+
+
+def vit_small(patch_size=16, **kwargs):
+    model = VisionTransformer(
+        patch_size=patch_size, embed_dim=384, depth=12, num_heads=6, mlp_ratio=4,
+        qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)
+    return model
+
+
+def vit_base(patch_size=16, **kwargs):
+    model = VisionTransformer(
+        patch_size=patch_size, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4,
+        qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)
+    return model
+
+
+class DINOHead(nn.Module):
+    def __init__(self, in_dim, out_dim, use_bn=False, norm_last_layer=True, nlayers=3, hidden_dim=2048, bottleneck_dim=256):
+        super().__init__()
+        nlayers = max(nlayers, 1)
+        if nlayers == 1:
+            self.mlp = nn.Linear(in_dim, bottleneck_dim)
+        else:
+            layers = [nn.Linear(in_dim, hidden_dim)]
+            if use_bn:
+                layers.append(nn.BatchNorm1d(hidden_dim))
+            layers.append(nn.GELU())
+            for _ in range(nlayers - 2):
+                layers.append(nn.Linear(hidden_dim, hidden_dim))
+                if use_bn:
+                    layers.append(nn.BatchNorm1d(hidden_dim))
+                layers.append(nn.GELU())
+            layers.append(nn.Linear(hidden_dim, bottleneck_dim))
+            self.mlp = nn.Sequential(*layers)
+        self.apply(self._init_weights)
+        self.last_layer = nn.utils.weight_norm(nn.Linear(bottleneck_dim, out_dim, bias=False))
+        self.last_layer.weight_g.data.fill_(1)
+        if norm_last_layer:
+            self.last_layer.weight_g.requires_grad = False
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+
+    def forward(self, x):
+        x = self.mlp(x)
+        x = nn.functional.normalize(x, dim=-1, p=2)
+        x = self.last_layer(x)
+        return x

biomap/helper.py

@@ -0,0 +1,179 @@
+import torch.multiprocessing
+import torchvision.transforms as T
+import numpy as np
+from utils import transform_to_pil, create_video
+from utils_gee import extract_img, transform_ee_img
+from dateutil.relativedelta import relativedelta
+import datetime
+from dateutil.relativedelta import relativedelta
+import cv2
+
+from joblib import Parallel, cpu_count, delayed
+
+def get_image(location, d1, d2):
+    print(f"getting image for {d1} to {d2}")
+    try:
+        img = extract_img(location, d1, d2)
+        img_test = transform_ee_img(
+            img, max=0.3
+        )
+        return img_test
+    except Exception as err:
+        print(err)
+        return 
+        
+
+def inference_on_location(model, latitude = 2.98, longitude = 48.81, start_date=2020, end_date=2022):
+    """Performe an inference on the latitude and longitude between the start date and the end date
+
+    Args:
+        latitude (float): the latitude of the landscape
+        longitude (float): the longitude of the landscape
+        start_date (str): the start date for our inference
+        end_date (str): the end date for our inference
+        model (_type_, optional): _description_. Defaults to model.
+
+    Returns:            
+        img, labeled_img,biodiv_score: the original landscape, the labeled landscape and the biodiversity score and the landscape
+    """
+    assert end_date > start_date, "end date must be stricly higher than start date"
+    location = [float(latitude), float(longitude)]
+    
+    # Extract img numpy from earth engine and transform it to PIL img
+    dates = [datetime.datetime(start_date, 1, 1, 0, 0, 0)]
+    while dates[-1] < datetime.datetime(end_date, 1, 1, 0, 0, 0):
+        dates.append(dates[-1] + relativedelta(months=1))
+    
+    dates = [d.strftime("%Y-%m-%d") for d in dates]
+
+    all_image = Parallel(n_jobs=cpu_count(), prefer="threads")(delayed(get_image)(location, d1,d2) for d1, d2 in zip(dates[:-1],dates[1:]))
+    all_image = [image for image in all_image if image is not None]
+
+    
+        
+
+    # tensorize & normalize img
+    preprocess = T.Compose(
+        [
+            T.ToPILImage(),
+            T.Resize((320, 320)),
+            #    T.CenterCrop(224),
+            T.ToTensor(),
+            T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        ]
+    )
+
+    # Preprocess opened img
+    x = torch.stack([preprocess(imag) for imag in all_image]).cpu()
+
+    # launch inference on cpu
+    # x = torch.unsqueeze(x, dim=0).cpu()
+    model = model.cpu()
+
+    with torch.no_grad():
+        feats, code = model.net(x)
+        linear_pred = model.linear_probe(x, code)
+        linear_pred = linear_pred.argmax(1)
+        outputs = [{
+            "img": torch.unsqueeze(img, dim=0).detach().cpu(),
+            "linear_preds": torch.unsqueeze(linear_pred, dim=0).detach().cpu(),
+        } for img, linear_pred in zip(x, linear_pred)]
+    all_img = []
+    all_label = []
+    all_labeled_img = []
+    for output in outputs:
+        img, label, labeled_img = transform_to_pil(output)
+        all_img.append(img)
+        all_label.append(label)
+        all_labeled_img.append(labeled_img)
+
+    all_labeled_img = [np.array(pil_image)[:, :, ::-1] for pil_image in all_labeled_img]
+    create_video(all_labeled_img, output_path='output/output.mp4')
+    
+    # all_labeled_img = [np.array(pil_image)[:, :, ::-1] for pil_image in all_img]
+    # create_video(all_labeled_img, output_path='raw.mp4')
+    
+    return 'output.mp4'
+
+def inference_on_location_and_month(model, latitude = 2.98, longitude = 48.81, start_date = '2020-03-20'):
+    """Performe an inference on the latitude and longitude between the start date and the end date
+
+    Args:
+        latitude (float): the latitude of the landscape
+        longitude (float): the longitude of the landscape
+        start_date (str): the start date for our inference
+        end_date (str): the end date for our inference
+        model (_type_, optional): _description_. Defaults to model.
+
+    Returns:
+        img, labeled_img,biodiv_score: the original landscape, the labeled landscape and the biodiversity score and the landscape
+    """
+    location = [float(latitude), float(longitude)]
+    
+    # Extract img numpy from earth engine and transform it to PIL img
+    end_date = datetime.datetime.strptime(start_date, "%Y-%m-%d") + relativedelta(months=1)
+    end_date = datetime.datetime.strftime(end_date, "%Y-%m-%d")
+    img = extract_img(location, start_date, end_date)
+    img_test = transform_ee_img(
+        img, max=0.3
+    )  # max value is the value from numpy file that will be equal to 255
+
+    # tensorize & normalize img
+    preprocess = T.Compose(
+        [
+            T.ToPILImage(),
+            T.Resize((320, 320)),
+            #    T.CenterCrop(224),
+            T.ToTensor(),
+            T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        ]
+    )
+
+    # Preprocess opened img
+    x = preprocess(img_test)
+
+    # launch inference on cpu
+    x = torch.unsqueeze(x, dim=0).cpu()
+    model = model.cpu()
+
+    with torch.no_grad():
+        feats, code = model.net(x)
+        linear_pred = model.linear_probe(x, code)
+        linear_pred = linear_pred.argmax(1)
+        output = {
+            "img": x[: model.cfg.n_images].detach().cpu(),
+            "linear_preds": linear_pred[: model.cfg.n_images].detach().cpu(),
+        }
+    nb_values = []
+    for i in range(7):
+        nb_values.append(np.count_nonzero(output['linear_preds'][0] == i+1))
+    scores_init = [2,3,4,3,1,4,0]
+    score = sum(x * y for x, y in zip(scores_init, nb_values)) / sum(nb_values) / max(scores_init)
+
+    img, label, labeled_img = transform_to_pil(output)
+    return img, labeled_img,score
+
+
+if __name__ == "__main__":
+    import logging
+    import hydra
+
+
+    from model import LitUnsupervisedSegmenter
+    logging.basicConfig(filename='example.log', encoding='utf-8', level=logging.INFO)
+    # Initialize hydra with configs
+    hydra.initialize(config_path="configs", job_name="corine")
+    cfg = hydra.compose(config_name="my_train_config.yml")
+    logging.info(f"config : {cfg}")
+    # Load the model
+
+    nbclasses = cfg.dir_dataset_n_classes
+    model = LitUnsupervisedSegmenter(nbclasses, cfg)
+    logging.info(f"Model Initialiazed")
+    
+    model_path = "checkpoint/model/model.pt"
+    saved_state_dict = torch.load(model_path, map_location=torch.device("cpu"))
+    logging.info(f"Model weights Loaded")
+    model.load_state_dict(saved_state_dict)
+    logging.info(f"Model Loaded")
+    inference_on_location(model)

biomap/inference.py

@@ -0,0 +1,261 @@
+import torch.multiprocessing
+import torchvision.transforms as T
+from utils import transform_to_pil
+
+def inference(image, model):
+    # tensorize & normalize img
+    preprocess = T.Compose(
+        [
+            T.ToPILImage(),
+            T.Resize((320, 320)),
+            #    T.CenterCrop(224),
+            T.ToTensor(),
+            T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        ]
+    )
+
+    # Preprocess opened img
+    x = preprocess(image)
+
+    # launch inference on cpu
+    x = torch.unsqueeze(x, dim=0).cpu()
+    model = model.cpu()
+
+    with torch.no_grad():
+        feats, code = model.net(x)
+        linear_pred = model.linear_probe(x, code)
+        linear_pred = linear_pred.argmax(1)
+        output = {
+            "img": x[: model.cfg.n_images].detach().cpu(),
+            "linear_preds": linear_pred[: model.cfg.n_images].detach().cpu(),
+        }
+
+    img, label, labeled_img = transform_to_pil(output)
+    return img, labeled_img, label
+
+
+if __name__ == "__main__":
+    import hydra
+    from model import LitUnsupervisedSegmenter
+    from utils_gee import extract_img, transform_ee_img
+    latitude = 2.98
+    longitude = 48.81
+    start_date = '2020-03-20'
+    end_date = '2020-04-20'
+
+    location = [float(latitude), float(longitude)]
+    # Extract img numpy from earth engine and transform it to PIL img
+    img = extract_img(location, start_date, end_date)
+    image = transform_ee_img(
+        img, max=0.3
+    )  # max value is the value from numpy file that will be equal to 255
+    print("image loaded")
+    # Initialize hydra with configs
+    hydra.initialize(config_path="configs", job_name="corine")
+    cfg = hydra.compose(config_name="my_train_config.yml")
+
+    # Load the model
+    model_path = "checkpoint/model/model.pt"
+    saved_state_dict = torch.load(model_path, map_location=torch.device("cpu"))
+
+    nbclasses = cfg.dir_dataset_n_classes
+
+    model = LitUnsupervisedSegmenter(nbclasses, cfg)
+    print("model initialized")
+    model.load_state_dict(saved_state_dict)
+    print("model loaded")
+    # img.save("output/image.png")
+    img, labeled_img, label = inference(image, model)
+    img.save("output/img.png")
+    label.save("output/label.png")
+    labeled_img.save("output/labeled_img.png")
+
+
+
+
+# def get_list_date(start_date, end_date):
+#     """Get all the date between the start date and the end date
+
+#     Args:
+#         start_date (str): start date at the format '%Y-%m-%d'
+#         end_date (str): end date at the format '%Y-%m-%d'
+
+#     Returns:
+#         list[str]: all the date between the start date and the end date
+#     """
+#     start_date = datetime.datetime.strptime(start_date, "%Y-%m-%d").date()
+#     end_date = datetime.datetime.strptime(end_date, "%Y-%m-%d").date()
+#     list_date = [start_date]
+#     date = start_date
+#     while date < end_date:
+#         date = date + datetime.timedelta(days=1)
+#         list_date.append(date)
+#     list_date.append(end_date)
+#     list_date2 = [x.strftime("%Y-%m-%d") for x in list_date]
+#     return list_date2
+
+
+# def get_length_interval(start_date, end_date):
+#     """Return how many days there is between the start date and the end date
+
+#     Args:
+#         start_date (str): start date at the format '%Y-%m-%d'
+#         end_date (str): end date at the format '%Y-%m-%d'
+
+#     Returns:
+#         int : number of days between start date and the end date
+#     """
+#     try:
+#         return len(get_list_date(start_date, end_date))
+#     except ValueError:
+#         return 0
+
+
+# def infer_unique_date(latitude, longitude, date, model=model):
+#     """Perform an inference on a latitude and a longitude at a specific date
+
+#     Args:
+#         latitude (float): the latitude of the landscape
+#         longitude (float): the longitude of the landscape
+#         date (str): date for the inference at the format '%Y-%m-%d'
+#         model (_type_, optional): _description_. Defaults to model.
+
+#     Returns:
+#         img, labeled_img,biodiv_score: the original landscape, the labeled landscape and the biodiversity score and the landscape
+#     """
+#     start_date = date
+#     end_date = date
+#     location = [float(latitude), float(longitude)]
+#     # Extract img numpy from earth engine and transform it to PIL img
+#     img = extract_img(location, start_date, end_date)
+#     img_test = transform_ee_img(
+#         img, max=0.3
+#     )  # max value is the value from numpy file that will be equal to 255
+
+#     # tensorize & normalize img
+#     preprocess = T.Compose(
+#         [
+#             T.ToPILImage(),
+#             T.Resize((320, 320)),
+#             #    T.CenterCrop(224),
+#             T.ToTensor(),
+#             T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+#         ]
+#     )
+
+#     # Preprocess opened img
+#     x = preprocess(img_test)
+
+#     # launch inference on cpu
+#     x = torch.unsqueeze(x, dim=0).cpu()
+#     model = model.cpu()
+
+#     with torch.no_grad():
+#         feats, code = model.net(x)
+#         linear_pred = model.linear_probe(x, code)
+#         linear_pred = linear_pred.argmax(1)
+#         output = {
+#             "img": x[: model.cfg.n_images].detach().cpu(),
+#             "linear_preds": linear_pred[: model.cfg.n_images].detach().cpu(),
+#         }
+
+#     img, label, labeled_img = transform_to_pil(output)
+#     biodiv_score = compute_biodiv_score(labeled_img)
+#     return img, labeled_img, biodiv_score
+
+
+# def get_img_array(start_date, end_date, latitude, longitude, model=model):
+#     list_date = get_list_date(start_date, end_date)
+#     list_img = []
+#     for date in list_date:
+#         list_img.append(img)
+#     return list_img
+
+
+# def variable_outputs(start_date, end_date, latitude, longitude, day, model=model):
+#     """Perform an inference on the day number day starting from the start at the latitude and longitude selected
+
+#     Args:
+#         latitude (float): the latitude of the landscape
+#         longitude (float): the longitude of the landscape
+#         start_date (str): the start date for our inference
+#         end_date (str): the end date for our inference
+#         model (_type_, optional): _description_. Defaults to model.
+
+#     Returns:
+#         img,labeled_img,biodiv_score: the original landscape, the labeled landscape and the biodiversity score and the landscape at the selected, longitude, latitude and date
+#     """
+#     list_date = get_list_date(start_date, end_date)
+#     k = int(day)
+#     date = list_date[k]
+#     img, labeled_img, biodiv_score = infer_unique_date(
+#         latitude, longitude, date, model=model
+#     )
+#     return img, labeled_img, biodiv_score
+
+
+# def variable_outputs2(
+#     start_date, end_date, latitude, longitude, day_number, model=model
+# ):
+#     """Perform an inference on the day number day starting from the start at the latitude and longitude selected
+
+#     Args:
+#         latitude (float): the latitude of the landscape
+#         longitude (float): the longitude of the landscape
+#         start_date (str): the start date for our inference
+#         end_date (str): the end date for our inference
+#         model (_type_, optional): _description_. Defaults to model.
+
+#     Returns:
+#         list[img,labeled_img,biodiv_score]: the original landscape, the labeled landscape and the biodiversity score and the landscape at the selected, longitude, latitude and date
+#     """
+#     list_date = get_list_date(start_date, end_date)
+#     k = int(day_number)
+#     date = list_date[k]
+#     img, labeled_img, biodiv_score = infer_unique_date(
+#         latitude, longitude, date, model=model
+#     )
+#     return [img, labeled_img, biodiv_score]
+
+
+# def gif_maker(img_array):
+#     output_file = "test2.mkv"
+#     image_test = img_array[0]
+#     size = (320, 320)
+#     print(size)
+#     out = cv2.VideoWriter(
+#         output_file, cv2.VideoWriter_fourcc(*"avc1"), 15, frameSize=size
+#     )
+#     for i in range(len(img_array)):
+#         image = img_array[i]
+#         pix = np.array(image.getdata())
+#         out.write(pix)
+#     out.release()
+#     return output_file
+
+
+# def infer_multiple_date(start_date, end_date, latitude, longitude, model=model):
+#     """Perform an inference on all the dates between the start date and the end date at the latitude and longitude
+
+#     Args:
+#         latitude (float): the latitude of the landscape
+#         longitude (float): the longitude of the landscape
+#         start_date (str): the start date for our inference
+#         end_date (str): the end date for our inference
+#         model (_type_, optional): _description_. Defaults to model.
+
+#     Returns:
+#         list_img,list_labeled_img,list_biodiv_score: list of the original landscape, the labeled landscape and the biodiversity score and the landscape
+#     """
+#     list_date = get_list_date(start_date, end_date)
+#     list_img = []
+#     list_labeled_img = []
+#     list_biodiv_score = []
+#     for date in list_date:
+#         img, labeled_img, biodiv_score = infer_unique_date(
+#             latitude, longitude, date, model=model
+#         )
+#         list_img.append(img)
+#         list_labeled_img.append(labeled_img)
+#         list_biodiv_score.append(biodiv_score)
+#     return gif_maker(list_img), gif_maker(list_labeled_img), list_biodiv_score[0]

biomap/model.py

@@ -0,0 +1,453 @@
+from utils import *
+from modules import *
+from data import *
+import torch.nn.functional as F
+import pytorch_lightning as pl
+import torch.multiprocessing
+import seaborn as sns
+import unet
+
+class LitUnsupervisedSegmenter(pl.LightningModule):
+    def __init__(self, n_classes, cfg):
+        super().__init__()
+        self.cfg = cfg
+        self.n_classes = n_classes
+
+        if not cfg.continuous:
+            dim = n_classes
+        else:
+            dim = cfg.dim
+
+        data_dir = join(cfg.output_root, "data")
+        if cfg.arch == "feature-pyramid":
+            cut_model = load_model(cfg.model_type, data_dir).cuda()
+            self.net = FeaturePyramidNet(
+                cfg.granularity, cut_model, dim, cfg.continuous
+            )
+        elif cfg.arch == "dino":
+            self.net = DinoFeaturizer(dim, cfg)
+        else:
+            raise ValueError("Unknown arch {}".format(cfg.arch))
+
+        self.train_cluster_probe = ClusterLookup(dim, n_classes)
+
+        self.cluster_probe = ClusterLookup(dim, n_classes + cfg.extra_clusters)
+        #         self.linear_probe = nn.Conv2d(dim, n_classes, (1, 1))
+        #         self.linear_probe = nn.Sequential(OrderedDict([
+        #           ('conv1', nn.Conv2d(dim, 2*n_classes, (7, 7), padding='same')),
+        #           ('relu1', nn.ReLU()),
+        #           ('conv2', nn.Conv2d(2*n_classes, n_classes, (3, 3), padding='same'))
+        #         ]))
+        self.linear_probe = unet.AuxUNet(
+            enc_chs=(3, 32, 64, 128, 256),
+            dec_chs=(256, 128, 64, 32),
+            aux_ch=70,
+            num_class=n_classes,
+        )
+
+        self.decoder = nn.Conv2d(dim, self.net.n_feats, (1, 1))
+
+        self.cluster_metrics = UnsupervisedMetrics(
+            "test/cluster/", n_classes, cfg.extra_clusters, True
+        )
+        self.linear_metrics = UnsupervisedMetrics("test/linear/", n_classes, 0, False)
+
+        self.test_cluster_metrics = UnsupervisedMetrics(
+            "final/cluster/", n_classes, cfg.extra_clusters, True
+        )
+        self.test_linear_metrics = UnsupervisedMetrics(
+            "final/linear/", n_classes, 0, False
+        )
+
+        self.linear_probe_loss_fn = torch.nn.CrossEntropyLoss()
+        self.crf_loss_fn = ContrastiveCRFLoss(
+            cfg.crf_samples, cfg.alpha, cfg.beta, cfg.gamma, cfg.w1, cfg.w2, cfg.shift
+        )
+
+        self.contrastive_corr_loss_fn = ContrastiveCorrelationLoss(cfg)
+        for p in self.contrastive_corr_loss_fn.parameters():
+            p.requires_grad = False
+
+        self.automatic_optimization = False
+
+        if self.cfg.dataset_name.startswith("cityscapes"):
+            self.label_cmap = create_cityscapes_colormap()
+        else:
+            self.label_cmap = create_pascal_label_colormap()
+
+        self.val_steps = 0
+        self.save_hyperparameters()
+
+    def forward(self, x):
+        # in lightning, forward defines the prediction/inference actions
+        return self.net(x)[1]
+
+    def training_step(self, batch, batch_idx):
+        # training_step defined the train loop.
+        # It is independent of forward
+        net_optim, linear_probe_optim, cluster_probe_optim = self.optimizers()
+
+        net_optim.zero_grad()
+        linear_probe_optim.zero_grad()
+        cluster_probe_optim.zero_grad()
+
+        with torch.no_grad():
+            ind = batch["ind"]
+            img = batch["img"]
+            img_aug = batch["img_aug"]
+            coord_aug = batch["coord_aug"]
+            img_pos = batch["img_pos"]
+            label = batch["label"]
+            label_pos = batch["label_pos"]
+
+        feats, code = self.net(img)
+        if self.cfg.correspondence_weight > 0:
+            feats_pos, code_pos = self.net(img_pos)
+        log_args = dict(sync_dist=False, rank_zero_only=True)
+
+        if self.cfg.use_true_labels:
+            signal = one_hot_feats(label + 1, self.n_classes + 1)
+            signal_pos = one_hot_feats(label_pos + 1, self.n_classes + 1)
+        else:
+            signal = feats
+            signal_pos = feats_pos
+
+        loss = 0
+
+        should_log_hist = (
+            (self.cfg.hist_freq is not None)
+            and (self.global_step % self.cfg.hist_freq == 0)
+            and (self.global_step > 0)
+        )
+        if self.cfg.use_salience:
+            salience = batch["mask"].to(torch.float32).squeeze(1)
+            salience_pos = batch["mask_pos"].to(torch.float32).squeeze(1)
+        else:
+            salience = None
+            salience_pos = None
+
+        if self.cfg.correspondence_weight > 0:
+            (
+                pos_intra_loss,
+                pos_intra_cd,
+                pos_inter_loss,
+                pos_inter_cd,
+                neg_inter_loss,
+                neg_inter_cd,
+            ) = self.contrastive_corr_loss_fn(
+                signal,
+                signal_pos,
+                salience,
+                salience_pos,
+                code,
+                code_pos,
+            )
+
+            if should_log_hist:
+                self.logger.experiment.add_histogram(
+                    "intra_cd", pos_intra_cd, self.global_step
+                )
+                self.logger.experiment.add_histogram(
+                    "inter_cd", pos_inter_cd, self.global_step
+                )
+                self.logger.experiment.add_histogram(
+                    "neg_cd", neg_inter_cd, self.global_step
+                )
+            neg_inter_loss = neg_inter_loss.mean()
+            pos_intra_loss = pos_intra_loss.mean()
+            pos_inter_loss = pos_inter_loss.mean()
+            self.log("loss/pos_intra", pos_intra_loss, **log_args)
+            self.log("loss/pos_inter", pos_inter_loss, **log_args)
+            self.log("loss/neg_inter", neg_inter_loss, **log_args)
+            self.log("cd/pos_intra", pos_intra_cd.mean(), **log_args)
+            self.log("cd/pos_inter", pos_inter_cd.mean(), **log_args)
+            self.log("cd/neg_inter", neg_inter_cd.mean(), **log_args)
+
+            loss += (
+                self.cfg.pos_inter_weight * pos_inter_loss
+                + self.cfg.pos_intra_weight * pos_intra_loss
+                + self.cfg.neg_inter_weight * neg_inter_loss
+            ) * self.cfg.correspondence_weight
+
+        if self.cfg.rec_weight > 0:
+            rec_feats = self.decoder(code)
+            rec_loss = -(norm(rec_feats) * norm(feats)).sum(1).mean()
+            self.log("loss/rec", rec_loss, **log_args)
+            loss += self.cfg.rec_weight * rec_loss
+
+        if self.cfg.aug_alignment_weight > 0:
+            orig_feats_aug, orig_code_aug = self.net(img_aug)
+            downsampled_coord_aug = resize(
+                coord_aug.permute(0, 3, 1, 2), orig_code_aug.shape[2]
+            ).permute(0, 2, 3, 1)
+            aug_alignment = -torch.einsum(
+                "bkhw,bkhw->bhw",
+                norm(sample(code, downsampled_coord_aug)),
+                norm(orig_code_aug),
+            ).mean()
+            self.log("loss/aug_alignment", aug_alignment, **log_args)
+            loss += self.cfg.aug_alignment_weight * aug_alignment
+
+        if self.cfg.crf_weight > 0:
+            crf = self.crf_loss_fn(resize(img, 56), norm(resize(code, 56))).mean()
+            self.log("loss/crf", crf, **log_args)
+            loss += self.cfg.crf_weight * crf
+
+        flat_label = label.reshape(-1)
+        mask = (flat_label >= 0) & (flat_label < self.n_classes)
+
+        detached_code = torch.clone(code.detach())
+
+        #         pdb.set_trace()
+
+        linear_logits = self.linear_probe(img, detached_code)
+        linear_logits = F.interpolate(
+            linear_logits, label.shape[-2:], mode="bilinear", align_corners=False
+        )
+        linear_logits = linear_logits.permute(0, 2, 3, 1).reshape(-1, self.n_classes)
+        linear_loss = self.linear_probe_loss_fn(
+            linear_logits[mask], flat_label[mask]
+        ).mean()
+        loss += linear_loss
+        self.log("loss/linear", linear_loss, **log_args)
+
+        cluster_loss, cluster_probs = self.cluster_probe(detached_code, None)
+        loss += cluster_loss
+        self.log("loss/cluster", cluster_loss, **log_args)
+        self.log("loss/total", loss, **log_args)
+
+        self.manual_backward(loss)
+        net_optim.step()
+        cluster_probe_optim.step()
+        linear_probe_optim.step()
+
+        if (
+            self.cfg.reset_probe_steps is not None
+            and self.global_step == self.cfg.reset_probe_steps
+        ):
+            print("RESETTING PROBES")
+            self.linear_probe.reset_parameters()
+            self.cluster_probe.reset_parameters()
+            self.trainer.optimizers[1] = torch.optim.Adam(
+                list(self.linear_probe.parameters()), lr=5e-3
+            )
+            self.trainer.optimizers[2] = torch.optim.Adam(
+                list(self.cluster_probe.parameters()), lr=5e-3
+            )
+
+        if self.global_step % 2000 == 0 and self.global_step > 0:
+            print("RESETTING TFEVENT FILE")
+            # Make a new tfevent file
+            self.logger.experiment.close()
+            self.logger.experiment._get_file_writer()
+
+        return loss
+
+    def on_train_start(self):
+        tb_metrics = {**self.linear_metrics.compute(), **self.cluster_metrics.compute()}
+        self.logger.log_hyperparams(self.cfg, tb_metrics)
+
+    def validation_step(self, batch, batch_idx):
+        img = batch["img"]
+        label = batch["label"]
+        self.net.eval()
+
+        with torch.no_grad():
+            feats, code = self.net(img)
+
+            #             code = F.interpolate(code, label.shape[-2:], mode='bilinear', align_corners=False)
+            # linear_preds = self.linear_probe(code)
+            linear_preds = self.linear_probe(img, code)
+            linear_preds = linear_preds.argmax(1)
+            self.linear_metrics.update(linear_preds, label)
+
+            code = F.interpolate(
+                code, label.shape[-2:], mode="bilinear", align_corners=False
+            )
+            cluster_loss, cluster_preds = self.cluster_probe(code, None)
+            cluster_preds = cluster_preds.argmax(1)
+            self.cluster_metrics.update(cluster_preds, label)
+
+            return {
+                "img": img[: self.cfg.n_images].detach().cpu(),
+                "linear_preds": linear_preds[: self.cfg.n_images].detach().cpu(),
+                "cluster_preds": cluster_preds[: self.cfg.n_images].detach().cpu(),
+                "label": label[: self.cfg.n_images].detach().cpu(),
+            }
+
+    def validation_epoch_end(self, outputs) -> None:
+        super().validation_epoch_end(outputs)
+        with torch.no_grad():
+            tb_metrics = {
+                **self.linear_metrics.compute(),
+                **self.cluster_metrics.compute(),
+            }
+
+            if self.trainer.is_global_zero and not self.cfg.submitting_to_aml:
+                # output_num = 0
+                output_num = random.randint(0, len(outputs) - 1)
+                output = {k: v.detach().cpu() for k, v in outputs[output_num].items()}
+
+                #                 pdb.set_trace()
+                alpha = 0.4
+                n_rows = 6
+                fig, ax = plt.subplots(
+                    n_rows,
+                    self.cfg.n_images,
+                    figsize=(self.cfg.n_images * 3, n_rows * 3),
+                )
+                for i in range(self.cfg.n_images):
+                    try:
+                        rbg_img = prep_for_plot(output["img"][i])
+                        true_label = output["label"].squeeze()[i]
+                        true_label[true_label == -1] = 7
+                    except:
+                        continue
+                    #                     ax[0, i].imshow(prep_for_plot(output["img"][i]))
+                    #                     ax[1, i].imshow(self.label_cmap[output["label"].squeeze()[i]])
+                    #                     ax[2, i].imshow(self.label_cmap[output["linear_preds"][i]])
+                    #                     ax[3, i].imshow(self.label_cmap[self.cluster_metrics.map_clusters(output["cluster_preds"][i])])
+                    ax[0, i].imshow(rbg_img)
+
+                    ax[1, i].imshow(rbg_img)
+                    ax[1, i].imshow(true_label, alpha=alpha, cmap=cmap, norm=norm)
+
+                    ax[2, i].imshow(rbg_img)
+                    pred_label = output["linear_preds"][i]
+                    ax[2, i].imshow(pred_label, alpha=alpha, cmap=cmap, norm=norm)
+
+                    ax[3, i].imshow(rbg_img)
+                    retouched_label = retouch_label(pred_label.numpy(), true_label)
+                    ax[3, i].imshow(retouched_label, alpha=alpha, cmap=cmap, norm=norm)
+
+                    ax[4, i].imshow(rbg_img)
+                    pred_label = self.cluster_metrics.map_clusters(
+                        output["cluster_preds"][i]
+                    )
+                    ax[4, i].imshow(pred_label, alpha=alpha, cmap=cmap, norm=norm)
+                    #             ax[3, i].imshow(map_clusters_with_label(true_label, pred_label), alpha=0.5, cmap=cmap, norm=norm)
+
+                    ax[5, i].imshow(rbg_img)
+                    retouched_label = retouch_label(pred_label.numpy(), true_label)
+                    ax[5, i].imshow(retouched_label, alpha=alpha, cmap=cmap, norm=norm)
+
+                ax[0, 0].set_ylabel("Image", fontsize=16)
+                ax[1, 0].set_ylabel("Label", fontsize=16)
+                ax[2, 0].set_ylabel("UNet Probe", fontsize=16)
+                ax[3, 0].set_ylabel("Retouched UNet Probe", fontsize=16)
+                ax[4, 0].set_ylabel("Cluster Probe", fontsize=16)
+                ax[5, 0].set_ylabel("Retouched cluster Probe", fontsize=16)
+                remove_axes(ax)
+                plt.tight_layout()
+                add_plot(self.logger.experiment, "plot_labels", self.global_step)
+
+                if self.cfg.has_labels:
+                    fig = plt.figure(figsize=(13, 10))
+                    ax = fig.gca()
+                    hist = (
+                        self.cluster_metrics.histogram.detach().cpu().to(torch.float32)
+                    )
+                    hist /= torch.clamp_min(hist.sum(dim=0, keepdim=True), 1)
+                    sns.heatmap(hist.t(), annot=False, fmt="g", ax=ax, cmap="Blues")
+                    ax.set_xlabel("Predicted labels")
+                    ax.set_ylabel("True labels")
+                    names = get_class_labels(self.cfg.dataset_name)
+                    if self.cfg.extra_clusters:
+                        names = names + ["Extra"]
+                    ax.set_xticks(np.arange(0, len(names)) + 0.5)
+                    ax.set_yticks(np.arange(0, len(names)) + 0.5)
+                    ax.xaxis.tick_top()
+                    ax.xaxis.set_ticklabels(names, fontsize=14)
+                    ax.yaxis.set_ticklabels(names, fontsize=14)
+                    colors = [self.label_cmap[i] / 255.0 for i in range(len(names))]
+                    [
+                        t.set_color(colors[i])
+                        for i, t in enumerate(ax.xaxis.get_ticklabels())
+                    ]
+                    [
+                        t.set_color(colors[i])
+                        for i, t in enumerate(ax.yaxis.get_ticklabels())
+                    ]
+                    # ax.yaxis.get_ticklabels()[-1].set_color(self.label_cmap[0] / 255.0)
+                    # ax.xaxis.get_ticklabels()[-1].set_color(self.label_cmap[0] / 255.0)
+                    plt.xticks(rotation=90)
+                    plt.yticks(rotation=0)
+                    ax.vlines(
+                        np.arange(0, len(names) + 1),
+                        color=[0.5, 0.5, 0.5],
+                        *ax.get_xlim()
+                    )
+                    ax.hlines(
+                        np.arange(0, len(names) + 1),
+                        color=[0.5, 0.5, 0.5],
+                        *ax.get_ylim()
+                    )
+                    plt.tight_layout()
+                    add_plot(self.logger.experiment, "conf_matrix", self.global_step)
+
+                    all_bars = torch.cat(
+                        [
+                            self.cluster_metrics.histogram.sum(0).cpu(),
+                            self.cluster_metrics.histogram.sum(1).cpu(),
+                        ],
+                        axis=0,
+                    )
+                    ymin = max(all_bars.min() * 0.8, 1)
+                    ymax = all_bars.max() * 1.2
+
+                    fig, ax = plt.subplots(1, 2, figsize=(2 * 5, 1 * 4))
+                    ax[0].bar(
+                        range(self.n_classes + self.cfg.extra_clusters),
+                        self.cluster_metrics.histogram.sum(0).cpu(),
+                        tick_label=names,
+                        color=colors,
+                    )
+                    ax[0].set_ylim(ymin, ymax)
+                    ax[0].set_title("Label Frequency")
+                    ax[0].set_yscale("log")
+                    ax[0].tick_params(axis="x", labelrotation=90)
+
+                    ax[1].bar(
+                        range(self.n_classes + self.cfg.extra_clusters),
+                        self.cluster_metrics.histogram.sum(1).cpu(),
+                        tick_label=names,
+                        color=colors,
+                    )
+                    ax[1].set_ylim(ymin, ymax)
+                    ax[1].set_title("Cluster Frequency")
+                    ax[1].set_yscale("log")
+                    ax[1].tick_params(axis="x", labelrotation=90)
+
+                    plt.tight_layout()
+                    add_plot(
+                        self.logger.experiment, "label frequency", self.global_step
+                    )
+
+            if self.global_step > 2:
+                self.log_dict(tb_metrics)
+
+                if self.trainer.is_global_zero and self.cfg.azureml_logging:
+                    from azureml.core.run import Run
+
+                    run_logger = Run.get_context()
+                    for metric, value in tb_metrics.items():
+                        run_logger.log(metric, value)
+
+            self.linear_metrics.reset()
+            self.cluster_metrics.reset()
+
+    def configure_optimizers(self):
+        main_params = list(self.net.parameters())
+
+        if self.cfg.rec_weight > 0:
+            main_params.extend(self.decoder.parameters())
+
+        net_optim = torch.optim.Adam(main_params, lr=self.cfg.lr)
+        linear_probe_optim = torch.optim.Adam(
+            list(self.linear_probe.parameters()), lr=5e-3
+        )
+        cluster_probe_optim = torch.optim.Adam(
+            list(self.cluster_probe.parameters()), lr=5e-3
+        )
+
+        return net_optim, linear_probe_optim, cluster_probe_optim

biomap/modules.py

@@ -0,0 +1,472 @@
+import torch
+
+from utils import *
+import torch.nn.functional as F
+import dino.vision_transformer as vits
+
+import pdb
+
+class LambdaLayer(nn.Module):
+    def __init__(self, lambd):
+        super(LambdaLayer, self).__init__()
+        self.lambd = lambd
+
+    def forward(self, x):
+        return self.lambd(x)
+
+
+class DinoFeaturizer(nn.Module):
+
+    def __init__(self, dim, cfg):
+        super().__init__()
+        self.cfg = cfg
+        self.dim = dim
+        patch_size = self.cfg.dino_patch_size
+        self.patch_size = patch_size
+        self.feat_type = self.cfg.dino_feat_type
+        arch = self.cfg.model_type
+        self.model = vits.__dict__[arch](
+            patch_size=patch_size,
+            num_classes=0)
+        for p in self.model.parameters():
+            p.requires_grad = False
+        # pdb.set_trace()
+        self.model=self.model.cpu()
+        self.model.eval()
+        self.dropout = torch.nn.Dropout2d(p=.1)
+
+        if arch == "vit_small" and patch_size == 16:
+            url = "dino_deitsmall16_pretrain/dino_deitsmall16_pretrain.pth"
+        elif arch == "vit_small" and patch_size == 8:
+            url = "dino_deitsmall8_300ep_pretrain/dino_deitsmall8_300ep_pretrain.pth"
+        elif arch == "vit_base" and patch_size == 16:
+            url = "dino_vitbase16_pretrain/dino_vitbase16_pretrain.pth"
+        elif arch == "vit_base" and patch_size == 8:
+            url = "dino_vitbase8_pretrain/dino_vitbase8_pretrain.pth"
+        else:
+            raise ValueError("Unknown arch and patch size")
+
+        if cfg.pretrained_weights is not None:
+            state_dict = torch.load(cfg.pretrained_weights, map_location="cpu")
+            state_dict = state_dict["teacher"]
+            # remove `module.` prefix
+            state_dict = {k.replace("module.", ""): v for k, v in state_dict.items()}
+            # remove `backbone.` prefix induced by multicrop wrapper
+            state_dict = {k.replace("backbone.", ""): v for k, v in state_dict.items()}
+
+            # state_dict = {k.replace("projection_head", "mlp"): v for k, v in state_dict.items()}
+            # state_dict = {k.replace("prototypes", "last_layer"): v for k, v in state_dict.items()}
+
+            msg = self.model.load_state_dict(state_dict, strict=False)
+            print('Pretrained weights found at {} and loaded with msg: {}'.format(cfg.pretrained_weights, msg))
+        else:
+            print("Since no pretrained weights have been provided, we load the reference pretrained DINO weights.")
+            state_dict = torch.hub.load_state_dict_from_url(url="https://dl.fbaipublicfiles.com/dino/" + url)
+            self.model.load_state_dict(state_dict, strict=True)
+
+        if arch == "vit_small":
+            self.n_feats = 384
+        else:
+            self.n_feats = 768
+        self.cluster1 = self.make_clusterer(self.n_feats)
+        self.proj_type = cfg.projection_type
+        if self.proj_type == "nonlinear":
+            self.cluster2 = self.make_nonlinear_clusterer(self.n_feats)
+
+    def make_clusterer(self, in_channels):
+        return torch.nn.Sequential(
+            torch.nn.Conv2d(in_channels, self.dim, (1, 1)))  # ,
+
+    def make_nonlinear_clusterer(self, in_channels):
+        return torch.nn.Sequential(
+            torch.nn.Conv2d(in_channels, in_channels, (1, 1)),
+            torch.nn.ReLU(),
+            torch.nn.Conv2d(in_channels, self.dim, (1, 1)))
+
+    def forward(self, img, n=1, return_class_feat=False):
+        self.model.eval()
+        with torch.no_grad():
+            assert (img.shape[2] % self.patch_size == 0)
+            assert (img.shape[3] % self.patch_size == 0)
+
+            # get selected layer activations
+            feat, attn, qkv = self.model.get_intermediate_feat(img, n=n)
+            feat, attn, qkv = feat[0], attn[0], qkv[0]
+
+            feat_h = img.shape[2] // self.patch_size
+            feat_w = img.shape[3] // self.patch_size
+
+            if self.feat_type == "feat":
+                image_feat = feat[:, 1:, :].reshape(feat.shape[0], feat_h, feat_w, -1).permute(0, 3, 1, 2)
+            elif self.feat_type == "KK":
+                image_k = qkv[1, :, :, 1:, :].reshape(feat.shape[0], 6, feat_h, feat_w, -1)
+                B, H, I, J, D = image_k.shape
+                image_feat = image_k.permute(0, 1, 4, 2, 3).reshape(B, H * D, I, J)
+            else:
+                raise ValueError("Unknown feat type:{}".format(self.feat_type))
+
+            if return_class_feat:
+                return feat[:, :1, :].reshape(feat.shape[0], 1, 1, -1).permute(0, 3, 1, 2)
+
+        if self.proj_type is not None:
+            code = self.cluster1(self.dropout(image_feat))
+            if self.proj_type == "nonlinear":
+                code += self.cluster2(self.dropout(image_feat))
+        else:
+            code = image_feat
+
+        if self.cfg.dropout:
+            return self.dropout(image_feat), code
+        else:
+            return image_feat, code
+
+
+class ResizeAndClassify(nn.Module):
+
+    def __init__(self, dim: int, size: int, n_classes: int):
+        super(ResizeAndClassify, self).__init__()
+        self.size = size
+        self.predictor = torch.nn.Sequential(
+            torch.nn.Conv2d(dim, n_classes, (1, 1)),
+            torch.nn.LogSoftmax(1))
+
+    def forward(self, x):
+        return F.interpolate(self.predictor.forward(x), self.size, mode="bilinear", align_corners=False)
+
+
+class ClusterLookup(nn.Module):
+
+    def __init__(self, dim: int, n_classes: int):
+        super(ClusterLookup, self).__init__()
+        self.n_classes = n_classes
+        self.dim = dim
+        self.clusters = torch.nn.Parameter(torch.randn(n_classes, dim))
+
+    def reset_parameters(self):
+        with torch.no_grad():
+            self.clusters.copy_(torch.randn(self.n_classes, self.dim))
+
+    def forward(self, x, alpha, log_probs=False):
+        normed_clusters = F.normalize(self.clusters, dim=1)
+        normed_features = F.normalize(x, dim=1)
+        inner_products = torch.einsum("bchw,nc->bnhw", normed_features, normed_clusters)
+
+        if alpha is None:
+            cluster_probs = F.one_hot(torch.argmax(inner_products, dim=1), self.clusters.shape[0]) \
+                .permute(0, 3, 1, 2).to(torch.float32)
+        else:
+            cluster_probs = nn.functional.softmax(inner_products * alpha, dim=1)
+
+        cluster_loss = -(cluster_probs * inner_products).sum(1).mean()
+        if log_probs:
+            return nn.functional.log_softmax(inner_products * alpha, dim=1)
+        else:
+            return cluster_loss, cluster_probs
+
+
+class FeaturePyramidNet(nn.Module):
+
+    @staticmethod
+    def _helper(x):
+        # TODO remove this hard coded 56
+        return F.interpolate(x, 56, mode="bilinear", align_corners=False).unsqueeze(-1)
+
+    def make_clusterer(self, in_channels):
+        return torch.nn.Sequential(
+            torch.nn.Conv2d(in_channels, self.dim, (1, 1)),
+            LambdaLayer(FeaturePyramidNet._helper))
+
+    def make_nonlinear_clusterer(self, in_channels):
+        return torch.nn.Sequential(
+            torch.nn.Conv2d(in_channels, in_channels, (1, 1)),
+            torch.nn.ReLU(),
+            torch.nn.Conv2d(in_channels, in_channels, (1, 1)),
+            torch.nn.ReLU(),
+            torch.nn.Conv2d(in_channels, self.dim, (1, 1)),
+            LambdaLayer(FeaturePyramidNet._helper))
+
+    def __init__(self, granularity, cut_model, dim, continuous):
+        super(FeaturePyramidNet, self).__init__()
+        self.layer_nums = [5, 6, 7]
+        self.spatial_resolutions = [7, 14, 28, 56]
+        self.feat_channels = [2048, 1024, 512, 3]
+        self.extra_channels = [128, 64, 32, 32]
+        self.granularity = granularity
+        self.encoder = NetWithActivations(cut_model, self.layer_nums)
+        self.dim = dim
+        self.continuous = continuous
+        self.n_feats = self.dim
+
+        self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False)
+
+        assert granularity in {1, 2, 3, 4}
+        self.cluster1 = self.make_clusterer(self.feat_channels[0])
+        self.cluster1_nl = self.make_nonlinear_clusterer(self.feat_channels[0])
+
+        if granularity >= 2:
+            # self.conv1 = DoubleConv(self.feat_channels[0], self.extra_channels[0])
+            # self.conv2 = DoubleConv(self.extra_channels[0] + self.feat_channels[1], self.extra_channels[1])
+            self.conv2 = DoubleConv(self.feat_channels[0] + self.feat_channels[1], self.extra_channels[1])
+            self.cluster2 = self.make_clusterer(self.extra_channels[1])
+        if granularity >= 3:
+            self.conv3 = DoubleConv(self.extra_channels[1] + self.feat_channels[2], self.extra_channels[2])
+            self.cluster3 = self.make_clusterer(self.extra_channels[2])
+        if granularity >= 4:
+            self.conv4 = DoubleConv(self.extra_channels[2] + self.feat_channels[3], self.extra_channels[3])
+            self.cluster4 = self.make_clusterer(self.extra_channels[3])
+
+    def c(self, x, y):
+        return torch.cat([x, y], dim=1)
+
+    def forward(self, x):
+        with torch.no_grad():
+            feats = self.encoder(x)
+        low_res_feats = feats[self.layer_nums[-1]]
+
+        all_clusters = []
+
+        # all_clusters.append(self.cluster1(low_res_feats) + self.cluster1_nl(low_res_feats))
+        all_clusters.append(self.cluster1(low_res_feats))
+
+        if self.granularity >= 2:
+            # f1 = self.conv1(low_res_feats)
+            # f1_up = self.up(f1)
+            f1_up = self.up(low_res_feats)
+            f2 = self.conv2(self.c(f1_up, feats[self.layer_nums[-2]]))
+            all_clusters.append(self.cluster2(f2))
+        if self.granularity >= 3:
+            f2_up = self.up(f2)
+            f3 = self.conv3(self.c(f2_up, feats[self.layer_nums[-3]]))
+            all_clusters.append(self.cluster3(f3))
+        if self.granularity >= 4:
+            f3_up = self.up(f3)
+            final_size = self.spatial_resolutions[-1]
+            f4 = self.conv4(self.c(f3_up, F.interpolate(
+                x, (final_size, final_size), mode="bilinear", align_corners=False)))
+            all_clusters.append(self.cluster4(f4))
+
+        avg_code = torch.cat(all_clusters, 4).mean(4)
+
+        if self.continuous:
+            clusters = avg_code
+        else:
+            clusters = torch.log_softmax(avg_code, 1)
+
+        return low_res_feats, clusters
+
+
+class DoubleConv(nn.Module):
+    """(convolution => [BN] => ReLU) * 2"""
+
+    def __init__(self, in_channels, out_channels, mid_channels=None):
+        super().__init__()
+        if not mid_channels:
+            mid_channels = out_channels
+        self.double_conv = nn.Sequential(
+            nn.Conv2d(in_channels, mid_channels, kernel_size=3, padding=1),
+            nn.BatchNorm2d(mid_channels),
+            nn.ReLU(),
+            nn.Conv2d(mid_channels, out_channels, kernel_size=3, padding=1),
+            nn.BatchNorm2d(out_channels),
+            nn.ReLU()
+        )
+
+    def forward(self, x):
+        return self.double_conv(x)
+
+
+def norm(t):
+    return F.normalize(t, dim=1, eps=1e-10)
+
+
+def average_norm(t):
+    return t / t.square().sum(1, keepdim=True).sqrt().mean()
+
+
+def tensor_correlation(a, b):
+    return torch.einsum("nchw,ncij->nhwij", a, b)
+
+
+def sample(t: torch.Tensor, coords: torch.Tensor):
+    return F.grid_sample(t, coords.permute(0, 2, 1, 3), padding_mode='border', align_corners=True)
+
+
+@torch.jit.script
+def super_perm(size: int, device: torch.device):
+    perm = torch.randperm(size, device=device, dtype=torch.long)
+    perm[perm == torch.arange(size, device=device)] += 1
+    return perm % size
+
+
+def sample_nonzero_locations(t, target_size):
+    nonzeros = torch.nonzero(t)
+    coords = torch.zeros(target_size, dtype=nonzeros.dtype, device=nonzeros.device)
+    n = target_size[1] * target_size[2]
+    for i in range(t.shape[0]):
+        selected_nonzeros = nonzeros[nonzeros[:, 0] == i]
+        if selected_nonzeros.shape[0] == 0:
+            selected_coords = torch.randint(t.shape[1], size=(n, 2), device=nonzeros.device)
+        else:
+            selected_coords = selected_nonzeros[torch.randint(len(selected_nonzeros), size=(n,)), 1:]
+        coords[i, :, :, :] = selected_coords.reshape(target_size[1], target_size[2], 2)
+    coords = coords.to(torch.float32) / t.shape[1]
+    coords = coords * 2 - 1
+    return torch.flip(coords, dims=[-1])
+
+
+class ContrastiveCorrelationLoss(nn.Module):
+
+    def __init__(self, cfg, ):
+        super(ContrastiveCorrelationLoss, self).__init__()
+        self.cfg = cfg
+
+    def standard_scale(self, t):
+        t1 = t - t.mean()
+        t2 = t1 / t1.std()
+        return t2
+
+    def helper(self, f1, f2, c1, c2, shift):
+        with torch.no_grad():
+            # Comes straight from backbone which is currently frozen. this saves mem.
+            fd = tensor_correlation(norm(f1), norm(f2))
+
+            if self.cfg.pointwise:
+                old_mean = fd.mean()
+                fd -= fd.mean([3, 4], keepdim=True)
+                fd = fd - fd.mean() + old_mean
+
+        cd = tensor_correlation(norm(c1), norm(c2))
+
+        if self.cfg.zero_clamp:
+            min_val = 0.0
+        else:
+            min_val = -9999.0
+
+        if self.cfg.stabalize:
+            loss = - cd.clamp(min_val, .8) * (fd - shift)
+        else:
+            loss = - cd.clamp(min_val) * (fd - shift)
+
+        return loss, cd
+
+    def forward(self,
+                orig_feats: torch.Tensor, orig_feats_pos: torch.Tensor,
+                orig_salience: torch.Tensor, orig_salience_pos: torch.Tensor,
+                orig_code: torch.Tensor, orig_code_pos: torch.Tensor,
+                ):
+
+        coord_shape = [orig_feats.shape[0], self.cfg.feature_samples, self.cfg.feature_samples, 2]
+
+        if self.cfg.use_salience:
+            coords1_nonzero = sample_nonzero_locations(orig_salience, coord_shape)
+            coords2_nonzero = sample_nonzero_locations(orig_salience_pos, coord_shape)
+            coords1_reg = torch.rand(coord_shape, device=orig_feats.device) * 2 - 1
+            coords2_reg = torch.rand(coord_shape, device=orig_feats.device) * 2 - 1
+            mask = (torch.rand(coord_shape[:-1], device=orig_feats.device) > .1).unsqueeze(-1).to(torch.float32)
+            coords1 = coords1_nonzero * mask + coords1_reg * (1 - mask)
+            coords2 = coords2_nonzero * mask + coords2_reg * (1 - mask)
+        else:
+            coords1 = torch.rand(coord_shape, device=orig_feats.device) * 2 - 1
+            coords2 = torch.rand(coord_shape, device=orig_feats.device) * 2 - 1
+
+        feats = sample(orig_feats, coords1)
+        code = sample(orig_code, coords1)
+
+        feats_pos = sample(orig_feats_pos, coords2)
+        code_pos = sample(orig_code_pos, coords2)
+
+        pos_intra_loss, pos_intra_cd = self.helper(
+            feats, feats, code, code, self.cfg.pos_intra_shift)
+        pos_inter_loss, pos_inter_cd = self.helper(
+            feats, feats_pos, code, code_pos, self.cfg.pos_inter_shift)
+
+        neg_losses = []
+        neg_cds = []
+        for i in range(self.cfg.neg_samples):
+            perm_neg = super_perm(orig_feats.shape[0], orig_feats.device)
+            feats_neg = sample(orig_feats[perm_neg], coords2)
+            code_neg = sample(orig_code[perm_neg], coords2)
+            neg_inter_loss, neg_inter_cd = self.helper(
+                feats, feats_neg, code, code_neg, self.cfg.neg_inter_shift)
+            neg_losses.append(neg_inter_loss)
+            neg_cds.append(neg_inter_cd)
+        neg_inter_loss = torch.cat(neg_losses, axis=0)
+        neg_inter_cd = torch.cat(neg_cds, axis=0)
+
+        return (pos_intra_loss.mean(),
+                pos_intra_cd,
+                pos_inter_loss.mean(),
+                pos_inter_cd,
+                neg_inter_loss,
+                neg_inter_cd)
+
+
+class Decoder(nn.Module):
+    def __init__(self, code_channels, feat_channels):
+        super().__init__()
+        self.linear = torch.nn.Conv2d(code_channels, feat_channels, (1, 1))
+        self.nonlinear = torch.nn.Sequential(
+            torch.nn.Conv2d(code_channels, code_channels, (1, 1)),
+            torch.nn.ReLU(),
+            torch.nn.Conv2d(code_channels, code_channels, (1, 1)),
+            torch.nn.ReLU(),
+            torch.nn.Conv2d(code_channels, feat_channels, (1, 1)))
+
+    def forward(self, x):
+        return self.linear(x) + self.nonlinear(x)
+
+
+class NetWithActivations(torch.nn.Module):
+    def __init__(self, model, layer_nums):
+        super(NetWithActivations, self).__init__()
+        self.layers = nn.ModuleList(model.children())
+        self.layer_nums = []
+        for l in layer_nums:
+            if l < 0:
+                self.layer_nums.append(len(self.layers) + l)
+            else:
+                self.layer_nums.append(l)
+        self.layer_nums = set(sorted(self.layer_nums))
+
+    def forward(self, x):
+        activations = {}
+        for ln, l in enumerate(self.layers):
+            x = l(x)
+            if ln in self.layer_nums:
+                activations[ln] = x
+        return activations
+
+
+class ContrastiveCRFLoss(nn.Module):
+
+    def __init__(self, n_samples, alpha, beta, gamma, w1, w2, shift):
+        super(ContrastiveCRFLoss, self).__init__()
+        self.alpha = alpha
+        self.beta = beta
+        self.gamma = gamma
+        self.w1 = w1
+        self.w2 = w2
+        self.n_samples = n_samples
+        self.shift = shift
+
+    def forward(self, guidance, clusters):
+        device = clusters.device
+        assert (guidance.shape[0] == clusters.shape[0])
+        assert (guidance.shape[2:] == clusters.shape[2:])
+        h = guidance.shape[2]
+        w = guidance.shape[3]
+
+        coords = torch.cat([
+            torch.randint(0, h, size=[1, self.n_samples], device=device),
+            torch.randint(0, w, size=[1, self.n_samples], device=device)], 0)
+
+        selected_guidance = guidance[:, :, coords[0, :], coords[1, :]]
+        coord_diff = (coords.unsqueeze(-1) - coords.unsqueeze(1)).square().sum(0).unsqueeze(0)
+        guidance_diff = (selected_guidance.unsqueeze(-1) - selected_guidance.unsqueeze(2)).square().sum(1)
+
+        sim_kernel = self.w1 * torch.exp(- coord_diff / (2 * self.alpha) - guidance_diff / (2 * self.beta)) + \
+                     self.w2 * torch.exp(- coord_diff / (2 * self.gamma)) - self.shift
+
+        selected_clusters = clusters[:, :, coords[0, :], coords[1, :]]
+        cluster_sims = torch.einsum("nka,nkb->nab", selected_clusters, selected_clusters)
+        return -(cluster_sims * sim_kernel)

biomap/plot_functions.py

@@ -0,0 +1,778 @@
+from PIL import Image
+
+import hydra
+import matplotlib as mpl
+from utils import prep_for_plot
+
+import torch.multiprocessing
+import torchvision.transforms as T
+# import matplotlib.pyplot as plt
+from model import LitUnsupervisedSegmenter
+colors = ('red', 'palegreen', 'green', 'steelblue', 'blue', 'yellow', 'lightgrey')
+class_names = ('Buildings', 'Cultivation', 'Natural green', 'Wetland', 'Water', 'Infrastructure', 'Background')
+cmap = mpl.colors.ListedColormap(colors)
+#from train_segmentation import LitUnsupervisedSegmenter, cmap
+
+from utils_gee import extract_img, transform_ee_img
+
+import plotly.graph_objects as go
+import plotly.express as px
+import numpy as np
+from plotly.subplots import make_subplots
+
+import os    
+os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
+
+
+colors = ('red', 'palegreen', 'green', 'steelblue', 'blue', 'yellow', 'lightgrey')
+class_names = ('Buildings', 'Cultivation', 'Natural green', 'Wetland', 'Water', 'Infrastructure', 'Background')
+scores_init = [2,3,4,3,1,4,0]
+
+# Import model configs
+hydra.initialize(config_path="configs",  job_name="corine")
+cfg = hydra.compose(config_name="my_train_config.yml")
+
+nbclasses = cfg.dir_dataset_n_classes
+
+# Load Model
+model_path = "checkpoint/model/model.pt"
+saved_state_dict = torch.load(model_path,map_location=torch.device('cpu'))
+
+model = LitUnsupervisedSegmenter(nbclasses, cfg)
+model.load_state_dict(saved_state_dict)
+
+from PIL import Image
+
+import hydra
+
+from utils import prep_for_plot
+
+import torch.multiprocessing
+import torchvision.transforms as T
+# import matplotlib.pyplot as plt   
+
+from model import LitUnsupervisedSegmenter
+
+from utils_gee import extract_img, transform_ee_img
+
+import plotly.graph_objects as go
+import plotly.express as px
+import numpy as np
+from plotly.subplots import make_subplots
+
+import os    
+os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
+
+
+colors = ('red', 'palegreen', 'green', 'steelblue', 'blue', 'yellow', 'lightgrey')
+cmap = mpl.colors.ListedColormap(colors)
+class_names = ('Buildings', 'Cultivation', 'Natural green', 'Wetland', 'Water', 'Infrastructure', 'Background')
+scores_init = [2,3,4,3,1,4,0]
+
+# Import model configs
+#hydra.initialize(config_path="configs",  job_name="corine")
+cfg = hydra.compose(config_name="my_train_config.yml")
+
+nbclasses = cfg.dir_dataset_n_classes
+
+# Load Model
+model_path = "checkpoint/model/model.pt"
+saved_state_dict = torch.load(model_path,map_location=torch.device('cpu'))
+
+model = LitUnsupervisedSegmenter(nbclasses, cfg)
+model.load_state_dict(saved_state_dict)
+
+
+#normalize img
+preprocess = T.Compose([
+   T.ToPILImage(),
+   T.Resize((320,320)),
+#    T.CenterCrop(224),
+   T.ToTensor(),
+   T.Normalize(
+       mean=[0.485, 0.456, 0.406],
+       std=[0.229, 0.224, 0.225]
+   )
+])
+
+# Function that look for img on EE and segment it
+# -- 3 ways possible to avoid cloudy environment -- monthly / bi-monthly / yearly meaned img
+
+def segment_loc(location, month, year, how = "month", month_end = '12', year_end = None) :
+    if how == 'month':
+        img = extract_img(location, year +'-'+ month +'-01', year +'-'+ month +'-28')
+    elif how == 'year' :
+        if year_end == None :
+            img = extract_img(location, year +'-'+ month +'-01', year +'-'+ month_end +'-28', width = 0.04 , len = 0.04)
+        else : 
+            img = extract_img(location, year +'-'+ month +'-01', year_end +'-'+ month_end +'-28', width = 0.04 , len = 0.04)
+            
+    
+    img_test= transform_ee_img(img, max = 0.25)
+
+    # Preprocess opened img
+    x = preprocess(img_test)
+    x = torch.unsqueeze(x, dim=0).cpu()
+    # model=model.cpu()
+
+    with torch.no_grad():
+        feats, code = model.net(x)
+        linear_preds = model.linear_probe(x, code)
+        linear_preds = linear_preds.argmax(1)
+        outputs = {
+            'img': x[:model.cfg.n_images].detach().cpu(),
+            'linear_preds': linear_preds[:model.cfg.n_images].detach().cpu()
+            }
+    return outputs
+
+
+# Function that look for all img on EE and extract all segments with the date as first output arg
+
+def segment_group(location, start_date, end_date, how = 'month') :
+    outputs = []
+    st_month = int(start_date[5:7])
+    end_month = int(end_date[5:7])
+    
+    st_year = int(start_date[0:4])
+    end_year = int(end_date[0:4])
+    
+
+
+    for year in range(st_year, end_year+1) : 
+        
+        if year != end_year :
+            last = 12
+        else :
+            last = end_month 
+
+        if year != st_year:
+            start = 1
+        else :
+            start = st_month
+
+        if how == 'month' :
+            for month in range(start, last + 1):
+                month_str = f"{month:0>2d}"
+                year_str = str(year)
+                
+                outputs.append((year_str + '-' + month_str, segment_loc(location, month_str, year_str)))
+        
+        elif how == 'year' :
+             outputs.append((str(year) + '-' + f"{start:0>2d}", segment_loc(location, f"{start:0>2d}", str(year), how = 'year', month_end=f"{last:0>2d}")))
+       
+        elif how == '2months' :
+            for month in range(start, last + 1):
+                month_str = f"{month:0>2d}"
+                year_str = str(year)
+                month_end = (month) % 12 +1
+                if month_end < month :
+                    year_end = year +1
+                else :
+                    year_end = year
+                month_end= f"{month_end:0>2d}"
+                year_end = str(year_end)
+                
+                outputs.append((year_str + '-' + month_str, segment_loc(location, month_str, year_str,how = 'year', month_end=month_end, year_end=year_end)))
+
+             
+    return outputs
+
+
+# Function that transforms an output to PIL images
+
+def transform_to_pil(outputs,alpha=0.3):
+    # Transform img with torch
+    img = torch.moveaxis(prep_for_plot(outputs['img'][0]),-1,0)
+    img=T.ToPILImage()(img)
+    
+    # Transform label by saving it then open it
+    # label = outputs['linear_preds'][0]
+    # plt.imsave('label.png',label,cmap=cmap)
+    # label = Image.open('label.png')
+
+    cmaplist = np.array([np.array(cmap(i)) for i in range(cmap.N)])
+    labels = np.array(outputs['linear_preds'][0])-1
+    label = T.ToPILImage()((cmaplist[labels]*255).astype(np.uint8))
+    
+
+    # Overlay labels with img wit alpha
+    background = img.convert("RGBA")
+    overlay = label.convert("RGBA")
+    
+    labeled_img = Image.blend(background, overlay, alpha)
+
+    return img, label, labeled_img
+
+
+
+# Function that extract labeled_img(PIL) and nb_values(number of pixels for each class) and the score for each observation
+
+def values_from_output(output):
+    imgs = transform_to_pil(output,alpha = 0.3)
+    
+    img = imgs[0]
+    img = np.array(img.convert('RGB'))
+
+    labeled_img = imgs[2]
+    labeled_img = np.array(labeled_img.convert('RGB'))
+    
+    nb_values = []
+    for i in range(7):
+        nb_values.append(np.count_nonzero(output['linear_preds'][0] == i+1))
+
+    score = sum(x * y for x, y in zip(scores_init, nb_values)) / sum(nb_values) / max(scores_init)    
+    
+    return img, labeled_img, nb_values, score
+
+
+# Function that extract from outputs (from segment_group function) all dates/ all images 
+def values_from_outputs(outputs) : 
+    months = []
+    imgs = []
+    imgs_label = []
+    nb_values = []
+    scores = []
+
+    for output in outputs:
+        img, labeled_img, nb_value, score = values_from_output(output[1])
+        months.append(output[0])
+        imgs.append(img)
+        imgs_label.append(labeled_img)
+        nb_values.append(nb_value)
+        scores.append(score)
+
+    return months, imgs, imgs_label, nb_values, scores
+
+
+
+def plot_imgs_labels(months, imgs, imgs_label, nb_values, scores) :       
+
+    fig2 = px.imshow(np.array(imgs), animation_frame=0, binary_string=True)
+    fig3 = px.imshow(np.array(imgs_label), animation_frame=0, binary_string=True)
+    
+    # Scores 
+    scatters = []
+    temp = []
+    for score in scores :
+        temp_score = []
+        temp_date = []
+        score = scores[i]
+        temp.append(score)
+        text_temp = ["" for i in temp]
+        text_temp[-1] = str(round(score,2))
+        scatters.append(go.Scatter(x=text_temp, y=temp, mode="lines+markers+text", marker_color="black", text = text_temp, textposition="top center"))
+        
+
+    # Scores 
+    fig = make_subplots(
+        rows=1, cols=4,
+        # specs=[[{"rowspan": 2}, {"rowspan": 2}, {"type": "pie"}, None]]
+        # row_heights=[0.8, 0.2],
+        column_widths = [0.6, 0.6,0.3, 0.3],
+        subplot_titles=("Localisation visualization", "labeled visualisation", "Segments repartition", "Biodiversity scores")
+    )
+
+    fig.add_trace(fig2["frames"][0]["data"][0], row=1, col=1)
+    fig.add_trace(fig3["frames"][0]["data"][0], row=1, col=2)
+
+    fig.add_trace(go.Pie(labels = class_names,
+                values = nb_values[0],
+                marker_colors = colors, 
+                name="Segment repartition",
+                textposition='inside',
+                texttemplate = "%{percent:.0%}",
+                textfont_size=14
+                ),
+                row=1, col=3)
+
+
+    fig.add_trace(scatters[0], row=1, col=4)
+    # fig.add_annotation(text='score:' + str(scores[0]), 
+    #                 showarrow=False,
+    #                 row=2, col=2)
+
+
+    number_frames = len(imgs)
+    frames = [dict(
+                name = k,
+                data = [ fig2["frames"][k]["data"][0],
+                        fig3["frames"][k]["data"][0],
+                        go.Pie(labels = class_names,
+                                values = nb_values[k],
+                                marker_colors = colors, 
+                                name="Segment repartition",
+                                textposition='inside',
+                                texttemplate = "%{percent:.0%}",
+                                textfont_size=14
+                                ),
+                        scatters[k]
+                        ],
+                traces=[0, 1,2,3] # the elements of the list [0,1,2] give info on the traces in fig.data
+                                        # that are updated by the above three go.Scatter instances
+                ) for k in range(number_frames)]
+
+    updatemenus = [dict(type='buttons',
+                        buttons=[dict(label='Play',
+                                    method='animate',
+                                    args=[[f'{k}' for k in range(number_frames)], 
+                                            dict(frame=dict(duration=500, redraw=False), 
+                                                transition=dict(duration=0),
+                                                easing='linear',
+                                                fromcurrent=True,
+                                                mode='immediate'
+                                                                    )])],
+                        direction= 'left', 
+                        pad=dict(r= 10, t=85), 
+                        showactive =True, x= 0.1, y= 0.13, xanchor= 'right', yanchor= 'top')
+                ]
+
+    sliders = [{'yanchor': 'top',
+                'xanchor': 'left', 
+                'currentvalue': {'font': {'size': 16}, 'prefix': 'Frame: ', 'visible': False, 'xanchor': 'right'},
+                'transition': {'duration': 500.0, 'easing': 'linear'},
+                'pad': {'b': 10, 't': 50}, 
+                'len': 0.9, 'x': 0.1, 'y': 0, 
+                'steps': [{'args': [[k], {'frame': {'duration': 500.0, 'easing': 'linear', 'redraw': False},
+                                        'transition': {'duration': 0, 'easing': 'linear'}}], 
+                        'label': months[k], 'method': 'animate'} for k in range(number_frames)       
+                        ]}]
+
+
+    fig.update(frames=frames)
+
+    for i,fr in enumerate(fig["frames"]):
+        fr.update(
+            layout={
+                "xaxis": {
+                            "range": [0,imgs[0].shape[1]+i/100000]
+                        },
+                "yaxis": {
+                            "range": [imgs[0].shape[0]+i/100000,0]
+                        },
+            })
+        
+        fr.update(layout_title_text= months[i])
+
+
+    fig.update(layout_title_text= 'tot')
+    fig.update(
+            layout={
+                "xaxis": {
+                            "range": [0,imgs[0].shape[1]+i/100000],
+                            'showgrid': False, # thin lines in the background
+                            'zeroline': False, # thick line at x=0
+                            'visible': False,  # numbers below
+                        },
+
+                "yaxis": {
+                            "range": [imgs[0].shape[0]+i/100000,0],
+                            'showgrid': False, # thin lines in the background
+                            'zeroline': False, # thick line at y=0
+                            'visible': False,},
+                
+                "xaxis3": {
+                            "range": [0,len(scores)+1],
+                            'autorange': False, # thin lines in the background
+                            'showgrid': False, # thin lines in the background
+                            'zeroline': False, # thick line at y=0
+                            'visible': False    
+                        },
+                
+                "yaxis3": {
+                            "range": [0,1.5],
+                            'autorange': False,
+                            'showgrid': False, # thin lines in the background
+                            'zeroline': False, # thick line at y=0
+                            'visible': False # thin lines in the background
+                         }   
+            },
+            legend=dict(
+                yanchor="bottom",
+                y=0.99,
+                xanchor="center",
+                x=0.01
+            )
+            )
+
+
+    fig.update_layout(updatemenus=updatemenus,
+                    sliders=sliders)
+
+    fig.update_layout(margin=dict(b=0, r=0))
+
+    # fig.show() #in jupyter notebook
+    
+    return fig
+
+
+
+# Last function (global one)
+# how = 'month' or '2months' or 'year' 
+
+def segment_region(location, start_date, end_date, how = 'month'):
+    
+    #extract the outputs for each image
+    outputs = segment_group(location, start_date, end_date, how = how)
+
+    #extract the intersting values from image
+    months, imgs, imgs_label, nb_values, scores = values_from_outputs(outputs)
+
+    #Create the figure
+    fig = plot_imgs_labels(months, imgs, imgs_label, nb_values, scores)
+    
+    return fig
+#normalize img
+preprocess = T.Compose([
+   T.ToPILImage(),
+   T.Resize((320,320)),
+#    T.CenterCrop(224),
+   T.ToTensor(),
+   T.Normalize(
+       mean=[0.485, 0.456, 0.406],
+       std=[0.229, 0.224, 0.225]
+   )
+])
+
+# Function that look for img on EE and segment it
+# -- 3 ways possible to avoid cloudy environment -- monthly / bi-monthly / yearly meaned img
+
+def segment_loc(location, month, year, how = "month", month_end = '12', year_end = None) :
+    if how == 'month':
+        img = extract_img(location, year +'-'+ month +'-01', year +'-'+ month +'-28')
+    elif how == 'year' :
+        if year_end == None :
+            img = extract_img(location, year +'-'+ month +'-01', year +'-'+ month_end +'-28', width = 0.04 , len = 0.04)
+        else : 
+            img = extract_img(location, year +'-'+ month +'-01', year_end +'-'+ month_end +'-28', width = 0.04 , len = 0.04)
+            
+    
+    img_test= transform_ee_img(img, max = 0.25)
+
+    # Preprocess opened img
+    x = preprocess(img_test)
+    x = torch.unsqueeze(x, dim=0).cpu()
+    # model=model.cpu()
+
+    with torch.no_grad():
+        feats, code = model.net(x)
+        linear_preds = model.linear_probe(x, code)
+        linear_preds = linear_preds.argmax(1)
+        outputs = {
+            'img': x[:model.cfg.n_images].detach().cpu(),
+            'linear_preds': linear_preds[:model.cfg.n_images].detach().cpu()
+            }
+    return outputs
+
+
+# Function that look for all img on EE and extract all segments with the date as first output arg
+
+def segment_group(location, start_date, end_date, how = 'month') :
+    outputs = []
+    st_month = int(start_date[5:7])
+    end_month = int(end_date[5:7])
+    
+    st_year = int(start_date[0:4])
+    end_year = int(end_date[0:4])
+
+
+
+    for year in range(st_year, end_year+1) : 
+    
+        if year != end_year :
+            last = 12
+        else :
+            last = end_month 
+
+        if year != st_year:
+            start = 1
+        else :
+            start = st_month
+
+        if how == 'month' :
+            for month in range(start, last + 1):
+                month_str = f"{month:0>2d}"
+                year_str = str(year)
+                
+                outputs.append((year_str + '-' + month_str, segment_loc(location, month_str, year_str)))
+        
+        elif how == 'year' :
+             outputs.append((str(year) + '-' + f"{start:0>2d}", segment_loc(location, f"{start:0>2d}", str(year), how = 'year', month_end=f"{last:0>2d}")))
+       
+        elif how == '2months' :
+            for month in range(start, last + 1):
+                month_str = f"{month:0>2d}"
+                year_str = str(year)
+                month_end = (month) % 12 +1
+                if month_end < month :
+                    year_end = year +1
+                else :
+                    year_end = year
+                month_end= f"{month_end:0>2d}"
+                year_end = str(year_end)
+                
+                outputs.append((year_str + '-' + month_str, segment_loc(location, month_str, year_str,how = 'year', month_end=month_end, year_end=year_end)))
+
+             
+    return outputs
+
+
+# Function that transforms an output to PIL images
+
+def transform_to_pil(outputs,alpha=0.3):
+    # Transform img with torch
+    img = torch.moveaxis(prep_for_plot(outputs['img'][0]),-1,0)
+    img=T.ToPILImage()(img)
+    
+    # Transform label by saving it then open it
+    # label = outputs['linear_preds'][0]
+    # plt.imsave('label.png',label,cmap=cmap)
+    # label = Image.open('label.png')
+
+    cmaplist = np.array([np.array(cmap(i)) for i in range(cmap.N)])
+    labels = np.array(outputs['linear_preds'][0])-1
+    label = T.ToPILImage()((cmaplist[labels]*255).astype(np.uint8))
+    
+
+    # Overlay labels with img wit alpha
+    background = img.convert("RGBA")
+    overlay = label.convert("RGBA")
+    
+    labeled_img = Image.blend(background, overlay, alpha)
+
+    return img, label, labeled_img
+
+def values_from_output(output):
+    imgs = transform_to_pil(output,alpha = 0.3)
+    
+    img = imgs[0]
+    img = np.array(img.convert('RGB'))
+
+    labeled_img = imgs[2]
+    labeled_img = np.array(labeled_img.convert('RGB'))
+    
+    nb_values = []
+    for i in range(7):
+        nb_values.append(np.count_nonzero(output['linear_preds'][0] == i+1))
+
+    score = sum(x * y for x, y in zip(scores_init, nb_values)) / sum(nb_values) / max(scores_init)    
+    
+    return img, labeled_img, nb_values, score
+
+
+# Function that extract labeled_img(PIL) and nb_values(number of pixels for each class) and the score for each observation
+
+
+
+# Function that extract from outputs (from segment_group function) all dates/ all images 
+def values_from_outputs(outputs) : 
+    months = []
+    imgs = []
+    imgs_label = []
+    nb_values = []
+    scores = []
+
+    for output in outputs:
+        img, labeled_img, nb_value, score = values_from_output(output[1])
+        months.append(output[0])
+        imgs.append(img)
+        imgs_label.append(labeled_img)
+        nb_values.append(nb_value)
+        scores.append(score)
+  
+    return months, imgs, imgs_label, nb_values, scores
+
+
+
+def plot_imgs_labels(months, imgs, imgs_label, nb_values, scores) :       
+
+    fig2 = px.imshow(np.array(imgs), animation_frame=0, binary_string=True)
+    fig3 = px.imshow(np.array(imgs_label), animation_frame=0, binary_string=True)
+    
+    # Scores 
+    scatters = []
+    temp = []
+    for score in scores :
+        temp_score = []
+        temp_date = []
+        #score = scores[i]
+        temp.append(score)
+        n = len(temp)
+        text_temp = ["" for i in temp]
+        text_temp[-1] = str(round(score,2))
+        scatters.append(go.Scatter(x=[0,1], y=temp, mode="lines+markers+text", marker_color="black", text = text_temp, textposition="top center"))
+        print(text_temp)
+
+    # Scores 
+    fig = make_subplots(
+        rows=1, cols=4,
+        specs=[[{"type": "image"},{"type": "image"}, {"type": "pie"}, {"type": "scatter"}]],
+        subplot_titles=("Localisation visualization", "Labeled visualisation", "Segments repartition", "Biodiversity scores")
+    )
+
+    fig.add_trace(fig2["frames"][0]["data"][0], row=1, col=1)
+    fig.add_trace(fig3["frames"][0]["data"][0], row=1, col=2)
+
+    fig.add_trace(go.Pie(labels = class_names,
+                values = nb_values[0],
+                marker_colors = colors, 
+                name="Segment repartition",
+                textposition='inside',
+                texttemplate = "%{percent:.0%}",
+                textfont_size=14
+                ),
+                row=1, col=3)
+
+
+    fig.add_trace(scatters[0], row=1, col=4)
+    fig.update_traces(showlegend=False, selector=dict(type='scatter'))
+    #fig.update_traces(, selector=dict(type='scatter'))
+    # fig.add_annotation(text='score:' + str(scores[0]), 
+    #                 showarrow=False,
+    #                 row=2, col=2)
+
+
+    number_frames = len(imgs)
+    frames = [dict(
+                name = k,
+                data = [ fig2["frames"][k]["data"][0],
+                        fig3["frames"][k]["data"][0],
+                        go.Pie(labels = class_names,
+                                values = nb_values[k],
+                                marker_colors = colors, 
+                                name="Segment repartition",
+                                textposition='inside',
+                                texttemplate = "%{percent:.0%}",
+                                textfont_size=14
+                                ),
+                        scatters[k]
+                        ],
+                traces=[0, 1,2,3] # the elements of the list [0,1,2] give info on the traces in fig.data
+                                        # that are updated by the above three go.Scatter instances
+                ) for k in range(number_frames)]
+
+    updatemenus = [dict(type='buttons',
+                        buttons=[dict(label='Play',
+                                    method='animate',
+                                    args=[[f'{k}' for k in range(number_frames)], 
+                                            dict(frame=dict(duration=500, redraw=False), 
+                                                transition=dict(duration=0),
+                                                easing='linear',
+                                                fromcurrent=True,
+                                                mode='immediate'
+                                                                    )])],
+                        direction= 'left', 
+                        pad=dict(r= 10, t=85), 
+                        showactive =True, x= 0.1, y= 0.13, xanchor= 'right', yanchor= 'top')
+                ]
+
+    sliders = [{'yanchor': 'top',
+                'xanchor': 'left', 
+                'currentvalue': {'font': {'size': 16}, 'prefix': 'Frame: ', 'visible': False, 'xanchor': 'right'},
+                'transition': {'duration': 500.0, 'easing': 'linear'},
+                'pad': {'b': 10, 't': 50}, 
+                'len': 0.9, 'x': 0.1, 'y': 0, 
+                'steps': [{'args': [[k], {'frame': {'duration': 500.0, 'easing': 'linear', 'redraw': False},
+                                        'transition': {'duration': 0, 'easing': 'linear'}}], 
+                        'label': months[k], 'method': 'animate'} for k in range(number_frames)       
+                        ]}]
+
+
+    fig.update(frames=frames)
+
+    for i,fr in enumerate(fig["frames"]):
+        fr.update(
+            layout={
+                "xaxis": {
+                            "range": [0,imgs[0].shape[1]+i/100000]
+                        },
+                "yaxis": {
+                            "range": [imgs[0].shape[0]+i/100000,0]
+                        },
+            })
+        
+        fr.update(layout_title_text= months[i])
+
+
+    fig.update(layout_title_text= months[0])
+    fig.update(
+            layout={
+                "xaxis": {
+                            "range": [0,imgs[0].shape[1]+i/100000],
+                            'showgrid': False, # thin lines in the background
+                            'zeroline': False, # thick line at x=0
+                            'visible': False,  # numbers below
+                        },
+
+                "yaxis": {
+                            "range": [imgs[0].shape[0]+i/100000,0],
+                            'showgrid': False, # thin lines in the background
+                            'zeroline': False, # thick line at y=0
+                            'visible': False,},
+                            
+                "xaxis2": {
+                            "range": [0,imgs[0].shape[1]+i/100000],
+                            'showgrid': False, # thin lines in the background
+                            'zeroline': False, # thick line at x=0
+                            'visible': False,  # numbers below
+                        },
+
+                "yaxis2": {
+                            "range": [imgs[0].shape[0]+i/100000,0],
+                            'showgrid': False, # thin lines in the background
+                            'zeroline': False, # thick line at y=0
+                            'visible': False,},
+                            
+                
+                "xaxis3": {
+                            "range": [0,len(scores)+1],
+                            'autorange': False, # thin lines in the background
+                            'showgrid': False, # thin lines in the background
+                            'zeroline': False, # thick line at y=0
+                            'visible': False    
+                        },
+                
+                "yaxis3": {
+                            "range": [0,1.5],
+                            'autorange': False,
+                            'showgrid': False, # thin lines in the background
+                            'zeroline': False, # thick line at y=0
+                            'visible': False # thin lines in the background
+                         }   
+            }
+            )
+
+
+    fig.update_layout(updatemenus=updatemenus,
+                    sliders=sliders,
+                    legend=dict(
+                        yanchor= 'top',
+                        xanchor= 'left', 
+                        orientation="h")
+    )
+
+
+    fig.update_layout(margin=dict(b=0, r=0))
+    
+    # fig.show() #in jupyter notebook
+    
+    return fig
+
+
+
+# Last function (global one)
+# how = 'month' or '2months' or 'year' 
+
+def segment_region(latitude, longitude, start_date, end_date, how = 'month'):
+    location = [float(latitude),float(longitude)]
+    how = how[0]
+    #extract the outputs for each image
+    outputs = segment_group(location, start_date, end_date, how = how)
+
+    #extract the intersting values from image
+    months, imgs, imgs_label, nb_values, scores = values_from_outputs(outputs)
+
+
+    #Create the figure
+    fig = plot_imgs_labels(months, imgs, imgs_label, nb_values, scores)
+    
+    return fig

biomap/train.py

@@ -0,0 +1,267 @@
+from utils import *
+from modules import *
+from data import *
+from torch.utils.data import DataLoader
+import torch.nn.functional as F
+from datetime import datetime
+import hydra
+from omegaconf import DictConfig, OmegaConf
+import pytorch_lightning as pl
+from pytorch_lightning import Trainer
+from pytorch_lightning.loggers import TensorBoardLogger
+from pytorch_lightning.utilities.seed import seed_everything
+import torch.multiprocessing
+import seaborn as sns
+from pytorch_lightning.callbacks import ModelCheckpoint
+import sys
+import pdb
+import matplotlib as mpl
+from skimage import measure
+from scipy.stats import mode as statsmode
+from collections import OrderedDict
+import unet
+import pdb
+
+torch.multiprocessing.set_sharing_strategy("file_system")
+colors = ("red", "palegreen", "green", "steelblue", "blue", "yellow", "lightgrey")
+class_names = (
+    "Buildings",
+    "Cultivation",
+    "Natural green",
+    "Wetland",
+    "Water",
+    "Infrastructure",
+    "Background",
+)
+bounds = list(np.arange(len(class_names) + 1) + 1)
+cmap = mpl.colors.ListedColormap(colors)
+norm = mpl.colors.BoundaryNorm(bounds, cmap.N)
+
+
+def retouch_label(pred_label, true_label):
+    retouched_label = pred_label + 0
+    blobs = measure.label(retouched_label)
+    for idx in np.unique(blobs):
+        #  most frequent label class in this blob
+        retouched_label[blobs == idx] = statsmode(true_label[blobs == idx])[0][0]
+    return retouched_label
+
+
+def get_class_labels(dataset_name):
+    if dataset_name.startswith("cityscapes"):
+        return [
+            "road",
+            "sidewalk",
+            "parking",
+            "rail track",
+            "building",
+            "wall",
+            "fence",
+            "guard rail",
+            "bridge",
+            "tunnel",
+            "pole",
+            "polegroup",
+            "traffic light",
+            "traffic sign",
+            "vegetation",
+            "terrain",
+            "sky",
+            "person",
+            "rider",
+            "car",
+            "truck",
+            "bus",
+            "caravan",
+            "trailer",
+            "train",
+            "motorcycle",
+            "bicycle",
+        ]
+    elif dataset_name == "cocostuff27":
+        return [
+            "electronic",
+            "appliance",
+            "food",
+            "furniture",
+            "indoor",
+            "kitchen",
+            "accessory",
+            "animal",
+            "outdoor",
+            "person",
+            "sports",
+            "vehicle",
+            "ceiling",
+            "floor",
+            "food",
+            "furniture",
+            "rawmaterial",
+            "textile",
+            "wall",
+            "window",
+            "building",
+            "ground",
+            "plant",
+            "sky",
+            "solid",
+            "structural",
+            "water",
+        ]
+    elif dataset_name == "voc":
+        return [
+            "background",
+            "aeroplane",
+            "bicycle",
+            "bird",
+            "boat",
+            "bottle",
+            "bus",
+            "car",
+            "cat",
+            "chair",
+            "cow",
+            "diningtable",
+            "dog",
+            "horse",
+            "motorbike",
+            "person",
+            "pottedplant",
+            "sheep",
+            "sofa",
+            "train",
+            "tvmonitor",
+        ]
+    elif dataset_name == "potsdam":
+        return ["roads and cars", "buildings and clutter", "trees and vegetation"]
+    else:
+        raise ValueError("Unknown Dataset {}".format(dataset_name))
+
+
+@hydra.main(config_path="configs", config_name="train_config.yml")
+def my_app(cfg: DictConfig) -> None:
+    OmegaConf.set_struct(cfg, False)
+    print(OmegaConf.to_yaml(cfg))
+    pytorch_data_dir = cfg.pytorch_data_dir
+    data_dir = join(cfg.output_root, "data")
+    log_dir = join(cfg.output_root, "logs")
+    checkpoint_dir = join(cfg.output_root, "checkpoints")
+
+    prefix = "{}/{}_{}".format(cfg.log_dir, cfg.dataset_name, cfg.experiment_name)
+    name = "{}_date_{}".format(prefix, datetime.now().strftime("%b%d_%H-%M-%S"))
+    cfg.full_name = prefix
+
+    os.makedirs(data_dir, exist_ok=True)
+    os.makedirs(log_dir, exist_ok=True)
+    os.makedirs(checkpoint_dir, exist_ok=True)
+
+    seed_everything(seed=0)
+
+    print(data_dir)
+    print(cfg.output_root)
+
+    geometric_transforms = T.Compose(
+        [T.RandomHorizontalFlip(), T.RandomResizedCrop(size=cfg.res, scale=(0.8, 1.0))]
+    )
+    photometric_transforms = T.Compose(
+        [
+            T.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0.1),
+            T.RandomGrayscale(0.2),
+            T.RandomApply([T.GaussianBlur((5, 5))]),
+        ]
+    )
+
+    sys.stdout.flush()
+
+    train_dataset = ContrastiveSegDataset(
+        pytorch_data_dir=pytorch_data_dir,
+        dataset_name=cfg.dataset_name,
+        crop_type=cfg.crop_type,
+        image_set="train",
+        transform=get_transform(cfg.res, False, cfg.loader_crop_type),
+        target_transform=get_transform(cfg.res, True, cfg.loader_crop_type),
+        cfg=cfg,
+        aug_geometric_transform=geometric_transforms,
+        aug_photometric_transform=photometric_transforms,
+        num_neighbors=cfg.num_neighbors,
+        mask=True,
+        pos_images=True,
+        pos_labels=True,
+    )
+
+    if cfg.dataset_name == "voc":
+        val_loader_crop = None
+    else:
+        val_loader_crop = "center"
+
+    val_dataset = ContrastiveSegDataset(
+        pytorch_data_dir=pytorch_data_dir,
+        dataset_name=cfg.dataset_name,
+        crop_type=None,
+        image_set="val",
+        transform=get_transform(320, False, val_loader_crop),
+        target_transform=get_transform(320, True, val_loader_crop),
+        mask=True,
+        cfg=cfg,
+    )
+
+    # val_dataset = MaterializedDataset(val_dataset)
+    train_loader = DataLoader(
+        train_dataset,
+        cfg.batch_size,
+        shuffle=True,
+        num_workers=cfg.num_workers,
+        pin_memory=True,
+    )
+
+    if cfg.submitting_to_aml:
+        val_batch_size = 16
+    else:
+        val_batch_size = cfg.batch_size
+
+    val_loader = DataLoader(
+        val_dataset,
+        val_batch_size,
+        shuffle=False,
+        num_workers=cfg.num_workers,
+        pin_memory=True,
+    )
+
+    model = LitUnsupervisedSegmenter(train_dataset.n_classes, cfg)
+
+    tb_logger = TensorBoardLogger(join(log_dir, name), default_hp_metric=False)
+
+    if cfg.submitting_to_aml:
+        gpu_args = dict(gpus=1, val_check_interval=250)
+
+        if gpu_args["val_check_interval"] > len(train_loader):
+            gpu_args.pop("val_check_interval")
+
+    else:
+        gpu_args = dict(gpus=-1, accelerator="ddp", val_check_interval=cfg.val_freq)
+        # gpu_args = dict(gpus=1, accelerator='ddp', val_check_interval=cfg.val_freq)
+
+        if gpu_args["val_check_interval"] > len(train_loader) // 4:
+            gpu_args.pop("val_check_interval")
+
+    trainer = Trainer(
+        log_every_n_steps=cfg.scalar_log_freq,
+        logger=tb_logger,
+        max_steps=cfg.max_steps,
+        callbacks=[
+            ModelCheckpoint(
+                dirpath=join(checkpoint_dir, name),
+                every_n_train_steps=400,
+                save_top_k=2,
+                monitor="test/cluster/mIoU",
+                mode="max",
+            )
+        ],
+        **gpu_args
+    )
+    trainer.fit(model, train_loader, val_loader)
+
+
+if __name__ == "__main__":
+    prep_args()
+    my_app()

biomap/unet.py

@@ -0,0 +1,80 @@
+import os
+import torch
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from torch.utils.data import DataLoader
+import torch.nn as nn
+from collections import defaultdict
+import torchvision
+import torch.nn.functional as F
+from torch.utils.data.sampler import Sampler
+
+class Block(nn.Module):
+    def __init__(self, in_ch, out_ch, padding='same'):
+        super().__init__()
+        self.conv1 = nn.Conv2d(in_ch, out_ch, 3, padding=padding)
+        self.relu  = nn.ReLU()
+        self.conv2 = nn.Conv2d(out_ch, out_ch, 3, padding=padding)
+    
+    def forward(self, x):
+        return self.conv2(self.relu(self.conv1(x)))
+
+
+class Encoder(nn.Module):
+    def __init__(self, chs=(3,32,64,128,256)):
+        super().__init__()
+        self.enc_blocks = nn.ModuleList([Block(chs[i], chs[i+1]) for i in range(len(chs)-1)])
+        self.pool       = nn.MaxPool2d(2)
+    
+    def forward(self, x):
+        ftrs = []
+        for block in self.enc_blocks:
+            x = block(x)
+            ftrs.append(x)
+            x = self.pool(x)
+        return ftrs
+
+
+class Decoder(nn.Module):
+    def __init__(self, chs=(256,128, 64, 32), aux_ch=70):
+        super().__init__()
+        upchs      = tuple([chs[i]+aux_ch if i == 0 else chs[i] for i in range(len(chs))])
+        self.chs         = chs
+        self.upchs      = upchs
+        self.upconvs    = nn.ModuleList([nn.ConvTranspose2d(upchs[i], upchs[i+1], 2, 2) for i in range(len(upchs)-1)])
+        self.dec_blocks = nn.ModuleList([Block(chs[i], chs[i+1]) for i in range(len(chs)-1)]) 
+        
+    def forward(self, x, encoder_features):
+        for i in range(len(self.chs)-1):
+#             pdb.set_trace()
+            x        = self.upconvs[i](x)
+            enc_ftrs = self.crop(encoder_features[i], x)
+            x        = torch.cat([x, enc_ftrs], dim=1)
+            x        = self.dec_blocks[i](x)
+        return x
+    
+    def crop(self, enc_ftrs, x):
+        _, _, H, W = x.shape
+        enc_ftrs   = torchvision.transforms.CenterCrop([H, W])(enc_ftrs)
+        return enc_ftrs
+
+
+class AuxUNet(nn.Module):
+    # UNet with auxiliary feature at the bottom
+    def __init__(self, enc_chs=(3,32,64,128,256), dec_chs=(256,128, 64, 32), aux_ch=70, num_class=7, retain_dim=False, out_sz=(224,224)):
+        super().__init__()
+        self.encoder     = Encoder(enc_chs)
+        self.decoder     = Decoder(dec_chs, aux_ch)
+        self.head        = nn.Conv2d(dec_chs[-1], num_class, 1)
+        self.retain_dim  = retain_dim
+
+    def forward(self, x, aux):
+        # aux: auxiliary feature at the bottom
+        enc_ftrs = self.encoder(x)
+        enc_ftrs[-1] = torch.cat((enc_ftrs[-1], aux), 1)
+        out      = self.decoder(enc_ftrs[::-1][0], enc_ftrs[::-1][1:])
+        out      = self.head(out)
+        if self.retain_dim:
+            out = F.interpolate(out, out_sz)
+        return out

biomap/utils.py

@@ -0,0 +1,390 @@
+import collections
+import os
+from os.path import join
+import io
+
+import matplotlib.pyplot as plt
+import numpy as np
+import torch.multiprocessing
+import torch.nn as nn
+import torch.nn.functional as F
+import wget
+from PIL import Image
+from scipy.optimize import linear_sum_assignment
+from torch._six import string_classes
+from torch.utils.data._utils.collate import np_str_obj_array_pattern, default_collate_err_msg_format
+from torchmetrics import Metric
+from torchvision import models
+from torchvision import transforms as T
+from torch.utils.tensorboard.summary import hparams
+import matplotlib as mpl
+torch.multiprocessing.set_sharing_strategy("file_system")
+colors = ("red", "palegreen", "green", "steelblue", "blue", "yellow", "lightgrey")
+class_names = (
+    "Buildings",
+    "Cultivation",
+    "Natural green",
+    "Wetland",
+    "Water",
+    "Infrastructure",
+    "Background",
+)
+bounds = list(np.arange(len(class_names) + 1) + 1)
+cmap = mpl.colors.ListedColormap(colors)
+norm = mpl.colors.BoundaryNorm(bounds, cmap.N)
+
+def compute_biodiv_score(image):
+    """Compute the biodiversity score of an image
+
+    Args:
+        image (_type_): _description_
+
+    Returns:
+        biodiversity_score: the biodiversity score associated to the landscape of the image
+    """
+    pix = np.array(image.getdata())
+    return np.mean(pix)
+
+import cv2
+def create_video(array_images, output_path="output.mp4"):
+    height, width, layers = array_images[0].shape
+    size = (width,height)
+    
+    fourcc = cv2.VideoWriter_fourcc(*'VP90')
+    out = cv2.VideoWriter('output.mp4', fourcc, 2, size)
+ 
+    for i in range(len(array_images)):
+        out.write(array_images[i])
+    out.release()
+    return out
+
+
+
+def transform_to_pil(outputs, alpha=0.3):
+    """Turn an ouput into a PIL
+
+    Args:
+        outputs (_type_): _description_
+        alpha (float, optional): _description_. Defaults to 0.3.
+
+    Returns:
+        _type_: _description_
+    """
+    
+    # Transform img with torch
+    img = torch.moveaxis(prep_for_plot(outputs["img"][0]), -1, 0)
+    img = T.ToPILImage()(img)
+    # Transform label by saving it then open it
+    label = outputs["linear_preds"][0].numpy()
+    # image_label = Image.fromarray(label, mode="P")
+    plt.imsave("output/label.png", label, cmap=cmap)
+    image_label = Image.open("output/label.png")
+    # Overlay labels with img wit alpha
+    background = img.convert("RGBA")
+    overlay = image_label.convert("RGBA")
+    labeled_img = Image.blend(background, overlay, alpha)
+    labeled_img = labeled_img.convert("RGB")
+    return img, image_label, labeled_img
+
+
+def prep_for_plot(img, rescale=True, resize=None):
+    if resize is not None:
+        img = F.interpolate(img.unsqueeze(0), resize, mode="bilinear")
+    else:
+        img = img.unsqueeze(0)
+
+    plot_img = unnorm(img).squeeze(0).cpu().permute(1, 2, 0)
+    if rescale:
+        plot_img = (plot_img - plot_img.min()) / (plot_img.max() - plot_img.min())
+    return plot_img
+
+
+def add_plot(writer, name, step):
+    buf = io.BytesIO()
+    plt.savefig(buf, format='jpeg', dpi=100)
+    buf.seek(0)
+    image = Image.open(buf)
+    image = T.ToTensor()(image)
+    writer.add_image(name, image, step)
+    plt.clf()
+    plt.close()
+
+
+@torch.jit.script
+def shuffle(x):
+    return x[torch.randperm(x.shape[0])]
+
+
+def add_hparams_fixed(writer, hparam_dict, metric_dict, global_step):
+    exp, ssi, sei = hparams(hparam_dict, metric_dict)
+    writer.file_writer.add_summary(exp)
+    writer.file_writer.add_summary(ssi)
+    writer.file_writer.add_summary(sei)
+    for k, v in metric_dict.items():
+        writer.add_scalar(k, v, global_step)
+
+
+@torch.jit.script
+def resize(classes: torch.Tensor, size: int):
+    return F.interpolate(classes, (size, size), mode="bilinear", align_corners=False)
+
+
+def one_hot_feats(labels, n_classes):
+    return F.one_hot(labels, n_classes).permute(0, 3, 1, 2).to(torch.float32)
+
+
+def load_model(model_type, data_dir):
+    if model_type == "robust_resnet50":
+        model = models.resnet50(pretrained=False)
+        model_file = join(data_dir, 'imagenet_l2_3_0.pt')
+        if not os.path.exists(model_file):
+            wget.download("http://6.869.csail.mit.edu/fa19/psets19/pset6/imagenet_l2_3_0.pt",
+                          model_file)
+        model_weights = torch.load(model_file)
+        model_weights_modified = {name.split('model.')[1]: value for name, value in model_weights['model'].items() if
+                                  'model' in name}
+        model.load_state_dict(model_weights_modified)
+        model = nn.Sequential(*list(model.children())[:-1])
+    elif model_type == "densecl":
+        model = models.resnet50(pretrained=False)
+        model_file = join(data_dir, 'densecl_r50_coco_1600ep.pth')
+        if not os.path.exists(model_file):
+            wget.download("https://cloudstor.aarnet.edu.au/plus/s/3GapXiWuVAzdKwJ/download",
+                          model_file)
+        model_weights = torch.load(model_file)
+        # model_weights_modified = {name.split('model.')[1]: value for name, value in model_weights['model'].items() if
+        #                          'model' in name}
+        model.load_state_dict(model_weights['state_dict'], strict=False)
+        model = nn.Sequential(*list(model.children())[:-1])
+    elif model_type == "resnet50":
+        model = models.resnet50(pretrained=True)
+        model = nn.Sequential(*list(model.children())[:-1])
+    elif model_type == "mocov2":
+        model = models.resnet50(pretrained=False)
+        model_file = join(data_dir, 'moco_v2_800ep_pretrain.pth.tar')
+        if not os.path.exists(model_file):
+            wget.download("https://dl.fbaipublicfiles.com/moco/moco_checkpoints/"
+                          "moco_v2_800ep/moco_v2_800ep_pretrain.pth.tar", model_file)
+        checkpoint = torch.load(model_file)
+        # rename moco pre-trained keys
+        state_dict = checkpoint['state_dict']
+        for k in list(state_dict.keys()):
+            # retain only encoder_q up to before the embedding layer
+            if k.startswith('module.encoder_q') and not k.startswith('module.encoder_q.fc'):
+                # remove prefix
+                state_dict[k[len("module.encoder_q."):]] = state_dict[k]
+            # delete renamed or unused k
+            del state_dict[k]
+        msg = model.load_state_dict(state_dict, strict=False)
+        assert set(msg.missing_keys) == {"fc.weight", "fc.bias"}
+        model = nn.Sequential(*list(model.children())[:-1])
+    elif model_type == "densenet121":
+        model = models.densenet121(pretrained=True)
+        model = nn.Sequential(*list(model.children())[:-1] + [nn.AdaptiveAvgPool2d((1, 1))])
+    elif model_type == "vgg11":
+        model = models.vgg11(pretrained=True)
+        model = nn.Sequential(*list(model.children())[:-1] + [nn.AdaptiveAvgPool2d((1, 1))])
+    else:
+        raise ValueError("No model: {} found".format(model_type))
+
+    model.eval()
+    model.cuda()
+    return model
+
+
+class UnNormalize(object):
+    def __init__(self, mean, std):
+        self.mean = mean
+        self.std = std
+
+    def __call__(self, image):
+        image2 = torch.clone(image)
+        for t, m, s in zip(image2, self.mean, self.std):
+            t.mul_(s).add_(m)
+        return image2
+
+
+normalize = T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
+unnorm = UnNormalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
+
+
+class ToTargetTensor(object):
+    def __call__(self, target):
+        return torch.as_tensor(np.array(target), dtype=torch.int64).unsqueeze(0)
+
+
+def prep_args():
+    import sys
+
+    old_args = sys.argv
+    new_args = [old_args.pop(0)]
+    while len(old_args) > 0:
+        arg = old_args.pop(0)
+        if len(arg.split("=")) == 2:
+            new_args.append(arg)
+        elif arg.startswith("--"):
+            new_args.append(arg[2:] + "=" + old_args.pop(0))
+        else:
+            raise ValueError("Unexpected arg style {}".format(arg))
+    sys.argv = new_args
+
+
+def get_transform(res, is_label, crop_type):
+    if crop_type == "center":
+        cropper = T.CenterCrop(res)
+    elif crop_type == "random":
+        cropper = T.RandomCrop(res)
+    elif crop_type is None:
+        cropper = T.Lambda(lambda x: x)
+        res = (res, res)
+    else:
+        raise ValueError("Unknown Cropper {}".format(crop_type))
+    if is_label:
+        return T.Compose([T.Resize(res, Image.NEAREST),
+                          cropper,
+                          ToTargetTensor()])
+    else:
+        return T.Compose([T.Resize(res, Image.NEAREST),
+                          cropper,
+                          T.ToTensor(),
+                          normalize])
+
+
+def _remove_axes(ax):
+    ax.xaxis.set_major_formatter(plt.NullFormatter())
+    ax.yaxis.set_major_formatter(plt.NullFormatter())
+    ax.set_xticks([])
+    ax.set_yticks([])
+
+
+def remove_axes(axes):
+    if len(axes.shape) == 2:
+        for ax1 in axes:
+            for ax in ax1:
+                _remove_axes(ax)
+    else:
+        for ax in axes:
+            _remove_axes(ax)
+
+
+class UnsupervisedMetrics(Metric):
+    def __init__(self, prefix: str, n_classes: int, extra_clusters: int, compute_hungarian: bool,
+                 dist_sync_on_step=True):
+        # call `self.add_state`for every internal state that is needed for the metrics computations
+        # dist_reduce_fx indicates the function that should be used to reduce
+        # state from multiple processes
+        super().__init__(dist_sync_on_step=dist_sync_on_step)
+
+        self.n_classes = n_classes
+        self.extra_clusters = extra_clusters
+        self.compute_hungarian = compute_hungarian
+        self.prefix = prefix
+        self.add_state("stats",
+                       default=torch.zeros(n_classes + self.extra_clusters, n_classes, dtype=torch.int64),
+                       dist_reduce_fx="sum")
+
+    def update(self, preds: torch.Tensor, target: torch.Tensor):
+        with torch.no_grad():
+            actual = target.reshape(-1)
+            preds = preds.reshape(-1)
+            mask = (actual >= 0) & (actual < self.n_classes) & (preds >= 0) & (preds < self.n_classes)
+            actual = actual[mask]
+            preds = preds[mask]
+            self.stats += torch.bincount(
+                (self.n_classes + self.extra_clusters) * actual + preds,
+                minlength=self.n_classes * (self.n_classes + self.extra_clusters)) \
+                .reshape(self.n_classes, self.n_classes + self.extra_clusters).t().to(self.stats.device)
+
+    def map_clusters(self, clusters):
+        if self.extra_clusters == 0:
+            return torch.tensor(self.assignments[1])[clusters]
+        else:
+            missing = sorted(list(set(range(self.n_classes + self.extra_clusters)) - set(self.assignments[0])))
+            cluster_to_class = self.assignments[1]
+            for missing_entry in missing:
+                if missing_entry == cluster_to_class.shape[0]:
+                    cluster_to_class = np.append(cluster_to_class, -1)
+                else:
+                    cluster_to_class = np.insert(cluster_to_class, missing_entry + 1, -1)
+            cluster_to_class = torch.tensor(cluster_to_class)
+            return cluster_to_class[clusters]
+
+    def compute(self):
+        if self.compute_hungarian:
+            self.assignments = linear_sum_assignment(self.stats.detach().cpu(), maximize=True)
+            # print(self.assignments)
+            if self.extra_clusters == 0:
+                self.histogram = self.stats[np.argsort(self.assignments[1]), :]
+            if self.extra_clusters > 0:
+                self.assignments_t = linear_sum_assignment(self.stats.detach().cpu().t(), maximize=True)
+                histogram = self.stats[self.assignments_t[1], :]
+                missing = list(set(range(self.n_classes + self.extra_clusters)) - set(self.assignments[0]))
+                new_row = self.stats[missing, :].sum(0, keepdim=True)
+                histogram = torch.cat([histogram, new_row], axis=0)
+                new_col = torch.zeros(self.n_classes + 1, 1, device=histogram.device)
+                self.histogram = torch.cat([histogram, new_col], axis=1)
+        else:
+            self.assignments = (torch.arange(self.n_classes).unsqueeze(1),
+                                torch.arange(self.n_classes).unsqueeze(1))
+            self.histogram = self.stats
+
+        tp = torch.diag(self.histogram)
+        fp = torch.sum(self.histogram, dim=0) - tp
+        fn = torch.sum(self.histogram, dim=1) - tp
+
+        iou = tp / (tp + fp + fn)
+        prc = tp / (tp + fn)
+        opc = torch.sum(tp) / torch.sum(self.histogram)
+
+        metric_dict = {self.prefix + "mIoU": iou[~torch.isnan(iou)].mean().item(),
+                       self.prefix + "Accuracy": opc.item()}
+        return {k: 100 * v for k, v in metric_dict.items()}
+
+
+def flexible_collate(batch):
+    r"""Puts each data field into a tensor with outer dimension batch size"""
+
+    elem = batch[0]
+    elem_type = type(elem)
+    if isinstance(elem, torch.Tensor):
+        out = None
+        if torch.utils.data.get_worker_info() is not None:
+            # If we're in a background process, concatenate directly into a
+            # shared memory tensor to avoid an extra copy
+            numel = sum([x.numel() for x in batch])
+            storage = elem.storage()._new_shared(numel)
+            out = elem.new(storage)
+        try:
+            return torch.stack(batch, 0, out=out)
+        except RuntimeError:
+            return batch
+    elif elem_type.__module__ == 'numpy' and elem_type.__name__ != 'str_' \
+            and elem_type.__name__ != 'string_':
+        if elem_type.__name__ == 'ndarray' or elem_type.__name__ == 'memmap':
+            # array of string classes and object
+            if np_str_obj_array_pattern.search(elem.dtype.str) is not None:
+                raise TypeError(default_collate_err_msg_format.format(elem.dtype))
+
+            return flexible_collate([torch.as_tensor(b) for b in batch])
+        elif elem.shape == ():  # scalars
+            return torch.as_tensor(batch)
+    elif isinstance(elem, float):
+        return torch.tensor(batch, dtype=torch.float64)
+    elif isinstance(elem, int):
+        return torch.tensor(batch)
+    elif isinstance(elem, string_classes):
+        return batch
+    elif isinstance(elem, collections.abc.Mapping):
+        return {key: flexible_collate([d[key] for d in batch]) for key in elem}
+    elif isinstance(elem, tuple) and hasattr(elem, '_fields'):  # namedtuple
+        return elem_type(*(flexible_collate(samples) for samples in zip(*batch)))
+    elif isinstance(elem, collections.abc.Sequence):
+        # check to make sure that the elements in batch have consistent size
+        it = iter(batch)
+        elem_size = len(next(it))
+        if not all(len(elem) == elem_size for elem in it):
+            raise RuntimeError('each element in list of batch should be of equal size')
+        transposed = zip(*batch)
+        return [flexible_collate(samples) for samples in transposed]
+
+    raise TypeError(default_collate_err_msg_format.format(elem_type))

biomap/utils_gee.py

@@ -0,0 +1,157 @@
+import io
+import requests
+import ee
+import numpy as np
+import matplotlib.pyplot as plt
+
+#Initialize
+service_account = '[email protected]'
+credentials = ee.ServiceAccountCredentials(service_account, '.private-key.json')
+ee.Initialize(credentials)
+
+#delete clouds
+def maskS2clouds(image):
+        qa = image.select('QA60');
+
+    #   // Bits 10 and 11 are clouds and cirrus, respectively.
+        cloudBitMask = 1 << 10;
+        cirrusBitMask = 1 << 11;
+
+    #   // Both flags should be set to zero, indicating clear conditions.
+        mask = (qa.bitwiseAnd(cloudBitMask).eq(0))and(qa.bitwiseAnd(cirrusBitMask).eq(0))
+
+        return image.updateMask(mask).divide(10000);
+
+
+#find ee_img
+def extract_ee_img(location,start_date,end_date, width = 0.01 , len = 0.01) :
+    """Extract the earth engine image
+
+    Args:
+        location (list[float]): 
+        start_date (str): the start date for finding an image
+        end_date (str): the end date for finding an image
+        width (float, optional): _description_. Defaults to 0.01.
+        len (float, optional): _description_. Defaults to 0.01.
+
+    Returns:
+        _type_: _description_
+    """
+    # define the polygone
+    polygone =[[[float(location[0])-0.01,float(location[1])+0.01],
+                [float(location[0])-0.01,float(location[1])-0.01],
+                [float(location[0])+0.01,float(location[1])-0.01],
+                [float(location[0])+0.01,float(location[1])+0.01],
+    ]]    
+
+    #define the ee geometry 
+    geometry = ee.Geometry.Polygon(polygone, None, False);
+
+    #extract the dataset
+    dataset = ee.ImageCollection('COPERNICUS/S2_SR_HARMONIZED')\
+        .filterDate(start_date, end_date)\
+        .filter(ee.Filter.lt('CLOUDY_PIXEL_PERCENTAGE',1))\
+        .map(maskS2clouds)
+    return dataset.mean(), geometry
+
+
+
+# Get URL
+def get_url(ee_img, geometry, scale=5):
+    """Get the url of a dataset and a geometry 
+
+    Args:
+        ee_img (ee.ImageCollection: meta data on the image 
+        geometry (ee.Geometry.Polygon): geometry of the desired landscape
+        scale (int, optional): _description_. Defaults to 5.
+
+    Returns:
+        str: the url to use to ask the server
+    """
+    region = geometry
+
+    # collectionList = ee_img.toList(ee_img.size())
+    # collectionSize = collectionList.size().getInfo()
+    # for i in xrange(collectionSize):
+    #     ee.batch.Export.image.toDrive(
+    #         image = ee.Image(collectionList.get(i)).clip(rectangle), 
+    #         fileNamePrefix = 'foo' + str(i + 1), 
+    #         dimensions = '128x128').start()
+
+    url = ee_img.getDownloadURL({
+        # 'min': 0.0,
+        # 'max': 0.3,
+        'bands': ['B4', 'B3', 'B2'],
+        'region' : region,
+        'scale' : scale,
+        'format' : 'NPY'
+    })
+    
+    return url
+
+def extract_np_from_url(url):
+    """extract a numpy array based on a url
+
+    Args:
+        url (str): _description_
+
+    Returns:
+        numpyarray: response from earth engine as numpy
+    """
+    #get the response from url
+    response = requests.get(url)
+
+    #transform it into numpy
+    data = np.load(io.BytesIO(response.content))
+    
+    #transform numpy of tuples to 3D numpy
+    temp1 = []
+
+    for x in data:
+        temp2 = []
+        for y in x :
+            temp2.append([z for z in y])
+        temp1.append(temp2)
+
+    data = np.array(temp1)
+
+    return data
+    
+#Fonction globale
+def extract_img(location,start_date,end_date, width = 0.01 , len = 0.01,scale=5):
+    """Extract an image of the landscape at the selected longitude and latitude with the selected width and length 
+
+    Args:
+        location (list[float]): [latitude of the center of the landscape, longitude of the center of the landscape]
+        start_date (str): the start date  
+        end_date (str): _description_
+        width (float, optional): _description_. Defaults to 0.01.
+        len (float, optional): _description_. Defaults to 0.01.
+        scale (int, optional): _description_. Defaults to 5.
+
+    Returns:
+        img: image as numpy array
+    """
+    ee_img, geometry = extract_ee_img(location, width,start_date,end_date , len)
+    url = get_url(ee_img, geometry, scale)
+    img = extract_np_from_url(url)
+    
+    return img
+
+# transform img from numpy to PIL
+def transform_ee_img(img, min = 0, max=0.3):
+    """Transform an img from numpy to PIL
+
+    Args:
+        img (numpy array): the original image as a numpy array 
+        min (int, optional): _description_. Defaults to 0.
+        max (float, optional): _description_. Defaults to 0.3.
+
+    Returns:
+        img_test: a PIL image 
+    """
+    img_test=img
+    img_test=np.minimum(img_test*255/max,np.ones(img.shape)*255)
+    img_test=np.uint8((np.rint(img_test)).astype(int))
+    plt.imshow(img_test)
+    return img_test

poetry.lock

@@ -0,0 +1,1625 @@
+[[package]]
+name = "absl-py"
+version = "1.4.0"
+description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py."
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[[package]]
+name = "aiofiles"
+version = "23.1.0"
+description = "File support for asyncio."
+category = "main"
+optional = false
+python-versions = ">=3.7,<4.0"
+
+[[package]]
+name = "aiohttp"
+version = "3.8.4"
+description = "Async http client/server framework (asyncio)"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+aiosignal = ">=1.1.2"
+async-timeout = ">=4.0.0a3,<5.0"
+attrs = ">=17.3.0"
+charset-normalizer = ">=2.0,<4.0"
+frozenlist = ">=1.1.1"
+multidict = ">=4.5,<7.0"
+yarl = ">=1.0,<2.0"
+
+[package.extras]
+speedups = ["aiodns", "brotli", "cchardet"]
+
+[[package]]
+name = "aiosignal"
+version = "1.3.1"
+description = "aiosignal: a list of registered asynchronous callbacks"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+frozenlist = ">=1.1.0"
+
+[[package]]
+name = "altair"
+version = "4.2.2"
+description = "Altair: A declarative statistical visualization library for Python."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+entrypoints = "*"
+jinja2 = "*"
+jsonschema = ">=3.0"
+numpy = "*"
+pandas = ">=0.18"
+toolz = "*"
+
+[package.extras]
+dev = ["black", "docutils", "ipython", "flake8", "pytest", "sphinx", "mistune (<2.0.0)", "m2r", "vega-datasets", "recommonmark"]
+
+[[package]]
+name = "antlr4-python3-runtime"
+version = "4.9.3"
+description = "ANTLR 4.9.3 runtime for Python 3.7"
+category = "main"
+optional = false
+python-versions = "*"
+
+[[package]]
+name = "anyio"
+version = "3.6.2"
+description = "High level compatibility layer for multiple asynchronous event loop implementations"
+category = "main"
+optional = false
+python-versions = ">=3.6.2"
+
+[package.dependencies]
+idna = ">=2.8"
+sniffio = ">=1.1"
+
+[package.extras]
+doc = ["packaging", "sphinx-rtd-theme", "sphinx-autodoc-typehints (>=1.2.0)"]
+test = ["coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "contextlib2", "uvloop (<0.15)", "mock (>=4)", "uvloop (>=0.15)"]
+trio = ["trio (>=0.16,<0.22)"]
+
+[[package]]
+name = "async-timeout"
+version = "4.0.2"
+description = "Timeout context manager for asyncio programs"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[[package]]
+name = "attrs"
+version = "19.3.0"
+description = "Classes Without Boilerplate"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+
+[package.extras]
+azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"]
+dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"]
+docs = ["sphinx", "zope.interface"]
+tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"]
+
+[[package]]
+name = "cachetools"
+version = "5.3.0"
+description = "Extensible memoizing collections and decorators"
+category = "main"
+optional = false
+python-versions = "~=3.7"
+
+[[package]]
+name = "certifi"
+version = "2022.12.7"
+description = "Python package for providing Mozilla's CA Bundle."
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[[package]]
+name = "charset-normalizer"
+version = "3.1.0"
+description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
+category = "main"
+optional = false
+python-versions = ">=3.7.0"
+
+[[package]]
+name = "click"
+version = "8.1.3"
+description = "Composable command line interface toolkit"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+colorama = {version = "*", markers = "platform_system == \"Windows\""}
+
+[[package]]
+name = "colorama"
+version = "0.4.6"
+description = "Cross-platform colored terminal text."
+category = "main"
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
+
+[[package]]
+name = "contourpy"
+version = "1.0.7"
+description = "Python library for calculating contours of 2D quadrilateral grids"
+category = "main"
+optional = false
+python-versions = ">=3.8"
+
+[package.dependencies]
+numpy = ">=1.16"
+
+[package.extras]
+bokeh = ["bokeh", "chromedriver", "selenium"]
+docs = ["furo", "sphinx-copybutton"]
+mypy = ["contourpy", "docutils-stubs", "mypy (==0.991)", "types-pillow"]
+test = ["matplotlib", "pillow", "pytest"]
+test-no-images = ["pytest"]
+
+[[package]]
+name = "cycler"
+version = "0.11.0"
+description = "Composable style cycles"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[[package]]
+name = "earthengine-api"
+version = "0.1.338"
+description = "Earth Engine Python API"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+google-api-python-client = ">=1.12.1"
+google-auth = ">=1.4.1"
+google-auth-httplib2 = ">=0.0.3"
+google-cloud-storage = "*"
+httplib2 = ">=0.9.2,<1dev"
+requests = "*"
+
+[[package]]
+name = "ee-extra"
+version = "0.0.15"
+description = "A ninja Python package behind rgee, rgeeExtra and eemont."
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+earthengine-api = "*"
+
+[[package]]
+name = "entrypoints"
+version = "0.4"
+description = "Discover and load entry points from installed packages."
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[[package]]
+name = "fastapi"
+version = "0.95.1"
+description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+pydantic = ">=1.6.2,<1.7 || >1.7,<1.7.1 || >1.7.1,<1.7.2 || >1.7.2,<1.7.3 || >1.7.3,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0"
+starlette = ">=0.26.1,<0.27.0"
+
+[package.extras]
+all = ["email-validator (>=1.1.1)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "python-multipart (>=0.0.5)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"]
+dev = ["pre-commit (>=2.17.0,<3.0.0)", "ruff (==0.0.138)", "uvicorn[standard] (>=0.12.0,<0.21.0)"]
+doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs-markdownextradata-plugin (>=0.1.7,<0.3.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "pyyaml (>=5.3.1,<7.0.0)", "typer-cli (>=0.0.13,<0.0.14)", "typer[all] (>=0.6.1,<0.8.0)"]
+test = ["anyio[trio] (>=3.2.1,<4.0.0)", "black (==23.1.0)", "coverage[toml] (>=6.5.0,<8.0)", "databases[sqlite] (>=0.3.2,<0.7.0)", "email-validator (>=1.1.1,<2.0.0)", "flask (>=1.1.2,<3.0.0)", "httpx (>=0.23.0,<0.24.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.982)", "orjson (>=3.2.1,<4.0.0)", "passlib[bcrypt] (>=1.7.2,<2.0.0)", "peewee (>=3.13.3,<4.0.0)", "pytest (>=7.1.3,<8.0.0)", "python-jose[cryptography] (>=3.3.0,<4.0.0)", "python-multipart (>=0.0.5,<0.0.7)", "pyyaml (>=5.3.1,<7.0.0)", "ruff (==0.0.138)", "sqlalchemy (>=1.3.18,<1.4.43)", "types-orjson (==3.6.2)", "types-ujson (==5.7.0.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,<6.0.0)"]
+
+[[package]]
+name = "ffmpy"
+version = "0.3.0"
+description = "A simple Python wrapper for ffmpeg"
+category = "main"
+optional = false
+python-versions = "*"
+
+[[package]]
+name = "filelock"
+version = "3.11.0"
+description = "A platform independent file lock."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.extras]
+docs = ["furo (>=2023.3.27)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)", "sphinx (>=6.1.3)"]
+testing = ["covdefaults (>=2.3)", "coverage (>=7.2.2)", "diff-cover (>=7.5)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)", "pytest (>=7.2.2)"]
+
+[[package]]
+name = "fonttools"
+version = "4.39.3"
+description = "Tools to manipulate font files"
+category = "main"
+optional = false
+python-versions = ">=3.8"
+
+[package.extras]
+all = ["fs (>=2.2.0,<3)", "lxml (>=4.0,<5)", "zopfli (>=0.1.4)", "lz4 (>=1.7.4.2)", "matplotlib", "sympy", "skia-pathops (>=0.5.0)", "uharfbuzz (>=0.23.0)", "brotlicffi (>=0.8.0)", "scipy", "brotli (>=1.0.1)", "munkres", "unicodedata2 (>=15.0.0)", "xattr"]
+graphite = ["lz4 (>=1.7.4.2)"]
+interpolatable = ["scipy", "munkres"]
+lxml = ["lxml (>=4.0,<5)"]
+pathops = ["skia-pathops (>=0.5.0)"]
+plot = ["matplotlib"]
+repacker = ["uharfbuzz (>=0.23.0)"]
+symfont = ["sympy"]
+type1 = ["xattr"]
+ufo = ["fs (>=2.2.0,<3)"]
+unicode = ["unicodedata2 (>=15.0.0)"]
+woff = ["zopfli (>=0.1.4)", "brotlicffi (>=0.8.0)", "brotli (>=1.0.1)"]
+
+[[package]]
+name = "frozenlist"
+version = "1.3.3"
+description = "A list-like structure which implements collections.abc.MutableSequence"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "fsspec"
+version = "2023.4.0"
+description = "File-system specification"
+category = "main"
+optional = false
+python-versions = ">=3.8"
+
+[package.dependencies]
+aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""}
+requests = {version = "*", optional = true, markers = "extra == \"http\""}
+
+[package.extras]
+abfs = ["adlfs"]
+adl = ["adlfs"]
+arrow = ["pyarrow (>=1)"]
+dask = ["dask", "distributed"]
+devel = ["pytest", "pytest-cov"]
+dropbox = ["dropboxdrivefs", "requests", "dropbox"]
+full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"]
+fuse = ["fusepy"]
+gcs = ["gcsfs"]
+git = ["pygit2"]
+github = ["requests"]
+gs = ["gcsfs"]
+gui = ["panel"]
+hdfs = ["pyarrow (>=1)"]
+http = ["requests", "aiohttp (!=4.0.0a0,!=4.0.0a1)"]
+libarchive = ["libarchive-c"]
+oci = ["ocifs"]
+s3 = ["s3fs"]
+sftp = ["paramiko"]
+smb = ["smbprotocol"]
+ssh = ["paramiko"]
+tqdm = ["tqdm"]
+
+[[package]]
+name = "google-api-core"
+version = "2.11.0"
+description = "Google API client core library"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+google-auth = ">=2.14.1,<3.0dev"
+googleapis-common-protos = ">=1.56.2,<2.0dev"
+protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev"
+requests = ">=2.18.0,<3.0.0dev"
+
+[package.extras]
+grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio-status (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.49.1,<2.0dev)"]
+grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"]
+grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"]
+
+[[package]]
+name = "google-api-python-client"
+version = "2.85.0"
+description = "Google API Client Library for Python"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+google-api-core = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev"
+google-auth = ">=1.19.0,<3.0.0dev"
+google-auth-httplib2 = ">=0.1.0"
+httplib2 = ">=0.15.0,<1dev"
+uritemplate = ">=3.0.1,<5"
+
+[[package]]
+name = "google-auth"
+version = "2.17.3"
+description = "Google Authentication Library"
+category = "main"
+optional = false
+python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*"
+
+[package.dependencies]
+cachetools = ">=2.0.0,<6.0"
+pyasn1-modules = ">=0.2.1"
+rsa = {version = ">=3.1.4,<5", markers = "python_version >= \"3.6\""}
+six = ">=1.9.0"
+
+[package.extras]
+aiohttp = ["requests (>=2.20.0,<3.0.0dev)", "aiohttp (>=3.6.2,<4.0.0dev)"]
+enterprise_cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"]
+pyopenssl = ["pyopenssl (>=20.0.0)", "cryptography (>=38.0.3)"]
+reauth = ["pyu2f (>=0.1.5)"]
+requests = ["requests (>=2.20.0,<3.0.0dev)"]
+
+[[package]]
+name = "google-auth-httplib2"
+version = "0.1.0"
+description = "Google Authentication Library: httplib2 transport"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+google-auth = "*"
+httplib2 = ">=0.15.0"
+six = "*"
+
+[[package]]
+name = "google-auth-oauthlib"
+version = "0.4.6"
+description = "Google Authentication Library"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+google-auth = ">=1.0.0"
+requests-oauthlib = ">=0.7.0"
+
+[package.extras]
+tool = ["click (>=6.0.0)"]
+
+[[package]]
+name = "google-cloud-core"
+version = "2.3.2"
+description = "Google Cloud API client core library"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+google-api-core = ">=1.31.6,<2.0.0 || >2.3.0,<3.0.0dev"
+google-auth = ">=1.25.0,<3.0dev"
+
+[package.extras]
+grpc = ["grpcio (>=1.38.0,<2.0dev)"]
+
+[[package]]
+name = "google-cloud-storage"
+version = "2.8.0"
+description = "Google Cloud Storage API client library"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+google-api-core = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev"
+google-auth = ">=1.25.0,<3.0dev"
+google-cloud-core = ">=2.3.0,<3.0dev"
+google-resumable-media = ">=2.3.2"
+requests = ">=2.18.0,<3.0.0dev"
+
+[package.extras]
+protobuf = ["protobuf (<5.0.0dev)"]
+
+[[package]]
+name = "google-crc32c"
+version = "1.5.0"
+description = "A python wrapper of the C library 'Google CRC32C'"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.extras]
+testing = ["pytest"]
+
+[[package]]
+name = "google-resumable-media"
+version = "2.4.1"
+description = "Utilities for Google Media Downloads and Resumable Uploads"
+category = "main"
+optional = false
+python-versions = ">= 3.7"
+
+[package.dependencies]
+google-crc32c = ">=1.0,<2.0dev"
+
+[package.extras]
+aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)"]
+requests = ["requests (>=2.18.0,<3.0.0dev)"]
+
+[[package]]
+name = "googleapis-common-protos"
+version = "1.59.0"
+description = "Common protobufs used in Google APIs"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev"
+
+[package.extras]
+grpc = ["grpcio (>=1.44.0,<2.0.0dev)"]
+
+[[package]]
+name = "gradio"
+version = "3.27.0"
+description = "Python library for easily interacting with trained machine learning models"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+aiofiles = "*"
+aiohttp = "*"
+altair = ">=4.2.0"
+fastapi = "*"
+ffmpy = "*"
+gradio-client = ">=0.1.3"
+httpx = "*"
+huggingface-hub = ">=0.13.0"
+jinja2 = "*"
+markdown-it-py = {version = ">=2.0.0", extras = ["linkify"]}
+markupsafe = "*"
+matplotlib = "*"
+mdit-py-plugins = "<=0.3.3"
+numpy = "*"
+orjson = "*"
+pandas = "*"
+pillow = "*"
+pydantic = "*"
+pydub = "*"
+python-multipart = "*"
+pyyaml = "*"
+requests = "*"
+semantic-version = "*"
+typing-extensions = "*"
+uvicorn = "*"
+websockets = ">=10.0"
+
+[[package]]
+name = "gradio-client"
+version = "0.1.3"
+description = "Python library for easily interacting with trained machine learning models"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+fsspec = "*"
+httpx = "*"
+huggingface-hub = ">=0.13.0"
+packaging = "*"
+requests = "*"
+typing-extensions = "*"
+websockets = "*"
+
+[[package]]
+name = "grpcio"
+version = "1.53.0"
+description = "HTTP/2-based RPC framework"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.extras]
+protobuf = ["grpcio-tools (>=1.53.0)"]
+
+[[package]]
+name = "h11"
+version = "0.14.0"
+description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "httpcore"
+version = "0.17.0"
+description = "A minimal low-level HTTP client."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+anyio = ">=3.0,<5.0"
+certifi = "*"
+h11 = ">=0.13,<0.15"
+sniffio = ">=1.0.0,<2.0.0"
+
+[package.extras]
+http2 = ["h2 (>=3,<5)"]
+socks = ["socksio (>=1.0.0,<2.0.0)"]
+
+[[package]]
+name = "httplib2"
+version = "0.22.0"
+description = "A comprehensive HTTP client library."
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+
+[package.dependencies]
+pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0.2,<3.0.3 || >3.0.3,<4", markers = "python_version > \"3.0\""}
+
+[[package]]
+name = "httpx"
+version = "0.24.0"
+description = "The next generation HTTP client."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+certifi = "*"
+httpcore = ">=0.15.0,<0.18.0"
+idna = "*"
+sniffio = "*"
+
+[package.extras]
+brotli = ["brotli", "brotlicffi"]
+cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<14)"]
+http2 = ["h2 (>=3,<5)"]
+socks = ["socksio (>=1.0.0,<2.0.0)"]
+
+[[package]]
+name = "huggingface-hub"
+version = "0.13.4"
+description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub"
+category = "main"
+optional = false
+python-versions = ">=3.7.0"
+
+[package.dependencies]
+filelock = "*"
+packaging = ">=20.9"
+pyyaml = ">=5.1"
+requests = "*"
+tqdm = ">=4.42.1"
+typing-extensions = ">=3.7.4.3"
+
+[package.extras]
+all = ["InquirerPy (==0.3.4)", "jedi", "jinja2", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile", "pillow", "black (>=23.1,<24.0)", "ruff (>=0.0.241)", "mypy (==0.982)", "types-pyyaml", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"]
+cli = ["InquirerPy (==0.3.4)"]
+dev = ["InquirerPy (==0.3.4)", "jedi", "jinja2", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile", "pillow", "black (>=23.1,<24.0)", "ruff (>=0.0.241)", "mypy (==0.982)", "types-pyyaml", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"]
+fastai = ["toml", "fastai (>=2.4)", "fastcore (>=1.3.27)"]
+quality = ["black (>=23.1,<24.0)", "ruff (>=0.0.241)", "mypy (==0.982)"]
+tensorflow = ["tensorflow", "pydot", "graphviz"]
+testing = ["InquirerPy (==0.3.4)", "jedi", "jinja2", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile", "pillow"]
+torch = ["torch"]
+typing = ["types-pyyaml", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"]
+
+[[package]]
+name = "hydra-client"
+version = "0.5.1"
+description = "Client library for ORY Hydra (OAuth2 and OpenID Connect provider)"
+category = "main"
+optional = false
+python-versions = ">=3.7,<4.0"
+
+[package.dependencies]
+attrs = ">=19.2,<20.0"
+python-dateutil = ">=2.8,<3.0"
+requests = ">=2.21,<3.0"
+requests-oauthlib = ">=1.0,<2.0"
+
+[[package]]
+name = "hydra-core"
+version = "1.3.1"
+description = "A framework for elegantly configuring complex applications"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+antlr4-python3-runtime = ">=4.9.0,<4.10.0"
+omegaconf = ">=2.2,<2.4"
+packaging = "*"
+
+[[package]]
+name = "idna"
+version = "3.4"
+description = "Internationalized Domain Names in Applications (IDNA)"
+category = "main"
+optional = false
+python-versions = ">=3.5"
+
+[[package]]
+name = "jinja2"
+version = "3.1.2"
+description = "A very fast and expressive template engine."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+MarkupSafe = ">=2.0"
+
+[package.extras]
+i18n = ["Babel (>=2.7)"]
+
+[[package]]
+name = "jsonschema"
+version = "4.17.3"
+description = "An implementation of JSON Schema validation for Python"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+attrs = ">=17.4.0"
+pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2"
+
+[package.extras]
+format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"]
+format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"]
+
+[[package]]
+name = "kiwisolver"
+version = "1.4.4"
+description = "A fast implementation of the Cassowary constraint solver"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "lightning-utilities"
+version = "0.8.0"
+description = "PyTorch Lightning Sample project."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+packaging = ">=17.1"
+typing-extensions = "*"
+
+[package.extras]
+cli = ["fire"]
+docs = ["sphinx (>=4.0,<5.0)"]
+test = ["coverage (==6.5.0)"]
+typing = ["mypy (>=1.0.0)"]
+
+[[package]]
+name = "linkify-it-py"
+version = "2.0.0"
+description = "Links recognition library with FULL unicode support."
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+uc-micro-py = "*"
+
+[package.extras]
+benchmark = ["pytest", "pytest-benchmark"]
+dev = ["pre-commit", "isort", "flake8", "black"]
+doc = ["sphinx", "sphinx-book-theme", "myst-parser"]
+test = ["coverage", "pytest", "pytest-cov"]
+
+[[package]]
+name = "markdown"
+version = "3.4.3"
+description = "Python implementation of John Gruber's Markdown."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.extras]
+testing = ["coverage", "pyyaml"]
+
+[[package]]
+name = "markdown-it-py"
+version = "2.2.0"
+description = "Python port of markdown-it. Markdown parsing, done right!"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+linkify-it-py = {version = ">=1,<3", optional = true, markers = "extra == \"linkify\""}
+mdurl = ">=0.1,<1.0"
+
+[package.extras]
+benchmarking = ["psutil", "pytest", "pytest-benchmark"]
+code_style = ["pre-commit (>=3.0,<4.0)"]
+compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"]
+linkify = ["linkify-it-py (>=1,<3)"]
+plugins = ["mdit-py-plugins"]
+profiling = ["gprof2dot"]
+rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx-book-theme"]
+testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"]
+
+[[package]]
+name = "markupsafe"
+version = "2.1.2"
+description = "Safely add untrusted strings to HTML/XML markup."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "matplotlib"
+version = "3.7.1"
+description = "Python plotting package"
+category = "main"
+optional = false
+python-versions = ">=3.8"
+
+[package.dependencies]
+contourpy = ">=1.0.1"
+cycler = ">=0.10"
+fonttools = ">=4.22.0"
+kiwisolver = ">=1.0.1"
+numpy = ">=1.20"
+packaging = ">=20.0"
+pillow = ">=6.2.0"
+pyparsing = ">=2.3.1"
+python-dateutil = ">=2.7"
+setuptools_scm = ">=7"
+
+[[package]]
+name = "mdit-py-plugins"
+version = "0.3.3"
+description = "Collection of plugins for markdown-it-py"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+markdown-it-py = ">=1.0.0,<3.0.0"
+
+[package.extras]
+code_style = ["pre-commit"]
+rtd = ["attrs", "myst-parser (>=0.16.1,<0.17.0)", "sphinx-book-theme (>=0.1.0,<0.2.0)"]
+testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"]
+
+[[package]]
+name = "mdurl"
+version = "0.1.2"
+description = "Markdown URL utilities"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "multidict"
+version = "6.0.4"
+description = "multidict implementation"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "numpy"
+version = "1.24.2"
+description = "Fundamental package for array computing in Python"
+category = "main"
+optional = false
+python-versions = ">=3.8"
+
+[[package]]
+name = "nvidia-cublas-cu11"
+version = "11.10.3.66"
+description = "CUBLAS native runtime libraries"
+category = "main"
+optional = false
+python-versions = ">=3"
+
+[[package]]
+name = "nvidia-cuda-nvrtc-cu11"
+version = "11.7.99"
+description = "NVRTC native runtime libraries"
+category = "main"
+optional = false
+python-versions = ">=3"
+
+[[package]]
+name = "nvidia-cuda-runtime-cu11"
+version = "11.7.99"
+description = "CUDA Runtime native Libraries"
+category = "main"
+optional = false
+python-versions = ">=3"
+
+[[package]]
+name = "nvidia-cudnn-cu11"
+version = "8.5.0.96"
+description = "cuDNN runtime libraries"
+category = "main"
+optional = false
+python-versions = ">=3"
+
+[[package]]
+name = "oauthlib"
+version = "3.2.2"
+description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.extras]
+rsa = ["cryptography (>=3.0.0)"]
+signals = ["blinker (>=1.4.0)"]
+signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"]
+
+[[package]]
+name = "omegaconf"
+version = "2.3.0"
+description = "A flexible configuration library"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+antlr4-python3-runtime = ">=4.9.0,<4.10.0"
+PyYAML = ">=5.1.0"
+
+[[package]]
+name = "opencv-python"
+version = "4.7.0.72"
+description = "Wrapper package for OpenCV python bindings."
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+numpy = [
+    {version = ">=1.21.2", markers = "python_version >= \"3.10\""},
+    {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""},
+    {version = ">=1.22.0", markers = "python_version >= \"3.11\""},
+    {version = ">=1.19.3", markers = "python_version >= \"3.6\" and platform_system == \"Linux\" and platform_machine == \"aarch64\" or python_version >= \"3.9\""},
+    {version = ">=1.17.0", markers = "python_version >= \"3.7\""},
+    {version = ">=1.17.3", markers = "python_version >= \"3.8\""},
+]
+
+[[package]]
+name = "orjson"
+version = "3.8.10"
+description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy"
+category = "main"
+optional = false
+python-versions = ">= 3.7"
+
+[[package]]
+name = "packaging"
+version = "23.1"
+description = "Core utilities for Python packages"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "pandas"
+version = "2.0.0"
+description = "Powerful data structures for data analysis, time series, and statistics"
+category = "main"
+optional = false
+python-versions = ">=3.8"
+
+[package.dependencies]
+numpy = [
+    {version = ">=1.21.0", markers = "python_version >= \"3.10\""},
+    {version = ">=1.23.2", markers = "python_version >= \"3.11\""},
+]
+python-dateutil = ">=2.8.2"
+pytz = ">=2020.1"
+tzdata = ">=2022.1"
+
+[package.extras]
+all = ["beautifulsoup4 (>=4.9.3)", "bottleneck (>=1.3.2)", "brotlipy (>=0.7.0)", "fastparquet (>=0.6.3)", "fsspec (>=2021.07.0)", "gcsfs (>=2021.07.0)", "html5lib (>=1.1)", "hypothesis (>=6.34.2)", "jinja2 (>=3.0.0)", "lxml (>=4.6.3)", "matplotlib (>=3.6.1)", "numba (>=0.53.1)", "numexpr (>=2.7.3)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pandas-gbq (>=0.15.0)", "psycopg2 (>=2.8.6)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "PyQt5 (>=5.15.1)", "pyreadstat (>=1.1.2)", "pytest (>=7.0.0)", "pytest-xdist (>=2.2.0)", "pytest-asyncio (>=0.17.0)", "python-snappy (>=0.6.0)", "pyxlsb (>=1.0.8)", "qtpy (>=2.2.0)", "scipy (>=1.7.1)", "s3fs (>=2021.08.0)", "SQLAlchemy (>=1.4.16)", "tables (>=3.6.1)", "tabulate (>=0.8.9)", "xarray (>=0.21.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)", "zstandard (>=0.15.2)"]
+aws = ["s3fs (>=2021.08.0)"]
+clipboard = ["PyQt5 (>=5.15.1)", "qtpy (>=2.2.0)"]
+compression = ["brotlipy (>=0.7.0)", "python-snappy (>=0.6.0)", "zstandard (>=0.15.2)"]
+computation = ["scipy (>=1.7.1)", "xarray (>=0.21.0)"]
+excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pyxlsb (>=1.0.8)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)"]
+feather = ["pyarrow (>=7.0.0)"]
+fss = ["fsspec (>=2021.07.0)"]
+gcp = ["gcsfs (>=2021.07.0)", "pandas-gbq (>=0.15.0)"]
+hdf5 = ["tables (>=3.6.1)"]
+html = ["beautifulsoup4 (>=4.9.3)", "html5lib (>=1.1)", "lxml (>=4.6.3)"]
+mysql = ["SQLAlchemy (>=1.4.16)", "pymysql (>=1.0.2)"]
+output_formatting = ["jinja2 (>=3.0.0)", "tabulate (>=0.8.9)"]
+parquet = ["pyarrow (>=7.0.0)"]
+performance = ["bottleneck (>=1.3.2)", "numba (>=0.53.1)", "numexpr (>=2.7.1)"]
+plot = ["matplotlib (>=3.6.1)"]
+postgresql = ["SQLAlchemy (>=1.4.16)", "psycopg2 (>=2.8.6)"]
+spss = ["pyreadstat (>=1.1.2)"]
+sql-other = ["SQLAlchemy (>=1.4.16)"]
+test = ["hypothesis (>=6.34.2)", "pytest (>=7.0.0)", "pytest-xdist (>=2.2.0)", "pytest-asyncio (>=0.17.0)"]
+xml = ["lxml (>=4.6.3)"]
+
+[[package]]
+name = "pillow"
+version = "8.4.0"
+description = "Python Imaging Library (Fork)"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[[package]]
+name = "plotly"
+version = "5.14.1"
+description = "An open-source, interactive data visualization library for Python"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+packaging = "*"
+tenacity = ">=6.2.0"
+
+[[package]]
+name = "protobuf"
+version = "3.20.3"
+description = "Protocol Buffers"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "pyasn1"
+version = "0.4.8"
+description = "ASN.1 types and codecs"
+category = "main"
+optional = false
+python-versions = "*"
+
+[[package]]
+name = "pyasn1-modules"
+version = "0.2.8"
+description = "A collection of ASN.1-based protocols modules."
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+pyasn1 = ">=0.4.6,<0.5.0"
+
+[[package]]
+name = "pydantic"
+version = "1.10.7"
+description = "Data validation and settings management using python type hints"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+typing-extensions = ">=4.2.0"
+
+[package.extras]
+dotenv = ["python-dotenv (>=0.10.4)"]
+email = ["email-validator (>=1.0.3)"]
+
+[[package]]
+name = "pydub"
+version = "0.25.1"
+description = "Manipulate audio with an simple and easy high level interface"
+category = "main"
+optional = false
+python-versions = "*"
+
+[[package]]
+name = "pyparsing"
+version = "3.0.9"
+description = "pyparsing module - Classes and methods to define and execute parsing grammars"
+category = "main"
+optional = false
+python-versions = ">=3.6.8"
+
+[package.extras]
+diagrams = ["railroad-diagrams", "jinja2"]
+
+[[package]]
+name = "pyrsistent"
+version = "0.19.3"
+description = "Persistent/Functional/Immutable data structures"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "python-dateutil"
+version = "2.8.2"
+description = "Extensions to the standard Python datetime module"
+category = "main"
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
+
+[package.dependencies]
+six = ">=1.5"
+
+[[package]]
+name = "python-multipart"
+version = "0.0.6"
+description = "A streaming multipart parser for Python"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.extras]
+dev = ["atomicwrites (==1.2.1)", "attrs (==19.2.0)", "coverage (==6.5.0)", "hatch", "invoke (==1.7.3)", "more-itertools (==4.3.0)", "pbr (==4.3.0)", "pluggy (==1.0.0)", "py (==1.11.0)", "pytest-cov (==4.0.0)", "pytest-timeout (==2.1.0)", "pytest (==7.2.0)", "pyyaml (==5.1)"]
+
+[[package]]
+name = "pytorch-lightning"
+version = "1.9.0"
+description = "PyTorch Lightning is the lightweight PyTorch wrapper for ML researchers. Scale your models. Write less boilerplate."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+fsspec = {version = ">2021.06.0", extras = ["http"]}
+lightning-utilities = ">=0.4.2"
+numpy = ">=1.17.2"
+packaging = ">=17.1"
+PyYAML = ">=5.4"
+torch = ">=1.10.0"
+torchmetrics = ">=0.7.0"
+tqdm = ">=4.57.0"
+typing-extensions = ">=4.0.0"
+
+[package.extras]
+all = ["matplotlib (>3.1)", "omegaconf (>=2.0.5)", "hydra-core (>=1.0.5)", "jsonargparse[signatures] (>=4.18.0)", "rich (>=10.14.0,!=10.15.0.a)", "tensorboardX (>=2.2)", "fairscale (>=0.4.5)", "deepspeed (>=0.6.0)", "horovod (>=0.21.2,!=0.24.0)", "torchvision (>=0.11.1)", "gym[classic_control] (>=0.17.0)", "ipython[all] (<8.7.1)", "hivemind (==1.1.5)"]
+deepspeed = ["deepspeed (>=0.6.0)"]
+dev = ["matplotlib (>3.1)", "omegaconf (>=2.0.5)", "hydra-core (>=1.0.5)", "jsonargparse[signatures] (>=4.18.0)", "rich (>=10.14.0,!=10.15.0.a)", "tensorboardX (>=2.2)", "fairscale (>=0.4.5)", "deepspeed (>=0.6.0)", "horovod (>=0.21.2,!=0.24.0)", "torchvision (>=0.11.1)", "gym[classic_control] (>=0.17.0)", "ipython[all] (<8.7.1)", "coverage (==6.5.0)", "codecov (==2.1.12)", "pytest (==7.2.0)", "pytest-cov (==4.0.0)", "pytest-forked (==1.4.0)", "pytest-rerunfailures (==10.3)", "pre-commit (==2.20.0)", "cloudpickle (>=1.3)", "scikit-learn (>0.22.1)", "onnxruntime (<1.14.0)", "psutil (<5.9.5)", "pandas (>1.0)", "fastapi (<0.87.0)", "uvicorn (<0.19.1)", "tensorboard (>=2.9.1)", "protobuf (<=3.20.1)", "hivemind (==1.1.5)"]
+examples = ["torchvision (>=0.11.1)", "gym[classic_control] (>=0.17.0)", "ipython[all] (<8.7.1)"]
+extra = ["matplotlib (>3.1)", "omegaconf (>=2.0.5)", "hydra-core (>=1.0.5)", "jsonargparse[signatures] (>=4.18.0)", "rich (>=10.14.0,!=10.15.0.a)", "tensorboardX (>=2.2)"]
+fairscale = ["fairscale (>=0.4.5)"]
+hivemind = ["hivemind (==1.1.5)"]
+horovod = ["horovod (>=0.21.2,!=0.24.0)"]
+strategies = ["fairscale (>=0.4.5)", "deepspeed (>=0.6.0)", "horovod (>=0.21.2,!=0.24.0)", "hivemind (==1.1.5)"]
+test = ["coverage (==6.5.0)", "codecov (==2.1.12)", "pytest (==7.2.0)", "pytest-cov (==4.0.0)", "pytest-forked (==1.4.0)", "pytest-rerunfailures (==10.3)", "pre-commit (==2.20.0)", "cloudpickle (>=1.3)", "scikit-learn (>0.22.1)", "onnxruntime (<1.14.0)", "psutil (<5.9.5)", "pandas (>1.0)", "fastapi (<0.87.0)", "uvicorn (<0.19.1)", "tensorboard (>=2.9.1)", "protobuf (<=3.20.1)"]
+
+[[package]]
+name = "pytz"
+version = "2023.3"
+description = "World timezone definitions, modern and historical"
+category = "main"
+optional = false
+python-versions = "*"
+
+[[package]]
+name = "pyyaml"
+version = "6.0"
+description = "YAML parser and emitter for Python"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[[package]]
+name = "requests"
+version = "2.28.2"
+description = "Python HTTP for Humans."
+category = "main"
+optional = false
+python-versions = ">=3.7, <4"
+
+[package.dependencies]
+certifi = ">=2017.4.17"
+charset-normalizer = ">=2,<4"
+idna = ">=2.5,<4"
+urllib3 = ">=1.21.1,<1.27"
+
+[package.extras]
+socks = ["PySocks (>=1.5.6,!=1.5.7)"]
+use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"]
+
+[[package]]
+name = "requests-oauthlib"
+version = "1.3.1"
+description = "OAuthlib authentication support for Requests."
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+
+[package.dependencies]
+oauthlib = ">=3.0.0"
+requests = ">=2.0.0"
+
+[package.extras]
+rsa = ["oauthlib[signedtoken] (>=3.0.0)"]
+
+[[package]]
+name = "rsa"
+version = "4.9"
+description = "Pure-Python RSA implementation"
+category = "main"
+optional = false
+python-versions = ">=3.6,<4"
+
+[package.dependencies]
+pyasn1 = ">=0.1.3"
+
+[[package]]
+name = "scipy"
+version = "1.10.1"
+description = "Fundamental algorithms for scientific computing in Python"
+category = "main"
+optional = false
+python-versions = "<3.12,>=3.8"
+
+[package.dependencies]
+numpy = ">=1.19.5,<1.27.0"
+
+[package.extras]
+test = ["pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "asv", "mpmath", "gmpy2", "threadpoolctl", "scikit-umfpack", "pooch"]
+doc = ["sphinx (!=4.1.0)", "pydata-sphinx-theme (==0.9.0)", "sphinx-design (>=0.2.0)", "matplotlib (>2)", "numpydoc"]
+dev = ["mypy", "typing-extensions", "pycodestyle", "flake8", "rich-click", "click", "doit (>=0.36.0)", "pydevtool"]
+
+[[package]]
+name = "seaborn"
+version = "0.12.2"
+description = "Statistical data visualization"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+matplotlib = ">=3.1,<3.6.1 || >3.6.1"
+numpy = ">=1.17,<1.24.0 || >1.24.0"
+pandas = ">=0.25"
+
+[package.extras]
+dev = ["pytest", "pytest-cov", "pytest-xdist", "flake8", "mypy", "pandas-stubs", "pre-commit", "flit"]
+docs = ["numpydoc", "nbconvert", "ipykernel", "sphinx-copybutton", "sphinx-issues", "sphinx-design", "pyyaml", "pydata_sphinx_theme (==0.10.0rc2)"]
+stats = ["scipy (>=1.3)", "statsmodels (>=0.10)"]
+
+[[package]]
+name = "semantic-version"
+version = "2.10.0"
+description = "A library implementing the 'SemVer' scheme."
+category = "main"
+optional = false
+python-versions = ">=2.7"
+
+[package.extras]
+dev = ["Django (>=1.11)", "nose2", "tox", "check-manifest", "coverage", "flake8", "wheel", "zest.releaser", "readme-renderer (<25.0)", "colorama (<=0.4.1)"]
+doc = ["sphinx", "sphinx-rtd-theme"]
+
+[[package]]
+name = "setuptools-scm"
+version = "7.1.0"
+description = "the blessed package to manage your versions by scm tags"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+packaging = ">=20.0"
+tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""}
+typing-extensions = "*"
+
+[package.extras]
+test = ["pytest (>=6.2)", "virtualenv (>20)"]
+toml = ["setuptools (>=42)"]
+
+[[package]]
+name = "six"
+version = "1.16.0"
+description = "Python 2 and 3 compatibility utilities"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
+
+[[package]]
+name = "sniffio"
+version = "1.3.0"
+description = "Sniff out which async library your code is running under"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "starlette"
+version = "0.26.1"
+description = "The little ASGI library that shines."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+anyio = ">=3.4.0,<5"
+
+[package.extras]
+full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"]
+
+[[package]]
+name = "tenacity"
+version = "8.2.2"
+description = "Retry code until it succeeds"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.extras]
+doc = ["reno", "sphinx", "tornado (>=4.5)"]
+
+[[package]]
+name = "tensorboard"
+version = "2.11.2"
+description = "TensorBoard lets you watch Tensors Flow"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+absl-py = ">=0.4"
+google-auth = ">=1.6.3,<3"
+google-auth-oauthlib = ">=0.4.1,<0.5"
+grpcio = ">=1.24.3"
+markdown = ">=2.6.8"
+numpy = ">=1.12.0"
+protobuf = ">=3.9.2,<4"
+requests = ">=2.21.0,<3"
+tensorboard-data-server = ">=0.6.0,<0.7.0"
+tensorboard-plugin-wit = ">=1.6.0"
+werkzeug = ">=1.0.1"
+
+[[package]]
+name = "tensorboard-data-server"
+version = "0.6.1"
+description = "Fast data loading for TensorBoard"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[[package]]
+name = "tensorboard-plugin-wit"
+version = "1.8.1"
+description = "What-If Tool TensorBoard plugin."
+category = "main"
+optional = false
+python-versions = "*"
+
+[[package]]
+name = "tomli"
+version = "2.0.1"
+description = "A lil' TOML parser"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "toolz"
+version = "0.12.0"
+description = "List processing tools and functional utilities"
+category = "main"
+optional = false
+python-versions = ">=3.5"
+
+[[package]]
+name = "torch"
+version = "1.13.1"
+description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration"
+category = "main"
+optional = false
+python-versions = ">=3.7.0"
+
+[package.dependencies]
+nvidia-cublas-cu11 = {version = "11.10.3.66", markers = "platform_system == \"Linux\""}
+nvidia-cuda-nvrtc-cu11 = {version = "11.7.99", markers = "platform_system == \"Linux\""}
+nvidia-cuda-runtime-cu11 = {version = "11.7.99", markers = "platform_system == \"Linux\""}
+nvidia-cudnn-cu11 = {version = "8.5.0.96", markers = "platform_system == \"Linux\""}
+typing-extensions = "*"
+
+[package.extras]
+opt-einsum = ["opt-einsum (>=3.3)"]
+
+[[package]]
+name = "torchmetrics"
+version = "0.11.0"
+description = "PyTorch native Metrics"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+numpy = ">=1.17.2"
+packaging = "*"
+torch = ">=1.8.1"
+
+[package.extras]
+all = ["pystoi", "torchvision (>=0.8)", "pycocotools", "scipy", "torch-fidelity", "torchvision", "lpips", "pytorch-lightning (>=1.5)", "transformers (>=4.10.0)", "regex (>=2021.9.24)", "nltk (>=3.6)", "tqdm (>=4.41.0)"]
+audio = ["pystoi"]
+detection = ["torchvision (>=0.8)", "pycocotools"]
+docs = ["sphinx-autodoc-typehints (>=1.0)", "nbsphinx (>=0.8)", "docutils (>=0.16)", "sphinx-togglebutton (>=0.2)", "pandoc (>=1.0)", "myst-parser", "sphinx-paramlinks (>=0.5.1)", "sphinxcontrib-fulltoc (>=1.0)", "sphinxcontrib-mockautodoc", "sphinx-copybutton (>=0.3)", "sphinx (>=4.0,<5.0)"]
+image = ["scipy", "torch-fidelity", "torchvision", "lpips"]
+integrate = ["pytorch-lightning (>=1.5)"]
+multimodal = ["transformers (>=4.10.0)"]
+test = ["types-protobuf", "rouge-score (>=0.0.4)", "bert-score (==0.3.10)", "requests", "mir-eval (>=0.6)", "jiwer (>=2.3.0)", "scikit-learn (>1.0,<1.1.1)", "check-manifest", "types-tabulate", "pytest-timeout", "types-emoji", "pycocotools", "coverage (>5.2)", "pytest (>=6.0.0,<7.0.0)", "types-six", "kornia (>=0.6.7)", "phmdoctest (>=1.1.1)", "pandas", "pytest-cov (>2.10)", "cloudpickle (>=1.3)", "pre-commit (>=1.0)", "scipy", "psutil", "mypy (==0.982)", "types-requests", "pytest-rerunfailures (>=10.0)", "types-pyyaml", "types-setuptools", "sacrebleu (>=2.0.0)", "netcal", "pytorch-msssim (==0.2.1)", "transformers (>4.4.0)", "fast-bss-eval (>=0.1.0)", "fire", "scikit-image (>0.17.1)", "dython", "torch-complex", "pytest-doctestplus (>=0.9.0)", "huggingface-hub (<0.7)", "pypesq (>1.2)"]
+text = ["regex (>=2021.9.24)", "nltk (>=3.6)", "tqdm (>=4.41.0)"]
+
+[[package]]
+name = "torchvision"
+version = "0.14.1"
+description = "image and video datasets and models for torch deep learning"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+numpy = "*"
+pillow = ">=5.3.0,<8.3.0 || >=8.4.0"
+requests = "*"
+torch = "1.13.1"
+typing-extensions = "*"
+
+[package.extras]
+scipy = ["scipy"]
+
+[[package]]
+name = "tqdm"
+version = "4.65.0"
+description = "Fast, Extensible Progress Meter"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+colorama = {version = "*", markers = "platform_system == \"Windows\""}
+
+[package.extras]
+dev = ["py-make (>=0.1.0)", "twine", "wheel"]
+notebook = ["ipywidgets (>=6)"]
+slack = ["slack-sdk"]
+telegram = ["requests"]
+
+[[package]]
+name = "typing-extensions"
+version = "4.5.0"
+description = "Backported and Experimental Type Hints for Python 3.7+"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "tzdata"
+version = "2023.3"
+description = "Provider of IANA time zone data"
+category = "main"
+optional = false
+python-versions = ">=2"
+
+[[package]]
+name = "uc-micro-py"
+version = "1.0.1"
+description = "Micro subset of unicode data files for linkify-it-py projects."
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.extras]
+test = ["coverage", "pytest", "pytest-cov"]
+
+[[package]]
+name = "uritemplate"
+version = "4.1.1"
+description = "Implementation of RFC 6570 URI Templates"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[[package]]
+name = "urllib3"
+version = "1.26.15"
+description = "HTTP library with thread-safe connection pooling, file post, and more."
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
+
+[package.extras]
+brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"]
+secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"]
+socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
+
+[[package]]
+name = "uvicorn"
+version = "0.21.1"
+description = "The lightning-fast ASGI server."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+click = ">=7.0"
+h11 = ">=0.8"
+
+[package.extras]
+standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"]
+
+[[package]]
+name = "websockets"
+version = "11.0.1"
+description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "werkzeug"
+version = "2.2.3"
+description = "The comprehensive WSGI web application library."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+MarkupSafe = ">=2.1.1"
+
+[package.extras]
+watchdog = ["watchdog"]
+
+[[package]]
+name = "wget"
+version = "3.2"
+description = "pure python download utility"
+category = "main"
+optional = false
+python-versions = "*"
+
+[[package]]
+name = "yarl"
+version = "1.8.2"
+description = "Yet another URL library"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+idna = ">=2.0"
+multidict = ">=4.0"
+
+[metadata]
+lock-version = "1.1"
+python-versions = ">=3.10,<3.12"
+content-hash = "17cec1f61fed3b070c0b744eeecc9dbaed1ea06d758238ac84f108545ab14a21"
+
+[metadata.files]
+absl-py = []
+aiofiles = []
+aiohttp = []
+aiosignal = []
+altair = []
+antlr4-python3-runtime = []
+anyio = []
+async-timeout = []
+attrs = []
+cachetools = []
+certifi = []
+charset-normalizer = []
+click = []
+colorama = []
+contourpy = []
+cycler = []
+earthengine-api = []
+ee-extra = []
+entrypoints = []
+fastapi = []
+ffmpy = []
+filelock = []
+fonttools = []
+frozenlist = []
+fsspec = []
+google-api-core = []
+google-api-python-client = []
+google-auth = []
+google-auth-httplib2 = []
+google-auth-oauthlib = []
+google-cloud-core = []
+google-cloud-storage = []
+google-crc32c = []
+google-resumable-media = []
+googleapis-common-protos = []
+gradio = []
+gradio-client = []
+grpcio = []
+h11 = []
+httpcore = []
+httplib2 = []
+httpx = []
+huggingface-hub = []
+hydra-client = []
+hydra-core = []
+idna = []
+jinja2 = []
+jsonschema = []
+kiwisolver = []
+lightning-utilities = []
+linkify-it-py = []
+markdown = []
+markdown-it-py = []
+markupsafe = []
+matplotlib = []
+mdit-py-plugins = []
+mdurl = []
+multidict = []
+numpy = []
+nvidia-cublas-cu11 = []
+nvidia-cuda-nvrtc-cu11 = []
+nvidia-cuda-runtime-cu11 = []
+nvidia-cudnn-cu11 = []
+oauthlib = []
+omegaconf = []
+opencv-python = []
+orjson = []
+packaging = []
+pandas = []
+pillow = []
+plotly = []
+protobuf = []
+pyasn1 = []
+pyasn1-modules = []
+pydantic = []
+pydub = []
+pyparsing = []
+pyrsistent = []
+python-dateutil = []
+python-multipart = []
+pytorch-lightning = []
+pytz = []
+pyyaml = []
+requests = []
+requests-oauthlib = []
+rsa = []
+scipy = []
+seaborn = []
+semantic-version = []
+setuptools-scm = []
+six = []
+sniffio = []
+starlette = []
+tenacity = []
+tensorboard = []
+tensorboard-data-server = []
+tensorboard-plugin-wit = []
+tomli = []
+toolz = []
+torch = []
+torchmetrics = []
+torchvision = []
+tqdm = []
+typing-extensions = []
+tzdata = []
+uc-micro-py = []
+uritemplate = []
+urllib3 = []
+uvicorn = []
+websockets = []
+werkzeug = []
+wget = []
+yarl = []

pyproject.toml

@@ -0,0 +1,31 @@
+[tool.poetry]
+name = "cv_app"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <[email protected]>"]
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.12"
+torch = "1.13.1"
+tensorboard = "2.11.2"
+pytorch-lightning = "1.9.0"
+torchmetrics = "0.11.0"
+Pillow = "8.4.0"
+torchvision = "0.14.1"
+matplotlib = "^3.7.1"
+hydra-client = "0.5.1"
+hydra-core = "1.3.1"
+wget = "^3.2"
+scipy = "^1.10.1"
+seaborn = "^0.12.2"
+earthengine-api = "0.1.338"
+ee-extra = "0.0.15"
+gradio = "^3.27.0"
+opencv-python = "^4.7.0"
+plotly = "^5.14.1"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -0,0 +1,133 @@
+
+aiohttp==3.8.3
+aiosignal==1.3.1
+antlr4-python3-runtime==4.9.3
+appdirs==1.4.4
+argh==0.26.2
+async-timeout==4.0.2
+atomicwrites==1.4.0
+attrs==19.3.0
+backports.weakref==1.0.post1
+bkcharts==0.2
+black==19.10b0
+boto==2.49.0
+bqplot==0.12.36
+branca==0.6.0
+brotlipy==0.7.0
+cachetools==5.3.0
+certifi==2021.10.8
+click==8.0.3
+colour==0.1.5
+comtypes==1.1.10
+cycler==0.10.0
+cytoolz==0.11.0
+daal4py==2021.3.0
+dask==2021.10.0
+earthengine-api==0.1.338
+ee-extra==0.0.15
+eerepr==0.0.4
+entrypoints==0.3
+et-xmlfile==1.1.0
+export==0.2.0
+ffmpeg-python==0.2.0
+folium==0.14.0
+fonttools==4.25.0
+frozenlist==1.3.3
+gdown==4.6.0
+geeadd==0.5.6
+geemap==0.19.6
+geocoder==1.38.1
+geojson==3.0.0
+google-api-core==2.11.0
+google-api-python-client==2.74.0
+google-auth==2.16.0
+google-auth-httplib2==0.1.0
+google-auth-oauthlib==0.4.6
+google-cloud-core==2.3.2
+google-cloud-storage==2.7.0
+google-crc32c==1.5.0
+google-resumable-media==2.4.1
+googleapis-common-protos==1.58.0
+grpcio==1.51.1
+httplib2==0.21.0
+hydra-client==0.5.1
+hydra-core==1.3.1
+inflection==0.5.1
+ipyevents==2.0.1
+ipyfilechooser==0.6.0
+ipyleaflet==0.17.2
+ipytree==0.2.2
+lightning-utilities==0.6.0.post0
+llvmlite==0.37.0
+locket==0.2.1
+logzero==1.7.0
+Markdown==3.4.1
+mccabe==0.6.1
+mkl-fft==1.3.1
+mkl-service==2.4.0
+mpmath==1.2.1
+multidict==6.0.4
+munkres==1.1.4
+mypy-extensions==0.4.3
+nltk==3.6.5
+oauthlib==3.2.2
+omegaconf==2.3.0
+pathspec==0.7.0
+patsy==0.5.2
+pep8==1.7.1
+Pillow==8.4.0
+pkginfo==1.7.1
+plotly==5.13.0
+ply==3.11
+protobuf==3.20.3
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycosat==0.6.3
+PyCRS==1.0.2
+pycurl==7.44.1
+pyls-spyder==0.4.0
+pyperclip==1.8.2
+pyreadline==2.1
+pyshp==2.3.1
+pytest==6.2.4
+python-box==6.1.0
+python-lsp-jsonrpc==1.0.0
+python-lsp-server==1.2.4
+pytorch-lightning==1.9.0
+pytz==2021.3
+PyYAML==6.0
+ratelim==0.1.6
+requests-oauthlib==1.3.1
+rsa==4.9
+sankee==0.2.1
+scikit-image==0.18.3
+scooby==0.7.1
+simplegeneric==0.8.1
+Sphinx==4.2.0
+statsmodels==0.12.2
+tables==3.6.1
+tenacity==8.1.0
+tensorboard==2.11.2
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+terminado==0.9.4
+torch==1.13.1
+torchaudio==0.13.1
+torchmetrics==0.11.0
+torchvision==0.14.1
+traittypes==0.2.1
+typing_extensions==4.4.0
+unicodecsv==0.14.1
+uritemplate==4.1.1
+urllib3==1.26.7
+webencodings==0.5.1
+wget==3.2
+whitebox==2.2.0
+whiteboxgui==2.2.0
+win-unicode-console==0.5
+wincertstore==0.2
+xlwt==1.3.0
+xyzservices==2022.9.0
+yarl==1.8.2
+zict==2.0.0
+zope.event==4.5.0