adding timesfm future metric prediction

app.py

@@ -58,8 +58,8 @@ def authenticate_user():
 
 
 def main():
-    for x in os.walk("."):
-         print(x[0])
+    # for x in os.walk("."):
+    #      print(x[0])
     
     if "authenticated" not in st.session_state:
         st.session_state.authenticated = False 

pag/monitor.py

@@ -5,13 +5,18 @@ import streamlit as st
 import geopandas as gpd
 from authentication import greeting, check_password
 from senHub import SenHub
-from datetime import datetime
+from datetime import datetime, timedelta
 from sentinelhub import  SHConfig
 import requests
 import process
 from zipfile import ZipFile
 import plotly.express as px
 import threading
+import pandas as pd
+import grpc
+import pb.timesfm_pb2_grpc
+import pb.timesfm_pb2
+import time
 
 def check_authentication():
     if not check_password():
@@ -20,9 +25,10 @@ def check_authentication():
 
 
 config = SHConfig()
-config.instance_id       = '20996045-deec-4830-85dc-ad539df944eb'
-config.sh_client_id      = 'bb286d9b-6ca5-4737-bd7c-8b19b63bd182'
-config.sh_client_secret  = 'LEyvAXIGIsHh6iFZaZ1u4uCmpYgcyWwf'
+config.instance_id       = '352670fb-2edf-4abd-90c8-437485a2403e'
+config.sh_client_id      = 'ca95f10f-443c-4c60-9a36-98950292bb9b'
+config.sh_client_secret  = 'rNFGRxGNiNFrXJfGyHIkVRyGOrdWNsfI'
+config.sh_timesfm_IP = "34.121.141.161"
 
 def select_field(gdf):
     st.markdown("""
@@ -78,35 +84,49 @@ def get_and_cache_available_dates(_df, field, year, start_date, end_date):
 #         curated_df = gpd.read_file(curated_date_path)
 #     return curated_df
 
-def get_cuarted_df_for_field(df, field, date, metric, clientName, dates):
+def get_cuarted_df_for_field(df, field, date, metric, clientName, dates=None):
     curated_date_path =  utils.get_curated_location_img_path(clientName, metric, date, field)
     if curated_date_path is not None:
         curated_df = gpd.read_file(curated_date_path)
 
     else:
-        download_date_date(df, field, [date], metric, clientName,)
+        download_date_data(df, field, [date], metric, clientName,)
         curated_date_path =  utils.get_curated_location_img_path(clientName, metric, date, field)
         print("curr selected date processed")
 
+        if dates:
+            old_dates = [prev_date for prev_date in dates if prev_date != date]
 
-        old_dates = [prev_date for prev_date in dates if prev_date != date]
-
-        download_thread = threading.Thread(target=download_date_date, name="Downloader", args=(df, field, old_dates, metric, clientName))
-        download_thread.start()
+            download_thread = threading.Thread(target=download_date_data, name="Downloader", args=(df, field, old_dates, metric, clientName))
+            download_thread.start()
 
         curated_date_path =  utils.get_curated_location_img_path(clientName, metric, date, field)
         curated_df = gpd.read_file(curated_date_path)
+
     return curated_df
 
-def download_date_date(df, field, dates, metric, clientName,):
+# def check_and_download_date_data(df, field, date, metric, clientName,):
+#     curated_date_path =  utils.get_curated_location_img_path(clientName, metric, date, field)
+#     if curated_date_path is not None:
+#         curated_df = gpd.read_file(curated_date_path)
+#     else:
+#         process.Download_image_in_given_date(clientName, metric, df, field, date)
+#         process.mask_downladed_image(clientName, metric, df, field, date)
+#         process.convert_maske_image_to_geodataframe(clientName, metric, df, field, date, df.crs)
+#         curated_date_path =  utils.get_curated_location_img_path(clientName, metric, date, field)
+#         curated_df = gpd.read_file(curated_date_path)
+#     return curated_df
+
+def download_date_data(df, field, dates, metric, clientName,):
     for date in dates:
         process.Download_image_in_given_date(clientName, metric, df, field, date)
         process.mask_downladed_image(clientName, metric, df, field, date)
         process.convert_maske_image_to_geodataframe(clientName, metric, df, field, date, df.crs)
-    print(f"finished downloading prev dates data")
+    # print(f"finished downloading prev dates data")
     return 
 
 
+
 def track(metric, field_name, src_df, client_name):
     st.title(":green[Select Date and Start Monitoring]")
     dates = []
@@ -160,9 +180,9 @@ def track(metric, field_name, src_df, client_name):
 
         # Get the field data at the selected date
         with st.spinner('Loading Field Data...'):
-            # Get the metric data and cloud cover data for the selected field and date
-            metric_data = get_cuarted_df_for_field(src_df, field_name, date, metric, client_name, dates)
-            cloud_cover_data = get_cuarted_df_for_field(src_df, field_name, date, 'CLP', client_name, dates)
+            # Get the metric data and cloud cover data for the selected field and date, to enable background download set dates=dates
+            metric_data = get_cuarted_df_for_field(src_df, field_name, date, metric, client_name, dates=None)
+            cloud_cover_data = get_cuarted_df_for_field(src_df, field_name, date, 'CLP', client_name, dates=None)
             
             #Merge the metric and cloud cover data on the geometry column
             field_data = metric_data.merge(cloud_cover_data, on='geometry')
@@ -280,6 +300,81 @@ def monitor_fields():
         else:
             st.info("No Fields Added Yet!")
             return
+        
+        if field_name != "Select Field":
+            st.title(":orange[Predict Metrics for Next Month]")
+            subcol1, subcol2, subcol3 = st.columns(3)
+            if subcol2.button(f'Predict {metric} for Next 3 Months'):
+                start_date = '2024-01-01'
+                today = datetime.today()
+                end_date = today.strftime('%Y-%m-%d')
+                year = '2024'
+
+                dates = get_and_cache_available_dates(gdf, field_name, year, start_date, end_date)
+                my_bar = st.progress(0, text= f"Downloading Data for the last {len(dates)//4} months ...")
+                counter = 0
+                downloaded_prev_metrics = []
+                for index, date in enumerate(dates):
+                    # time.sleep(0.1)
+                    metric_data = get_cuarted_df_for_field(gdf, field_name, date, metric, current_user, dates = None)
+                    cloud_cover_data = get_cuarted_df_for_field(gdf, field_name, date, 'CLP', current_user, dates = None)
+                    field_data = metric_data.merge(cloud_cover_data, on='geometry')
+                    avg_metric = field_data[f'{metric}_{date}'].mean() 
+                    downloaded_prev_metrics.append((date, avg_metric))
+                    counter = counter + 100/(len(dates))
+                    my_bar.progress(round(counter), text=f"Downloading Data for the last {len(dates)//4} months: {round(counter)}%")
+
+                # chart_data = pd.DataFrame(
+                # {
+                #     "date": [metric[0] for metric in downloaded_prev_metrics],
+                #     f"{metric}": [metric[1] for metric in downloaded_prev_metrics],
+                # }
+                # )
+
+                # st.area_chart(chart_data, x="date", y=f"{metric}")
+                channel = grpc.insecure_channel(f"{config.sh_timesfm_IP}:50051")
+                print("runing client request")
+                stub = pb.timesfm_pb2_grpc.PredictAgriStub(channel)
+                features = stub.predict_metric(iter([pb.timesfm_pb2.prev_values(value=metric[1], date=metric[0]) for metric in downloaded_prev_metrics]))
+                print("server streaming:")
+                predictions = []
+                for feature in features:
+                    predictions.append(feature.value)
+                # do something with the returned output
+                # print(predictions)
+                future_dates = []
+                # print(dates[0])
+                curr_date = datetime.today()
+                for pred in predictions:
+                    curr_date = curr_date + timedelta(days=7)
+                    future_dates.append(curr_date.strftime('%Y-%m-%d'))
+
+                prev_dates = [metric[0] for metric in downloaded_prev_metrics]
+                history_metric_data = [metric[1] for metric in downloaded_prev_metrics]
+                future_metric_data = predictions
+                interval_dates = prev_dates
+                interval_dates.extend(future_dates)
+                history_metric_data.extend([0 for i in range(len(predictions))])
+                masked_future_metric_data = [0 for i in range(len([metric[1] for metric in downloaded_prev_metrics]))]
+                masked_future_metric_data.extend(future_metric_data)
+                # print(f"interval_dates:{len(interval_dates)}")
+                # print(f"history_metric_data:{len(history_metric_data)}")
+                # print(f"masked_future_metric_data:{len(masked_future_metric_data)}")
+                print(predictions)
+            
+                print(interval_dates)
+                prediction_chart_data = pd.DataFrame(
+                {
+                    f"history_{metric}_values": history_metric_data,
+                    f"predicted_{metric}_values":masked_future_metric_data,
+                    f"date": interval_dates,
+                }
+                )
+
+                # print(prediction_chart_data)
+
+                st.area_chart(prediction_chart_data, x="date", y=[f"history_{metric}_values", f"predicted_{metric}_values"])
+
 
         
 

pb/timesfm_pb2.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: timesfm.proto
+# Protobuf Python Version: 4.25.1
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rtimesfm.proto\"*\n\x0bprev_values\x12\r\n\x05value\x18\x01 \x01(\x02\x12\x0c\n\x04\x64\x61te\x18\x02 \x01(\t\",\n\rfuture_values\x12\r\n\x05value\x18\x01 \x01(\x02\x12\x0c\n\x04\x64\x61te\x18\x02 \x01(\t2A\n\x0bPredictAgri\x12\x32\n\x0epredict_metric\x12\x0c.prev_values\x1a\x0e.future_values(\x01\x30\x01\x62\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'timesfm_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+  DESCRIPTOR._options = None
+  _globals['_PREV_VALUES']._serialized_start=17
+  _globals['_PREV_VALUES']._serialized_end=59
+  _globals['_FUTURE_VALUES']._serialized_start=61
+  _globals['_FUTURE_VALUES']._serialized_end=105
+  _globals['_PREDICTAGRI']._serialized_start=107
+  _globals['_PREDICTAGRI']._serialized_end=172
+# @@protoc_insertion_point(module_scope)

pb/timesfm_pb2_grpc.py

@@ -0,0 +1,66 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+import pb.timesfm_pb2 as timesfm__pb2
+
+
+class PredictAgriStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.predict_metric = channel.stream_stream(
+                '/PredictAgri/predict_metric',
+                request_serializer=timesfm__pb2.prev_values.SerializeToString,
+                response_deserializer=timesfm__pb2.future_values.FromString,
+                )
+
+
+class PredictAgriServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def predict_metric(self, request_iterator, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_PredictAgriServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'predict_metric': grpc.stream_stream_rpc_method_handler(
+                    servicer.predict_metric,
+                    request_deserializer=timesfm__pb2.prev_values.FromString,
+                    response_serializer=timesfm__pb2.future_values.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'PredictAgri', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class PredictAgri(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def predict_metric(request_iterator,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.stream_stream(request_iterator, target, '/PredictAgri/predict_metric',
+            timesfm__pb2.prev_values.SerializeToString,
+            timesfm__pb2.future_values.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

process.py

@@ -6,9 +6,10 @@ from sentinelhub import  SHConfig, MimeType
 
 
 config = SHConfig()
-config.instance_id       = '20996045-deec-4830-85dc-ad539df944eb'
-config.sh_client_id      = 'bb286d9b-6ca5-4737-bd7c-8b19b63bd182'
-config.sh_client_secret  = 'LEyvAXIGIsHh6iFZaZ1u4uCmpYgcyWwf'
+config.instance_id       = '352670fb-2edf-4abd-90c8-437485a2403e'
+config.sh_client_id      = 'ca95f10f-443c-4c60-9a36-98950292bb9b'
+config.sh_client_secret  = 'rNFGRxGNiNFrXJfGyHIkVRyGOrdWNsfI'
+config.sh_timesfm_IP = "34.121.141.161"
 
 def Download_image_in_given_date(clientName, metric, df, field, date, mime_type = MimeType.TIFF):
     sen_obj = SenHub(config, mime_type = mime_type)

requirements.txt

@@ -9,4 +9,6 @@ sentinelhub==3.9.1
 shapely==2.0.3
 streamlit-folium==0.12.0 
 streamlit==1.29.0
-geopy==2.4.1
+geopy==2.4.1
+grpcio
+grpcio-tools

senHub.py

@@ -41,9 +41,17 @@ class SenHub:
         '''
         client_id = self.config.sh_client_id
         client_secret = self.config.sh_client_secret
+        # client_id = "ca95f10f-443c-4c60-9a36-98950292bb9b"
+        # client_secret = "rNFGRxGNiNFrXJfGyHIkVRyGOrdWNsfI"
+
         client = BackendApplicationClient(client_id=client_id)
+        print(f"{client_id=}")
+        print(f"{client_secret=}")
+        # client = BackendApplicationClient(client_id="35fd54ac-3084-423d-8c2c-a8c1309e4770")
+
         oauth = OAuth2Session(client=client)
         token = oauth.fetch_token(token_url='https://services.sentinel-hub.com/oauth/token',client_secret=client_secret)
+        # token = oauth.fetch_token(token_url='https://services.sentinel-hub.com/oauth/token',client_secret="s0a9ltSVkVi3aqhmmdE7kHWPF4p3Muus")
         self.token =  token['access_token']
 
     def get_input_data(self, date):