removed files

huggingface-spaces-iris-monitor/app.py

@@ -1,31 +0,0 @@
-import gradio as gr
-from PIL import Image
-import hopsworks
-
-project = hopsworks.login()
-fs = project.get_feature_store()
-
-dataset_api = project.get_dataset_api()
-
-dataset_api.download("Resources/images/latest_iris.png")
-dataset_api.download("Resources/images/actual_iris.png")
-dataset_api.download("Resources/images/df_recent.png")
-dataset_api.download("Resources/images/confusion_matrix.png")
-
-with gr.Blocks() as demo:
-    with gr.Row():
-      with gr.Column():
-          gr.Label("Today's Predicted Image")
-          input_img = gr.Image("latest_iris.png", elem_id="predicted-img")
-      with gr.Column():          
-          gr.Label("Today's Actual Image")
-          input_img = gr.Image("actual_iris.png", elem_id="actual-img")        
-    with gr.Row():
-      with gr.Column():
-          gr.Label("Recent Prediction History")
-          input_img = gr.Image("df_recent.png", elem_id="recent-predictions")
-      with gr.Column():          
-          gr.Label("Confusion Maxtrix with Historical Prediction Performance")
-          input_img = gr.Image("confusion_matrix.png", elem_id="confusion-matrix")        
-
-demo.launch()

iris-batch-inference-pipeline.py

@@ -1,107 +0,0 @@
-import os
-import modal
-    
-LOCAL=True
-
-if LOCAL == False:
-   stub = modal.Stub()
-   hopsworks_image = modal.Image.debian_slim().pip_install(["hopsworks","joblib","seaborn","sklearn","dataframe-image"])
-   @stub.function(image=hopsworks_image, schedule=modal.Period(days=1), secret=modal.Secret.from_name("scalable_ml"))
-   def f():
-       g()
-
-def g():
-    import pandas as pd
-    import hopsworks
-    import joblib
-    import datetime
-    from PIL import Image
-    from datetime import datetime
-    import dataframe_image as dfi
-    from sklearn.metrics import confusion_matrix
-    from matplotlib import pyplot
-    import seaborn as sns
-    import requests
-
-    project = hopsworks.login()
-    fs = project.get_feature_store()
-    
-    mr = project.get_model_registry()
-    model = mr.get_model("iris_modal", version=1)
-    model_dir = model.download()
-    model = joblib.load(model_dir + "/iris_model.pkl")
-    
-    feature_view = fs.get_feature_view(name="iris_modal", version=1)
-    batch_data = feature_view.get_batch_data()
-    
-    y_pred = model.predict(batch_data)
-    # print(y_pred)
-    flower = y_pred[y_pred.size-1]
-    flower_url = "https://raw.githubusercontent.com/featurestoreorg/serverless-ml-course/main/src/01-module/assets/" + flower + ".png"
-    print("Flower predicted: " + flower)
-    img = Image.open(requests.get(flower_url, stream=True).raw)            
-    img.save("./latest_iris.png")
-    dataset_api = project.get_dataset_api()    
-    dataset_api.upload("./latest_iris.png", "Resources/images", overwrite=True)
-    
-    iris_fg = fs.get_feature_group(name="iris_modal", version=1)
-    df = iris_fg.read()
-    # print(df["variety"])
-    label = df.iloc[-1]["variety"]
-    label_url = "https://raw.githubusercontent.com/featurestoreorg/serverless-ml-course/main/src/01-module/assets/" + label + ".png"
-    print("Flower actual: " + label)
-    img = Image.open(requests.get(label_url, stream=True).raw)            
-    img.save("./actual_iris.png")
-    dataset_api.upload("./actual_iris.png", "Resources/images", overwrite=True)
-    
-    monitor_fg = fs.get_or_create_feature_group(name="iris_predictions",
-                                                version=1,
-                                                primary_key=["datetime"],
-                                                description="Iris flower Prediction/Outcome Monitoring"
-                                                )
-    
-    now = datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
-    data = {
-        'prediction': [flower],
-        'label': [label],
-        'datetime': [now],
-       }
-    monitor_df = pd.DataFrame(data)
-    monitor_fg.insert(monitor_df, write_options={"wait_for_job" : False})
-    
-    history_df = monitor_fg.read()
-    # Add our prediction to the history, as the history_df won't have it - 
-    # the insertion was done asynchronously, so it will take ~1 min to land on App
-    history_df = pd.concat([history_df, monitor_df])
-
-
-    df_recent = history_df.tail(5)
-    dfi.export(df_recent, './df_recent.png', table_conversion = 'matplotlib')
-    dataset_api.upload("./df_recent.png", "Resources/images", overwrite=True)
-    
-    predictions = history_df[['prediction']]
-    labels = history_df[['label']]
-
-    # Only create the confusion matrix when our iris_predictions feature group has examples of all 3 iris flowers
-    print("Number of different flower predictions to date: " + str(predictions.value_counts().count()))
-    if predictions.value_counts().count() == 3:
-        results = confusion_matrix(labels, predictions)
-    
-        df_cm = pd.DataFrame(results, ['True Setosa', 'True Versicolor', 'True Virginica'],
-                             ['Pred Setosa', 'Pred Versicolor', 'Pred Virginica'])
-    
-        cm = sns.heatmap(df_cm, annot=True)
-        fig = cm.get_figure()
-        fig.savefig("./confusion_matrix.png")
-        dataset_api.upload("./confusion_matrix.png", "Resources/images", overwrite=True)
-    else:
-        print("You need 3 different flower predictions to create the confusion matrix.")
-        print("Run the batch inference pipeline more times until you get 3 different iris flower predictions") 
-
-
-if __name__ == "__main__":
-    if LOCAL == True :
-        g()
-    else:
-        with stub.run():
-            f()

iris-feature-pipeline-daily.py

@@ -1,84 +0,0 @@
-import os
-import modal
-    
-BACKFILL=False
-LOCAL=True
-
-if LOCAL == False:
-   stub = modal.Stub()
-   image = modal.Image.debian_slim().pip_install(["hopsworks","joblib","seaborn","sklearn","dataframe-image"]) 
-
-   @stub.function(image=image, schedule=modal.Period(days=1), secret=modal.Secret.from_name("HOPSWORKS_API_KEY"))
-   def f():
-       g()
-
-
-def generate_flower(name, sepal_len_max, sepal_len_min, sepal_width_max, sepal_width_min, 
-                    petal_len_max, petal_len_min, petal_width_max, petal_width_min):
-    """
-    Returns a single iris flower as a single row in a DataFrame
-    """
-    import pandas as pd
-    import random
-
-    df = pd.DataFrame({ "sepal_length": [random.uniform(sepal_len_max, sepal_len_min)],
-                       "sepal_width": [random.uniform(sepal_width_max, sepal_width_min)],
-                       "petal_length": [random.uniform(petal_len_max, petal_len_min)],
-                       "petal_width": [random.uniform(petal_width_max, petal_width_min)]
-                      })
-    df['variety'] = name
-    return df
-
-
-def get_random_iris_flower():
-    """
-    Returns a DataFrame containing one random iris flower
-    """
-    import pandas as pd
-    import random
-
-    virginica_df = generate_flower("Virginica", 8, 5.5, 3.8, 2.2, 7, 4.5, 2.5, 1.4)
-    versicolor_df = generate_flower("Versicolor", 7.5, 4.5, 3.5, 2.1, 3.1, 5.5, 1.8, 1.0)
-    setosa_df =  generate_flower("Setosa", 6, 4.5, 4.5, 2.3, 1.2, 2, 0.7, 0.3)
-
-    # randomly pick one of these 3 and write it to the featurestore
-    pick_random = random.uniform(0,3)
-    if pick_random >= 2:
-        iris_df = virginica_df
-        print("Virginica added")
-    elif pick_random >= 1:
-        iris_df = versicolor_df
-        print("Versicolor added")
-    else:
-        iris_df = setosa_df
-        print("Setosa added")
-
-    return iris_df
-
-
-
-def g():
-    import hopsworks
-    import pandas as pd
-
-    project = hopsworks.login()
-    fs = project.get_feature_store()
-
-    if BACKFILL == True:
-        iris_df = pd.read_csv("https://repo.hops.works/master/hopsworks-tutorials/data/iris.csv")
-    else:
-        iris_df = get_random_iris_flower()
-
-    iris_fg = fs.get_or_create_feature_group(
-        name="iris_modal",
-        version=1,
-        primary_key=["sepal_length","sepal_width","petal_length","petal_width"], 
-        description="Iris flower dataset")
-    iris_fg.insert(iris_df, write_options={"wait_for_job" : False})
-
-if __name__ == "__main__":
-    if LOCAL == True :
-        g()
-    else:
-        with stub.run():
-            f()

iris-feature-pipeline.py

@@ -1,33 +0,0 @@
-import os
-import modal
-    
-LOCAL=True
-
-if LOCAL == False:
-   stub = modal.Stub()
-   image = modal.Image.debian_slim().pip_install(["hopsworks","joblib","seaborn","sklearn","dataframe-image"]) 
-
-   @stub.function(image=image, schedule=modal.Period(days=1), secret=modal.Secret.from_name("scalable_ml"))
-   def f():
-       g()
-
-def g():
-    import hopsworks
-    import pandas as pd
-
-    project = hopsworks.login()
-    fs = project.get_feature_store()
-    iris_df = pd.read_csv("https://repo.hops.works/master/hopsworks-tutorials/data/iris.csv")
-    iris_fg = fs.get_or_create_feature_group(
-        name="iris_modal",
-        version=1,
-        primary_key=["sepal_length","sepal_width","petal_length","petal_width"], 
-        description="Iris flower dataset")
-    iris_fg.insert(iris_df, write_options={"wait_for_job" : False})
-
-if __name__ == "__main__":
-    if LOCAL == True :
-        g()
-    else:
-        with stub.run():
-            f()

iris-training-pipeline.py

@@ -1,100 +0,0 @@
-import os
-import modal
-
-LOCAL=True
-
-if LOCAL == False:
-   stub = modal.Stub()
-   image = modal.Image.debian_slim().apt_install(["libgomp1"]).pip_install(["hopsworks", "seaborn", "joblib", "scikit-learn"])
-
-   @stub.function(image=image, schedule=modal.Period(days=1), secret=modal.Secret.from_name("scalable_ml"))
-   def f():
-       g()
-
-
-def g():
-    import hopsworks
-    import pandas as pd
-    from sklearn.neighbors import KNeighborsClassifier
-    from sklearn.metrics import accuracy_score
-    from sklearn.metrics import confusion_matrix
-    from sklearn.metrics import classification_report
-    import seaborn as sns
-    from matplotlib import pyplot
-    from hsml.schema import Schema
-    from hsml.model_schema import ModelSchema
-    import joblib
-
-    # You have to set the environment variable 'HOPSWORKS_API_KEY' for login to succeed
-    project = hopsworks.login()
-    # fs is a reference to the Hopsworks Feature Store
-    fs = project.get_feature_store()
-
-    # The feature view is the input set of features for your model. The features can come from different feature groups.    
-    # You can select features from different feature groups and join them together to create a feature view
-    try: 
-        feature_view = fs.get_feature_view(name="iris_modal", version=1)
-    except:
-        iris_fg = fs.get_feature_group(name="iris_modal", version=1)
-        query = iris_fg.select_all()
-        feature_view = fs.create_feature_view(name="iris_modal",
-                                          version=1,
-                                          description="Read from Iris flower dataset",
-                                          labels=["variety"],
-                                          query=query)    
-
-    # You can read training data, randomly split into train/test sets of features (X) and labels (y)        
-    X_train, X_test, y_train, y_test = feature_view.train_test_split(0.2)
-
-    # Train our model with the Scikit-learn K-nearest-neighbors algorithm using our features (X_train) and labels (y_train)
-    model = KNeighborsClassifier(n_neighbors=2)
-    model.fit(X_train, y_train.values.ravel())
-
-    # Evaluate model performance using the features from the test set (X_test)
-    y_pred = model.predict(X_test)
-
-    # Compare predictions (y_pred) with the labels in the test set (y_test)
-    metrics = classification_report(y_test, y_pred, output_dict=True)
-    results = confusion_matrix(y_test, y_pred)
-
-    # Create the confusion matrix as a figure, we will later store it as a PNG image file
-    df_cm = pd.DataFrame(results, ['True Setosa', 'True Versicolor', 'True Virginica'],
-                         ['Pred Setosa', 'Pred Versicolor', 'Pred Virginica'])
-    cm = sns.heatmap(df_cm, annot=True)
-    fig = cm.get_figure()
-
-    # We will now upload our model to the Hopsworks Model Registry. First get an object for the model registry.
-    mr = project.get_model_registry()
-    
-    # The contents of the 'iris_model' directory will be saved to the model registry. Create the dir, first.
-    model_dir="iris_model"
-    if os.path.isdir(model_dir) == False:
-        os.mkdir(model_dir)
-
-    # Save both our model and the confusion matrix to 'model_dir', whose contents will be uploaded to the model registry
-    joblib.dump(model, model_dir + "/iris_model.pkl")
-    fig.savefig(model_dir + "/confusion_matrix.png")    
-
-
-    # Specify the schema of the model's input/output using the features (X_train) and labels (y_train)
-    input_schema = Schema(X_train)
-    output_schema = Schema(y_train)
-    model_schema = ModelSchema(input_schema, output_schema)
-
-    # Create an entry in the model registry that includes the model's name, desc, metrics
-    iris_model = mr.python.create_model(
-        name="iris_modal", 
-        metrics={"accuracy" : metrics['accuracy']},
-        model_schema=model_schema,
-        description="Iris Flower Predictor"
-    )
-    
-    # Upload the model to the model registry, including all files in 'model_dir'
-    iris_model.save(model_dir)
-    
-if __name__ == "__main__":
-    if LOCAL == True :
-        g()
-    else:
-        with stub.run():
-            f()