# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Tensorflow Load and Benchmark Tests
#
# Using a pretrained model for [Tensorflow flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers)
#
#  * Load test the model at fixed rate
#  * Benchmark the model to find maximum throughput and saturation handling
#  
#  ## Setup
#  
#   * Create a 3 node GCP cluster with n1-standard-8 node
#   * Install Seldon Core
#   
#  ## TODO
#  
#   * gRPC
#   * Run vegeta on separate node to model servers using affinity/taints

# !kubectl create namespace seldon

# !kubectl config set-context $(kubectl config current-context) --namespace=seldon

# +
import sys

sys.path.append("../")
from vegeta_utils import *
# -

# ## Put Taint on Nodes

# raw=!kubectl get nodes -o jsonpath='{.items[0].metadata.name}'
firstNode = raw[0]
# raw=!kubectl get nodes -o jsonpath='{.items[1].metadata.name}'
secondNode = raw[0]
# raw=!kubectl get nodes -o jsonpath='{.items[2].metadata.name}'
thirdNode = raw[0]
# !kubectl taint nodes '{firstNode}' loadtester=active:NoSchedule
# !kubectl taint nodes '{secondNode}' model=active:NoSchedule 
# !kubectl taint nodes '{thirdNode}' model=active:NoSchedule     

# ## Benchmark with Saturation Test

# %%writefile tf_flowers.yaml
apiVersion: machinelearning.seldon.io/v1alpha2
kind: SeldonDeployment
metadata:
  name: tf-flowers
spec:
  protocol: tensorflow
  transport: rest
  predictors:
  - graph:
      implementation: TENSORFLOW_SERVER
      modelUri: gs://kfserving-samples/models/tensorflow/flowers
      name:  flowers
      parameters:
        - name: model_name
          type: STRING
          value: flowers
    componentSpecs:
    - spec:
        containers:
        - name: flowers
          resources:
            requests:
              cpu: '2'
        tolerations:
        - key: model
          operator: Exists
          effect: NoSchedule
    name: default
    replicas: 1

run_model("tf_flowers.yaml")

# Run test to gather the max throughput of the model

results = run_vegeta_test("tf_vegeta_cfg.yaml", "vegeta_max.yaml", "11m")
print(json.dumps(results, indent=4))
saturation_throughput = int(results["throughput"])

print("Max Throughtput=", saturation_throughput)

# ## Load Tests with HPA
#
# Run with an HPA at saturation rate to check:
#   * Latencies affected by scaling
#

# %%writefile tf_flowers.yaml
apiVersion: machinelearning.seldon.io/v1alpha2
kind: SeldonDeployment
metadata:
  name: tf-flowers
spec:
  protocol: tensorflow
  transport: rest
  predictors:
  - graph:
      implementation: TENSORFLOW_SERVER
      modelUri: gs://kfserving-samples/models/tensorflow/flowers
      name:  flowers
      parameters:
        - name: model_name
          type: STRING
          value: flowers
    componentSpecs:
    - hpaSpec:
        minReplicas: 1
        maxReplicas: 5
        metrics:
        - resource:
            name: cpu
            targetAverageUtilization: 70
          type: Resource
      spec:
        containers:
        - name: flowers
          resources:
            requests:
              cpu: '1'
          livenessProbe:
            failureThreshold: 3
            initialDelaySeconds: 60
            periodSeconds: 5
            successThreshold: 1
            tcpSocket:
              port: http
            timeoutSeconds: 5
          readinessProbe:
            failureThreshold: 3
            initialDelaySeconds: 20
            periodSeconds: 5
            successThreshold: 1
            tcpSocket:
              port: http
            timeoutSeconds: 5
        tolerations:
        - key: model
          operator: Exists
          effect: NoSchedule
    name: default
    replicas: 1

run_model("tf_flowers.yaml")

rate = saturation_throughput
duration = "10m"
# %env DURATION=$duration
# %env RATE=$rate/1s
# !cat vegeta_cfg.tmpl.yaml | envsubst > vegeta.tmp.yaml
# !cat vegeta.tmp.yaml

results = run_vegeta_test("tf_vegeta_cfg.yaml", "vegeta.tmp.yaml", "11m")
print(json.dumps(results, indent=4))

print_vegeta_results(results)



StarcoderdataJupyterScriptsDedupFiltered