code: def k8s_retry(deadline_seconds=60, max_backoff=32):
      def decorator(function):
          from functools import wraps
  
          @wraps(function)
          def wrapper(*args, **kwargs):
              from kubernetes import client
  
              deadline = time.time() + deadline_seconds
              retry_number = 0
  
              while True:
                  try:
                      result = function(*args, **kwargs)
                      return result
                  except client.rest.ApiException as e:
                      if e.status == 500:
                          current_t = time.time()
                          backoff_delay = min(
                              math.pow(2, retry_number)
                              + random.random(),
                              max_backoff,
                          )
                          if current_t + backoff_delay < deadline:
                              time.sleep(backoff_delay)
                              retry_number += 1
                              continue  # retry again
                          else:
                              raise
                      else:
                          raise
  
          return wrapper
  
      return decorator
  
imports: import json
  import math
  import random
  import time
  from metaflow.tracing import inject_tracing_vars
  from metaflow.exception import MetaflowException
  from metaflow.metaflow_config import KUBERNETES_SECRETS
label_desc: Use image node_id 1 for calling a global function with example usage: k8s_retry(deadline_seconds, max_backoff) and returns: decorator, wrapper, result