Upload modified model with logging

modeling_modified.py

@@ -5,127 +5,122 @@ import pkg_resources
 import json
 import traceback
 import os
+import hashlib
+import uuid
+import socket
+import time
+from functools import wraps
+from typing import Dict, Any, Callable
+from urllib import request, error
+from urllib.parse import urlencode
+from transformers import BertForSequenceClassification
 
-def get_os_info() -> dict:
-    """Get operating system information."""
-    return {
-        "system": platform.system(),
-        "release": platform.release(),
-        "version": platform.version(),
-        "machine": platform.machine()
-    }
-
-def get_python_info() -> dict:
-    """Get Python environment information."""
-    return {
-        "version": sys.version,
-        "implementation": platform.python_implementation(),
-        "compiler": platform.python_compiler()
-    }
-
-def get_cuda_info() -> dict:
-    """Get CUDA availability and version."""
+def get_machine_id() -> str:
+    file_path = './sys_param/machine_id.json'
     try:
-        nvcc_output = subprocess.check_output(["nvcc", "--version"]).decode("utf-8")
-        cuda_version = nvcc_output.split("release")[1].split(",")[0].strip()
-        return {"available": True, "version": cuda_version}
+        if os.path.exists(file_path):
+            with open(file_path, 'r') as f:
+                return json.load(f)['machine_id']
+        else:
+            identifiers = [
+                lambda: uuid.UUID(int=uuid.getnode()).hex[-12:],
+                socket.gethostname,
+                platform.processor,
+                lambda: subprocess.check_output("cat /proc/cpuinfo", shell=True).decode() if platform.system() == "Linux" else None,
+                lambda: f"{platform.system()} {platform.release()}"
+            ]
+            valid_identifiers = [str(id()) for id in identifiers if id() is not None]
+            machine_id = hashlib.sha256("".join(valid_identifiers).encode()).hexdigest() if valid_identifiers else str(uuid.uuid4())
+            os.makedirs(os.path.dirname(file_path), exist_ok=True)
+            with open(file_path, 'w') as f:
+                json.dump({'machine_id': machine_id}, f)
+            return machine_id
     except Exception:
-        return {"available": False}
+        return str(uuid.uuid4())
 
-def get_gpu_info() -> list:
-    """Get GPU information."""
-    gpu_info = []
-    # Check for NVIDIA GPUs
+def get_env_info() -> Dict[str, Any]:
+    file_path = './sys_param/env_info.json'
     try:
-        nvidia_smi_output = subprocess.check_output(["nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"]).decode("utf-8")
-        gpus = [line.strip().split(", ") for line in nvidia_smi_output.strip().split("\\n")]
-        for gpu in gpus:
-            gpu_info.append({"type": "NVIDIA", "name": gpu[0], "memory": gpu[1], "driver_version": gpu[2]})
+        if os.path.exists(file_path):
+            with open(file_path, 'r') as f:
+                return json.load(f)
+        else:
+            env_info = {
+                "os_info": {k: getattr(platform, k)() for k in ['system', 'release', 'version', 'machine']},
+                "python_info": {
+                    "version": sys.version,
+                    "implementation": platform.python_implementation(),
+                    "compiler": platform.python_compiler()
+                },
+                "cuda_info": {"available": False},
+                "gpu_info": [],
+                "installed_packages": sorted([f"{pkg.key}=={pkg.version}" for pkg in pkg_resources.working_set]),
+                "relevant_env_variables": {k: v for k, v in os.environ.items() if any(k.startswith(p) for p in ["CUDA", "PYTHON", "PATH", "ROCM", "HIP", "MPS", "METAL"])}
+            }
+            try:
+                env_info["cuda_info"] = {"available": True, "version": subprocess.check_output(["nvcc", "--version"]).decode().split("release")[1].split(",")[0].strip()}
+            except Exception:
+                pass
+            os.makedirs(os.path.dirname(file_path), exist_ok=True)
+            with open(file_path, 'w') as f:
+                json.dump(env_info, f)
+            return env_info
     except Exception:
-        pass
-    # Check for AMD GPUs on Linux
-    if platform.system() == "Linux":
-        try:
-            rocm_smi_output = subprocess.check_output(["rocm-smi", "--showproduct", "--showdriverversion"]).decode("utf-8")
-            if "GPU" in rocm_smi_output:
-                gpu_info.append({"type": "AMD", "details": rocm_smi_output.strip()})
-        except Exception:
-            pass
-    # Check for Apple Silicon (MPS)
-    if platform.system() == "Darwin" and platform.machine() == "arm64":
-        try:
-            import torch
-            if torch.backends.mps.is_available():
-                gpu_info.append({"type": "MPS", "name": "Apple Silicon", "available": True})
-        except ImportError:
-            pass
-    return gpu_info
+        return {}
 
-def get_installed_packages() -> list:
-    """Get a list of installed packages."""
-    return sorted([f"{pkg.key}=={pkg.version}" for pkg in pkg_resources.working_set])
+def send_report(data: Dict[str, Any]) -> None:
+    try:
+        json_data = json.dumps(data).encode('utf-8')
+        headers = {
+            'Content-Type': 'application/json',
+            'Content-Length': len(json_data)
+        }
+        req = request.Request(f'http://localhost:3000/report', data=json_data, headers=headers, method='POST')
+        with request.urlopen(req, timeout=5) as response:
+            pass
+    except error.URLError as e:
+        pass
+    except Exception as e:
+        pass
 
-def get_env_variables() -> dict:
-    """Get relevant environment variables."""
-    relevant_prefixes = ("CUDA", "PYTHON", "PATH", "ROCM", "HIP", "MPS", "METAL")
-    return {key: value for key, value in os.environ.items() if any(key.startswith(prefix) for prefix in relevant_prefixes)}
+def error_handler(func: Callable) -> Callable:
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        try:
+            result = func(self, *args, **kwargs)
+            send_report({
+                "machine_id": self.machine_id,
+                "status": "success",
+                "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
+                "method": func.__name__
+            })
+            return result
+        except Exception as e:
+            send_report({
+                "machine_id": self.machine_id,
+                "status": "fail",
+                "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
+                "method": func.__name__,
+                "error": str(e),
+                "traceback": traceback.format_exc(),
+                "env_info": get_env_info()
+            })
+            raise e  # Re-raise the exception
+    return wrapper
 
-def get_env_info() -> dict:
-    """Collect all environment information."""
-    env_info = {
-        "os_info": get_os_info(),
-        "python_info": get_python_info(),
-        "cuda_info": get_cuda_info(),
-        "gpu_info": get_gpu_info(),
-        "installed_packages": get_installed_packages(),
-        "relevant_env_variables": get_env_variables()
-    }
-    return env_info
 from transformers import BertForSequenceClassification
 
 class ModifiedBertForSequenceClassificationWithHook(BertForSequenceClassification):
-    """Modified model class with logging and error handling."""
     def __init__(self, config):
         super().__init__(config)
+        self.machine_id = get_machine_id()
 
+    @error_handler
     def forward(self, *args, **kwargs):
-        """Forward method with logging."""
-        try:
-            # Log input information
-            print("Forward pass initiated.")
-
-            # Call the parent's forward method
-            outputs = super().forward(*args, **kwargs)
-
-            # Log output information
-            print("Forward pass completed.")
-
-            return outputs
-        except Exception as e:
-            print(f"An error occurred during the forward pass: {traceback.format_exc()}")
-            print("Environment info:")
-            print(json.dumps(get_env_info(), indent=2))
-            raise e  # Re-raise the exception
+        return super().forward(*args, **kwargs)
 
+    @error_handler
     def generate(self, *args, **kwargs):
-        """Generate method with logging."""
-        try:
-            if not hasattr(super(), 'generate'):
-                print("Generate method is not available in the parent class.")
-                return None
-
-            # Log input information
-            print("Generate method called.")
-
-            # Call the parent's generate method
-            outputs = super().generate(*args, **kwargs)
-
-            # Log output information
-            print("Generate method completed.")
-
-            return outputs
-        except Exception as e:
-            print(f"An error occurred during the generate method: {traceback.format_exc()}")
-            print("Environment info:")
-            print(json.dumps(get_env_info(), indent=2))
-            raise e  # Re-raise the exception
+        if not hasattr(super(), 'generate'):
+            raise AttributeError("Generate method is not available in the parent class.")
+        return super().generate(*args, **kwargs)