Updated logging format for timestamps to be compatible with AWS. Added load_dynamo_logs.py example file.

load_dynamo_logs.py

@@ -0,0 +1,56 @@
+import boto3
+import csv
+from decimal import Decimal
+from boto3.dynamodb.conditions import Key
+
+from tools.config import AWS_REGION, ACCESS_LOG_DYNAMODB_TABLE_NAME, FEEDBACK_LOG_DYNAMODB_TABLE_NAME, USAGE_LOG_DYNAMODB_TABLE_NAME, OUTPUT_FOLDER
+
+# Replace with your actual table name and region
+TABLE_NAME = USAGE_LOG_DYNAMODB_TABLE_NAME # Choose as appropriate
+REGION = AWS_REGION
+CSV_OUTPUT = OUTPUT_FOLDER + 'dynamodb_logs_export.csv'
+
+# Create DynamoDB resource
+dynamodb = boto3.resource('dynamodb', region_name=REGION)
+table = dynamodb.Table(TABLE_NAME)
+
+# Helper function to convert Decimal to float or int
+def convert_types(item):
+    for key, value in item.items():
+        if isinstance(value, Decimal):
+            # Convert to int if no decimal places, else float
+            item[key] = int(value) if value % 1 == 0 else float(value)
+    return item
+
+# Paginated scan
+def scan_table():
+    items = []
+    response = table.scan()
+    items.extend(response['Items'])
+
+    while 'LastEvaluatedKey' in response:
+        response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
+        items.extend(response['Items'])
+
+    return items
+
+# Export to CSV
+def export_to_csv(items, output_path):
+    if not items:
+        print("No items found.")
+        return
+
+    fieldnames = sorted(items[0].keys())
+
+    with open(output_path, 'w', newline='', encoding='utf-8') as csvfile:
+        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+        writer.writeheader()
+
+        for item in items:
+            writer.writerow(convert_types(item))
+
+    print(f"Exported {len(items)} items to {output_path}")
+
+# Run export
+items = scan_table()
+export_to_csv(items, CSV_OUTPUT)

load_s3_logs.py

@@ -2,7 +2,7 @@ import boto3
 import pandas as pd
 from io import StringIO
 from datetime import datetime
-from tools.config import DOCUMENT_REDACTION_BUCKET, AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION
+from tools.config import DOCUMENT_REDACTION_BUCKET, AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, OUTPUT_FOLDER
 
 # Combine together log files that can be then used for e.g. dashboarding and financial tracking.
 
@@ -71,7 +71,7 @@ if df_list:
     concatenated_df = pd.concat(df_list, ignore_index=True)
 
     # Save the concatenated DataFrame to a CSV file
-    concatenated_df.to_csv('consolidated_logs.csv', index=False)
-    print("Consolidated CSV saved as 'consolidated_logs.csv'")
+    concatenated_df.to_csv(OUTPUT_FOLDER + 'consolidated_s3_logs.csv', index=False)
+    print("Consolidated CSV saved as 'consolidated_s3_logs.csv'")
 else:
     print("No log files found in the given date range.")

tools/custom_csvlogger.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 import contextlib
 import csv
 import datetime
+from datetime import datetime
 import os
 import re
 import boto3
@@ -177,7 +178,7 @@ class CSVLogger_custom(FlaggingCallback):
             csv_data.append(username)
 
 
-        timestamp = str(datetime.datetime.now())
+        timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3] # Correct format for Amazon Athena
         csv_data.append(timestamp)
 
         generated_id = str(uuid.uuid4())