fix: temporarily disable routers to isolate Pydantic validation issue and add agent endpoints to working routers
- Comment out most routers in main.py to isolate Pydantic issue - Keep only blockchain router enabled for testing - Fix database warmup to use get_session() instead of SessionDep() - Add blockchain router to __init__.py exports - Add test endpoint to agent_router for verification - Duplicate agent network and execution receipt endpoints in client and exchange routers as temporary workaround
This commit is contained in:
oib
209
scripts/performance_test.py
Normal file
209
scripts/performance_test.py
Normal file
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Performance Testing Suite for AITBC Platform
|
||||
Tests API endpoints, load handling, and system performance
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import time
|
||||
import json
|
||||
import statistics
|
||||
from typing import List, Dict, Any
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
class PerformanceTester:
|
||||
def __init__(self, base_url: str = "https://aitbc.bubuit.net/api/v1"):
|
||||
self.base_url = base_url
|
||||
self.api_key = "test_key_16_characters"
|
||||
self.results = []
|
||||
|
||||
async def single_request(self, session: aiohttp.ClientSession,
|
||||
method: str, endpoint: str, **kwargs) -> Dict[str, Any]:
|
||||
"""Execute a single API request and measure performance"""
|
||||
start_time = time.time()
|
||||
|
||||
headers = kwargs.pop('headers', {})
|
||||
headers['X-Api-Key'] = self.api_key
|
||||
|
||||
try:
|
||||
async with session.request(method, f"{self.base_url}{endpoint}",
|
||||
headers=headers, **kwargs) as response:
|
||||
content = await response.text()
|
||||
end_time = time.time()
|
||||
|
||||
return {
|
||||
'endpoint': endpoint,
|
||||
'method': method,
|
||||
'status_code': response.status,
|
||||
'response_time': end_time - start_time,
|
||||
'content_length': len(content),
|
||||
'success': response.status < 400
|
||||
}
|
||||
except Exception as e:
|
||||
end_time = time.time()
|
||||
return {
|
||||
'endpoint': endpoint,
|
||||
'method': method,
|
||||
'status_code': 0,
|
||||
'response_time': end_time - start_time,
|
||||
'content_length': 0,
|
||||
'success': False,
|
||||
'error': str(e)
|
||||
}
|
||||
|
||||
async def load_test_endpoint(self, endpoint: str, method: str = "GET",
|
||||
concurrent_users: int = 10, requests_per_user: int = 5,
|
||||
**kwargs) -> Dict[str, Any]:
|
||||
"""Perform load testing on a specific endpoint"""
|
||||
print(f"🧪 Load testing {method} {endpoint} - {concurrent_users} users × {requests_per_user} requests")
|
||||
|
||||
connector = aiohttp.TCPConnector(limit=100, limit_per_host=100)
|
||||
timeout = aiohttp.ClientTimeout(total=30)
|
||||
|
||||
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
|
||||
tasks = []
|
||||
for user in range(concurrent_users):
|
||||
for req in range(requests_per_user):
|
||||
task = self.single_request(session, method, endpoint, **kwargs)
|
||||
tasks.append(task)
|
||||
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
# Filter out exceptions and calculate metrics
|
||||
valid_results = [r for r in results if isinstance(r, dict)]
|
||||
successful_results = [r for r in valid_results if r['success']]
|
||||
|
||||
response_times = [r['response_time'] for r in successful_results]
|
||||
|
||||
return {
|
||||
'endpoint': endpoint,
|
||||
'total_requests': len(valid_results),
|
||||
'successful_requests': len(successful_results),
|
||||
'failed_requests': len(valid_results) - len(successful_results),
|
||||
'success_rate': len(successful_results) / len(valid_results) * 100 if valid_results else 0,
|
||||
'avg_response_time': statistics.mean(response_times) if response_times else 0,
|
||||
'min_response_time': min(response_times) if response_times else 0,
|
||||
'max_response_time': max(response_times) if response_times else 0,
|
||||
'median_response_time': statistics.median(response_times) if response_times else 0,
|
||||
'p95_response_time': statistics.quantiles(response_times, n=20)[18] if len(response_times) > 20 else 0,
|
||||
'requests_per_second': len(successful_results) / (max(response_times) - min(response_times)) if len(response_times) > 1 else 0
|
||||
}
|
||||
|
||||
async def run_performance_tests(self):
|
||||
"""Run comprehensive performance tests"""
|
||||
print("🚀 Starting AITBC Platform Performance Tests")
|
||||
print("=" * 60)
|
||||
|
||||
test_endpoints = [
|
||||
# Health check (baseline)
|
||||
{'endpoint': '/health', 'method': 'GET', 'users': 20, 'requests': 10},
|
||||
|
||||
# Client endpoints
|
||||
{'endpoint': '/client/jobs', 'method': 'GET', 'users': 5, 'requests': 5},
|
||||
|
||||
# Miner endpoints
|
||||
{'endpoint': '/miners/register', 'method': 'POST', 'users': 3, 'requests': 3,
|
||||
'json': {'capabilities': {'gpu': {'model': 'RTX 4090'}}},
|
||||
'headers': {'Content-Type': 'application/json', 'X-Miner-ID': 'perf-test-miner'}},
|
||||
|
||||
# Blockchain endpoints
|
||||
{'endpoint': '/blockchain/info', 'method': 'GET', 'users': 5, 'requests': 5},
|
||||
]
|
||||
|
||||
results = []
|
||||
|
||||
for test_config in test_endpoints:
|
||||
endpoint = test_config.pop('endpoint')
|
||||
method = test_config.pop('method')
|
||||
|
||||
result = await self.load_test_endpoint(endpoint, method, **test_config)
|
||||
results.append(result)
|
||||
|
||||
# Print immediate results
|
||||
print(f"📊 {method} {endpoint}:")
|
||||
print(f" ✅ Success Rate: {result['success_rate']:.1f}%")
|
||||
print(f" ⏱️ Avg Response: {result['avg_response_time']:.3f}s")
|
||||
print(f" 📈 RPS: {result['requests_per_second']:.1f}")
|
||||
print(f" 📏 P95: {result['p95_response_time']:.3f}s")
|
||||
print()
|
||||
|
||||
return results
|
||||
|
||||
def generate_report(self, results: List[Dict[str, Any]]):
|
||||
"""Generate performance test report"""
|
||||
print("📋 PERFORMANCE TEST REPORT")
|
||||
print("=" * 60)
|
||||
|
||||
total_requests = sum(r['total_requests'] for r in results)
|
||||
total_successful = sum(r['successful_requests'] for r in results)
|
||||
overall_success_rate = (total_successful / total_requests * 100) if total_requests > 0 else 0
|
||||
|
||||
print(f"📊 Overall Statistics:")
|
||||
print(f" Total Requests: {total_requests}")
|
||||
print(f" Successful Requests: {total_successful}")
|
||||
print(f" Overall Success Rate: {overall_success_rate:.1f}%")
|
||||
print()
|
||||
|
||||
print(f"🎯 Endpoint Performance:")
|
||||
for result in results:
|
||||
status = "✅" if result['success_rate'] >= 95 else "⚠️" if result['success_rate'] >= 80 else "❌"
|
||||
print(f" {status} {result['method']} {result['endpoint']}")
|
||||
print(f" Success: {result['success_rate']:.1f}% | "
|
||||
f"Avg: {result['avg_response_time']:.3f}s | "
|
||||
f"P95: {result['p95_response_time']:.3f}s | "
|
||||
f"RPS: {result['requests_per_second']:.1f}")
|
||||
|
||||
print()
|
||||
print("🏆 Performance Benchmarks:")
|
||||
print(" ✅ Excellent: <100ms response time, >95% success rate")
|
||||
print(" ⚠️ Good: <500ms response time, >80% success rate")
|
||||
print(" ❌ Needs Improvement: >500ms or <80% success rate")
|
||||
|
||||
# Recommendations
|
||||
print()
|
||||
print("💡 Recommendations:")
|
||||
|
||||
slow_endpoints = [r for r in results if r['avg_response_time'] > 0.5]
|
||||
if slow_endpoints:
|
||||
print(" 🐌 Slow endpoints detected - consider optimization:")
|
||||
for r in slow_endpoints:
|
||||
print(f" - {r['endpoint']} ({r['avg_response_time']:.3f}s avg)")
|
||||
|
||||
unreliable_endpoints = [r for r in results if r['success_rate'] < 95]
|
||||
if unreliable_endpoints:
|
||||
print(" 🔧 Unreliable endpoints detected - check for errors:")
|
||||
for r in unreliable_endpoints:
|
||||
print(f" - {r['endpoint']} ({r['success_rate']:.1f}% success)")
|
||||
|
||||
if not slow_endpoints and not unreliable_endpoints:
|
||||
print(" 🎉 All endpoints performing well - ready for production!")
|
||||
|
||||
async def main():
|
||||
"""Main performance testing execution"""
|
||||
tester = PerformanceTester()
|
||||
|
||||
try:
|
||||
results = await tester.run_performance_tests()
|
||||
tester.generate_report(results)
|
||||
|
||||
# Return exit code based on performance
|
||||
avg_success_rate = statistics.mean([r['success_rate'] for r in results])
|
||||
avg_response_time = statistics.mean([r['avg_response_time'] for r in results])
|
||||
|
||||
if avg_success_rate >= 95 and avg_response_time < 0.5:
|
||||
print("\n🎉 PERFORMANCE TESTS PASSED - Ready for production!")
|
||||
return 0
|
||||
else:
|
||||
print("\n⚠️ PERFORMANCE TESTS COMPLETED - Review recommendations")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Performance test failed: {e}")
|
||||
return 1
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = asyncio.run(main())
|
||||
sys.exit(exit_code)
|
||||
291
scripts/production_monitoring.sh
Executable file
291
scripts/production_monitoring.sh
Executable file
@@ -0,0 +1,291 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Production Monitoring Setup for AITBC Platform
|
||||
# Configures monitoring, alerting, and observability
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
BLUE='\033[0;34m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
log() { echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"; }
|
||||
success() { echo -e "${GREEN}✅ $1${NC}"; }
|
||||
warning() { echo -e "${YELLOW}⚠️ $1${NC}"; }
|
||||
|
||||
# Create monitoring directory
|
||||
MONITORING_DIR="/opt/aitbc/monitoring"
|
||||
mkdir -p "$MONITORING_DIR"
|
||||
|
||||
# Setup system metrics collection
|
||||
setup_system_metrics() {
|
||||
log "Setting up system metrics collection..."
|
||||
|
||||
# Create metrics collection script
|
||||
cat > "$MONITORING_DIR/collect_metrics.sh" << 'EOF'
|
||||
#!/bin/bash
|
||||
# System metrics collection for AITBC platform
|
||||
|
||||
METRICS_FILE="/opt/aitbc/monitoring/metrics.log"
|
||||
TIMESTAMP=$(date -Iseconds)
|
||||
|
||||
# System metrics
|
||||
CPU_USAGE=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | sed 's/%us,//')
|
||||
MEM_USAGE=$(free | grep Mem | awk '{printf "%.1f", $3/$2 * 100.0}')
|
||||
DISK_USAGE=$(df -h / | awk 'NR==2{print $5}' | sed 's/%//')
|
||||
|
||||
# Service metrics
|
||||
COORDINATOR_STATUS=$(systemctl is-active aitbc-coordinator)
|
||||
BLOCKCHAIN_STATUS=$(systemctl is-active blockchain-node)
|
||||
|
||||
# API metrics
|
||||
API_RESPONSE_TIME=$(curl -o /dev/null -s -w '%{time_total}' https://aitbc.bubuit.net/api/v1/health 2>/dev/null || echo "0")
|
||||
API_STATUS=$(curl -o /dev/null -s -w '%{http_code}' https://aitbc.bubuit.net/api/v1/health 2>/dev/null || echo "000")
|
||||
|
||||
# Write metrics
|
||||
echo "$TIMESTAMP,cpu:$CPU_USAGE,memory:$MEM_USAGE,disk:$DISK_USAGE,coordinator:$COORDINATOR_STATUS,blockchain:$BLOCKCHAIN_STATUS,api_time:$API_RESPONSE_TIME,api_status:$API_STATUS" >> "$METRICS_FILE"
|
||||
|
||||
# Keep only last 1000 lines
|
||||
tail -n 1000 "$METRICS_FILE" > "$METRICS_FILE.tmp" && mv "$METRICS_FILE.tmp" "$METRICS_FILE"
|
||||
EOF
|
||||
|
||||
chmod +x "$MONITORING_DIR/collect_metrics.sh"
|
||||
|
||||
# Add to crontab (every 2 minutes)
|
||||
(crontab -l 2>/dev/null; echo "*/2 * * * * $MONITORING_DIR/collect_metrics.sh") | crontab -
|
||||
|
||||
success "System metrics collection configured"
|
||||
}
|
||||
|
||||
# Setup alerting system
|
||||
setup_alerting() {
|
||||
log "Setting up alerting system..."
|
||||
|
||||
# Create alerting script
|
||||
cat > "$MONITORING_DIR/check_alerts.sh" << 'EOF'
|
||||
#!/bin/bash
|
||||
# Alert checking for AITBC platform
|
||||
|
||||
ALERT_LOG="/opt/aitbc/monitoring/alerts.log"
|
||||
TIMESTAMP=$(date -Iseconds)
|
||||
ALERT_TRIGGERED=false
|
||||
|
||||
# Check service status
|
||||
check_service() {
|
||||
local service=$1
|
||||
local status=$(systemctl is-active "$service" 2>/dev/null || echo "failed")
|
||||
|
||||
if [[ "$status" != "active" ]]; then
|
||||
echo "$TIMESTAMP,SERVICE,$service is $status" >> "$ALERT_LOG"
|
||||
echo "🚨 ALERT: Service $service is $status"
|
||||
ALERT_TRIGGERED=true
|
||||
fi
|
||||
}
|
||||
|
||||
# Check API health
|
||||
check_api() {
|
||||
local response=$(curl -s -o /dev/null -w '%{http_code}' https://aitbc.bubuit.net/api/v1/health 2>/dev/null || echo "000")
|
||||
|
||||
if [[ "$response" != "200" ]]; then
|
||||
echo "$TIMESTAMP,API,Health endpoint returned $response" >> "$ALERT_LOG"
|
||||
echo "🚨 ALERT: API health check failed (HTTP $response)"
|
||||
ALERT_TRIGGERED=true
|
||||
fi
|
||||
}
|
||||
|
||||
# Check disk space
|
||||
check_disk() {
|
||||
local usage=$(df / | awk 'NR==2{print $5}' | sed 's/%//')
|
||||
|
||||
if [[ $usage -gt 80 ]]; then
|
||||
echo "$TIMESTAMP,DISK,Disk usage is ${usage}%" >> "$ALERT_LOG"
|
||||
echo "🚨 ALERT: Disk usage is ${usage}%"
|
||||
ALERT_TRIGGERED=true
|
||||
fi
|
||||
}
|
||||
|
||||
# Check memory usage
|
||||
check_memory() {
|
||||
local usage=$(free | grep Mem | awk '{printf "%.0f", $3/$2 * 100.0}')
|
||||
|
||||
if [[ $usage -gt 90 ]]; then
|
||||
echo "$TIMESTAMP,MEMORY,Memory usage is ${usage}%" >> "$ALERT_LOG"
|
||||
echo "🚨 ALERT: Memory usage is ${usage}%"
|
||||
ALERT_TRIGGERED=true
|
||||
fi
|
||||
}
|
||||
|
||||
# Run checks
|
||||
check_service "aitbc-coordinator"
|
||||
check_service "blockchain-node"
|
||||
check_api
|
||||
check_disk
|
||||
check_memory
|
||||
|
||||
# If no alerts, log all clear
|
||||
if [[ "$ALERT_TRIGGERED" == "false" ]]; then
|
||||
echo "$TIMESTAMP,ALL_CLEAR,All systems operational" >> "$ALERT_LOG"
|
||||
fi
|
||||
EOF
|
||||
|
||||
chmod +x "$MONITORING_DIR/check_alerts.sh"
|
||||
|
||||
# Add to crontab (every 5 minutes)
|
||||
(crontab -l 2>/dev/null; echo "*/5 * * * * $MONITORING_DIR/check_alerts.sh") | crontab -
|
||||
|
||||
success "Alerting system configured"
|
||||
}
|
||||
|
||||
# Setup performance dashboard
|
||||
setup_dashboard() {
|
||||
log "Setting up performance dashboard..."
|
||||
|
||||
# Create dashboard script
|
||||
cat > "$MONITORING_DIR/dashboard.sh" << 'EOF'
|
||||
#!/bin/bash
|
||||
# Performance dashboard for AITBC platform
|
||||
|
||||
clear
|
||||
echo "🔍 AITBC Platform Performance Dashboard"
|
||||
echo "========================================"
|
||||
echo "Last Updated: $(date)"
|
||||
echo ""
|
||||
|
||||
# System Status
|
||||
echo "📊 System Status:"
|
||||
echo "CPU: $(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | sed 's/%us,//')% used"
|
||||
echo "Memory: $(free -h | grep Mem | awk '{print $3"/"$2}')"
|
||||
echo "Disk: $(df -h / | awk 'NR==2{print $3"/"$2" ("$5")"}')"
|
||||
echo ""
|
||||
|
||||
# Service Status
|
||||
echo "🔧 Service Status:"
|
||||
systemctl is-active aitbc-coordinator && echo "✅ Coordinator API: Active" || echo "❌ Coordinator API: Inactive"
|
||||
systemctl is-active blockchain-node && echo "✅ Blockchain Node: Active" || echo "❌ Blockchain Node: Inactive"
|
||||
systemctl is-active nginx && echo "✅ Nginx: Active" || echo "❌ Nginx: Inactive"
|
||||
echo ""
|
||||
|
||||
# API Performance
|
||||
echo "🌐 API Performance:"
|
||||
API_TIME=$(curl -o /dev/null -s -w '%{time_total}' https://aitbc.bubuit.net/api/v1/health 2>/dev/null || echo "0.000")
|
||||
echo "Health Endpoint: ${API_TIME}s"
|
||||
echo ""
|
||||
|
||||
# Recent Alerts (last 10)
|
||||
echo "🚨 Recent Alerts:"
|
||||
if [[ -f /opt/aitbc/monitoring/alerts.log ]]; then
|
||||
tail -n 10 /opt/aitbc/monitoring/alerts.log | while IFS=',' read -r timestamp type message; do
|
||||
echo " $timestamp: $message"
|
||||
done
|
||||
else
|
||||
echo " No alerts logged"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Quick Stats
|
||||
echo "📈 Quick Stats:"
|
||||
if [[ -f /opt/aitbc/monitoring/metrics.log ]]; then
|
||||
echo " Metrics collected: $(wc -l < /opt/aitbc/monitoring/metrics.log) entries"
|
||||
echo " Alerts triggered: $(grep -c "ALERT" /opt/aitbc/monitoring/alerts.log 2>/dev/null || echo "0")"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Press Ctrl+C to exit, or refresh in 30 seconds..."
|
||||
sleep 30
|
||||
exec "$0"
|
||||
EOF
|
||||
|
||||
chmod +x "$MONITORING_DIR/dashboard.sh"
|
||||
|
||||
success "Performance dashboard created"
|
||||
}
|
||||
|
||||
# Setup log analysis
|
||||
setup_log_analysis() {
|
||||
log "Setting up log analysis..."
|
||||
|
||||
# Create log analysis script
|
||||
cat > "$MONITORING_DIR/analyze_logs.sh" << 'EOF'
|
||||
#!/bin/bash
|
||||
# Log analysis for AITBC platform
|
||||
|
||||
LOG_DIR="/var/log"
|
||||
ANALYSIS_FILE="/opt/aitbc/monitoring/log_analysis.txt"
|
||||
TIMESTAMP=$(date -Iseconds)
|
||||
|
||||
echo "=== Log Analysis - $TIMESTAMP ===" >> "$ANALYSIS_FILE"
|
||||
|
||||
# Analyze nginx logs
|
||||
if [[ -f "$LOG_DIR/nginx/access.log" ]]; then
|
||||
echo "" >> "$ANALYSIS_FILE"
|
||||
echo "NGINX Access Analysis:" >> "$ANALYSIS_FILE"
|
||||
|
||||
# Top 10 endpoints
|
||||
echo "Top 10 endpoints:" >> "$ANALYSIS_FILE"
|
||||
awk '{print $7}' "$LOG_DIR/nginx/access.log" | sort | uniq -c | sort -nr | head -10 >> "$ANALYSIS_FILE"
|
||||
|
||||
# HTTP status codes
|
||||
echo "" >> "$ANALYSIS_FILE"
|
||||
echo "HTTP Status Codes:" >> "$ANALYSIS_FILE"
|
||||
awk '{print $9}' "$LOG_DIR/nginx/access.log" | sort | uniq -c | sort -nr >> "$ANALYSIS_FILE"
|
||||
|
||||
# Error rate
|
||||
local total=$(wc -l < "$LOG_DIR/nginx/access.log")
|
||||
local errors=$(awk '$9 >= 400 {print}' "$LOG_DIR/nginx/access.log" | wc -l)
|
||||
local error_rate=$(echo "scale=2; $errors * 100 / $total" | bc)
|
||||
echo "" >> "$ANALYSIS_FILE"
|
||||
echo "Error Rate: ${error_rate}%" >> "$ANALYSIS_FILE"
|
||||
fi
|
||||
|
||||
# Analyze application logs
|
||||
if journalctl -u aitbc-coordinator --since "1 hour ago" | grep -q "ERROR"; then
|
||||
echo "" >> "$ANALYSIS_FILE"
|
||||
echo "Application Errors (last hour):" >> "$ANALYSIS_FILE"
|
||||
journalctl -u aitbc-coordinator --since "1 hour ago" | grep "ERROR" | tail -5 >> "$ANALYSIS_FILE"
|
||||
fi
|
||||
|
||||
echo "Analysis complete" >> "$ANALYSIS_FILE"
|
||||
EOF
|
||||
|
||||
chmod +x "$MONITORING_DIR/analyze_logs.sh"
|
||||
|
||||
# Add to crontab (hourly)
|
||||
(crontab -l 2>/dev/null; echo "0 * * * * $MONITORING_DIR/analyze_logs.sh") | crontab -
|
||||
|
||||
success "Log analysis configured"
|
||||
}
|
||||
|
||||
# Main execution
|
||||
main() {
|
||||
log "Setting up AITBC Production Monitoring..."
|
||||
|
||||
setup_system_metrics
|
||||
setup_alerting
|
||||
setup_dashboard
|
||||
setup_log_analysis
|
||||
|
||||
success "Production monitoring setup complete!"
|
||||
|
||||
echo
|
||||
echo "📊 MONITORING SUMMARY:"
|
||||
echo " ✅ System metrics collection (every 2 minutes)"
|
||||
echo " ✅ Alert checking (every 5 minutes)"
|
||||
echo " ✅ Performance dashboard"
|
||||
echo " ✅ Log analysis (hourly)"
|
||||
echo
|
||||
echo "🔧 MONITORING COMMANDS:"
|
||||
echo " Dashboard: $MONITORING_DIR/dashboard.sh"
|
||||
echo " Metrics: $MONITORING_DIR/collect_metrics.sh"
|
||||
echo " Alerts: $MONITORING_DIR/check_alerts.sh"
|
||||
echo " Log Analysis: $MONITORING_DIR/analyze_logs.sh"
|
||||
echo
|
||||
echo "📁 MONITORING FILES:"
|
||||
echo " Metrics: $MONITORING_DIR/metrics.log"
|
||||
echo " Alerts: $MONITORING_DIR/alerts.log"
|
||||
echo " Analysis: $MONITORING_DIR/log_analysis.txt"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
31
scripts/quick_test.py
Normal file
31
scripts/quick_test.py
Normal file
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Quick Performance Test
|
||||
"""
|
||||
|
||||
import requests
|
||||
import time
|
||||
|
||||
def test_endpoint(url, headers=None):
|
||||
start = time.time()
|
||||
try:
|
||||
resp = requests.get(url, headers=headers, timeout=5)
|
||||
end = time.time()
|
||||
print(f"✅ {url}: {resp.status_code} in {end-start:.3f}s")
|
||||
return True
|
||||
except Exception as e:
|
||||
end = time.time()
|
||||
print(f"❌ {url}: Error in {end-start:.3f}s - {e}")
|
||||
return False
|
||||
|
||||
print("🧪 Quick Performance Test")
|
||||
print("=" * 30)
|
||||
|
||||
# Test health endpoint
|
||||
test_endpoint("https://aitbc.bubuit.net/api/v1/health")
|
||||
|
||||
# Test with API key
|
||||
headers = {"X-Api-Key": "test_key_16_characters"}
|
||||
test_endpoint("https://aitbc.bubuit.net/api/v1/client/jobs", headers)
|
||||
|
||||
print("\n✅ Basic connectivity test complete")
|
||||
315
scripts/scalability_validation.py
Executable file
315
scripts/scalability_validation.py
Executable file
@@ -0,0 +1,315 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Scalability Validation for AITBC Platform
|
||||
Tests system performance under load and validates scalability
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import time
|
||||
import statistics
|
||||
import json
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import List, Dict, Any
|
||||
|
||||
class ScalabilityValidator:
|
||||
def __init__(self, base_url="https://aitbc.bubuit.net/api/v1"):
|
||||
self.base_url = base_url
|
||||
self.api_key = "test_key_16_characters"
|
||||
self.results = []
|
||||
|
||||
async def measure_endpoint_performance(self, session, endpoint, method="GET", **kwargs):
|
||||
"""Measure performance of a single endpoint"""
|
||||
start_time = time.time()
|
||||
|
||||
headers = kwargs.pop('headers', {})
|
||||
headers['X-Api-Key'] = self.api_key
|
||||
|
||||
try:
|
||||
async with session.request(method, f"{self.base_url}{endpoint}",
|
||||
headers=headers, timeout=30, **kwargs) as response:
|
||||
content = await response.text()
|
||||
end_time = time.time()
|
||||
|
||||
return {
|
||||
'endpoint': endpoint,
|
||||
'method': method,
|
||||
'status_code': response.status,
|
||||
'response_time': end_time - start_time,
|
||||
'content_length': len(content),
|
||||
'success': response.status < 400
|
||||
}
|
||||
except Exception as e:
|
||||
end_time = time.time()
|
||||
return {
|
||||
'endpoint': endpoint,
|
||||
'method': method,
|
||||
'status_code': 0,
|
||||
'response_time': end_time - start_time,
|
||||
'content_length': 0,
|
||||
'success': False,
|
||||
'error': str(e)
|
||||
}
|
||||
|
||||
async def load_test_endpoint(self, endpoint, method="GET", concurrent_users=10,
|
||||
requests_per_user=5, ramp_up_time=5, **kwargs):
|
||||
"""Perform load testing with gradual ramp-up"""
|
||||
print(f"🧪 Load Testing {method} {endpoint}")
|
||||
print(f" Users: {concurrent_users}, Requests/User: {requests_per_user}")
|
||||
print(f" Total Requests: {concurrent_users * requests_per_user}")
|
||||
|
||||
connector = aiohttp.TCPConnector(limit=100, limit_per_host=100)
|
||||
timeout = aiohttp.ClientTimeout(total=30)
|
||||
|
||||
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
|
||||
tasks = []
|
||||
|
||||
# Gradual ramp-up
|
||||
for user in range(concurrent_users):
|
||||
# Add delay for ramp-up
|
||||
if user > 0:
|
||||
await asyncio.sleep(ramp_up_time / concurrent_users)
|
||||
|
||||
# Create requests for this user
|
||||
for req in range(requests_per_user):
|
||||
task = self.measure_endpoint_performance(session, method, endpoint, **kwargs)
|
||||
tasks.append(task)
|
||||
|
||||
# Wait for all tasks to complete
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
# Filter valid results
|
||||
valid_results = [r for r in results if isinstance(r, dict)]
|
||||
successful_results = [r for r in valid_results if r['success']]
|
||||
|
||||
# Calculate metrics
|
||||
response_times = [r['response_time'] for r in successful_results]
|
||||
|
||||
return {
|
||||
'endpoint': endpoint,
|
||||
'total_requests': len(valid_results),
|
||||
'successful_requests': len(successful_results),
|
||||
'failed_requests': len(valid_results) - len(successful_results),
|
||||
'success_rate': len(successful_results) / len(valid_results) * 100 if valid_results else 0,
|
||||
'avg_response_time': statistics.mean(response_times) if response_times else 0,
|
||||
'min_response_time': min(response_times) if response_times else 0,
|
||||
'max_response_time': max(response_times) if response_times else 0,
|
||||
'median_response_time': statistics.median(response_times) if response_times else 0,
|
||||
'p95_response_time': statistics.quantiles(response_times, n=20)[18] if len(response_times) > 20 else 0,
|
||||
'p99_response_time': statistics.quantiles(response_times, n=100)[98] if len(response_times) > 100 else 0,
|
||||
'requests_per_second': len(successful_results) / (max(response_times) - min(response_time)) if len(response_times) > 1 else 0
|
||||
}
|
||||
|
||||
def get_system_metrics(self):
|
||||
"""Get current system metrics"""
|
||||
try:
|
||||
# CPU usage
|
||||
cpu_result = subprocess.run(['top', '-bn1', '|', 'grep', 'Cpu(s)', '|', "awk", "'{print $2}'"],
|
||||
capture_output=True, text=True, shell=True)
|
||||
cpu_usage = cpu_result.stdout.strip().replace('%us,', '')
|
||||
|
||||
# Memory usage
|
||||
mem_result = subprocess.run(['free', '|', 'grep', 'Mem', '|', "awk", "'{printf \"%.1f\", $3/$2 * 100.0}'"],
|
||||
capture_output=True, text=True, shell=True)
|
||||
memory_usage = mem_result.stdout.strip()
|
||||
|
||||
# Disk usage
|
||||
disk_result = subprocess.run(['df', '/', '|', 'awk', 'NR==2{print $5}'],
|
||||
capture_output=True, text=True, shell=True)
|
||||
disk_usage = disk_result.stdout.strip().replace('%', '')
|
||||
|
||||
return {
|
||||
'cpu_usage': float(cpu_usage) if cpu_usage else 0,
|
||||
'memory_usage': float(memory_usage) if memory_usage else 0,
|
||||
'disk_usage': float(disk_usage) if disk_usage else 0
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"⚠️ Could not get system metrics: {e}")
|
||||
return {'cpu_usage': 0, 'memory_usage': 0, 'disk_usage': 0}
|
||||
|
||||
async def run_scalability_tests(self):
|
||||
"""Run comprehensive scalability tests"""
|
||||
print("🚀 AITBC Platform Scalability Validation")
|
||||
print("=" * 60)
|
||||
|
||||
# Record initial system metrics
|
||||
initial_metrics = self.get_system_metrics()
|
||||
print(f"📊 Initial System Metrics:")
|
||||
print(f" CPU: {initial_metrics['cpu_usage']:.1f}%")
|
||||
print(f" Memory: {initial_metrics['memory_usage']:.1f}%")
|
||||
print(f" Disk: {initial_metrics['disk_usage']:.1f}%")
|
||||
print()
|
||||
|
||||
# Test scenarios with increasing load
|
||||
test_scenarios = [
|
||||
# Light load
|
||||
{'endpoint': '/health', 'method': 'GET', 'users': 5, 'requests': 5, 'name': 'Light Load'},
|
||||
|
||||
# Medium load
|
||||
{'endpoint': '/health', 'method': 'GET', 'users': 20, 'requests': 10, 'name': 'Medium Load'},
|
||||
|
||||
# Heavy load
|
||||
{'endpoint': '/health', 'method': 'GET', 'users': 50, 'requests': 10, 'name': 'Heavy Load'},
|
||||
|
||||
# Stress test
|
||||
{'endpoint': '/health', 'method': 'GET', 'users': 100, 'requests': 5, 'name': 'Stress Test'},
|
||||
]
|
||||
|
||||
results = []
|
||||
|
||||
for scenario in test_scenarios:
|
||||
print(f"🎯 Scenario: {scenario['name']}")
|
||||
|
||||
endpoint = scenario['endpoint']
|
||||
method = scenario['method']
|
||||
users = scenario['users']
|
||||
requests = scenario['requests']
|
||||
|
||||
# Get metrics before test
|
||||
before_metrics = self.get_system_metrics()
|
||||
|
||||
# Run load test
|
||||
result = await self.load_test_endpoint(endpoint, method, users, requests)
|
||||
result['scenario'] = scenario['name']
|
||||
result['concurrent_users'] = users
|
||||
result['requests_per_user'] = requests
|
||||
|
||||
# Get metrics after test
|
||||
after_metrics = self.get_system_metrics()
|
||||
|
||||
# Calculate resource impact
|
||||
result['cpu_impact'] = after_metrics['cpu_usage'] - before_metrics['cpu_usage']
|
||||
result['memory_impact'] = after_metrics['memory_usage'] - before_metrics['memory_usage']
|
||||
|
||||
results.append(result)
|
||||
|
||||
# Print scenario results
|
||||
self.print_scenario_results(result)
|
||||
|
||||
# Wait between tests
|
||||
await asyncio.sleep(2)
|
||||
|
||||
return results
|
||||
|
||||
def print_scenario_results(self, result):
|
||||
"""Print results for a single scenario"""
|
||||
status = "✅" if result['success_rate'] >= 95 else "⚠️" if result['success_rate'] >= 80 else "❌"
|
||||
|
||||
print(f" {status} {result['scenario']}:")
|
||||
print(f" Success Rate: {result['success_rate']:.1f}%")
|
||||
print(f" Avg Response: {result['avg_response_time']:.3f}s")
|
||||
print(f" P95 Response: {result['p95_response_time']:.3f}s")
|
||||
print(f" P99 Response: {result['p99_response_time']:.3f}s")
|
||||
print(f" Requests/Second: {result['requests_per_second']:.1f}")
|
||||
print(f" CPU Impact: +{result['cpu_impact']:.1f}%")
|
||||
print(f" Memory Impact: +{result['memory_impact']:.1f}%")
|
||||
print()
|
||||
|
||||
def generate_scalability_report(self, results):
|
||||
"""Generate comprehensive scalability report"""
|
||||
print("📋 SCALABILITY VALIDATION REPORT")
|
||||
print("=" * 60)
|
||||
|
||||
# Overall statistics
|
||||
total_requests = sum(r['total_requests'] for r in results)
|
||||
total_successful = sum(r['successful_requests'] for r in results)
|
||||
overall_success_rate = (total_successful / total_requests * 100) if total_requests > 0 else 0
|
||||
|
||||
print(f"📊 Overall Performance:")
|
||||
print(f" Total Requests: {total_requests}")
|
||||
print(f" Successful Requests: {total_successful}")
|
||||
print(f" Overall Success Rate: {overall_success_rate:.1f}%")
|
||||
print()
|
||||
|
||||
# Performance by scenario
|
||||
print(f"🎯 Performance by Scenario:")
|
||||
for result in results:
|
||||
status = "✅" if result['success_rate'] >= 95 else "⚠️" if result['success_rate'] >= 80 else "❌"
|
||||
print(f" {status} {result['scenario']} ({result['concurrent_users']} users)")
|
||||
print(f" Success: {result['success_rate']:.1f}% | "
|
||||
f"Avg: {result['avg_response_time']:.3f}s | "
|
||||
f"P95: {result['p95_response_time']:.3f}s | "
|
||||
f"RPS: {result['requests_per_second']:.1f}")
|
||||
print()
|
||||
|
||||
# Scalability analysis
|
||||
print(f"📈 Scalability Analysis:")
|
||||
|
||||
# Response time scalability
|
||||
response_times = [(r['concurrent_users'], r['avg_response_time']) for r in results]
|
||||
print(f" Response Time Scalability:")
|
||||
for users, avg_time in response_times:
|
||||
print(f" {users} users: {avg_time:.3f}s avg")
|
||||
|
||||
# Success rate scalability
|
||||
success_rates = [(r['concurrent_users'], r['success_rate']) for r in results]
|
||||
print(f" Success Rate Scalability:")
|
||||
for users, success_rate in success_rates:
|
||||
print(f" {users} users: {success_rate:.1f}% success")
|
||||
|
||||
# Resource impact analysis
|
||||
cpu_impacts = [r['cpu_impact'] for r in results]
|
||||
memory_impacts = [r['memory_impact'] for r in results]
|
||||
|
||||
print(f" Resource Impact:")
|
||||
print(f" Max CPU Impact: +{max(cpu_impacts):.1f}%")
|
||||
print(f" Max Memory Impact: +{max(memory_impacts):.1f}%")
|
||||
print()
|
||||
|
||||
# Recommendations
|
||||
print(f"💡 Scalability Recommendations:")
|
||||
|
||||
# Check if performance degrades significantly
|
||||
max_response_time = max(r['avg_response_time'] for r in results)
|
||||
min_success_rate = min(r['success_rate'] for r in results)
|
||||
|
||||
if max_response_time < 0.5 and min_success_rate >= 95:
|
||||
print(" 🎉 Excellent scalability - system handles load well!")
|
||||
print(" ✅ Ready for production deployment")
|
||||
elif max_response_time < 1.0 and min_success_rate >= 90:
|
||||
print(" ✅ Good scalability - suitable for production")
|
||||
print(" 💡 Consider optimization for higher loads")
|
||||
else:
|
||||
print(" ⚠️ Scalability concerns detected:")
|
||||
if max_response_time >= 1.0:
|
||||
print(" - Response times exceed 1s under load")
|
||||
if min_success_rate < 90:
|
||||
print(" - Success rate drops below 90% under load")
|
||||
print(" 🔧 Performance optimization recommended before production")
|
||||
|
||||
print()
|
||||
print("🏆 Scalability Benchmarks:")
|
||||
print(" ✅ Excellent: <500ms response, >95% success at 100+ users")
|
||||
print(" ⚠️ Good: <1s response, >90% success at 50+ users")
|
||||
print(" ❌ Needs Work: >1s response or <90% success rate")
|
||||
|
||||
async def main():
|
||||
"""Main scalability validation"""
|
||||
validator = ScalabilityValidator()
|
||||
|
||||
try:
|
||||
results = await validator.run_scalability_tests()
|
||||
validator.generate_scalability_report(results)
|
||||
|
||||
# Determine if system is production-ready
|
||||
min_success_rate = min(r['success_rate'] for r in results)
|
||||
max_response_time = max(r['avg_response_time'] for r in results)
|
||||
|
||||
if min_success_rate >= 90 and max_response_time < 1.0:
|
||||
print("\n✅ SCALABILITY VALIDATION PASSED")
|
||||
print("🚀 System is ready for production deployment!")
|
||||
return 0
|
||||
else:
|
||||
print("\n⚠️ SCALABILITY VALIDATION NEEDS REVIEW")
|
||||
print("🔧 Performance optimization recommended")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Scalability validation failed: {e}")
|
||||
return 1
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = asyncio.run(main())
|
||||
sys.exit(exit_code)
|
||||
294
scripts/security_hardening.sh
Executable file
294
scripts/security_hardening.sh
Executable file
@@ -0,0 +1,294 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Production Security Hardening Script for AITBC Platform
|
||||
# This script implements security measures for production deployment
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
PRODUCTION_ENV="/opt/aitbc/apps/coordinator-api/.env.production"
|
||||
SERVICE_NAME="aitbc-coordinator"
|
||||
LOG_FILE="/var/log/aitbc-security-hardening.log"
|
||||
|
||||
# Logging function
|
||||
log() {
|
||||
echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
success() {
|
||||
echo -e "${GREEN}✅ $1${NC}" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
warning() {
|
||||
echo -e "${YELLOW}⚠️ $1${NC}" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
error() {
|
||||
echo -e "${RED}❌ $1${NC}" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Check if running as root
|
||||
check_root() {
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
error "This script must be run as root for system-level changes"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Generate secure API keys
|
||||
generate_api_keys() {
|
||||
log "Generating secure production API keys..."
|
||||
|
||||
# Generate 32-character secure keys
|
||||
CLIENT_KEY=$(openssl rand -hex 16)
|
||||
MINER_KEY=$(openssl rand -hex 16)
|
||||
ADMIN_KEY=$(openssl rand -hex 16)
|
||||
|
||||
log "Generated secure API keys"
|
||||
success "API keys generated successfully"
|
||||
|
||||
# Save keys securely
|
||||
cat > /opt/aitbc/secure/api_keys.txt << EOF
|
||||
# AITBC Production API Keys - Generated $(date)
|
||||
# Keep this file secure and restricted!
|
||||
CLIENT_API_KEYS=["$CLIENT_KEY"]
|
||||
MINER_API_KEYS=["$MINER_KEY"]
|
||||
ADMIN_API_KEYS=["$ADMIN_KEY"]
|
||||
EOF
|
||||
|
||||
chmod 600 /opt/aitbc/secure/api_keys.txt
|
||||
success "API keys saved to /opt/aitbc/secure/api_keys.txt"
|
||||
}
|
||||
|
||||
# Update production environment
|
||||
update_production_env() {
|
||||
log "Updating production environment configuration..."
|
||||
|
||||
if [[ ! -f "$PRODUCTION_ENV" ]]; then
|
||||
warning "Production env file not found, creating from template..."
|
||||
cp /opt/aitbc/apps/coordinator-api/.env "$PRODUCTION_ENV"
|
||||
fi
|
||||
|
||||
# Update API keys in production env
|
||||
if [[ -f /opt/aitbc/secure/api_keys.txt ]]; then
|
||||
source /opt/aitbc/secure/api_keys.txt
|
||||
|
||||
sed -i "s/CLIENT_API_KEYS=.*/CLIENT_API_KEYS=$CLIENT_API_KEYS/" "$PRODUCTION_ENV"
|
||||
sed -i "s/MINER_API_KEYS=.*/MINER_API_KEYS=$MINER_API_KEYS/" "$PRODUCTION_ENV"
|
||||
sed -i "s/ADMIN_API_KEYS=.*/ADMIN_API_KEYS=$ADMIN_API_KEYS/" "$PRODUCTION_ENV"
|
||||
|
||||
success "Production environment updated with secure API keys"
|
||||
fi
|
||||
|
||||
# Set production-specific settings
|
||||
cat >> "$PRODUCTION_ENV" << EOF
|
||||
|
||||
# Production Security Settings
|
||||
ENV=production
|
||||
DEBUG=false
|
||||
LOG_LEVEL=INFO
|
||||
RATE_LIMIT_ENABLED=true
|
||||
RATE_LIMIT_MINER_HEARTBEAT=60
|
||||
RATE_LIMIT_CLIENT_SUBMIT=30
|
||||
CORS_ORIGINS=["https://aitbc.bubuit.net"]
|
||||
EOF
|
||||
|
||||
success "Production security settings applied"
|
||||
}
|
||||
|
||||
# Configure firewall rules
|
||||
configure_firewall() {
|
||||
log "Configuring firewall rules..."
|
||||
|
||||
# Check if ufw is available
|
||||
if command -v ufw &> /dev/null; then
|
||||
# Allow SSH
|
||||
ufw allow 22/tcp
|
||||
|
||||
# Allow HTTP/HTTPS
|
||||
ufw allow 80/tcp
|
||||
ufw allow 443/tcp
|
||||
|
||||
# Allow internal services (restricted to localhost)
|
||||
ufw allow from 127.0.0.1 to any port 8000
|
||||
ufw allow from 127.0.0.1 to any port 8082
|
||||
|
||||
# Enable firewall
|
||||
ufw --force enable
|
||||
|
||||
success "Firewall configured with ufw"
|
||||
else
|
||||
warning "ufw not available, please configure firewall manually"
|
||||
fi
|
||||
}
|
||||
|
||||
# Setup SSL/TLS security
|
||||
setup_ssl_security() {
|
||||
log "Configuring SSL/TLS security..."
|
||||
|
||||
# Check SSL certificate
|
||||
if [[ -f "/etc/letsencrypt/live/aitbc.bubuit.net/fullchain.pem" ]]; then
|
||||
success "SSL certificate found and valid"
|
||||
|
||||
# Configure nginx security headers
|
||||
cat > /etc/nginx/snippets/security-headers.conf << EOF
|
||||
# Security Headers
|
||||
add_header X-Frame-Options "SAMEORIGIN" always;
|
||||
add_header X-XSS-Protection "1; mode=block" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header Referrer-Policy "no-referrer-when-downgrade" always;
|
||||
add_header Content-Security-Policy "default-src 'self' http: https: data: blob: 'unsafe-inline'" always;
|
||||
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
|
||||
EOF
|
||||
|
||||
# Include security headers in nginx config
|
||||
if grep -q "security-headers.conf" /etc/nginx/sites-available/aitbc-proxy.conf; then
|
||||
success "Security headers already configured"
|
||||
else
|
||||
# Add security headers to nginx config
|
||||
sed -i '/server_name/a\\n include snippets/security-headers.conf;' /etc/nginx/sites-available/aitbc-proxy.conf
|
||||
success "Security headers added to nginx configuration"
|
||||
fi
|
||||
|
||||
# Test and reload nginx
|
||||
nginx -t && systemctl reload nginx
|
||||
success "Nginx reloaded with security headers"
|
||||
else
|
||||
error "SSL certificate not found - please obtain certificate first"
|
||||
fi
|
||||
}
|
||||
|
||||
# Setup log rotation
|
||||
setup_log_rotation() {
|
||||
log "Configuring log rotation..."
|
||||
|
||||
cat > /etc/logrotate.d/aitbc << EOF
|
||||
/var/log/aitbc*.log {
|
||||
daily
|
||||
missingok
|
||||
rotate 30
|
||||
compress
|
||||
delaycompress
|
||||
notifempty
|
||||
create 644 aitbc aitbc
|
||||
postrotate
|
||||
systemctl reload rsyslog || true
|
||||
endscript
|
||||
}
|
||||
EOF
|
||||
|
||||
success "Log rotation configured"
|
||||
}
|
||||
|
||||
# Setup monitoring alerts
|
||||
setup_monitoring() {
|
||||
log "Setting up basic monitoring..."
|
||||
|
||||
# Create monitoring script
|
||||
cat > /opt/aitbc/scripts/health-check.sh << 'EOF'
|
||||
#!/bin/bash
|
||||
# Health check script for AITBC services
|
||||
|
||||
SERVICES=("aitbc-coordinator" "blockchain-node")
|
||||
WEB_URL="https://aitbc.bubuit.net/api/v1/health"
|
||||
|
||||
# Check systemd services
|
||||
for service in "${SERVICES[@]}"; do
|
||||
if systemctl is-active --quiet "$service"; then
|
||||
echo "✅ $service is running"
|
||||
else
|
||||
echo "❌ $service is not running"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Check web endpoint
|
||||
if curl -s -f "$WEB_URL" > /dev/null; then
|
||||
echo "✅ Web endpoint is responding"
|
||||
else
|
||||
echo "❌ Web endpoint is not responding"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ All health checks passed"
|
||||
EOF
|
||||
|
||||
chmod +x /opt/aitbc/scripts/health-check.sh
|
||||
|
||||
# Create cron job for health checks
|
||||
(crontab -l 2>/dev/null; echo "*/5 * * * * /opt/aitbc/scripts/health-check.sh >> /var/log/aitbc-health.log 2>&1") | crontab -
|
||||
|
||||
success "Health monitoring configured"
|
||||
}
|
||||
|
||||
# Security audit
|
||||
security_audit() {
|
||||
log "Performing security audit..."
|
||||
|
||||
# Check for open ports
|
||||
log "Open ports:"
|
||||
netstat -tuln | grep LISTEN | head -10
|
||||
|
||||
# Check running services
|
||||
log "Running services:"
|
||||
systemctl list-units --type=service --state=running | grep -E "(aitbc|nginx|ssh)" | head -10
|
||||
|
||||
# Check file permissions
|
||||
log "Critical file permissions:"
|
||||
ls -la /opt/aitbc/secure/ 2>/dev/null || echo "No secure directory found"
|
||||
ls -la /opt/aitbc/apps/coordinator-api/.env*
|
||||
|
||||
success "Security audit completed"
|
||||
}
|
||||
|
||||
# Main execution
|
||||
main() {
|
||||
log "Starting AITBC Production Security Hardening..."
|
||||
|
||||
# Create directories
|
||||
mkdir -p /opt/aitbc/secure
|
||||
mkdir -p /opt/aitbc/scripts
|
||||
|
||||
# Execute security measures
|
||||
check_root
|
||||
generate_api_keys
|
||||
update_production_env
|
||||
configure_firewall
|
||||
setup_ssl_security
|
||||
setup_log_rotation
|
||||
setup_monitoring
|
||||
security_audit
|
||||
|
||||
log "Security hardening completed successfully!"
|
||||
success "AITBC platform is now production-ready with enhanced security"
|
||||
|
||||
echo
|
||||
echo "🔐 SECURITY SUMMARY:"
|
||||
echo " ✅ Secure API keys generated"
|
||||
echo " ✅ Production environment configured"
|
||||
echo " ✅ Firewall rules applied"
|
||||
echo " ✅ SSL/TLS security enhanced"
|
||||
echo " ✅ Log rotation configured"
|
||||
echo " ✅ Health monitoring setup"
|
||||
echo
|
||||
echo "📋 NEXT STEPS:"
|
||||
echo " 1. Restart services: systemctl restart $SERVICE_NAME"
|
||||
echo " 2. Update CLI config with new API keys"
|
||||
echo " 3. Run production tests"
|
||||
echo " 4. Monitor system performance"
|
||||
echo
|
||||
echo "🔑 API Keys Location: /opt/aitbc/secure/api_keys.txt"
|
||||
echo "📊 Health Logs: /var/log/aitbc-health.log"
|
||||
echo "🔒 Security Log: $LOG_FILE"
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
168
scripts/simple_performance_test.py
Normal file
168
scripts/simple_performance_test.py
Normal file
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Simple Performance Testing for AITBC Platform
|
||||
"""
|
||||
|
||||
import time
|
||||
import requests
|
||||
import statistics
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import json
|
||||
|
||||
class SimplePerformanceTester:
|
||||
def __init__(self, base_url="https://aitbc.bubuit.net/api/v1"):
|
||||
self.base_url = base_url
|
||||
self.api_key = "test_key_16_characters"
|
||||
|
||||
def test_endpoint(self, method, endpoint, **kwargs):
|
||||
"""Test a single endpoint"""
|
||||
start_time = time.time()
|
||||
|
||||
headers = kwargs.pop('headers', {})
|
||||
headers['X-Api-Key'] = self.api_key
|
||||
|
||||
try:
|
||||
response = requests.request(method, f"{self.base_url}{endpoint}",
|
||||
headers=headers, timeout=10, **kwargs)
|
||||
end_time = time.time()
|
||||
|
||||
return {
|
||||
'endpoint': endpoint,
|
||||
'method': method,
|
||||
'status_code': response.status_code,
|
||||
'response_time': end_time - start_time,
|
||||
'success': response.status_code < 400,
|
||||
'content_length': len(response.text)
|
||||
}
|
||||
except Exception as e:
|
||||
end_time = time.time()
|
||||
return {
|
||||
'endpoint': endpoint,
|
||||
'method': method,
|
||||
'status_code': 0,
|
||||
'response_time': end_time - start_time,
|
||||
'success': False,
|
||||
'error': str(e)
|
||||
}
|
||||
|
||||
def load_test_endpoint(self, method, endpoint, concurrent_users=5, requests_per_user=3, **kwargs):
|
||||
"""Load test an endpoint"""
|
||||
print(f"🧪 Testing {method} {endpoint} - {concurrent_users} users × {requests_per_user} requests")
|
||||
|
||||
def make_request():
|
||||
return self.test_endpoint(method, endpoint, **kwargs)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=concurrent_users) as executor:
|
||||
futures = []
|
||||
for _ in range(concurrent_users * requests_per_user):
|
||||
future = executor.submit(make_request)
|
||||
futures.append(future)
|
||||
|
||||
results = []
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
results.append(result)
|
||||
|
||||
successful_results = [r for r in results if r['success']]
|
||||
response_times = [r['response_time'] for r in successful_results]
|
||||
|
||||
return {
|
||||
'endpoint': endpoint,
|
||||
'total_requests': len(results),
|
||||
'successful_requests': len(successful_results),
|
||||
'failed_requests': len(results) - len(successful_results),
|
||||
'success_rate': len(successful_results) / len(results) * 100 if results else 0,
|
||||
'avg_response_time': statistics.mean(response_times) if response_times else 0,
|
||||
'min_response_time': min(response_times) if response_times else 0,
|
||||
'max_response_time': max(response_times) if response_times else 0,
|
||||
'median_response_time': statistics.median(response_times) if response_times else 0,
|
||||
}
|
||||
|
||||
def run_tests(self):
|
||||
"""Run performance tests"""
|
||||
print("🚀 AITBC Platform Performance Tests")
|
||||
print("=" * 50)
|
||||
|
||||
test_cases = [
|
||||
# Health check
|
||||
{'method': 'GET', 'endpoint': '/health', 'users': 10, 'requests': 5},
|
||||
|
||||
# Client endpoints
|
||||
{'method': 'GET', 'endpoint': '/client/jobs', 'users': 5, 'requests': 3},
|
||||
|
||||
# Miner endpoints
|
||||
{'method': 'POST', 'endpoint': '/miners/register', 'users': 3, 'requests': 2,
|
||||
'json': {'capabilities': {'gpu': {'model': 'RTX 4090'}}},
|
||||
'headers': {'Content-Type': 'application/json', 'X-Miner-ID': 'perf-test-miner'}},
|
||||
]
|
||||
|
||||
results = []
|
||||
|
||||
for test_case in test_cases:
|
||||
method = test_case.pop('method')
|
||||
endpoint = test_case.pop('endpoint')
|
||||
|
||||
result = self.load_test_endpoint(method, endpoint, **test_case)
|
||||
results.append(result)
|
||||
|
||||
# Print results
|
||||
status = "✅" if result['success_rate'] >= 80 else "⚠️" if result['success_rate'] >= 50 else "❌"
|
||||
print(f"{status} {method} {endpoint}:")
|
||||
print(f" Success Rate: {result['success_rate']:.1f}%")
|
||||
print(f" Avg Response: {result['avg_response_time']:.3f}s")
|
||||
print(f" Requests: {result['successful_requests']}/{result['total_requests']}")
|
||||
print()
|
||||
|
||||
# Generate report
|
||||
self.generate_report(results)
|
||||
return results
|
||||
|
||||
def generate_report(self, results):
|
||||
"""Generate performance report"""
|
||||
print("📋 PERFORMANCE REPORT")
|
||||
print("=" * 50)
|
||||
|
||||
total_requests = sum(r['total_requests'] for r in results)
|
||||
total_successful = sum(r['successful_requests'] for r in results)
|
||||
overall_success_rate = (total_successful / total_requests * 100) if total_requests > 0 else 0
|
||||
|
||||
print(f"📊 Overall:")
|
||||
print(f" Total Requests: {total_requests}")
|
||||
print(f" Successful: {total_successful}")
|
||||
print(f" Success Rate: {overall_success_rate:.1f}%")
|
||||
print()
|
||||
|
||||
print(f"🎯 Endpoint Performance:")
|
||||
for result in results:
|
||||
status = "✅" if result['success_rate'] >= 80 else "⚠️" if result['success_rate'] >= 50 else "❌"
|
||||
print(f" {status} {result['method']} {result['endpoint']}")
|
||||
print(f" Success: {result['success_rate']:.1f}% | "
|
||||
f"Avg: {result['avg_response_time']:.3f}s | "
|
||||
f"Requests: {result['successful_requests']}/{result['total_requests']}")
|
||||
|
||||
print()
|
||||
print("💡 Recommendations:")
|
||||
|
||||
if overall_success_rate >= 80:
|
||||
print(" 🎉 Good performance - ready for production!")
|
||||
else:
|
||||
print(" ⚠️ Performance issues detected - review endpoints")
|
||||
|
||||
slow_endpoints = [r for r in results if r['avg_response_time'] > 1.0]
|
||||
if slow_endpoints:
|
||||
print(" 🐌 Slow endpoints:")
|
||||
for r in slow_endpoints:
|
||||
print(f" - {r['endpoint']} ({r['avg_response_time']:.3f}s)")
|
||||
|
||||
if __name__ == "__main__":
|
||||
tester = SimplePerformanceTester()
|
||||
results = tester.run_tests()
|
||||
|
||||
# Exit code based on performance
|
||||
avg_success_rate = statistics.mean([r['success_rate'] for r in results])
|
||||
if avg_success_rate >= 80:
|
||||
print("\n✅ PERFORMANCE TESTS PASSED")
|
||||
exit(0)
|
||||
else:
|
||||
print("\n⚠️ PERFORMANCE TESTS NEED REVIEW")
|
||||
exit(1)
|
||||
Reference in New Issue
Block a user