diff --git a/.windsurf/workflows/multi-node-blockchain-setup.md b/.windsurf/workflows/multi-node-blockchain-setup.md index 4cea9707..9a0d3ed5 100644 --- a/.windsurf/workflows/multi-node-blockchain-setup.md +++ b/.windsurf/workflows/multi-node-blockchain-setup.md @@ -178,57 +178,36 @@ The newly created wallet on aitbc will: ### 11. Complete Sync (Optional - for full demonstration) ```bash -# If aitbc is still behind, complete the sync -AITBC1_HEIGHT=$(curl -s http://localhost:8006/rpc/head | jq .height) -AITBC_HEIGHT=$(ssh aitbc 'curl -s http://localhost:8006/rpc/head | jq .height') - -echo "aitbc1 height: $AITBC1_HEIGHT" -echo "aitbc height: $AITBC_HEIGHT" - -if [ "$AITBC_HEIGHT" -lt "$((AITBC1_HEIGHT - 5))" ]; then - echo "Completing sync from aitbc1..." - for height in $(seq $((AITBC_HEIGHT + 1)) $AITBC1_HEIGHT); do - echo "Importing block $height..." - curl -s "http://10.1.223.40:8006/rpc/blocks-range?start=$height&end=$height" | \ - jq '.blocks[0]' > /tmp/block$height.json - curl -X POST http://localhost:8006/rpc/importBlock \ - -H "Content-Type: application/json" -d @/tmp/block$height.json - sleep 1 # Brief pause between imports - done - echo "Sync completed!" -fi - -# Final balance verification -echo "=== Final balance verification ===" -ssh aitbc "curl -s \"http://localhost:8006/rpc/getBalance/$WALLET_ADDR\" | jq ." +# Complete blockchain synchronization between nodes +/opt/aitbc/scripts/workflow/12_complete_sync.sh ``` -### 13. Legacy Environment File Cleanup +### 12. Legacy Environment File Cleanup ```bash # Remove all legacy .env.production and .env references from systemd services -/opt/aitbc/scripts/workflow/01_preflight_setup.sh +/opt/aitbc/scripts/workflow/13_maintenance_automation.sh ``` -### 14. Final Configuration Verification +### 13. Final Configuration Verification ```bash # Verify all configurations are using centralized files -/opt/aitbc/scripts/workflow/06_final_verification.sh +/opt/aitbc/scripts/workflow/13_maintenance_automation.sh ``` -### 15. Cross-Node Code Synchronization +### 14. Cross-Node Code Synchronization ```bash # Ensure aitbc node stays synchronized with aitbc1 after code changes -ssh aitbc 'cd /opt/aitbc && git pull origin main' +/opt/aitbc/scripts/workflow/13_maintenance_automation.sh ``` -### 16. Complete Workflow Execution +### 15. Complete Workflow Execution ```bash # Execute the complete multi-node blockchain setup workflow -/opt/aitbc/scripts/workflow/setup_multinode_blockchain.sh +/opt/aitbc/scripts/workflow/14_production_ready.sh ``` ### 🔍 Configuration Overview @@ -342,26 +321,14 @@ PYTHONPATH=/opt/aitbc/apps/blockchain-node/src:/opt/aitbc/apps/blockchain-node/s #### **Keystore Issues** ```bash -# Create keystore password file -echo 'aitbc123' > /var/lib/aitbc/keystore/.password -chmod 600 /var/lib/aitbc/keystore/.password - -# Check keystore permissions -ls -la /var/lib/aitbc/keystore/ +# Create keystore password file and check permissions +/opt/aitbc/scripts/workflow/01_preflight_setup.sh ``` #### **Sync Issues** ```bash -# Check network connectivity between nodes -ping 10.1.223.40 # aitbc1 from aitbc -ping 10.1.223.93 # aitbc from aitbc1 - -# Check Redis connectivity -redis-cli -h 10.1.223.40 ping - -# Compare blockchain heights -curl -s http://localhost:8006/rpc/head | jq .height -ssh aitbc 'curl -s http://localhost:8006/rpc/head | jq .height' +# Check and fix blockchain synchronization issues +/opt/aitbc/scripts/workflow/08_blockchain_sync_fix.sh ``` ### General Troubleshooting diff --git a/scripts/workflow/12_complete_sync.sh b/scripts/workflow/12_complete_sync.sh new file mode 100755 index 00000000..2a9d2634 --- /dev/null +++ b/scripts/workflow/12_complete_sync.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# AITBC Complete Blockchain Sync Script +# Handles complete synchronization between nodes + +echo "=== AITBC Complete Blockchain Sync ===" + +# Configuration +WALLET_ADDR="ait11c02342d4fec502240c20d609a8bb839ccd23838" + +# Check current heights +echo "1. Current blockchain status:" +AITBC1_HEIGHT=$(curl -s http://localhost:8006/rpc/head | jq .height 2>/dev/null || echo "0") +AITBC_HEIGHT=$(ssh aitbc 'curl -s http://localhost:8006/rpc/head | jq .height 2>/dev/null || echo "0") + +echo "aitbc1 height: $AITBC1_HEIGHT" +echo "aitbc height: $AITBC_HEIGHT" + +# Check if sync is needed +if [ "$AITBC_HEIGHT" -ge "$((AITBC1_HEIGHT - 5))" ]; then + echo "✅ Nodes are already synchronized (diff: $((AITBC1_HEIGHT - AITBC_HEIGHT)) blocks)" + exit 0 +fi + +echo "2. Performing complete sync from aitbc1..." +echo " Syncing from block $((AITBC_HEIGHT + 1)) to $AITBC1_HEIGHT" + +# Get proposer address for proper block format +PROPOSER_ADDR=$(cat /opt/aitbc/apps/blockchain-node/keystore/aitbc1genesis.json | jq -r '.address') + +# Sync blocks in batches +BATCH_SIZE=10 +CURRENT_HEIGHT=$((AITBC_HEIGHT + 1)) + +while [ $CURRENT_HEIGHT -le $AITBC1_HEIGHT ]; do + END_HEIGHT=$((CURRENT_HEIGHT + BATCH_SIZE - 1)) + if [ $END_HEIGHT -gt $AITBC1_HEIGHT ]; then + END_HEIGHT=$AITBC1_HEIGHT + fi + + echo " Syncing batch: blocks $CURRENT_HEIGHT to $END_HEIGHT" + + # Import blocks in batch + for height in $(seq $CURRENT_HEIGHT $END_HEIGHT); do + echo " Importing block $height..." + + # Get block with proper proposer field + curl -s "http://localhost:8006/rpc/blocks-range?start=$height&end=$height" | \ + jq '.blocks[0] + {"proposer": "'$PROPOSER_ADDR'"}' > /tmp/block$height.json + + # Import to aitbc + scp /tmp/block$height.json aitbc:/tmp/ 2>/dev/null + ssh aitbc "curl -X POST http://localhost:8006/rpc/importBlock -H 'Content-Type: application/json' -d @/tmp/block$height.json" > /dev/null 2>&1 + + sleep 0.5 + done + + # Check progress + CURRENT_HEIGHT=$(ssh aitbc 'curl -s http://localhost:8006/rpc/head | jq .height 2>/dev/null || echo "0"') + PROGRESS=$((CURRENT_HEIGHT * 100 / AITBC1_HEIGHT)) + echo " Progress: $PROGRESS% ($CURRENT_HEIGHT/$AITBC1_HEIGHT blocks)" + + # Brief pause between batches + sleep 2 +done + +echo "3. Final verification:" +FINAL_AITBC_HEIGHT=$(ssh aitbc 'curl -s http://localhost:8006/rpc/head | jq .height 2>/dev/null || echo "0"') +FINAL_DIFF=$((AITBC1_HEIGHT - FINAL_AITBC_HEIGHT)) + +echo " aitbc1 final height: $AITBC1_HEIGHT" +echo " aitbc final height: $FINAL_AITBC_HEIGHT" +echo " Height difference: $FINAL_DIFF blocks" + +if [ $FINAL_DIFF -le 2 ]; then + echo "✅ Complete sync successful!" +else + echo "⚠️ Sync may need additional time" +fi + +# Final balance verification +echo "4. Final balance verification:" +if [ -n "$WALLET_ADDR" ]; then + FINAL_BALANCE=$(ssh aitbc "curl -s \"http://localhost:8006/rpc/getBalance/$WALLET_ADDR\" | jq .balance 2>/dev/null || echo "0"") + echo " Wallet balance: $FINAL_BALANCE AIT" +fi + +echo "=== Complete Blockchain Sync Finished ===" diff --git a/scripts/workflow/13_maintenance_automation.sh b/scripts/workflow/13_maintenance_automation.sh new file mode 100755 index 00000000..fb6c02f6 --- /dev/null +++ b/scripts/workflow/13_maintenance_automation.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# AITBC Maintenance Automation Script +# Handles environment cleanup, configuration verification, and code synchronization + +echo "=== AITBC Maintenance Automation ===" + +# Step 1: Legacy Environment File Cleanup +echo "1. Legacy Environment File Cleanup..." +echo " Removing .env.production and .env references from systemd services..." + +# Find and update systemd service files +find /etc/systemd/system/aitbc-blockchain-*.service.d/ -name "*.conf" -exec grep -l "EnvironmentFile.*env" {} \; 2>/dev/null | while read file; do + echo " Updating: $file" + sed -i 's|EnvironmentFile=/opt/aitbc/.env|EnvironmentFile=/etc/aitbc/blockchain.env|g' "$file" + sed -i 's|EnvironmentFile=.*\.env\.production|EnvironmentFile=/etc/aitbc/blockchain.env|g' "$file" +done + +# Remove any remaining old references +find /etc/systemd/system/aitbc-blockchain-*.service.d/ -name "*.conf" -exec grep -l "EnvironmentFile.*\.env\.production" {} \; 2>/dev/null | while read file; do + echo " Removing .env.production references from: $file" + sed -i 's|EnvironmentFile.*\.env\.production|EnvironmentFile=/etc/aitbc/blockchain.env|g' "$file" +done + +echo " ✅ Legacy environment file cleanup completed" + +# Step 2: Final Configuration Verification +echo "2. Final Configuration Verification..." +echo " Checking systemd service configurations..." + +# Verify all services use correct environment file +SERVICES_USING_CORRECT_ENV=$(find /etc/systemd/system/aitbc-blockchain-*.service.d/ -name "*.conf" -exec grep -l "EnvironmentFile=/etc/aitbc/blockchain.env" {} \; 2>/dev/null | wc -l) +echo " Services using correct environment file: $SERVICES_USING_CORRECT_ENV" + +# Verify the centralized environment file exists +if [ -f "/etc/aitbc/blockchain.env" ]; then + echo " ✅ Centralized environment file exists" + echo " Key configurations:" + grep -E "^(chain_id|enable_block_production|proposer_id|keystore_path|db_path)" /etc/aitbc/blockchain.env | head -5 +else + echo " ❌ Centralized environment file missing" + exit 1 +fi + +# Check for legacy environment files +if [ -f "/opt/aitbc/.env" ]; then + echo " ⚠️ Legacy /opt/aitbc/.env still exists" +else + echo " ✅ No legacy environment files found" +fi + +# Test that services can access the centralized configuration +echo " Testing service configuration access..." +systemctl daemon-reload + +# Step 3: Cross-Node Code Synchronization +echo "3. Cross-Node Code Synchronization..." +echo " Ensuring aitbc node stays synchronized with aitbc1..." + +# Check current git status on aitbc1 +echo " Current git status on aitbc1:" +cd /opt/aitbc +git status --porcelain + +# Push any local changes to remote +if [ -n "$(git status --porcelain)" ]; then + echo " Pushing local changes to remote..." + git add -A + git commit -m "Automated maintenance update - $(date)" + git push origin main +else + echo " No local changes to push" +fi + +# Sync code to aitbc +echo " Syncing code to aitbc..." +ssh aitbc ' + cd /opt/aitbc + echo " Current branch: $(git branch --show-current)" + echo " Pulling latest changes..." + git pull origin main + echo " Latest commit: $(git log --oneline -1)" + + # Restart services if code changed + echo " Restarting services to apply code changes..." + systemctl restart aitbc-blockchain-node aitbc-blockchain-rpc + + # Verify services are running + echo " Verifying services..." + systemctl is-active aitbc-blockchain-node aitbc-blockchain-rpc +' + +# Verify both nodes are on the same commit +echo "4. Verification Results..." +AITBC1_COMMIT=$(cd /opt/aitbc && git rev-parse --short HEAD) +AITBC_COMMIT=$(ssh aitbc 'cd /opt/aitbc && git rev-parse --short HEAD') + +echo " aitbc1 commit: $AITBC1_COMMIT" +echo " aitbc commit: $AITBC_COMMIT" + +if [ "$AITBC1_COMMIT" = "$AITBC_COMMIT" ]; then + echo " ✅ Code synchronized between nodes" +else + echo " ❌ Code not synchronized - manual intervention required" +fi + +# Test that both nodes are operational +echo "5. Cross-node functionality test..." +AITBC1_HEIGHT=$(curl -s http://localhost:8006/rpc/head | jq .height 2>/dev/null || echo "0") +AITBC_HEIGHT=$(ssh aitbc 'curl -s http://localhost:8006/rpc/head | jq .height 2>/dev/null || echo "0") + +echo " aitbc1 height: $AITBC1_HEIGHT" +echo " aitbc height: $AITBC_HEIGHT" + +HEIGHT_DIFF=$((AITBC1_HEIGHT - AITBC_HEIGHT)) +if [ $HEIGHT_DIFF -le 5 ]; then + echo " ✅ Cross-node functionality working" +else + echo " ⚠️ Cross-node sync issues detected (diff: $HEIGHT_DIFF blocks)" +fi + +# Service health check +echo "6. Service Health Check..." +AITBC1_SERVICES=$(systemctl is-active aitbc-blockchain-node aitbc-blockchain-rpc | grep -c "active") +AITBC_SERVICES=$(ssh aitbc 'systemctl is-active aitbc-blockchain-node aitbc-blockchain-rpc | grep -c "active"') + +echo " aitbc1 active services: $AITBC1_SERVICES/2" +echo " aitbc active services: $AITBC_SERVICES/2" + +if [ "$AITBC1_SERVICES" -eq 2 ] && [ "$AITBC_SERVICES" -eq 2 ]; then + echo " ✅ All services operational" +else + echo " ❌ Some services not running" +fi + +echo "=== Maintenance Automation Complete ===" diff --git a/scripts/workflow/14_production_ready.sh b/scripts/workflow/14_production_ready.sh new file mode 100755 index 00000000..f1490c6a --- /dev/null +++ b/scripts/workflow/14_production_ready.sh @@ -0,0 +1,170 @@ +#!/bin/bash +# AITBC Production Ready Script +# Complete production deployment and verification + +echo "=== AITBC Production Ready Deployment ===" + +# Check prerequisites +echo "1. Prerequisites Check..." +if [ "$(hostname)" != "aitbc1" ]; then + echo "❌ Error: This script must be run on aitbc1 (genesis authority node)" + exit 1 +fi + +# Define production workflow +PRODUCTION_STEPS=( + "01_preflight_setup.sh:Pre-Flight Setup" + "02_genesis_authority_setup.sh:Genesis Authority Setup" + "03_follower_node_setup.sh:Follower Node Setup" + "08_blockchain_sync_fix.sh:Blockchain Sync Fix" + "04_create_wallet.sh:Wallet Creation" + "09_transaction_manager.sh:Transaction Manager" + "12_complete_sync.sh:Complete Sync" + "11_network_optimizer.sh:Network Optimization" + "13_maintenance_automation.sh:Maintenance Automation" + "06_final_verification.sh:Final Verification" +) + +# Execute production workflow +echo "2. Executing Production Workflow..." +FAILED_STEPS=() + +for step in "${PRODUCTION_STEPS[@]}"; do + SCRIPT=$(echo "$step" | cut -d: -f1) + DESCRIPTION=$(echo "$step" | cut -d: -f2) + + echo + echo "==========================================" + echo "PRODUCTION STEP: $DESCRIPTION" + echo "SCRIPT: $SCRIPT" + echo "==========================================" + + if [ -f "/opt/aitbc/scripts/workflow/$SCRIPT" ]; then + echo "Executing $SCRIPT..." + bash "/opt/aitbc/scripts/workflow/$SCRIPT" + + if [ $? -eq 0 ]; then + echo "✅ $DESCRIPTION completed successfully" + else + echo "❌ $DESCRIPTION failed" + FAILED_STEPS+=("$DESCRIPTION") + fi + else + echo "❌ Script not found: $SCRIPT" + FAILED_STEPS+=("$DESCRIPTION (script missing)") + fi +done + +# Production verification +echo +echo "==========================================" +echo "PRODUCTION VERIFICATION" +echo "==========================================" + +# Service status +echo "3. Service Status Verification:" +AITBC1_SERVICES=$(systemctl is-active aitbc-blockchain-node aitbc-blockchain-rpc | grep -c "active") +AITBC_SERVICES=$(ssh aitbc 'systemctl is-active aitbc-blockchain-node aitbc-blockchain-rpc 2>/dev/null | grep -c "active"') + +echo " aitbc1 services: $AITBC1_SERVICES/2 active" +echo " aitbc services: $AITBC_SERVICES/2 active" + +# Blockchain status +echo "4. Blockchain Status:" +AITBC1_HEIGHT=$(curl -s http://localhost:8006/rpc/head | jq .height 2>/dev/null || echo "0") +AITBC_HEIGHT=$(ssh aitbc 'curl -s http://localhost:8006/rpc/head | jq .height 2>/dev/null || echo "0") + +echo " aitbc1 height: $AITBC1_HEIGHT" +echo " aitbc height: $AITBC_HEIGHT" +echo " Sync difference: $((AITBC1_HEIGHT - AITBC_HEIGHT)) blocks" + +# Network performance +echo "5. Network Performance:" +AITBC1_RPC_TIME=$(curl -w "%{time_total}" -s -o /dev/null http://localhost:8006/rpc/head) +AITBC_RPC_TIME=$(ssh aitbc 'curl -w "%{time_total}" -s -o /dev/null http://localhost:8006/rpc/head') +NETWORK_LATENCY=$(ping -c 1 10.1.223.93 | grep "time=" | cut -d= -f2 | cut -d" " -f1) + +echo " aitbc1 RPC time: ${AITBC1_RPC_TIME}s" +echo " aitbc RPC time: ${AITBC_RPC_TIME}s" +echo " Network latency: ${NETWORK_LATENCY}ms" + +# Security check +echo "6. Security Configuration:" +if [ -f "/etc/aitbc/blockchain.env" ]; then + ENV_PERMISSIONS=$(stat -c "%a" /etc/aitbc/blockchain.env) + echo " Environment file permissions: $ENV_PERMISSIONS" +else + echo " ❌ Environment file missing" +fi + +KEYSTORE_PERMISSIONS=$(stat -c "%a" /var/lib/aitbc/keystore 2>/dev/null || echo "missing") +echo " Keystore permissions: $KEYSTORE_PERMISSIONS" + +# Production readiness assessment +echo +echo "==========================================" +echo "PRODUCTION READINESS ASSESSMENT" +echo "==========================================" + +SERVICES_OK=false +SYNC_OK=false +PERFORMANCE_OK=false +SECURITY_OK=false + +# Check services +if [ "$AITBC1_SERVICES" -eq 2 ] && [ "$AITBC_SERVICES" -eq 2 ]; then + SERVICES_OK=true + echo "✅ Services: All operational" +else + echo "❌ Services: Some services not running" +fi + +# Check sync +HEIGHT_DIFF=$((AITBC1_HEIGHT - AITBC_HEIGHT)) +if [ $HEIGHT_DIFF -le 2 ]; then + SYNC_OK=true + echo "✅ Sync: Nodes synchronized" +else + echo "❌ Sync: Nodes not synchronized (diff: $HEIGHT_DIFF blocks)" +fi + +# Check performance +if (( $(echo "$AITBC1_RPC_TIME < 1.0" | bc -l) )) && (( $(echo "$AITBC_RPC_TIME < 1.0" | bc -l) )); then + PERFORMANCE_OK=true + echo "✅ Performance: RPC times acceptable" +else + echo "❌ Performance: RPC times too slow" +fi + +# Check security +if [ "$ENV_PERMISSIONS" = "644" ] && [ "$KEYSTORE_PERMISSIONS" = "700" ]; then + SECURITY_OK=true + echo "✅ Security: Permissions correct" +else + echo "❌ Security: Check permissions" +fi + +# Final assessment +echo +echo "7. Final Production Status:" +if [ "$SERVICES_OK" = true ] && [ "$SYNC_OK" = true ] && [ "$PERFORMANCE_OK" = true ] && [ "$SECURITY_OK" = true ]; then + echo "🎉 PRODUCTION READY!" + echo " All systems operational and optimized" + echo " Ready for production deployment" +else + echo "⚠️ NOT PRODUCTION READY" + echo " Some issues need to be resolved" + + if [ ${#FAILED_STEPS[@]} -gt 0 ]; then + echo " Failed steps: ${FAILED_STEPS[*]}" + fi +fi + +echo +echo "8. Next Steps:" +echo " • Monitor with: /opt/aitbc/scripts/health_check.sh" +echo " • Test with: /opt/aitbc/tests/integration_test.sh" +echo " • Scale with: /opt/aitbc/scripts/provision_node.sh " +echo " • Enterprise features: /opt/aitbc/cli/enterprise_cli.py" + +echo "=== Production Ready Deployment Complete ==="