refactor: consolidate workflow steps and replace inline commands with script references

📋 Workflow Consolidation: • Renumbered steps 11-16 to 11-15 for better organization • Replaced inline sync commands with 12_complete_sync.sh script • Updated maintenance steps to use 13_maintenance_automation.sh • Updated production readiness to use 14_production_ready.sh • Removed duplicate script references and consolidated functionality 🔧 Troubleshooting Updates: • Replaced keystore setup commands with 01_preflight_setup.sh reference
2026-03-29 17:01:26 +02:00
parent 0d9ef9b5b7
commit 45cc1c8ddb
4 changed files with 406 additions and 47 deletions
--- a/scripts/workflow/12_complete_sync.sh
+++ b/scripts/workflow/12_complete_sync.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+# AITBC Complete Blockchain Sync Script
+# Handles complete synchronization between nodes
+
+echo "=== AITBC Complete Blockchain Sync ==="
+
+# Configuration
+WALLET_ADDR="ait11c02342d4fec502240c20d609a8bb839ccd23838"
+
+# Check current heights
+echo "1. Current blockchain status:"
+AITBC1_HEIGHT=$(curl -s http://localhost:8006/rpc/head | jq .height 2>/dev/null || echo "0")
+AITBC_HEIGHT=$(ssh aitbc 'curl -s http://localhost:8006/rpc/head | jq .height 2>/dev/null || echo "0")
+
+echo "aitbc1 height: $AITBC1_HEIGHT"
+echo "aitbc height: $AITBC_HEIGHT"
+
+# Check if sync is needed
+if [ "$AITBC_HEIGHT" -ge "$((AITBC1_HEIGHT - 5))" ]; then
+  echo "✅ Nodes are already synchronized (diff: $((AITBC1_HEIGHT - AITBC_HEIGHT)) blocks)"
+  exit 0
+fi
+
+echo "2. Performing complete sync from aitbc1..."
+echo "   Syncing from block $((AITBC_HEIGHT + 1)) to $AITBC1_HEIGHT"
+
+# Get proposer address for proper block format
+PROPOSER_ADDR=$(cat /opt/aitbc/apps/blockchain-node/keystore/aitbc1genesis.json | jq -r '.address')
+
+# Sync blocks in batches
+BATCH_SIZE=10
+CURRENT_HEIGHT=$((AITBC_HEIGHT + 1))
+
+while [ $CURRENT_HEIGHT -le $AITBC1_HEIGHT ]; do
+  END_HEIGHT=$((CURRENT_HEIGHT + BATCH_SIZE - 1))
+  if [ $END_HEIGHT -gt $AITBC1_HEIGHT ]; then
+    END_HEIGHT=$AITBC1_HEIGHT
+  fi
+  
+  echo "   Syncing batch: blocks $CURRENT_HEIGHT to $END_HEIGHT"
+  
+  # Import blocks in batch
+  for height in $(seq $CURRENT_HEIGHT $END_HEIGHT); do
+    echo "     Importing block $height..."
+    
+    # Get block with proper proposer field
+    curl -s "http://localhost:8006/rpc/blocks-range?start=$height&end=$height" | \
+      jq '.blocks[0] + {"proposer": "'$PROPOSER_ADDR'"}' > /tmp/block$height.json
+    
+    # Import to aitbc
+    scp /tmp/block$height.json aitbc:/tmp/ 2>/dev/null
+    ssh aitbc "curl -X POST http://localhost:8006/rpc/importBlock -H 'Content-Type: application/json' -d @/tmp/block$height.json" > /dev/null 2>&1
+    
+    sleep 0.5
+  done
+  
+  # Check progress
+  CURRENT_HEIGHT=$(ssh aitbc 'curl -s http://localhost:8006/rpc/head | jq .height 2>/dev/null || echo "0"')
+  PROGRESS=$((CURRENT_HEIGHT * 100 / AITBC1_HEIGHT))
+  echo "   Progress: $PROGRESS% ($CURRENT_HEIGHT/$AITBC1_HEIGHT blocks)"
+  
+  # Brief pause between batches
+  sleep 2
+done
+
+echo "3. Final verification:"
+FINAL_AITBC_HEIGHT=$(ssh aitbc 'curl -s http://localhost:8006/rpc/head | jq .height 2>/dev/null || echo "0"')
+FINAL_DIFF=$((AITBC1_HEIGHT - FINAL_AITBC_HEIGHT))
+
+echo "   aitbc1 final height: $AITBC1_HEIGHT"
+echo "   aitbc final height: $FINAL_AITBC_HEIGHT"
+echo "   Height difference: $FINAL_DIFF blocks"
+
+if [ $FINAL_DIFF -le 2 ]; then
+  echo "✅ Complete sync successful!"
+else
+  echo "⚠️ Sync may need additional time"
+fi
+
+# Final balance verification
+echo "4. Final balance verification:"
+if [ -n "$WALLET_ADDR" ]; then
+  FINAL_BALANCE=$(ssh aitbc "curl -s \"http://localhost:8006/rpc/getBalance/$WALLET_ADDR\" | jq .balance 2>/dev/null || echo "0"")
+  echo "   Wallet balance: $FINAL_BALANCE AIT"
+fi
+
+echo "=== Complete Blockchain Sync Finished ==="
--- a/scripts/workflow/13_maintenance_automation.sh
+++ b/scripts/workflow/13_maintenance_automation.sh
@@ -0,0 +1,135 @@
+#!/bin/bash
+# AITBC Maintenance Automation Script
+# Handles environment cleanup, configuration verification, and code synchronization
+
+echo "=== AITBC Maintenance Automation ==="
+
+# Step 1: Legacy Environment File Cleanup
+echo "1. Legacy Environment File Cleanup..."
+echo "   Removing .env.production and .env references from systemd services..."
+
+# Find and update systemd service files
+find /etc/systemd/system/aitbc-blockchain-*.service.d/ -name "*.conf" -exec grep -l "EnvironmentFile.*env" {} \; 2>/dev/null | while read file; do
+  echo "   Updating: $file"
+  sed -i 's|EnvironmentFile=/opt/aitbc/.env|EnvironmentFile=/etc/aitbc/blockchain.env|g' "$file"
+  sed -i 's|EnvironmentFile=.*\.env\.production|EnvironmentFile=/etc/aitbc/blockchain.env|g' "$file"
+done
+
+# Remove any remaining old references
+find /etc/systemd/system/aitbc-blockchain-*.service.d/ -name "*.conf" -exec grep -l "EnvironmentFile.*\.env\.production" {} \; 2>/dev/null | while read file; do
+  echo "   Removing .env.production references from: $file"
+  sed -i 's|EnvironmentFile.*\.env\.production|EnvironmentFile=/etc/aitbc/blockchain.env|g' "$file"
+done
+
+echo "   ✅ Legacy environment file cleanup completed"
+
+# Step 2: Final Configuration Verification
+echo "2. Final Configuration Verification..."
+echo "   Checking systemd service configurations..."
+
+# Verify all services use correct environment file
+SERVICES_USING_CORRECT_ENV=$(find /etc/systemd/system/aitbc-blockchain-*.service.d/ -name "*.conf" -exec grep -l "EnvironmentFile=/etc/aitbc/blockchain.env" {} \; 2>/dev/null | wc -l)
+echo "   Services using correct environment file: $SERVICES_USING_CORRECT_ENV"
+
+# Verify the centralized environment file exists
+if [ -f "/etc/aitbc/blockchain.env" ]; then
+  echo "   ✅ Centralized environment file exists"
+  echo "   Key configurations:"
+  grep -E "^(chain_id|enable_block_production|proposer_id|keystore_path|db_path)" /etc/aitbc/blockchain.env | head -5
+else
+  echo "   ❌ Centralized environment file missing"
+  exit 1
+fi
+
+# Check for legacy environment files
+if [ -f "/opt/aitbc/.env" ]; then
+  echo "   ⚠️ Legacy /opt/aitbc/.env still exists"
+else
+  echo "   ✅ No legacy environment files found"
+fi
+
+# Test that services can access the centralized configuration
+echo "   Testing service configuration access..."
+systemctl daemon-reload
+
+# Step 3: Cross-Node Code Synchronization
+echo "3. Cross-Node Code Synchronization..."
+echo "   Ensuring aitbc node stays synchronized with aitbc1..."
+
+# Check current git status on aitbc1
+echo "   Current git status on aitbc1:"
+cd /opt/aitbc
+git status --porcelain
+
+# Push any local changes to remote
+if [ -n "$(git status --porcelain)" ]; then
+  echo "   Pushing local changes to remote..."
+  git add -A
+  git commit -m "Automated maintenance update - $(date)"
+  git push origin main
+else
+  echo "   No local changes to push"
+fi
+
+# Sync code to aitbc
+echo "   Syncing code to aitbc..."
+ssh aitbc '
+  cd /opt/aitbc
+  echo "   Current branch: $(git branch --show-current)"
+  echo "   Pulling latest changes..."
+  git pull origin main
+  echo "   Latest commit: $(git log --oneline -1)"
+  
+  # Restart services if code changed
+  echo "   Restarting services to apply code changes..."
+  systemctl restart aitbc-blockchain-node aitbc-blockchain-rpc
+  
+  # Verify services are running
+  echo "   Verifying services..."
+  systemctl is-active aitbc-blockchain-node aitbc-blockchain-rpc
+'
+
+# Verify both nodes are on the same commit
+echo "4. Verification Results..."
+AITBC1_COMMIT=$(cd /opt/aitbc && git rev-parse --short HEAD)
+AITBC_COMMIT=$(ssh aitbc 'cd /opt/aitbc && git rev-parse --short HEAD')
+
+echo "   aitbc1 commit: $AITBC1_COMMIT"
+echo "   aitbc commit: $AITBC_COMMIT"
+
+if [ "$AITBC1_COMMIT" = "$AITBC_COMMIT" ]; then
+  echo "   ✅ Code synchronized between nodes"
+else
+  echo "   ❌ Code not synchronized - manual intervention required"
+fi
+
+# Test that both nodes are operational
+echo "5. Cross-node functionality test..."
+AITBC1_HEIGHT=$(curl -s http://localhost:8006/rpc/head | jq .height 2>/dev/null || echo "0")
+AITBC_HEIGHT=$(ssh aitbc 'curl -s http://localhost:8006/rpc/head | jq .height 2>/dev/null || echo "0")
+
+echo "   aitbc1 height: $AITBC1_HEIGHT"
+echo "   aitbc height: $AITBC_HEIGHT"
+
+HEIGHT_DIFF=$((AITBC1_HEIGHT - AITBC_HEIGHT))
+if [ $HEIGHT_DIFF -le 5 ]; then
+  echo "   ✅ Cross-node functionality working"
+else
+  echo "   ⚠️ Cross-node sync issues detected (diff: $HEIGHT_DIFF blocks)"
+fi
+
+# Service health check
+echo "6. Service Health Check..."
+AITBC1_SERVICES=$(systemctl is-active aitbc-blockchain-node aitbc-blockchain-rpc | grep -c "active")
+AITBC_SERVICES=$(ssh aitbc 'systemctl is-active aitbc-blockchain-node aitbc-blockchain-rpc | grep -c "active"')
+
+echo "   aitbc1 active services: $AITBC1_SERVICES/2"
+echo "   aitbc active services: $AITBC_SERVICES/2"
+
+if [ "$AITBC1_SERVICES" -eq 2 ] && [ "$AITBC_SERVICES" -eq 2 ]; then
+  echo "   ✅ All services operational"
+else
+  echo "   ❌ Some services not running"
+fi
+
+echo "=== Maintenance Automation Complete ==="
--- a/scripts/workflow/14_production_ready.sh
+++ b/scripts/workflow/14_production_ready.sh
@@ -0,0 +1,170 @@
+#!/bin/bash
+# AITBC Production Ready Script
+# Complete production deployment and verification
+
+echo "=== AITBC Production Ready Deployment ==="
+
+# Check prerequisites
+echo "1. Prerequisites Check..."
+if [ "$(hostname)" != "aitbc1" ]; then
+  echo "❌ Error: This script must be run on aitbc1 (genesis authority node)"
+  exit 1
+fi
+
+# Define production workflow
+PRODUCTION_STEPS=(
+  "01_preflight_setup.sh:Pre-Flight Setup"
+  "02_genesis_authority_setup.sh:Genesis Authority Setup"
+  "03_follower_node_setup.sh:Follower Node Setup"
+  "08_blockchain_sync_fix.sh:Blockchain Sync Fix"
+  "04_create_wallet.sh:Wallet Creation"
+  "09_transaction_manager.sh:Transaction Manager"
+  "12_complete_sync.sh:Complete Sync"
+  "11_network_optimizer.sh:Network Optimization"
+  "13_maintenance_automation.sh:Maintenance Automation"
+  "06_final_verification.sh:Final Verification"
+)
+
+# Execute production workflow
+echo "2. Executing Production Workflow..."
+FAILED_STEPS=()
+
+for step in "${PRODUCTION_STEPS[@]}"; do
+  SCRIPT=$(echo "$step" | cut -d: -f1)
+  DESCRIPTION=$(echo "$step" | cut -d: -f2)
+  
+  echo
+  echo "=========================================="
+  echo "PRODUCTION STEP: $DESCRIPTION"
+  echo "SCRIPT: $SCRIPT"
+  echo "=========================================="
+  
+  if [ -f "/opt/aitbc/scripts/workflow/$SCRIPT" ]; then
+    echo "Executing $SCRIPT..."
+    bash "/opt/aitbc/scripts/workflow/$SCRIPT"
+    
+    if [ $? -eq 0 ]; then
+      echo "✅ $DESCRIPTION completed successfully"
+    else
+      echo "❌ $DESCRIPTION failed"
+      FAILED_STEPS+=("$DESCRIPTION")
+    fi
+  else
+    echo "❌ Script not found: $SCRIPT"
+    FAILED_STEPS+=("$DESCRIPTION (script missing)")
+  fi
+done
+
+# Production verification
+echo
+echo "=========================================="
+echo "PRODUCTION VERIFICATION"
+echo "=========================================="
+
+# Service status
+echo "3. Service Status Verification:"
+AITBC1_SERVICES=$(systemctl is-active aitbc-blockchain-node aitbc-blockchain-rpc | grep -c "active")
+AITBC_SERVICES=$(ssh aitbc 'systemctl is-active aitbc-blockchain-node aitbc-blockchain-rpc 2>/dev/null | grep -c "active"')
+
+echo "   aitbc1 services: $AITBC1_SERVICES/2 active"
+echo "   aitbc services: $AITBC_SERVICES/2 active"
+
+# Blockchain status
+echo "4. Blockchain Status:"
+AITBC1_HEIGHT=$(curl -s http://localhost:8006/rpc/head | jq .height 2>/dev/null || echo "0")
+AITBC_HEIGHT=$(ssh aitbc 'curl -s http://localhost:8006/rpc/head | jq .height 2>/dev/null || echo "0")
+
+echo "   aitbc1 height: $AITBC1_HEIGHT"
+echo "   aitbc height: $AITBC_HEIGHT"
+echo "   Sync difference: $((AITBC1_HEIGHT - AITBC_HEIGHT)) blocks"
+
+# Network performance
+echo "5. Network Performance:"
+AITBC1_RPC_TIME=$(curl -w "%{time_total}" -s -o /dev/null http://localhost:8006/rpc/head)
+AITBC_RPC_TIME=$(ssh aitbc 'curl -w "%{time_total}" -s -o /dev/null http://localhost:8006/rpc/head')
+NETWORK_LATENCY=$(ping -c 1 10.1.223.93 | grep "time=" | cut -d= -f2 | cut -d" " -f1)
+
+echo "   aitbc1 RPC time: ${AITBC1_RPC_TIME}s"
+echo "   aitbc RPC time: ${AITBC_RPC_TIME}s"
+echo "   Network latency: ${NETWORK_LATENCY}ms"
+
+# Security check
+echo "6. Security Configuration:"
+if [ -f "/etc/aitbc/blockchain.env" ]; then
+  ENV_PERMISSIONS=$(stat -c "%a" /etc/aitbc/blockchain.env)
+  echo "   Environment file permissions: $ENV_PERMISSIONS"
+else
+  echo "   ❌ Environment file missing"
+fi
+
+KEYSTORE_PERMISSIONS=$(stat -c "%a" /var/lib/aitbc/keystore 2>/dev/null || echo "missing")
+echo "   Keystore permissions: $KEYSTORE_PERMISSIONS"
+
+# Production readiness assessment
+echo
+echo "=========================================="
+echo "PRODUCTION READINESS ASSESSMENT"
+echo "=========================================="
+
+SERVICES_OK=false
+SYNC_OK=false
+PERFORMANCE_OK=false
+SECURITY_OK=false
+
+# Check services
+if [ "$AITBC1_SERVICES" -eq 2 ] && [ "$AITBC_SERVICES" -eq 2 ]; then
+  SERVICES_OK=true
+  echo "✅ Services: All operational"
+else
+  echo "❌ Services: Some services not running"
+fi
+
+# Check sync
+HEIGHT_DIFF=$((AITBC1_HEIGHT - AITBC_HEIGHT))
+if [ $HEIGHT_DIFF -le 2 ]; then
+  SYNC_OK=true
+  echo "✅ Sync: Nodes synchronized"
+else
+  echo "❌ Sync: Nodes not synchronized (diff: $HEIGHT_DIFF blocks)"
+fi
+
+# Check performance
+if (( $(echo "$AITBC1_RPC_TIME < 1.0" | bc -l) )) && (( $(echo "$AITBC_RPC_TIME < 1.0" | bc -l) )); then
+  PERFORMANCE_OK=true
+  echo "✅ Performance: RPC times acceptable"
+else
+  echo "❌ Performance: RPC times too slow"
+fi
+
+# Check security
+if [ "$ENV_PERMISSIONS" = "644" ] && [ "$KEYSTORE_PERMISSIONS" = "700" ]; then
+  SECURITY_OK=true
+  echo "✅ Security: Permissions correct"
+else
+  echo "❌ Security: Check permissions"
+fi
+
+# Final assessment
+echo
+echo "7. Final Production Status:"
+if [ "$SERVICES_OK" = true ] && [ "$SYNC_OK" = true ] && [ "$PERFORMANCE_OK" = true ] && [ "$SECURITY_OK" = true ]; then
+  echo "🎉 PRODUCTION READY!"
+  echo "   All systems operational and optimized"
+  echo "   Ready for production deployment"
+else
+  echo "⚠️  NOT PRODUCTION READY"
+  echo "   Some issues need to be resolved"
+  
+  if [ ${#FAILED_STEPS[@]} -gt 0 ]; then
+    echo "   Failed steps: ${FAILED_STEPS[*]}"
+  fi
+fi
+
+echo
+echo "8. Next Steps:"
+echo "   • Monitor with: /opt/aitbc/scripts/health_check.sh"
+echo "   • Test with: /opt/aitbc/tests/integration_test.sh"
+echo "   • Scale with: /opt/aitbc/scripts/provision_node.sh <new-node>"
+echo "   • Enterprise features: /opt/aitbc/cli/enterprise_cli.py"
+
+echo "=== Production Ready Deployment Complete ==="