aitbc/scripts/blockchain/fix_sync_optimization.sh

#!/bin/bash

# Blockchain Synchronization Optimization Script
# Fixes common sync issues and optimizes cross-site synchronization

set -e

echo "🔧 Blockchain Synchronization Optimization"
echo "=========================================="

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Function to print colored output
print_status() {
    echo -e "${GREEN}✅ $1${NC}"
}

print_warning() {
    echo -e "${YELLOW}⚠️  $1${NC}"
}

print_error() {
    echo -e "${RED}❌ $1${NC}"
}

# Function to check if service is running
check_service() {
    local service=$1
    local host=$2
    
    if [ -z "$host" ]; then
        if systemctl is-active --quiet "$service"; then
            return 0
        else
            return 1
        fi
    else
        if ssh "$host" "systemctl is-active --quiet '$service'"; then
            return 0
        else
            return 1
        fi
    fi
}

# Function to restart service
restart_service() {
    local service=$1
    local host=$2
    
    if [ -z "$host" ]; then
        sudo systemctl restart "$service"
    else
        ssh "$host" "sudo systemctl restart '$service'"
    fi
}

# Function to get blockchain height
get_height() {
    local url=$1
    local host=$2
    
    if [ -z "$host" ]; then
        curl -s "$url/head" 2>/dev/null | grep -o '"height":[^,]*' | cut -d'"' -f2
    else
        ssh "$host" "curl -s '$url/head' 2>/dev/null | grep -o '\"height\":[^,]*' | cut -d'\"' -f2"
    fi
}

echo ""
echo "📊 Current Sync Status Analysis"
echo "=============================="

# Get current heights
echo "Checking current blockchain heights..."
NODE1_HEIGHT=$(get_height "http://localhost:8082/rpc" "aitbc-cascade")
NODE2_HEIGHT=$(get_height "http://localhost:8081/rpc" "aitbc-cascade")
NODE3_HEIGHT=$(get_height "http://192.168.100.10:8082/rpc" "ns3-root")

echo "Node 1 (aitbc-cascade): $NODE1_HEIGHT"
echo "Node 2 (aitbc-cascade): $NODE2_HEIGHT"
echo "Node 3 (ns3): $NODE3_HEIGHT"

# Calculate height differences
if [ -n "$NODE1_HEIGHT" ] && [ -n "$NODE2_HEIGHT" ]; then
    DIFF12=$((NODE2_HEIGHT - NODE1_HEIGHT))
    echo "Height difference (Node2 - Node1): $DIFF12"
fi

if [ -n "$NODE2_HEIGHT" ] && [ -n "$NODE3_HEIGHT" ]; then
    DIFF23=$((NODE2_HEIGHT - NODE3_HEIGHT))
    echo "Height difference (Node2 - Node3): $DIFF23"
fi

echo ""
echo "🔧 Step 1: Fix Node 1 Endpoint Configuration"
echo "============================================="

# Check Node 1 config for wrong endpoint
echo "Checking Node 1 configuration..."
NODE1_CONFIG=$(ssh aitbc-cascade "grep -n 'aitbc.bubuit.net/rpc2' /opt/blockchain-node/src/aitbc_chain/config.py 2>/dev/null || true")

if [ -n "$NODE1_CONFIG" ]; then
    print_warning "Found wrong endpoint /rpc2 in Node 1 config"
    echo "Fixing endpoint configuration..."
    
    # Backup original config
    ssh aitbc-cascade "sudo cp /opt/blockchain-node/src/aitbc_chain/config.py /opt/blockchain-node/src/aitbc_chain/config.py.backup"
    
    # Fix the endpoint
    ssh aitbc-cascade "sudo sed -i 's|https://aitbc.bubuit.net/rpc2|https://aitbc.bubuit.net/rpc|g' /opt/blockchain-node/src/aitbc_chain/config.py"
    
    print_status "Fixed Node 1 endpoint configuration"
    
    # Restart Node 1
    echo "Restarting Node 1 service..."
    restart_service "aitbc-blockchain-node-1.service" "aitbc-cascade"
    sleep 5
    
    if check_service "aitbc-blockchain-node-1.service" "aitbc-cascade"; then
        print_status "Node 1 service restarted successfully"
    else
        print_error "Node 1 service failed to restart"
    fi
else
    print_status "Node 1 endpoint configuration is correct"
fi

echo ""
echo "🔧 Step 2: Fix Node 3 Services"
echo "=============================="

# Check Node 3 service status
echo "Checking Node 3 services..."
NODE3_STATUS=$(ssh ns3-root "systemctl is-active blockchain-node-3.service 2>/dev/null || echo 'failed'")

if [ "$NODE3_STATUS" = "failed" ] || [ "$NODE3_STATUS" = "activating" ]; then
    print_warning "Node 3 service is in $NODE3_STATUS state"
    
    echo "Checking Node 3 service logs..."
    ssh ns3-root "journalctl -u blockchain-node-3.service --no-pager -n 10"
    
    echo "Attempting to fix Node 3 service..."
    
    # Stop and restart Node 3
    ssh ns3-root "sudo systemctl stop blockchain-node-3.service || true"
    sleep 2
    ssh ns3-root "sudo systemctl start blockchain-node-3.service"
    sleep 5
    
    # Check status again
    NODE3_NEW_STATUS=$(ssh ns3-root "systemctl is-active blockchain-node-3.service 2>/dev/null || echo 'failed'")
    
    if [ "$NODE3_NEW_STATUS" = "active" ]; then
        print_status "Node 3 service fixed and running"
    else
        print_error "Node 3 service still not working: $NODE3_NEW_STATUS"
        echo "Manual intervention required for Node 3"
    fi
else
    print_status "Node 3 service is running"
fi

echo ""
echo "🔧 Step 3: Optimize Sync Configuration"
echo "======================================"

# Function to optimize sync config
optimize_sync_config() {
    local host=$1
    local config_path=$2
    
    echo "Optimizing sync configuration on $host..."
    
    # Backup config
    ssh "$host" "sudo cp '$config_path' '$config_path.backup' 2>/dev/null || true"
    
    # Add/modify sync settings
    ssh "$host" "sudo tee -a '$config_path' > /dev/null << 'EOF'

# Sync optimization settings
sync_interval_seconds: int = 5  # Reduced from 10s
sync_retry_attempts: int = 3
sync_retry_delay_seconds: int = 2
sync_timeout_seconds: int = 10
max_sync_height_diff: int = 1000  # Alert if difference exceeds this
EOF"
    
    print_status "Sync configuration optimized on $host"
}

# Optimize sync configs
optimize_sync_config "aitbc-cascade" "/opt/blockchain-node/src/aitbc_chain/config.py"
optimize_sync_config "aitbc-cascade" "/opt/blockchain-node-2/src/aitbc_chain/config.py"
optimize_sync_config "ns3-root" "/opt/blockchain-node/src/aitbc_chain/config.py"

echo ""
echo "🔧 Step 4: Restart Services with New Config"
echo "=========================================="

# Restart all services
echo "Restarting blockchain services..."

for service in "aitbc-blockchain-node-1.service" "aitbc-blockchain-node-2.service"; do
    echo "Restarting $service on aitbc-cascade..."
    restart_service "$service" "aitbc-cascade"
    sleep 3
done

for service in "blockchain-node-3.service"; do
    echo "Restarting $service on ns3..."
    restart_service "$service" "ns3-root"
    sleep 3
done

echo ""
echo "📊 Step 5: Verify Sync Optimization"
echo "==================================="

# Wait for services to stabilize
echo "Waiting for services to stabilize..."
sleep 10

# Check new heights
echo "Checking new blockchain heights..."
NEW_NODE1_HEIGHT=$(get_height "http://localhost:8082/rpc" "aitbc-cascade")
NEW_NODE2_HEIGHT=$(get_height "http://localhost:8081/rpc" "aitbc-cascade")
NEW_NODE3_HEIGHT=$(get_height "http://192.168.100.10:8082/rpc" "ns3-root")

echo "New heights:"
echo "Node 1: $NEW_NODE1_HEIGHT"
echo "Node 2: $NEW_NODE2_HEIGHT"
echo "Node 3: $NEW_NODE3_HEIGHT"

# Calculate improvements
if [ -n "$NEW_NODE1_HEIGHT" ] && [ -n "$NEW_NODE2_HEIGHT" ] && [ -n "$NODE1_HEIGHT" ] && [ -n "$NODE2_HEIGHT" ]; then
    OLD_DIFF=$((NODE2_HEIGHT - NODE1_HEIGHT))
    NEW_DIFF=$((NEW_NODE2_HEIGHT - NEW_NODE1_HEIGHT))
    
    echo "Height difference improvement:"
    echo "Before: $OLD_DIFF"
    echo "After:  $NEW_DIFF"
    
    if [ $NEW_DIFF -lt $OLD_DIFF ]; then
        IMPROVEMENT=$((OLD_DIFF - NEW_DIFF))
        print_status "Sync improved by $IMPROVEMENT blocks"
    else
        print_warning "Sync did not improve or got worse"
    fi
fi

echo ""
echo "🔧 Step 6: Create Sync Monitoring Script"
echo "========================================="

# Create monitoring script
cat > /tmp/sync_monitor.sh << 'EOF'
#!/bin/bash

# Blockchain Sync Monitor
# Run this periodically to check sync health

echo "🔍 Blockchain Sync Monitor - $(date)"
echo "===================================="

# Get heights
NODE1=$(curl -s http://localhost:8082/rpc/head 2>/dev/null | grep -o '"height":[^,]*' | cut -d'"' -f2)
NODE2=$(curl -s http://localhost:8081/rpc/head 2>/dev/null | grep -o '"height":[^,]*' | cut -d'"' -f2)
NODE3=$(ssh ns3-root "curl -s http://192.168.100.10:8082/rpc/head 2>/dev/null | grep -o '\"height\":[^,]*' | cut -d'\"' -f2")

echo "Node 1: $NODE1"
echo "Node 2: $NODE2"
echo "Node 3: $NODE3"

# Check for issues
if [ -n "$NODE1" ] && [ -n "$NODE2" ]; then
    DIFF=$((NODE2 - NODE1))
    if [ $DIFF -gt 100 ]; then
        echo "⚠️  WARNING: Node 1 and Node 2 height difference: $DIFF"
    fi
fi

if [ -n "$NODE2" ] && [ -n "$NODE3" ]; then
    DIFF=$((NODE2 - NODE3))
    if [ $DIFF -gt 1000 ]; then
        echo "⚠️  WARNING: Node 2 and Node 3 height difference: $DIFF"
    fi
fi

echo "Sync check completed."
EOF

chmod +x /tmp/sync_monitor.sh
print_status "Created sync monitoring script: /tmp/sync_monitor.sh"

echo ""
echo "🎉 Sync Optimization Complete!"
echo "=============================="

echo ""
echo "📋 Summary of actions taken:"
echo "• Fixed Node 1 endpoint configuration"
echo "• Restarted problematic services"
echo "• Optimized sync intervals and retry logic"
echo "• Created monitoring script"
echo ""
echo "📊 Next steps:"
echo "1. Monitor sync performance with: /tmp/sync_monitor.sh"
echo "2. Set up cron job for periodic monitoring"
echo "3. Check logs for any remaining issues"
echo "4. Consider implementing P2P sync for better performance"
echo ""
echo "🔧 If issues persist:"
echo "• Check individual service logs: journalctl -u [service-name]"
echo "• Verify network connectivity between sites"
echo "• Consider manual block import for severely lagging nodes"
echo "• Review firewall and security group settings"

print_status "Blockchain synchronization optimization completed!"