Fix chain isolation violations and add monitoring

- Fix supported_chains configuration: aitbc now only supports ait-mainnet, aitbc1 only supports ait-testnet
- Clean up contaminated databases: removed cross-chain blocks and accounts
- Add transaction chain_id validation with audit logging in state_transition.py
- Fix cross-chain bridge import errors in cross_chain_integration.py
- Add bridge request validation whitelist to prevent unauthorized cross-chain transfers
- Create chain isolation verification script for ongoing monitoring
- Add Prometheus metrics for chain isolation violations
- Create alert rules for chain isolation violations
- Create systemd service and timer for ongoing monitoring
- Deploy fixes to aitbc1 node

scripts/monitoring/verify_chain_isolation.sh

@@ -0,0 +1,146 @@
+#!/bin/bash
+# Chain Isolation Verification Script
+# Checks for chain isolation violations across AITBC blockchain nodes
+
+set -e
+
+DATA_DIR="/var/lib/aitbc/data"
+LOG_FILE="/var/log/aitbc/chain-isolation-verification.log"
+VIOLATION_COUNT=0
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+log() {
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR] $1${NC}" | tee -a "$LOG_FILE"
+}
+
+log_success() {
+    echo -e "${GREEN}[OK] $1${NC}" | tee -a "$LOG_FILE"
+}
+
+log_warning() {
+    echo -e "${YELLOW}[WARN] $1${NC}" | tee -a "$LOG_FILE"
+}
+
+# Check database for cross-chain blocks
+check_database_isolation() {
+    local chain_db="$1"
+    local expected_chain="$2"
+    
+    log "Checking database: $chain_db (expected chain: $expected_chain)"
+    
+    if [ ! -f "$chain_db" ]; then
+        log_warning "Database file not found: $chain_db"
+        return 0
+    fi
+    
+    # Check for blocks from other chains
+    cross_chain_blocks=$(sqlite3 "$chain_db" "SELECT chain_id, COUNT(*) FROM block GROUP BY chain_id HAVING chain_id != '$expected_chain';" 2>/dev/null || echo "")
+    
+    if [ -n "$cross_chain_blocks" ]; then
+        log_error "Cross-chain blocks found in $chain_db:"
+        echo "$cross_chain_blocks" | while read -r line; do
+            log_error "  $line"
+        done
+        ((VIOLATION_COUNT++))
+    else
+        log_success "No cross-chain blocks in $chain_db"
+    fi
+    
+    # Check for accounts from other chains
+    cross_chain_accounts=$(sqlite3 "$chain_db" "SELECT chain_id, COUNT(*) FROM account GROUP BY chain_id HAVING chain_id != '$expected_chain';" 2>/dev/null || echo "")
+    
+    if [ -n "$cross_chain_accounts" ]; then
+        log_error "Cross-chain accounts found in $chain_db:"
+        echo "$cross_chain_accounts" | while read -r line; do
+            log_error "  $line"
+        done
+        ((VIOLATION_COUNT++))
+    else
+        log_success "No cross-chain accounts in $chain_db"
+    fi
+    
+    # Check for transactions from other chains
+    cross_chain_txs=$(sqlite3 "$chain_db" "SELECT chain_id, COUNT(*) FROM \"transaction\" GROUP BY chain_id HAVING chain_id != '$expected_chain';" 2>/dev/null || echo "")
+    
+    if [ -n "$cross_chain_txs" ]; then
+        log_error "Cross-chain transactions found in $chain_db:"
+        echo "$cross_chain_txs" | while read -r line; do
+            log_error "  $line"
+        done
+        ((VIOLATION_COUNT++))
+    else
+        log_success "No cross-chain transactions in $chain_db"
+    fi
+}
+
+# Check node configuration
+check_node_configuration() {
+    local node_name="$1"
+    local blockchain_env="$2"
+    local expected_chain="$3"
+    
+    log "Checking $node_name configuration (expected chain: $expected_chain)"
+    
+    if [ ! -f "$blockchain_env" ]; then
+        log_warning "Blockchain env file not found: $blockchain_env"
+        return 0
+    fi
+    
+    supported_chains=$(grep "^supported_chains=" "$blockchain_env" | cut -d'=' -f2)
+    
+    if [ "$supported_chains" != "$expected_chain" ]; then
+        log_error "$node_name supported_chains=$supported_chains (expected: $expected_chain)"
+        ((VIOLATION_COUNT++))
+    else
+        log_success "$node_name supported_chains=$supported_chains"
+    fi
+}
+
+# Main verification
+main() {
+    log "=== Chain Isolation Verification Started ==="
+    
+    # Check aitbc (mainnet) node
+    check_node_configuration "aitbc" "/etc/aitbc/blockchain.env" "ait-mainnet"
+    check_database_isolation "$DATA_DIR/ait-mainnet/chain.db" "ait-mainnet"
+    
+    # Check aitbc1 (testnet) node if accessible
+    if ssh aitbc1 test -f "/etc/aitbc/blockchain.env" 2>/dev/null; then
+        REMOTE_CHAINS=$(ssh aitbc1 'cat /etc/aitbc/blockchain.env | grep "^supported_chains=" | cut -d"=" -f2')
+        if [ "$REMOTE_CHAINS" != "ait-testnet" ]; then
+            log_error "aitbc1 supported_chains=$REMOTE_CHAINS (expected: ait-testnet)"
+            ((VIOLATION_COUNT++))
+        else
+            log_success "aitbc1 supported_chains=$REMOTE_CHAINS"
+        fi
+        check_database_isolation "$DATA_DIR/ait-testnet/chain.db" "ait-testnet"
+    else
+        log_warning "aitbc1 not accessible, skipping remote checks"
+    fi
+    
+    log "=== Chain Isolation Verification Completed ==="
+    log "Total violations found: $VIOLATION_COUNT"
+    
+    if [ $VIOLATION_COUNT -gt 0 ]; then
+        log_error "CHAIN ISOLATION VIOLATIONS DETECTED"
+        exit 1
+    else
+        log_success "No chain isolation violations detected"
+        exit 0
+    fi
+}
+
+# Create log directory if needed
+mkdir -p "$(dirname "$LOG_FILE")"
+
+# Run verification
+main