Fix chain isolation violations and add monitoring

- Fix supported_chains configuration: aitbc now only supports ait-mainnet, aitbc1 only supports ait-testnet
- Clean up contaminated databases: removed cross-chain blocks and accounts
- Add transaction chain_id validation with audit logging in state_transition.py
- Fix cross-chain bridge import errors in cross_chain_integration.py
- Add bridge request validation whitelist to prevent unauthorized cross-chain transfers
- Create chain isolation verification script for ongoing monitoring
- Add Prometheus metrics for chain isolation violations
- Create alert rules for chain isolation violations
- Create systemd service and timer for ongoing monitoring
- Deploy fixes to aitbc1 node

apps/blockchain-node/src/aitbc_chain/metrics/chain_isolation.py

@@ -0,0 +1,94 @@
+"""
+Chain Isolation Metrics for Prometheus
+Metrics for monitoring chain isolation violations and cross-chain transaction attempts
+"""
+
+from prometheus_client import Counter, Histogram
+from prometheus_client.registry import CollectorRegistry
+
+# Registry for chain isolation metrics
+registry = CollectorRegistry()
+
+# Counters for chain isolation violations
+chain_isolation_violations_total = Counter(
+    'chain_isolation_violations_total',
+    'Total number of chain isolation violations detected',
+    ['violation_type', 'node'],
+    registry=registry
+)
+
+cross_chain_transaction_attempts = Counter(
+    'cross_chain_transaction_attempts_total',
+    'Total number of cross-chain transaction attempts',
+    ['source_chain', 'target_chain', 'node'],
+    registry=registry
+)
+
+chain_id_validation_failures = Counter(
+    'chain_id_validation_failures_total',
+    'Total number of chain_id validation failures',
+    ['expected_chain', 'actual_chain', 'node'],
+    registry=registry
+)
+
+bridge_request_chain_mismatches = Counter(
+    'bridge_request_chain_mismatches_total',
+    'Total number of bridge requests with chain mismatches',
+    ['source_chain', 'target_chain', 'node'],
+    registry=registry
+)
+
+# Histogram for transaction validation time
+transaction_validation_duration = Histogram(
+    'transaction_validation_duration_seconds',
+    'Time spent validating transactions',
+    ['chain_id'],
+    registry=registry
+)
+
+
+def record_chain_isolation_violation(violation_type: str, node: str = 'unknown') -> None:
+    """Record a chain isolation violation"""
+    chain_isolation_violations_total.labels(
+        violation_type=violation_type,
+        node=node
+    ).inc()
+
+
+def record_cross_chain_transaction_attempt(
+    source_chain: str,
+    target_chain: str,
+    node: str = 'unknown'
+) -> None:
+    """Record a cross-chain transaction attempt"""
+    cross_chain_transaction_attempts.labels(
+        source_chain=source_chain,
+        target_chain=target_chain,
+        node=node
+    ).inc()
+
+
+def record_chain_id_validation_failure(
+    expected_chain: str,
+    actual_chain: str,
+    node: str = 'unknown'
+) -> None:
+    """Record a chain_id validation failure"""
+    chain_id_validation_failures.labels(
+        expected_chain=expected_chain,
+        actual_chain=actual_chain,
+        node=node
+    ).inc()
+
+
+def record_bridge_request_chain_mismatch(
+    source_chain: str,
+    target_chain: str,
+    node: str = 'unknown'
+) -> None:
+    """Record a bridge request chain mismatch"""
+    bridge_request_chain_mismatches.labels(
+        source_chain=source_chain,
+        target_chain=target_chain,
+        node=node
+    ).inc()

apps/blockchain-node/src/aitbc_chain/state/state_transition.py

@@ -42,18 +42,28 @@ class StateTransition:
     ) -> Tuple[bool, str]:
         """
         Validate a transaction before applying state changes.
-        
+
         Args:
             session: Database session
             chain_id: Chain identifier
             tx_data: Transaction data
             tx_hash: Transaction hash
-            
+
         Returns:
             Tuple of (is_valid, error_message)
         """
+        # Explicit chain_id validation - reject transactions with mismatched chain_id
+        tx_chain_id = tx_data.get("chain_id")
+        if tx_chain_id and tx_chain_id != chain_id:
+            logger.warning(
+                f"Chain isolation violation: Transaction {tx_hash} has chain_id={tx_chain_id} "
+                f"but node is configured for chain_id={chain_id}. Rejecting cross-chain transaction."
+            )
+            return False, f"Chain isolation violation: transaction chain_id={tx_chain_id} does not match node chain_id={chain_id}"
+
         # Check for replay attacks
         if tx_hash in self._processed_tx_hashes:
+            logger.warning(f"Replay attack detected: Transaction {tx_hash} already processed")
             return False, f"Transaction {tx_hash} already processed (replay attack)"
         
         # Get sender account

apps/coordinator-api/src/app/contexts/cross_chain/routers/cross_chain_integration.py

@@ -12,8 +12,8 @@ from sqlmodel import Session
 
 from aitbc.rate_limiting import rate_limit
 
-from ..agent_identity.manager import AgentIdentityManager
-from ..agent_identity.wallet_adapter_enhanced import (
+from ...agent_identity.manager import AgentIdentityManager
+from ...agent_identity.wallet_adapter_enhanced import (
     SecurityLevel,
     TransactionStatus,
     WalletAdapterFactory,
@@ -25,7 +25,7 @@ from ..services.cross_chain.bridge_enhanced import (
     BridgeSecurityLevel,
     CrossChainBridgeService,
 )
-from ....services.multi_chain_transaction_manager import (
+from ...services.multi_chain_transaction_manager import (
     MultiChainTransactionManager,
     RoutingStrategy,
     TransactionPriority,

apps/coordinator-api/src/app/contexts/cross_chain/services/cross_chain/bridge_enhanced.py

@@ -57,6 +57,9 @@ class CrossChainBridgeService:
         self.bridge_protocols: dict[str, Any] = {}
         self.liquidity_pools: dict[tuple[int, int], Any] = {}
         self.reputation_engine = CrossChainReputationEngine(session)
+        # Whitelist for allowed cross-chain transfers (source_chain_id -> target_chain_id)
+        # By default, all transfers are disallowed unless explicitly enabled
+        self.allowed_transfers: set[tuple[int, int]] = set()
 
     async def initialize_bridge(self, chain_configs: dict[int, dict[str, Any]]) -> None:
         """Initialize bridge service with chain configurations"""
@@ -113,6 +116,19 @@ class CrossChainBridgeService:
             if source_chain_id == target_chain_id:
                 raise ValueError("Source and target chains must be different")
 
+            # Validate chain transfer whitelist (chain isolation enforcement)
+            transfer_key = (source_chain_id, target_chain_id)
+            if transfer_key not in self.allowed_transfers:
+                logger.warning(
+                    f"Chain isolation violation: Bridge request from chain {source_chain_id} "
+                    f"to chain {target_chain_id} not in allowed_transfers whitelist. "
+                    f"Rejecting cross-chain transfer for address {user_address}"
+                )
+                raise ValueError(
+                    f"Cross-chain transfer from chain {source_chain_id} to {target_chain_id} "
+                    "is not permitted (chain isolation policy)"
+                )
+
             # Validate amount
             amount_float = float(amount)
             source_config = self.bridge_protocols[str(source_chain_id)]

scripts/monitoring/verify_chain_isolation.sh

@@ -0,0 +1,146 @@
+#!/bin/bash
+# Chain Isolation Verification Script
+# Checks for chain isolation violations across AITBC blockchain nodes
+
+set -e
+
+DATA_DIR="/var/lib/aitbc/data"
+LOG_FILE="/var/log/aitbc/chain-isolation-verification.log"
+VIOLATION_COUNT=0
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+log() {
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR] $1${NC}" | tee -a "$LOG_FILE"
+}
+
+log_success() {
+    echo -e "${GREEN}[OK] $1${NC}" | tee -a "$LOG_FILE"
+}
+
+log_warning() {
+    echo -e "${YELLOW}[WARN] $1${NC}" | tee -a "$LOG_FILE"
+}
+
+# Check database for cross-chain blocks
+check_database_isolation() {
+    local chain_db="$1"
+    local expected_chain="$2"
+    
+    log "Checking database: $chain_db (expected chain: $expected_chain)"
+    
+    if [ ! -f "$chain_db" ]; then
+        log_warning "Database file not found: $chain_db"
+        return 0
+    fi
+    
+    # Check for blocks from other chains
+    cross_chain_blocks=$(sqlite3 "$chain_db" "SELECT chain_id, COUNT(*) FROM block GROUP BY chain_id HAVING chain_id != '$expected_chain';" 2>/dev/null || echo "")
+    
+    if [ -n "$cross_chain_blocks" ]; then
+        log_error "Cross-chain blocks found in $chain_db:"
+        echo "$cross_chain_blocks" | while read -r line; do
+            log_error "  $line"
+        done
+        ((VIOLATION_COUNT++))
+    else
+        log_success "No cross-chain blocks in $chain_db"
+    fi
+    
+    # Check for accounts from other chains
+    cross_chain_accounts=$(sqlite3 "$chain_db" "SELECT chain_id, COUNT(*) FROM account GROUP BY chain_id HAVING chain_id != '$expected_chain';" 2>/dev/null || echo "")
+    
+    if [ -n "$cross_chain_accounts" ]; then
+        log_error "Cross-chain accounts found in $chain_db:"
+        echo "$cross_chain_accounts" | while read -r line; do
+            log_error "  $line"
+        done
+        ((VIOLATION_COUNT++))
+    else
+        log_success "No cross-chain accounts in $chain_db"
+    fi
+    
+    # Check for transactions from other chains
+    cross_chain_txs=$(sqlite3 "$chain_db" "SELECT chain_id, COUNT(*) FROM \"transaction\" GROUP BY chain_id HAVING chain_id != '$expected_chain';" 2>/dev/null || echo "")
+    
+    if [ -n "$cross_chain_txs" ]; then
+        log_error "Cross-chain transactions found in $chain_db:"
+        echo "$cross_chain_txs" | while read -r line; do
+            log_error "  $line"
+        done
+        ((VIOLATION_COUNT++))
+    else
+        log_success "No cross-chain transactions in $chain_db"
+    fi
+}
+
+# Check node configuration
+check_node_configuration() {
+    local node_name="$1"
+    local blockchain_env="$2"
+    local expected_chain="$3"
+    
+    log "Checking $node_name configuration (expected chain: $expected_chain)"
+    
+    if [ ! -f "$blockchain_env" ]; then
+        log_warning "Blockchain env file not found: $blockchain_env"
+        return 0
+    fi
+    
+    supported_chains=$(grep "^supported_chains=" "$blockchain_env" | cut -d'=' -f2)
+    
+    if [ "$supported_chains" != "$expected_chain" ]; then
+        log_error "$node_name supported_chains=$supported_chains (expected: $expected_chain)"
+        ((VIOLATION_COUNT++))
+    else
+        log_success "$node_name supported_chains=$supported_chains"
+    fi
+}
+
+# Main verification
+main() {
+    log "=== Chain Isolation Verification Started ==="
+    
+    # Check aitbc (mainnet) node
+    check_node_configuration "aitbc" "/etc/aitbc/blockchain.env" "ait-mainnet"
+    check_database_isolation "$DATA_DIR/ait-mainnet/chain.db" "ait-mainnet"
+    
+    # Check aitbc1 (testnet) node if accessible
+    if ssh aitbc1 test -f "/etc/aitbc/blockchain.env" 2>/dev/null; then
+        REMOTE_CHAINS=$(ssh aitbc1 'cat /etc/aitbc/blockchain.env | grep "^supported_chains=" | cut -d"=" -f2')
+        if [ "$REMOTE_CHAINS" != "ait-testnet" ]; then
+            log_error "aitbc1 supported_chains=$REMOTE_CHAINS (expected: ait-testnet)"
+            ((VIOLATION_COUNT++))
+        else
+            log_success "aitbc1 supported_chains=$REMOTE_CHAINS"
+        fi
+        check_database_isolation "$DATA_DIR/ait-testnet/chain.db" "ait-testnet"
+    else
+        log_warning "aitbc1 not accessible, skipping remote checks"
+    fi
+    
+    log "=== Chain Isolation Verification Completed ==="
+    log "Total violations found: $VIOLATION_COUNT"
+    
+    if [ $VIOLATION_COUNT -gt 0 ]; then
+        log_error "CHAIN ISOLATION VIOLATIONS DETECTED"
+        exit 1
+    else
+        log_success "No chain isolation violations detected"
+        exit 0
+    fi
+}
+
+# Create log directory if needed
+mkdir -p "$(dirname "$LOG_FILE")"
+
+# Run verification
+main