feat: add multi-node blockchain monitoring workflows for 3-node network
- Create multi-node blockchain health monitoring workflow - Create P2P network verification workflow for all 3 nodes - Create blockchain synchronization verification workflow - Update blockchain-communication-test.sh to include aitbc2 (gitea-runner) - Add shared scripts directory with health check, P2P verification, and sync verification scripts - All workflows trigger on git push to main/develop branches - Workflows run on gitea-runner (has SSH access to all nodes) - Include automatic remediation for failed services and sync issues - Sync threshold set to 10 blocks - Logging to /var/log/aitbc/ and alerts in Gitea UI
This commit is contained in:
aitbc
288
scripts/multi-node/blockchain-health-check.sh
Executable file
288
scripts/multi-node/blockchain-health-check.sh
Executable file
@@ -0,0 +1,288 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Multi-Node Blockchain Health Check Script
|
||||
# Checks health of all 3 blockchain nodes (aitbc, aitbc1, aitbc2)
|
||||
# Provides automatic remediation for failed services
|
||||
#
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||
LOG_DIR="/var/log/aitbc"
|
||||
LOG_FILE="${LOG_DIR}/multi-node-health.log"
|
||||
|
||||
# Node Configuration
|
||||
NODES=(
|
||||
"aitbc:10.1.223.93"
|
||||
"aitbc1:10.1.223.40"
|
||||
"aitbc2:10.1.223.98"
|
||||
)
|
||||
|
||||
RPC_PORT=8006
|
||||
REDIS_HOST="10.1.223.93"
|
||||
REDIS_PORT=6379
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
# Logging functions
|
||||
log() {
|
||||
local level="$1"
|
||||
shift
|
||||
local message="$@"
|
||||
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
echo "[${timestamp}] [${level}] ${message}" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
log "SUCCESS" "$@"
|
||||
echo -e "${GREEN}$@${NC}"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
log "ERROR" "$@"
|
||||
echo -e "${RED}$@${NC}"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
log "WARNING" "$@"
|
||||
echo -e "${YELLOW}$@${NC}"
|
||||
}
|
||||
|
||||
# SSH execution helper
|
||||
ssh_exec() {
|
||||
local node="$1"
|
||||
local command="$2"
|
||||
ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$node" "$command" 2>&1 || return 1
|
||||
}
|
||||
|
||||
# Check RPC endpoint health
|
||||
check_rpc_health() {
|
||||
local node_name="$1"
|
||||
local node_ip="$2"
|
||||
|
||||
log "Checking RPC health for ${node_name} (${node_ip}:${RPC_PORT})"
|
||||
|
||||
if curl -f -s --max-time 5 "http://${node_ip}:${RPC_PORT}/health" > /dev/null 2>&1; then
|
||||
log_success "RPC endpoint healthy on ${node_name}"
|
||||
return 0
|
||||
else
|
||||
log_error "RPC endpoint unhealthy on ${node_name}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check systemd service status
|
||||
check_service_status() {
|
||||
local node="$1"
|
||||
local service="$2"
|
||||
|
||||
log "Checking ${service} status on ${node}"
|
||||
|
||||
status=$(ssh_exec "$node" "systemctl is-active ${service}" 2>&1 || echo "inactive")
|
||||
|
||||
if [ "$status" = "active" ]; then
|
||||
log_success "${service} is active on ${node}"
|
||||
return 0
|
||||
else
|
||||
log_error "${service} is ${status} on ${node}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check resource usage
|
||||
check_resource_usage() {
|
||||
local node="$1"
|
||||
|
||||
log "Checking resource usage on ${node}"
|
||||
|
||||
memory=$(ssh_exec "$node" "free | grep Mem | awk '{printf \"%.1f\", (\$3/\$2)*100}'" 2>&1 || echo "0")
|
||||
cpu=$(ssh_exec "$node" "top -bn1 | grep 'Cpu(s)' | awk '{print \$2}' | cut -d'%' -f1" 2>&1 || echo "0")
|
||||
disk=$(ssh_exec "$node" "df /var/lib/aitbc | tail -1 | awk '{print \$5}' | cut -d'%' -f1" 2>&1 || echo "0")
|
||||
|
||||
log "Resource usage on ${node}: CPU ${cpu}%, Memory ${memory}%, Disk ${disk}%"
|
||||
|
||||
# Check thresholds
|
||||
if [ "${disk%.*}" -gt 90 ]; then
|
||||
log_warning "Disk usage critical on ${node}: ${disk}%"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ "${memory%.*}" -gt 90 ]; then
|
||||
log_warning "Memory usage critical on ${node}: ${memory}%"
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# Check Redis connectivity
|
||||
check_redis_connectivity() {
|
||||
log "Checking Redis connectivity (${REDIS_HOST}:${REDIS_PORT})"
|
||||
|
||||
if redis-cli -h "${REDIS_HOST}" -p "${REDIS_PORT}" ping > /dev/null 2>&1; then
|
||||
log_success "Redis connectivity OK"
|
||||
return 0
|
||||
else
|
||||
log_error "Redis connectivity failed"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Remediation functions
|
||||
restart_rpc_service() {
|
||||
local node="$1"
|
||||
log "Attempting to restart aitbc-blockchain-rpc on ${node}"
|
||||
|
||||
ssh_exec "$node" "systemctl restart aitbc-blockchain-rpc" 2>&1 | tee -a "${LOG_FILE}"
|
||||
sleep 5
|
||||
|
||||
if ssh_exec "$node" "systemctl is-active aitbc-blockchain-rpc" 2>&1 | grep -q "active"; then
|
||||
log_success "Successfully restarted aitbc-blockchain-rpc on ${node}"
|
||||
return 0
|
||||
else
|
||||
log_error "Failed to restart aitbc-blockchain-rpc on ${node}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
restart_p2p_service() {
|
||||
local node="$1"
|
||||
log "Attempting to restart aitbc-blockchain-p2p on ${node}"
|
||||
|
||||
ssh_exec "$node" "systemctl restart aitbc-blockchain-p2p" 2>&1 | tee -a "${LOG_FILE}"
|
||||
sleep 5
|
||||
|
||||
if ssh_exec "$node" "systemctl is-active aitbc-blockchain-p2p" 2>&1 | grep -q "active"; then
|
||||
log_success "Successfully restarted aitbc-blockchain-p2p on ${node}"
|
||||
return 0
|
||||
else
|
||||
log_error "Failed to restart aitbc-blockchain-p2p on ${node}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
restart_node_service() {
|
||||
local node="$1"
|
||||
log "Attempting to restart aitbc-blockchain-node on ${node}"
|
||||
|
||||
ssh_exec "$node" "systemctl restart aitbc-blockchain-node" 2>&1 | tee -a "${LOG_FILE}"
|
||||
sleep 10
|
||||
|
||||
if ssh_exec "$node" "systemctl is-active aitbc-blockchain-node" 2>&1 | grep -q "active"; then
|
||||
log_success "Successfully restarted aitbc-blockchain-node on ${node}"
|
||||
return 0
|
||||
else
|
||||
log_error "Failed to restart aitbc-blockchain-node on ${node}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Main health check for a node
|
||||
check_node_health() {
|
||||
local node_name="$1"
|
||||
local node_ip="$2"
|
||||
local node="${node_name}"
|
||||
|
||||
local failures=0
|
||||
|
||||
# Check RPC health
|
||||
if ! check_rpc_health "$node_name" "$node_ip"; then
|
||||
((failures++))
|
||||
log "Attempting remediation for RPC on ${node_name}"
|
||||
if restart_rpc_service "$node"; then
|
||||
# Retry RPC check
|
||||
if ! check_rpc_health "$node_name" "$node_ip"; then
|
||||
log_error "RPC remediation failed on ${node_name}"
|
||||
else
|
||||
log_success "RPC remediation successful on ${node_name}"
|
||||
((failures--))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check blockchain node service
|
||||
if ! check_service_status "$node" "aitbc-blockchain-node"; then
|
||||
((failures++))
|
||||
log "Attempting remediation for blockchain node on ${node_name}"
|
||||
if restart_node_service "$node"; then
|
||||
# Retry service check
|
||||
if check_service_status "$node" "aitbc-blockchain-node"; then
|
||||
log_success "Blockchain node remediation successful on ${node_name}"
|
||||
((failures--))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check P2P service
|
||||
if ! check_service_status "$node" "aitbc-blockchain-p2p"; then
|
||||
((failures++))
|
||||
log "Attempting remediation for P2P on ${node_name}"
|
||||
if restart_p2p_service "$node"; then
|
||||
# Retry service check
|
||||
if check_service_status "$node" "aitbc-blockchain-p2p"; then
|
||||
log_success "P2P remediation successful on ${node_name}"
|
||||
((failures--))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check resource usage
|
||||
if ! check_resource_usage "$node"; then
|
||||
((failures++))
|
||||
log_warning "Resource usage issues on ${node_name}"
|
||||
fi
|
||||
|
||||
return $failures
|
||||
}
|
||||
|
||||
# Main execution
|
||||
main() {
|
||||
log "=== Multi-Node Blockchain Health Check Started ==="
|
||||
|
||||
# Create log directory if it doesn't exist
|
||||
mkdir -p "${LOG_DIR}"
|
||||
|
||||
local total_failures=0
|
||||
|
||||
# Check Redis connectivity (shared resource)
|
||||
if ! check_redis_connectivity; then
|
||||
log_error "Redis connectivity failed - this affects all nodes"
|
||||
((total_failures++))
|
||||
fi
|
||||
|
||||
# Check each node
|
||||
for node_config in "${NODES[@]}"; do
|
||||
IFS=':' read -r node_name node_ip <<< "$node_config"
|
||||
|
||||
log "=== Checking node: ${node_name} (${node_ip}) ==="
|
||||
|
||||
if check_node_health "$node_name" "$node_ip"; then
|
||||
log_success "Node ${node_name} is healthy"
|
||||
else
|
||||
failures=$?
|
||||
log_error "Node ${node_name} has ${failures} health issues"
|
||||
((total_failures+=failures))
|
||||
fi
|
||||
|
||||
echo "" | tee -a "${LOG_FILE}"
|
||||
done
|
||||
|
||||
log "=== Multi-Node Blockchain Health Check Completed ==="
|
||||
log "Total failures: ${total_failures}"
|
||||
|
||||
if [ ${total_failures} -eq 0 ]; then
|
||||
log_success "All nodes are healthy"
|
||||
exit 0
|
||||
else
|
||||
log_error "Health check completed with ${total_failures} failures"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
287
scripts/multi-node/p2p-verification.sh
Executable file
287
scripts/multi-node/p2p-verification.sh
Executable file
@@ -0,0 +1,287 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# P2P Network Verification Script
|
||||
# Verifies P2P network connectivity across all 3 blockchain nodes
|
||||
#
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||
LOG_DIR="/var/log/aitbc"
|
||||
LOG_FILE="${LOG_DIR}/p2p-verification.log"
|
||||
|
||||
# Node Configuration
|
||||
NODES=(
|
||||
"aitbc:10.1.223.93"
|
||||
"aitbc1:10.1.223.40"
|
||||
"aitbc2:10.1.223.98"
|
||||
)
|
||||
|
||||
P2P_PORT=7070
|
||||
REDIS_HOST="10.1.223.93"
|
||||
REDIS_PORT=6379
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
# Logging functions
|
||||
log() {
|
||||
local level="$1"
|
||||
shift
|
||||
local message="$@"
|
||||
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
echo "[${timestamp}] [${level}] ${message}" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
log "SUCCESS" "$@"
|
||||
echo -e "${GREEN}$@${NC}"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
log "ERROR" "$@"
|
||||
echo -e "${RED}$@${NC}"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
log "WARNING" "$@"
|
||||
echo -e "${YELLOW}$@${NC}"
|
||||
}
|
||||
|
||||
# SSH execution helper
|
||||
ssh_exec() {
|
||||
local node="$1"
|
||||
local command="$2"
|
||||
ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$node" "$command" 2>&1 || return 1
|
||||
}
|
||||
|
||||
# Check P2P peer list on a node
|
||||
check_p2p_peers() {
|
||||
local node="$1"
|
||||
local node_name="$2"
|
||||
|
||||
log "Checking P2P peers on ${node_name}"
|
||||
|
||||
# Read node.env to get expected peers
|
||||
peers=$(ssh_exec "$node" "grep '^p2p_peers=' /etc/aitbc/node.env | cut -d'=' -f2" 2>&1 || echo "")
|
||||
|
||||
if [ -z "$peers" ]; then
|
||||
log_error "No p2p_peers configured on ${node_name}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Expected peers on ${node_name}: ${peers}"
|
||||
|
||||
# Check P2P service status
|
||||
if ! ssh_exec "$node" "systemctl is-active aitbc-blockchain-p2p" | grep -q "active"; then
|
||||
log_error "P2P service not active on ${node_name}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_success "P2P peers configured on ${node_name}"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Check P2P connectivity between nodes
|
||||
check_p2p_connectivity() {
|
||||
local source_node="$1"
|
||||
local source_name="$2"
|
||||
local target_node="$3"
|
||||
local target_name="$4"
|
||||
|
||||
log "Checking P2P connectivity from ${source_name} to ${target_name}"
|
||||
|
||||
# Try to connect to target P2P port
|
||||
if ssh_exec "$source_node" "timeout 5 bash -c '</dev/tcp/${target_node#*:}/${P2P_PORT}'" 2>&1; then
|
||||
log_success "P2P connectivity OK from ${source_name} to ${target_name}"
|
||||
return 0
|
||||
else
|
||||
log_error "P2P connectivity FAILED from ${source_name} to ${target_name}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check Redis gossip backend connectivity
|
||||
check_gossip_backend() {
|
||||
log "Checking Redis gossip backend connectivity (${REDIS_HOST}:${REDIS_PORT})"
|
||||
|
||||
if redis-cli -h "${REDIS_HOST}" -p "${REDIS_PORT}" ping > /dev/null 2>&1; then
|
||||
log_success "Redis gossip backend connectivity OK"
|
||||
return 0
|
||||
else
|
||||
log_error "Redis gossip backend connectivity failed"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check for P2P handshake errors in logs
|
||||
check_p2p_logs() {
|
||||
local node="$1"
|
||||
local node_name="$2"
|
||||
|
||||
log "Checking P2P logs for errors on ${node_name}"
|
||||
|
||||
# Check for handshake errors
|
||||
errors=$(ssh_exec "$node" "journalctl -u aitbc-blockchain-p2p --since '1 hour ago' | grep -i 'handshake\|error\|failed' | tail -5" 2>&1 || echo "")
|
||||
|
||||
if [ -n "$errors" ]; then
|
||||
log_warning "P2P errors found on ${node_name}:"
|
||||
echo "$errors" | tee -a "${LOG_FILE}"
|
||||
return 1
|
||||
else
|
||||
log_success "No P2P errors found on ${node_name}"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# Remediation: Restart P2P service
|
||||
remediate_p2p_service() {
|
||||
local node="$1"
|
||||
local node_name="$2"
|
||||
|
||||
log "Attempting P2P remediation on ${node_name}"
|
||||
|
||||
ssh_exec "$node" "systemctl restart aitbc-blockchain-p2p" 2>&1 | tee -a "${LOG_FILE}"
|
||||
sleep 5
|
||||
|
||||
if ssh_exec "$node" "systemctl is-active aitbc-blockchain-p2p" | grep -q "active"; then
|
||||
log_success "P2P service remediation successful on ${node_name}"
|
||||
return 0
|
||||
else
|
||||
log_error "P2P service remediation failed on ${node_name}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Update p2p_peers configuration if needed
|
||||
update_p2p_peers() {
|
||||
local node="$1"
|
||||
local node_name="$2"
|
||||
|
||||
log "Updating p2p_peers configuration on ${node_name}"
|
||||
|
||||
# Determine correct peers based on node name
|
||||
case "$node_name" in
|
||||
"aitbc")
|
||||
peers="aitbc1:7070,aitbc2:7070"
|
||||
;;
|
||||
"aitbc1")
|
||||
peers="aitbc:7070,aitbc2:7070"
|
||||
;;
|
||||
"aitbc2")
|
||||
peers="aitbc:7070,aitbc1:7070"
|
||||
;;
|
||||
*)
|
||||
log_error "Unknown node name: ${node_name}"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# Update node.env
|
||||
ssh_exec "$node" "sed -i 's/^p2p_peers=.*/p2p_peers=${peers}/' /etc/aitbc/node.env" 2>&1 | tee -a "${LOG_FILE}"
|
||||
|
||||
# Restart P2P service to apply changes
|
||||
ssh_exec "$node" "systemctl restart aitbc-blockchain-p2p" 2>&1 | tee -a "${LOG_FILE}"
|
||||
sleep 5
|
||||
|
||||
log_success "Updated p2p_peers on ${node_name} to: ${peers}"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Main verification for a node
|
||||
verify_node_p2p() {
|
||||
local node_name="$1"
|
||||
local node_ip="$2"
|
||||
local node="${node_name}"
|
||||
|
||||
local failures=0
|
||||
|
||||
# Check P2P peers configuration
|
||||
if ! check_p2p_peers "$node" "$node_name"; then
|
||||
((failures++))
|
||||
log "Attempting remediation for P2P peers on ${node_name}"
|
||||
update_p2p_peers "$node" "$node_name" || true
|
||||
fi
|
||||
|
||||
# Check P2P logs for errors
|
||||
if ! check_p2p_logs "$node" "$node_name"; then
|
||||
((failures++))
|
||||
log "Attempting remediation for P2P errors on ${node_name}"
|
||||
remediate_p2p_service "$node" "$node_name" || true
|
||||
fi
|
||||
|
||||
return $failures
|
||||
}
|
||||
|
||||
# Main execution
|
||||
main() {
|
||||
log "=== P2P Network Verification Started ==="
|
||||
|
||||
# Create log directory if it doesn't exist
|
||||
mkdir -p "${LOG_DIR}"
|
||||
|
||||
local total_failures=0
|
||||
|
||||
# Check Redis gossip backend
|
||||
if ! check_gossip_backend; then
|
||||
log_error "Gossip backend connectivity failed"
|
||||
((total_failures++))
|
||||
fi
|
||||
|
||||
# Check each node's P2P configuration
|
||||
for node_config in "${NODES[@]}"; do
|
||||
IFS=':' read -r node_name node_ip <<< "$node_config"
|
||||
|
||||
log "=== Verifying P2P on node: ${node_name} (${node_ip}) ==="
|
||||
|
||||
if verify_node_p2p "$node_name" "$node_ip"; then
|
||||
log_success "P2P verification passed for ${node_name}"
|
||||
else
|
||||
failures=$?
|
||||
log_error "P2P verification failed for ${node_name} with ${failures} issues"
|
||||
((total_failures+=failures))
|
||||
fi
|
||||
|
||||
echo "" | tee -a "${LOG_FILE}"
|
||||
done
|
||||
|
||||
# Check P2P connectivity between all node pairs
|
||||
log "=== Checking P2P connectivity between node pairs ==="
|
||||
|
||||
for source_config in "${NODES[@]}"; do
|
||||
IFS=':' read -r source_name source_ip <<< "$source_config"
|
||||
|
||||
for target_config in "${NODES[@]}"; do
|
||||
IFS=':' read -r target_name target_ip <<< "$target_config"
|
||||
|
||||
# Skip self-connectivity check
|
||||
if [ "$source_name" = "$target_name" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if ! check_p2p_connectivity "$source_name" "$source_name" "$target_ip" "$target_name"; then
|
||||
((total_failures++))
|
||||
log "Attempting remediation for P2P connectivity"
|
||||
remediate_p2p_service "$source_name" "$source_name" || true
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
log "=== P2P Network Verification Completed ==="
|
||||
log "Total failures: ${total_failures}"
|
||||
|
||||
if [ ${total_failures} -eq 0 ]; then
|
||||
log_success "P2P network verification passed"
|
||||
exit 0
|
||||
else
|
||||
log_error "P2P network verification failed with ${total_failures} failures"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
348
scripts/multi-node/sync-verification.sh
Executable file
348
scripts/multi-node/sync-verification.sh
Executable file
@@ -0,0 +1,348 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Blockchain Synchronization Verification Script
|
||||
# Verifies blockchain synchronization across all 3 nodes
|
||||
# Provides automatic remediation by forcing sync from healthy node
|
||||
#
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||
LOG_DIR="/var/log/aitbc"
|
||||
LOG_FILE="${LOG_DIR}/sync-verification.log"
|
||||
|
||||
# Node Configuration
|
||||
NODES=(
|
||||
"aitbc:10.1.223.93"
|
||||
"aitbc1:10.1.223.40"
|
||||
"aitbc2:10.1.223.98"
|
||||
)
|
||||
|
||||
RPC_PORT=8006
|
||||
SYNC_THRESHOLD=10
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
# Logging functions
|
||||
log() {
|
||||
local level="$1"
|
||||
shift
|
||||
local message="$@"
|
||||
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
echo "[${timestamp}] [${level}] ${message}" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
log "SUCCESS" "$@"
|
||||
echo -e "${GREEN}$@${NC}"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
log "ERROR" "$@"
|
||||
echo -e "${RED}$@${NC}"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
log "WARNING" "$@"
|
||||
echo -e "${YELLOW}$@${NC}"
|
||||
}
|
||||
|
||||
# SSH execution helper
|
||||
ssh_exec() {
|
||||
local node="$1"
|
||||
local command="$2"
|
||||
ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$node" "$command" 2>&1 || return 1
|
||||
}
|
||||
|
||||
# Get block height from RPC endpoint
|
||||
get_block_height() {
|
||||
local node_ip="$1"
|
||||
|
||||
# Try to get block height from RPC
|
||||
height=$(curl -s --max-time 5 "http://${node_ip}:${RPC_PORT}/blockchain/height" 2>/dev/null | grep -o '[0-9]*' || echo "0")
|
||||
|
||||
if [ -z "$height" ] || [ "$height" = "0" ]; then
|
||||
# Try alternative endpoint
|
||||
height=$(curl -s --max-time 5 "http://${node_ip}:${RPC_PORT}/height" 2>/dev/null | grep -o '[0-9]*' || echo "0")
|
||||
fi
|
||||
|
||||
echo "$height"
|
||||
}
|
||||
|
||||
# Get chain ID from RPC endpoint
|
||||
get_chain_id() {
|
||||
local node_ip="$1"
|
||||
|
||||
chain_id=$(curl -s --max-time 5 "http://${node_ip}:${RPC_PORT}/blockchain/chain-id" 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$chain_id" ]; then
|
||||
chain_id=$(curl -s --max-time 5 "http://${node_ip}:${RPC_PORT}/chain-id" 2>/dev/null || echo "")
|
||||
fi
|
||||
|
||||
echo "$chain_id"
|
||||
}
|
||||
|
||||
# Get block hash at specific height
|
||||
get_block_hash() {
|
||||
local node_ip="$1"
|
||||
local height="$2"
|
||||
|
||||
hash=$(curl -s --max-time 5 "http://${node_ip}:${RPC_PORT}/blockchain/block/${height}/hash" 2>/dev/null || echo "")
|
||||
echo "$hash"
|
||||
}
|
||||
|
||||
# Check chain ID consistency
|
||||
check_chain_id_consistency() {
|
||||
log "Checking chain ID consistency across nodes"
|
||||
|
||||
local first_chain_id=""
|
||||
local consistent=true
|
||||
|
||||
for node_config in "${NODES[@]}"; do
|
||||
IFS=':' read -r node_name node_ip <<< "$node_config"
|
||||
|
||||
chain_id=$(get_chain_id "$node_ip")
|
||||
|
||||
if [ -z "$chain_id" ]; then
|
||||
log_error "Could not get chain ID from ${node_name}"
|
||||
consistent=false
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Chain ID on ${node_name}: ${chain_id}"
|
||||
|
||||
if [ -z "$first_chain_id" ]; then
|
||||
first_chain_id="$chain_id"
|
||||
elif [ "$chain_id" != "$first_chain_id" ]; then
|
||||
log_error "Chain ID mismatch on ${node_name}: ${chain_id} vs ${first_chain_id}"
|
||||
consistent=false
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$consistent" = true ]; then
|
||||
log_success "Chain ID consistent across all nodes"
|
||||
return 0
|
||||
else
|
||||
log_error "Chain ID inconsistent across nodes"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check block synchronization
|
||||
check_block_sync() {
|
||||
log "Checking block synchronization across nodes"
|
||||
|
||||
local heights=()
|
||||
local max_height=0
|
||||
local min_height=999999
|
||||
|
||||
for node_config in "${NODES[@]}"; do
|
||||
IFS=':' read -r node_name node_ip <<< "$node_config"
|
||||
|
||||
height=$(get_block_height "$node_ip")
|
||||
|
||||
if [ -z "$height" ] || [ "$height" = "0" ]; then
|
||||
log_error "Could not get block height from ${node_name}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
heights+=("${node_name}:${height}")
|
||||
log "Block height on ${node_name}: ${height}"
|
||||
|
||||
if [ "$height" -gt "$max_height" ]; then
|
||||
max_height=$height
|
||||
max_node="${node_name}"
|
||||
max_ip="${node_ip}"
|
||||
fi
|
||||
|
||||
if [ "$height" -lt "$min_height" ]; then
|
||||
min_height=$height
|
||||
fi
|
||||
done
|
||||
|
||||
local height_diff=$((max_height - min_height))
|
||||
|
||||
log "Max height: ${max_height} (${max_node}), Min height: ${min_height}, Diff: ${height_diff}"
|
||||
|
||||
if [ "$height_diff" -le "$SYNC_THRESHOLD" ]; then
|
||||
log_success "Block synchronization within threshold (diff: ${height_diff})"
|
||||
return 0
|
||||
else
|
||||
log_error "Block synchronization exceeds threshold (diff: ${height_diff})"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check block hash consistency at current height
|
||||
check_block_hash_consistency() {
|
||||
log "Checking block hash consistency"
|
||||
|
||||
local target_height=""
|
||||
|
||||
# Find the minimum height to compare at
|
||||
for node_config in "${NODES[@]}"; do
|
||||
IFS=':' read -r node_name node_ip <<< "$node_config"
|
||||
height=$(get_block_height "$node_ip")
|
||||
|
||||
if [ -z "$target_height" ] || [ "$height" -lt "$target_height" ]; then
|
||||
target_height=$height
|
||||
fi
|
||||
done
|
||||
|
||||
log "Comparing block hashes at height ${target_height}"
|
||||
|
||||
local first_hash=""
|
||||
local consistent=true
|
||||
|
||||
for node_config in "${NODES[@]}"; do
|
||||
IFS=':' read -r node_name node_ip <<< "$node_config"
|
||||
|
||||
hash=$(get_block_hash "$node_ip" "$target_height")
|
||||
|
||||
if [ -z "$hash" ]; then
|
||||
log_warning "Could not get block hash from ${node_name} at height ${target_height}"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Block hash on ${node_name} at height ${target_height}: ${hash}"
|
||||
|
||||
if [ -z "$first_hash" ]; then
|
||||
first_hash="$hash"
|
||||
elif [ "$hash" != "$first_hash" ]; then
|
||||
log_error "Block hash mismatch on ${node_name} at height ${target_height}"
|
||||
consistent=false
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$consistent" = true ]; then
|
||||
log_success "Block hashes consistent at height ${target_height}"
|
||||
return 0
|
||||
else
|
||||
log_error "Block hashes inconsistent"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Remediation: Force sync from healthy node
|
||||
force_sync_from_source() {
|
||||
local target_node="$1"
|
||||
local target_name="$2"
|
||||
local source_node="$3"
|
||||
local source_name="$4"
|
||||
|
||||
log "Forcing sync from ${source_name} to ${target_name}"
|
||||
|
||||
# Stop blockchain service on target
|
||||
log "Stopping blockchain service on ${target_name}"
|
||||
ssh_exec "$target_node" "systemctl stop aitbc-blockchain-node" 2>&1 | tee -a "${LOG_FILE}"
|
||||
sleep 5
|
||||
|
||||
# Copy chain.db from source to target
|
||||
log "Copying chain.db from ${source_name} to ${target_name}"
|
||||
ssh_exec "$source_node" "cat /var/lib/aitbc/data/chain.db" | ssh_exec "$target_node" "cat > /var/lib/aitbc/data/chain.db" 2>&1 | tee -a "${LOG_FILE}"
|
||||
|
||||
# Start blockchain service on target
|
||||
log "Starting blockchain service on ${target_name}"
|
||||
ssh_exec "$target_node" "systemctl start aitbc-blockchain-node" 2>&1 | tee -a "${LOG_FILE}"
|
||||
sleep 10
|
||||
|
||||
# Verify service is running
|
||||
if ssh_exec "$target_node" "systemctl is-active aitbc-blockchain-node" | grep -q "active"; then
|
||||
log_success "Sync completed successfully on ${target_name}"
|
||||
return 0
|
||||
else
|
||||
log_error "Failed to start blockchain service on ${target_name} after sync"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Main sync verification
|
||||
main() {
|
||||
log "=== Blockchain Synchronization Verification Started ==="
|
||||
|
||||
# Create log directory if it doesn't exist
|
||||
mkdir -p "${LOG_DIR}"
|
||||
|
||||
local total_failures=0
|
||||
|
||||
# Check chain ID consistency
|
||||
if ! check_chain_id_consistency; then
|
||||
log_error "Chain ID inconsistency detected - this is critical"
|
||||
((total_failures++))
|
||||
fi
|
||||
|
||||
# Check block synchronization
|
||||
if ! check_block_sync; then
|
||||
log_error "Block synchronization issue detected"
|
||||
((total_failures++))
|
||||
|
||||
# Determine source and target nodes for remediation
|
||||
local max_height=0
|
||||
local max_node=""
|
||||
local max_ip=""
|
||||
local min_height=999999
|
||||
local min_node=""
|
||||
local min_ip=""
|
||||
|
||||
for node_config in "${NODES[@]}"; do
|
||||
IFS=':' read -r node_name node_ip <<< "$node_config"
|
||||
height=$(get_block_height "$node_ip")
|
||||
|
||||
if [ "$height" -gt "$max_height" ]; then
|
||||
max_height=$height
|
||||
max_node="${node_name}"
|
||||
max_ip="${node_ip}"
|
||||
fi
|
||||
|
||||
if [ "$height" -lt "$min_height" ]; then
|
||||
min_height=$height
|
||||
min_node="${node_name}"
|
||||
min_ip="${node_ip}"
|
||||
fi
|
||||
done
|
||||
|
||||
# Attempt remediation if difference exceeds threshold
|
||||
local height_diff=$((max_height - min_height))
|
||||
if [ "$height_diff" -gt "$SYNC_THRESHOLD" ]; then
|
||||
log "Attempting remediation: sync from ${max_node} to ${min_node}"
|
||||
if force_sync_from_source "$min_ip" "$min_node" "$max_ip" "$max_node"; then
|
||||
log_success "Remediation successful"
|
||||
# Re-check sync after remediation
|
||||
if check_block_sync; then
|
||||
log_success "Sync verification passed after remediation"
|
||||
else
|
||||
log_error "Sync still fails after remediation"
|
||||
((total_failures++))
|
||||
fi
|
||||
else
|
||||
log_error "Remediation failed"
|
||||
((total_failures++))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check block hash consistency
|
||||
if ! check_block_hash_consistency; then
|
||||
log_error "Block hash inconsistency detected"
|
||||
((total_failures++))
|
||||
fi
|
||||
|
||||
log "=== Blockchain Synchronization Verification Completed ==="
|
||||
log "Total failures: ${total_failures}"
|
||||
|
||||
if [ ${total_failures} -eq 0 ]; then
|
||||
log_success "Blockchain synchronization verification passed"
|
||||
exit 0
|
||||
else
|
||||
log_error "Blockchain synchronization verification failed with ${total_failures} failures"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user