Move blockchain app READMEs to centralized documentation

- Relocate blockchain-event-bridge README content to docs/apps/blockchain/blockchain-event-bridge.md - Relocate blockchain-explorer README content to docs/apps/blockchain/blockchain-explorer.md - Replace app READMEs with redirect notices pointing to new documentation location - Consolidate documentation in central docs/ directory for better organization
2026-04-23 12:24:48 +02:00
parent cd240485c6
commit 522655ef92
55 changed files with 7033 additions and 1536 deletions
--- a/scripts/workflow/44_comprehensive_multi_node_scenario.sh
+++ b/scripts/workflow/44_comprehensive_multi_node_scenario.sh
@@ -0,0 +1,461 @@
+#!/bin/bash
+
+# AITBC Comprehensive Multi-Node Scenario Orchestration
+# Executes end-to-end scenarios across all 3 nodes (aitbc1, aitbc, gitea-runner)
+# Using all AITBC apps with real execution, verbose logging, and health checks
+
+set -e
+
+# Configuration
+AITBC1_HOST="aitbc1"
+AITBC_HOST="localhost"
+GITEA_RUNNER_HOST="gitea-runner"
+GENESIS_PORT="8006"
+LOG_DIR="/var/log/aitbc"
+LOG_FILE="$LOG_DIR/comprehensive_scenario_$(date +%Y%m%d_%H%M%S).log"
+ERROR_LOG="$LOG_DIR/comprehensive_scenario_errors_$(date +%Y%m%d_%H%M%S).log"
+
+# Debug mode flags
+DEBUG_MODE=true
+VERBOSE_LOGGING=true
+HEALTH_CHECKS=true
+ERROR_DETAILED=true
+
+# Colors for output
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Ensure log directory exists
+mkdir -p "$LOG_DIR"
+
+# Helper functions
+log_debug() {
+    if [ "$DEBUG_MODE" = true ]; then
+        echo "[DEBUG] $(date '+%Y-%m-%d %H:%M:%S'): $1" | tee -a "$LOG_FILE"
+    fi
+}
+
+log_info() {
+    echo "[INFO] $(date '+%Y-%m-%d %H:%M:%S'): $1" | tee -a "$LOG_FILE"
+}
+
+log_error() {
+    echo "[ERROR] $(date '+%Y-%m-%d %H:%M:%S'): $1" | tee -a "$LOG_FILE" | tee -a "$ERROR_LOG"
+}
+
+log_success() {
+    echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S'): $1" | tee -a "$LOG_FILE"
+}
+
+log_warning() {
+    echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S'): $1" | tee -a "$LOG_FILE"
+}
+
+# Health check function
+health_check() {
+    local node=$1
+    local service=$2
+    local port=$3
+    
+    log_info "Checking $service on $node:$port"
+    
+    if [ "$node" = "localhost" ]; then
+        local health_url="http://localhost:$port/health"
+    else
+        local health_url="http://$node:$port/health"
+    fi
+    
+    if [ "$node" = "localhost" ]; then
+        local health_status=$(curl -s -o /dev/null -w "%{http_code}" "$health_url" 2>/dev/null || echo "000")
+    else
+        local health_status=$(ssh -o ConnectTimeout=5 "$node" "curl -s -o /dev/null -w '%{http_code}' $health_url" 2>/dev/null || echo "000")
+    fi
+    
+    if [ "$health_status" = "200" ]; then
+        log_success "$service on $node is healthy"
+        return 0
+    else
+        log_error "$service on $node is unhealthy (HTTP $health_status)"
+        return 1
+    fi
+}
+
+# Execute command on remote node
+execute_on_node() {
+    local node=$1
+    local command=$2
+    local quiet=${3:-false}
+    
+    if [ "$quiet" = true ]; then
+        # Quiet mode - no debug logging
+        if [ "$node" = "localhost" ]; then
+            eval "$command" 2>/dev/null
+        else
+            ssh -o ConnectTimeout=10 "$node" "cd /opt/aitbc && $command" 2>/dev/null
+        fi
+    else
+        log_debug "Executing on $node: $command"
+        if [ "$node" = "localhost" ]; then
+            eval "$command"
+        else
+            ssh -o ConnectTimeout=10 "$node" "cd /opt/aitbc && $command"
+        fi
+    fi
+}
+
+# Check SSH connectivity
+check_ssh_connectivity() {
+    local node=$1
+    
+    log_info "Checking SSH connectivity to $node"
+    
+    if ssh -o ConnectTimeout=5 -o BatchMode=yes "$node" "echo 'SSH OK'" >/dev/null 2>&1; then
+        log_success "SSH connectivity to $node OK"
+        return 0
+    else
+        log_error "SSH connectivity to $node FAILED"
+        return 1
+    fi
+}
+
+# Phase 1: Pre-Flight Health Checks
+phase1_preflight_checks() {
+    log_info "=== PHASE 1: PRE-FLIGHT HEALTH CHECKS ==="
+    
+    # Check SSH connectivity
+    log_info "Checking SSH connectivity to all nodes"
+    check_ssh_connectivity "$AITBC1_HOST" || return 1
+    check_ssh_connectivity "$GITEA_RUNNER_HOST" || return 1
+    log_success "SSH connectivity verified for all nodes"
+    
+    # Check AITBC services on aitbc1
+    log_info "Checking AITBC services on aitbc1"
+    health_check "$AITBC1_HOST" "blockchain-node" "8006" || log_warning "Blockchain node on aitbc1 may not be healthy"
+    health_check "$AITBC1_HOST" "coordinator-api" "8011" || log_warning "Coordinator API on aitbc1 may not be healthy"
+    
+    # Check AITBC services on localhost
+    log_info "Checking AITBC services on localhost"
+    health_check "localhost" "blockchain-node" "8006" || log_warning "Blockchain node on localhost may not be healthy"
+    health_check "localhost" "coordinator-api" "8011" || log_warning "Coordinator API on localhost may not be healthy"
+    health_check "localhost" "blockchain-event-bridge" "8204" || log_warning "Blockchain event bridge on localhost may not be healthy"
+    
+    # Check AITBC services on gitea-runner
+    log_info "Checking AITBC services on gitea-runner"
+    health_check "$GITEA_RUNNER_HOST" "blockchain-node" "8006" || log_warning "Blockchain node on gitea-runner may not be healthy"
+    health_check "$GITEA_RUNNER_HOST" "blockchain-node" "8007" || log_warning "Blockchain node on gitea-runner may not be healthy"
+    
+    # Verify blockchain sync status
+    log_info "Checking blockchain sync status across nodes"
+    local aitbc1_height=$(execute_on_node "$AITBC1_HOST" "curl -s http://localhost:8006/rpc/head | jq -r .height" true 2>/dev/null || echo "0")
+    local aitbc_height=$(execute_on_node "localhost" "curl -s http://localhost:8006/rpc/head | jq -r .height" true 2>/dev/null || echo "0")
+    local gitea_height=$(execute_on_node "$GITEA_RUNNER_HOST" "curl -s http://localhost:8006/rpc/head | jq -r .height" true 2>/dev/null || echo "0")
+    
+    log_info "Blockchain heights - aitbc1: $aitbc1_height, aitbc: $aitbc_height, gitea-runner: $gitea_height"
+    
+    # Check CLI tools
+    log_info "Checking CLI tools installation"
+    if command -v /opt/aitbc/aitbc-cli >/dev/null 2>&1; then
+        log_success "CLI tool found on localhost"
+    else
+        log_error "CLI tool not found on localhost"
+        return 1
+    fi
+    
+    log_success "Phase 1: Pre-flight checks completed"
+}
+
+# Phase 2: Complete Transaction Flow
+phase2_transaction_flow() {
+    log_info "=== PHASE 2: COMPLETE TRANSACTION FLOW ==="
+    
+    # Check if genesis wallet exists
+    log_info "Checking genesis wallet on aitbc1"
+    local genesis_wallets=$(execute_on_node "$AITBC1_HOST" "/opt/aitbc/aitbc-cli wallet list" 2>/dev/null || echo "")
+    
+    if echo "$genesis_wallets" | grep -q "aitbc1genesis"; then
+        log_success "Genesis wallet exists on aitbc1"
+    else
+        log_warning "Genesis wallet may not exist, creating..."
+        execute_on_node "$AITBC1_HOST" "/opt/aitbc/aitbc-cli wallet create aitbc1genesis aitbc123" || log_error "Failed to create genesis wallet"
+    fi
+    
+    # Check user wallet on localhost
+    log_info "Checking user wallet on localhost"
+    local user_wallets=$(/opt/aitbc/aitbc-cli wallet list 2>/dev/null || echo "")
+    
+    if echo "$user_wallets" | grep -q "scenario_user"; then
+        log_success "User wallet exists on localhost"
+    else
+        log_info "Creating user wallet on localhost"
+        /opt/aitbc/aitbc-cli wallet create scenario_user scenario123 || log_error "Failed to create user wallet"
+    fi
+    
+    # Get addresses
+    local genesis_addr=$(execute_on_node "$AITBC1_HOST" "cat /var/lib/aitbc/keystore/aitbc1genesis.json" true 2>/dev/null | jq -r .address 2>/dev/null || echo "")
+    local user_addr=$(cat /var/lib/aitbc/keystore/scenario_user.json 2>/dev/null | jq -r .address 2>/dev/null || echo "")
+    
+    log_info "Genesis address: $genesis_addr"
+    log_info "User address: $user_addr"
+    
+    # Check genesis balance via RPC (quiet mode to avoid debug output in variable)
+    local genesis_balance=$(execute_on_node "$AITBC1_HOST" "curl -s http://localhost:8006/rpc/getBalance/$genesis_addr" true 2>/dev/null | jq -r '.balance // 0' 2>/dev/null || echo "0")
+    
+    # Handle null or non-numeric balance
+    if [ "$genesis_balance" = "null" ] || ! [[ "$genesis_balance" =~ ^[0-9]+$ ]]; then
+        genesis_balance=0
+    fi
+    
+    log_info "Genesis balance: $genesis_balance AIT"
+    
+    if [ "$genesis_balance" -lt 100 ]; then
+        log_warning "Genesis balance is low ($genesis_balance AIT), mining some blocks first..."
+        # Mine some blocks to fund the genesis wallet
+        log_info "Mining 5 blocks to fund genesis wallet..."
+        for i in {1..5}; do
+            log_debug "Mining block $i..."
+            execute_on_node "$AITBC1_HOST" "curl -s -X POST http://localhost:8006/rpc/mineBlock -H 'Content-Type: application/json' -d '{}'" >/dev/null 2>&1 || log_warning "Failed to mine block $i"
+            sleep 1
+        done
+        
+        # Check balance again after mining
+        genesis_balance=$(execute_on_node "$AITBC1_HOST" "curl -s http://localhost:8006/rpc/getBalance/$genesis_addr" true 2>/dev/null | jq -r '.balance // 0' 2>/dev/null || echo "0")
+        if [ "$genesis_balance" = "null" ] || ! [[ "$genesis_balance" =~ ^[0-9]+$ ]]; then
+            genesis_balance=0
+        fi
+        log_info "Genesis balance after mining: $genesis_balance AIT"
+    fi
+    
+    # Skip transaction if balance is still too low
+    if [ "$genesis_balance" -lt 50 ]; then
+        log_warning "Genesis balance still too low ($genesis_balance AIT), skipping transaction phase"
+        log_success "Phase 2: Transaction flow skipped (insufficient balance)"
+        return 0
+    fi
+    
+    # Send transaction from genesis to user
+    log_info "Sending transaction from genesis to user"
+    local tx_result=$(execute_on_node "$AITBC1_HOST" "/opt/aitbc/aitbc-cli wallet send aitbc1genesis $user_addr 50 --fee 5" 2>/dev/null || echo "")
+    
+    if echo "$tx_result" | grep -q "tx_hash"; then
+        log_success "Transaction sent successfully"
+    else
+        log_error "Failed to send transaction"
+        return 1
+    fi
+    
+    # Wait for confirmation
+    log_info "Waiting for transaction confirmation..."
+    sleep 5
+    
+    # Verify balance updates via RPC
+    local user_balance=$(curl -s "http://localhost:8006/rpc/getBalance/$user_addr" | jq -r .balance 2>/dev/null || echo "0")
+    log_info "User balance after transaction: $user_balance AIT"
+    
+    log_success "Phase 2: Transaction flow completed"
+}
+
+# Phase 3: AI Job Submission Flow
+phase3_ai_job_submission() {
+    log_info "=== PHASE 3: AI JOB SUBMISSION FLOW ==="
+    
+    # Get GPU info
+    log_info "Getting GPU information"
+    local gpu_info=$(execute_on_node "localhost" "nvidia-smi --query-gpu=name,memory.total --format=csv,noheader" 2>/dev/null || echo "Unknown,0")
+    log_info "GPU: $gpu_info"
+    
+    # Submit AI job
+    log_info "Submitting AI job"
+    local ai_result=$(/opt/aitbc/aitbc-cli ai submit --wallet scenario_user --type text-generation --prompt "Test prompt for comprehensive scenario" --payment 10 2>/dev/null || echo "")
+    
+    if echo "$ai_result" | grep -q "job_id\|success"; then
+        log_success "AI job submitted successfully"
+    else
+        log_warning "AI job submission may have failed, continuing..."
+    fi
+    
+    # List AI jobs
+    log_info "Listing AI jobs"
+    /opt/aitbc/aitbc-cli ai jobs || log_warning "Failed to list AI jobs"
+    
+    # Check marketplace listings
+    log_info "Checking marketplace listings"
+    /opt/aitbc/aitbc-cli market list || log_warning "Failed to list marketplace items"
+    
+    log_success "Phase 3: AI job submission flow completed"
+}
+
+# Phase 4: Multi-Node Blockchain Sync with Event Bridge
+phase4_blockchain_sync_event_bridge() {
+    log_info "=== PHASE 4: BLOCKCHAIN SYNC WITH EVENT BRIDGE ==="
+    
+    # Check event bridge health
+    log_info "Checking blockchain event bridge health"
+    /opt/aitbc/aitbc-cli bridge health || log_warning "Event bridge health check failed"
+    
+    # Check event bridge metrics
+    log_info "Getting event bridge metrics"
+    /opt/aitbc/aitbc-cli bridge metrics || log_warning "Failed to get event bridge metrics"
+    
+    # Get current heights (quiet mode to avoid debug output in variables)
+    local aitbc1_height=$(execute_on_node "$AITBC1_HOST" "curl -s http://localhost:8006/rpc/head | jq -r .height" true 2>/dev/null || echo "0")
+    local aitbc_height=$(execute_on_node "localhost" "curl -s http://localhost:8006/rpc/head | jq -r .height" true 2>/dev/null || echo "0")
+    local gitea_height=$(execute_on_node "$GITEA_RUNNER_HOST" "curl -s http://localhost:8006/rpc/head | jq -r .height" true 2>/dev/null || echo "0")
+    
+    log_info "Current heights - aitbc1: $aitbc1_height, aitbc: $aitbc_height, gitea-runner: $gitea_height"
+    
+    # Trigger sync if needed
+    local height_diff=$((aitbc1_height - aitbc_height))
+    if [ "$height_diff" -gt 5 ]; then
+        log_info "Height difference detected ($height_diff blocks), triggering sync"
+        execute_on_node "localhost" "/opt/aitbc/scripts/workflow/12_complete_sync.sh" || log_warning "Sync script execution failed"
+    else
+        log_success "Nodes are synchronized"
+    fi
+    
+    # Check event bridge status
+    log_info "Checking event bridge status"
+    /opt/aitbc/aitbc-cli bridge status || log_warning "Failed to get event bridge status"
+    
+    log_success "Phase 4: Blockchain sync with event bridge completed"
+}
+
+# Phase 5: Agent Coordination via OpenClaw
+phase5_agent_coordination() {
+    log_info "=== PHASE 5: AGENT COORDINATION VIA OPENCLAW ==="
+    
+    # Check if OpenClaw is available
+    if command -v openclaw >/dev/null 2>&1; then
+        log_success "OpenClaw CLI found"
+        
+        # Create a test session
+        local session_id="scenario_$(date +%s)"
+        log_info "Creating OpenClaw session: $session_id"
+        
+        # Send coordination message
+        log_info "Sending coordination message to agent network"
+        openclaw agent --agent main --session-id "$session_id" --message "Multi-node scenario coordination: All nodes operational" --thinking low 2>/dev/null || log_warning "OpenClaw agent command failed"
+        
+        log_success "Phase 5: Agent coordination completed"
+    else
+        log_warning "OpenClaw CLI not found, skipping agent coordination"
+    fi
+}
+
+# Phase 6: Pool-Hub SLA and Billing
+phase6_pool_hub_sla_billing() {
+    log_info "=== PHASE 6: POOL-HUB SLA AND BILLING ==="
+    
+    # Trigger SLA metrics collection
+    log_info "Triggering SLA metrics collection"
+    /opt/aitbc/aitbc-cli pool-hub collect-metrics --test-mode || log_warning "Pool-hub metrics collection failed"
+    
+    # Get capacity snapshots
+    log_info "Getting capacity planning snapshots"
+    /opt/aitbc/aitbc-cli pool-hub capacity-snapshots --test-mode || log_warning "Failed to get capacity snapshots"
+    
+    # Get capacity forecast
+    log_info "Getting capacity forecast"
+    /opt/aitbc/aitbc-cli pool-hub capacity-forecast --test-mode || log_warning "Failed to get capacity forecast"
+    
+    # Check SLA violations
+    log_info "Checking SLA violations"
+    /opt/aitbc/aitbc-cli pool-hub sla-violations --test-mode || log_warning "Failed to check SLA violations"
+    
+    # Get billing usage
+    log_info "Getting billing usage data"
+    /opt/aitbc/aitbc-cli pool-hub billing-usage --test-mode || log_warning "Failed to get billing usage"
+    
+    log_success "Phase 6: Pool-hub SLA and billing completed"
+}
+
+# Phase 7: Exchange Integration
+phase7_exchange_integration() {
+    log_info "=== PHASE 7: EXCHANGE INTEGRATION ==="
+    
+    log_info "Exchange integration phase - checking exchange status"
+    
+    # Note: Exchange integration would require actual exchange API setup
+    # This phase is informational for now
+    log_warning "Exchange integration requires external API configuration, skipping for now"
+    
+    log_success "Phase 7: Exchange integration completed (skipped - requires external config)"
+}
+
+# Phase 8: Plugin System
+phase8_plugin_system() {
+    log_info "=== PHASE 8: PLUGIN SYSTEM ==="
+    
+    log_info "Checking plugin system"
+    
+    # Browse plugin marketplace
+    log_info "Browsing plugin marketplace"
+    # Plugin marketplace would require coordinator-api plugin service
+    log_warning "Plugin marketplace requires coordinator-api plugin service, skipping for now"
+    
+    log_success "Phase 8: Plugin system completed (skipped - requires plugin service)"
+}
+
+# Phase 9: Final Verification
+phase9_final_verification() {
+    log_info "=== PHASE 9: FINAL VERIFICATION ==="
+    
+    # Check blockchain heights consistency (quiet mode)
+    log_info "Final blockchain height check"
+    local aitbc1_height=$(execute_on_node "$AITBC1_HOST" "curl -s http://localhost:8006/rpc/head | jq -r .height" true 2>/dev/null || echo "0")
+    local aitbc_height=$(execute_on_node "localhost" "curl -s http://localhost:8006/rpc/head | jq -r .height" true 2>/dev/null || echo "0")
+    local gitea_height=$(execute_on_node "$GITEA_RUNNER_HOST" "curl -s http://localhost:8006/rpc/head | jq -r .height" true 2>/dev/null || echo "0")
+    
+    log_info "Final heights - aitbc1: $aitbc1_height, aitbc: $aitbc_height, gitea-runner: $gitea_height"
+    
+    # Check service health
+    log_info "Final service health check"
+    health_check "localhost" "blockchain-node" "8006" || log_error "Blockchain node unhealthy"
+    health_check "localhost" "coordinator-api" "8011" || log_warning "Coordinator API unhealthy"
+    health_check "localhost" "blockchain-event-bridge" "8204" || log_warning "Event bridge unhealthy"
+    
+    # Generate comprehensive report
+    log_info "Generating comprehensive scenario report"
+    
+    echo "========================================" | tee -a "$LOG_FILE"
+    echo "COMPREHENSIVE SCENARIO REPORT" | tee -a "$LOG_FILE"
+    echo "========================================" | tee -a "$LOG_FILE"
+    echo "Timestamp: $(date)" | tee -a "$LOG_FILE"
+    echo "" | tee -a "$LOG_FILE"
+    echo "Blockchain Heights:" | tee -a "$LOG_FILE"
+    echo "  aitbc1: $aitbc1_height" | tee -a "$LOG_FILE"
+    echo "  aitbc: $aitbc_height" | tee -a "$LOG_FILE"
+    echo "  gitea-runner: $gitea_height" | tee -a "$LOG_FILE"
+    echo "" | tee -a "$LOG_FILE"
+    echo "Log file: $LOG_FILE" | tee -a "$LOG_FILE"
+    echo "Error log: $ERROR_LOG" | tee -a "$LOG_FILE"
+    echo "========================================" | tee -a "$LOG_FILE"
+    
+    log_success "Phase 9: Final verification completed"
+}
+
+# Main execution
+main() {
+    log_info "Starting comprehensive multi-node scenario"
+    log_info "Log file: $LOG_FILE"
+    log_info "Error log: $ERROR_LOG"
+    
+    # Execute phases
+    phase1_preflight_checks || { log_error "Phase 1 failed"; exit 1; }
+    phase2_transaction_flow || { log_error "Phase 2 failed"; exit 1; }
+    phase3_ai_job_submission || { log_warning "Phase 3 had issues"; }
+    phase4_blockchain_sync_event_bridge || { log_error "Phase 4 failed"; exit 1; }
+    phase5_agent_coordination || { log_warning "Phase 5 had issues"; }
+    phase6_pool_hub_sla_billing || { log_warning "Phase 6 had issues"; }
+    phase7_exchange_integration || { log_warning "Phase 7 skipped"; }
+    phase8_plugin_system || { log_warning "Phase 8 skipped"; }
+    phase9_final_verification || { log_error "Phase 9 failed"; exit 1; }
+    
+    log_success "Comprehensive multi-node scenario completed successfully"
+    log_info "Full log available at: $LOG_FILE"
+}
+
+# Run main function
+main