feat: add multi-node blockchain monitoring workflows for 3-node network

- Create multi-node blockchain health monitoring workflow
- Create P2P network verification workflow for all 3 nodes
- Create blockchain synchronization verification workflow
- Update blockchain-communication-test.sh to include aitbc2 (gitea-runner)
- Add shared scripts directory with health check, P2P verification, and sync verification scripts
- All workflows trigger on git push to main/develop branches
- Workflows run on gitea-runner (has SSH access to all nodes)
- Include automatic remediation for failed services and sync issues
- Sync threshold set to 10 blocks
- Logging to /var/log/aitbc/ and alerts in Gitea UI

.gitea/workflows/blockchain-sync-verification.yml

@@ -0,0 +1,67 @@
+name: Blockchain Synchronization Verification
+
+on:
+  push:
+    branches: [main, develop]
+    paths:
+      - 'apps/blockchain-node/**'
+      - 'scripts/multi-node/**'
+      - '.gitea/workflows/blockchain-sync-verification.yml'
+  pull_request:
+    branches: [main, develop]
+  workflow_dispatch:
+  schedule:
+    - cron: '0 */6 * * *'  # Every 6 hours
+
+concurrency:
+  group: blockchain-sync-verification-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  sync-verification:
+    runs-on: debian
+    timeout-minutes: 20
+
+    steps:
+      - name: Clone repository
+        run: |
+          WORKSPACE="/var/lib/aitbc-workspaces/blockchain-sync-verification"
+          rm -rf "$WORKSPACE"
+          mkdir -p "$WORKSPACE"
+          cd "$WORKSPACE"
+          git clone --depth 1 http://gitea.bubuit.net:3000/oib/aitbc.git repo
+
+      - name: Initialize job logging
+        run: |
+          cd /var/lib/aitbc-workspaces/blockchain-sync-verification/repo
+          bash scripts/ci/setup-job-logging.sh
+
+      - name: Setup Python environment
+        run: |
+          cd /var/lib/aitbc-workspaces/blockchain-sync-verification/repo
+          
+          # Remove any existing venv to avoid cache corruption issues
+          rm -rf venv
+          
+          bash scripts/ci/setup-python-venv.sh \
+            --repo-dir "$PWD" \
+            --venv-dir "$PWD/venv" \
+            --skip-requirements \
+            --extra-packages "requests psutil"
+
+      - name: Run blockchain synchronization verification
+        run: |
+          cd /var/lib/aitbc-workspaces/blockchain-sync-verification/repo
+          bash scripts/multi-node/sync-verification.sh
+
+      - name: Sync verification report
+        if: always()
+        run: |
+          echo "=== Blockchain Synchronization Verification Report ==="
+          if [ -f /var/log/aitbc/sync-verification.log ]; then
+            tail -50 /var/log/aitbc/sync-verification.log
+          fi
+
+      - name: Cleanup
+        if: always()
+        run: rm -rf /var/lib/aitbc-workspaces/blockchain-sync-verification

.gitea/workflows/multi-node-health.yml

@@ -0,0 +1,67 @@
+name: Multi-Node Blockchain Health Monitoring
+
+on:
+  push:
+    branches: [main, develop]
+    paths:
+      - 'apps/blockchain-node/**'
+      - 'scripts/multi-node/**'
+      - '.gitea/workflows/multi-node-health.yml'
+  pull_request:
+    branches: [main, develop]
+  workflow_dispatch:
+  schedule:
+    - cron: '0 */2 * * *'  # Every 2 hours
+
+concurrency:
+  group: multi-node-health-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  health-check:
+    runs-on: debian
+    timeout-minutes: 15
+
+    steps:
+      - name: Clone repository
+        run: |
+          WORKSPACE="/var/lib/aitbc-workspaces/multi-node-health"
+          rm -rf "$WORKSPACE"
+          mkdir -p "$WORKSPACE"
+          cd "$WORKSPACE"
+          git clone --depth 1 http://gitea.bubuit.net:3000/oib/aitbc.git repo
+
+      - name: Initialize job logging
+        run: |
+          cd /var/lib/aitbc-workspaces/multi-node-health/repo
+          bash scripts/ci/setup-job-logging.sh
+
+      - name: Setup Python environment
+        run: |
+          cd /var/lib/aitbc-workspaces/multi-node-health/repo
+          
+          # Remove any existing venv to avoid cache corruption issues
+          rm -rf venv
+          
+          bash scripts/ci/setup-python-venv.sh \
+            --repo-dir "$PWD" \
+            --venv-dir "$PWD/venv" \
+            --skip-requirements \
+            --extra-packages "requests psutil"
+
+      - name: Run multi-node health check
+        run: |
+          cd /var/lib/aitbc-workspaces/multi-node-health/repo
+          bash scripts/multi-node/blockchain-health-check.sh
+
+      - name: Health check report
+        if: always()
+        run: |
+          echo "=== Multi-Node Health Check Report ==="
+          if [ -f /var/log/aitbc/multi-node-health.log ]; then
+            tail -50 /var/log/aitbc/multi-node-health.log
+          fi
+
+      - name: Cleanup
+        if: always()
+        run: rm -rf /var/lib/aitbc-workspaces/multi-node-health

.gitea/workflows/p2p-network-verification.yml

@@ -0,0 +1,67 @@
+name: P2P Network Verification
+
+on:
+  push:
+    branches: [main, develop]
+    paths:
+      - 'apps/blockchain-node/**'
+      - 'scripts/multi-node/**'
+      - '.gitea/workflows/p2p-network-verification.yml'
+  pull_request:
+    branches: [main, develop]
+  workflow_dispatch:
+  schedule:
+    - cron: '0 */4 * * *'  # Every 4 hours
+
+concurrency:
+  group: p2p-network-verification-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  p2p-verification:
+    runs-on: debian
+    timeout-minutes: 15
+
+    steps:
+      - name: Clone repository
+        run: |
+          WORKSPACE="/var/lib/aitbc-workspaces/p2p-network-verification"
+          rm -rf "$WORKSPACE"
+          mkdir -p "$WORKSPACE"
+          cd "$WORKSPACE"
+          git clone --depth 1 http://gitea.bubuit.net:3000/oib/aitbc.git repo
+
+      - name: Initialize job logging
+        run: |
+          cd /var/lib/aitbc-workspaces/p2p-network-verification/repo
+          bash scripts/ci/setup-job-logging.sh
+
+      - name: Setup Python environment
+        run: |
+          cd /var/lib/aitbc-workspaces/p2p-network-verification/repo
+          
+          # Remove any existing venv to avoid cache corruption issues
+          rm -rf venv
+          
+          bash scripts/ci/setup-python-venv.sh \
+            --repo-dir "$PWD" \
+            --venv-dir "$PWD/venv" \
+            --skip-requirements \
+            --extra-packages "requests psutil"
+
+      - name: Run P2P network verification
+        run: |
+          cd /var/lib/aitbc-workspaces/p2p-network-verification/repo
+          bash scripts/multi-node/p2p-verification.sh
+
+      - name: P2P verification report
+        if: always()
+        run: |
+          echo "=== P2P Network Verification Report ==="
+          if [ -f /var/log/aitbc/p2p-verification.log ]; then
+            tail -50 /var/log/aitbc/p2p-verification.log
+          fi
+
+      - name: Cleanup
+        if: always()
+        run: rm -rf /var/lib/aitbc-workspaces/p2p-network-verification