ci: add daily failover simulation schedule and standardize service configurations

Add daily 2 AM cron schedule for node failover simulation workflow. Relax AITBC address validation to support variable-length addresses. Add missing logging import to chain_sync. Make coordinator database initialization non-fatal to allow startup even if init_db fails. Replace Ethereum address validation with AITBC-specific format checks in multisig transactions. Standardize PYTHONPATH across all systemd services to include

.gitea/workflows/node-failover-simulation.yml

@@ -1,6 +1,8 @@
 name: Node Failover Simulation
 
 on:
+  schedule:
+    - cron: '0 2 * * *'  # Daily at 2 AM
   workflow_dispatch:
 
 concurrency:

aitbc/validation.py

@@ -24,8 +24,8 @@ def validate_address(address: str) -> bool:
     if not address:
         raise ValidationError("Address cannot be empty")
     
-    # AITBC addresses typically start with 'ait' and are alphanumeric
-    pattern = r'^ait[a-z0-9]{40}$'
+    # AITBC addresses typically start with 'ait' and are alphanumeric (variable length)
+    pattern = r'^ait[a-z0-9]+$'
     if not re.match(pattern, address):
         raise ValidationError(f"Invalid address format: {address}")
     

apps/blockchain-node/src/aitbc_chain/chain_sync.py

@@ -6,6 +6,7 @@ Keeps blockchain nodes synchronized by sharing blocks via P2P and Redis gossip
 
 import asyncio
 import json
+import logging
 import time
 from typing import Dict, Any, Optional, List
 

apps/coordinator-api/src/app/main.py

@@ -104,8 +104,12 @@ async def lifespan(app: FastAPI):
 
     try:
         # Initialize database
-        init_db()
-        logger.info("Database initialized successfully")
+        try:
+            init_db()
+            logger.info("Database initialized successfully")
+        except Exception as e:
+            logger.warning(f"Database initialization failed (non-fatal): {e}")
+            # Continue startup even if init_db fails
 
         # Warmup database connections
         logger.info("Warming up database connections...")

cli/utils/crypto_utils.py

@@ -123,11 +123,15 @@ def validate_multisig_transaction(tx_data: Dict) -> Tuple[bool, str]:
         if field not in tx_data:
             return False, f"Missing required field: {field}"
     
-    # Validate address format
-    try:
-        to_checksum_address(tx_data["to"])
-    except Exception:
-        return False, "Invalid recipient address format"
+    # Validate address format (AITBC addresses start with 'ait')
+    to_address = tx_data["to"]
+    if not to_address.startswith("ait"):
+        return False, "Invalid recipient address format: must start with 'ait'"
+    if len(to_address) < 50 or len(to_address) > 70:
+        return False, "Invalid recipient address format: invalid length"
+    # Check that the rest is hex-like (after 'ait' prefix)
+    if not all(c.lower() in '0123456789abcdef' for c in to_address[3:]):
+        return False, "Invalid recipient address format: invalid characters"
     
     # Validate amount
     try:

docs/development/PRE_COMMIT_HOOKS.md

@@ -0,0 +1,107 @@
+# Pre-Commit Hooks
+
+This project uses pre-commit hooks to ensure code quality and consistency before commits.
+
+## Installation
+
+```bash
+# Install pre-commit (if not already installed)
+pip install pre-commit
+
+# Install the hooks
+pre-commit install
+
+# Install pre-commit hooks for all files (optional)
+pre-commit install --hook-type pre-push
+```
+
+## Usage
+
+### Running hooks manually
+
+```bash
+# Run on all files
+pre-commit run --all-files
+
+# Run on staged files only (what happens during commit)
+pre-commit run
+
+# Run specific hooks
+pre-commit run black flake8 mypy
+```
+
+### Automatic execution
+
+Hooks run automatically on `git commit` for staged files. If a hook fails, the commit will be blocked. Fix the issues and try again.
+
+To bypass hooks (not recommended):
+```bash
+git commit --no-verify
+```
+
+## Available Hooks
+
+### Python
+- **black**: Code formatting
+- **flake8**: Linting (max line length: 120)
+- **mypy**: Type checking
+- **bandit**: Security scanning
+
+### General
+- **trailing-whitespace**: Remove trailing whitespace
+- **end-of-file-fixer**: Ensure newline at end of file
+- **check-yaml**: Validate YAML syntax
+- **check-toml**: Validate TOML syntax
+- **check-json**: Validate JSON syntax
+- **check-added-large-files**: Prevent large files (>1MB)
+- **detect-private-key**: Detect private keys in code
+- **mixed-line-ending**: Ensure consistent line endings (LF)
+
+### Configuration Files
+- **yamllint**: YAML linting with custom config
+- **markdownlint**: Markdown linting (excludes docs/archive)
+
+### JavaScript/TypeScript
+- **eslint**: JavaScript/TypeScript linting for packages/js and cli
+
+### Shell Scripts
+- **shellcheck**: Shell script linting
+
+## Configuration
+
+- **.pre-commit-config.yaml**: Main pre-commit configuration
+- **.yamllint.yaml**: YAML linting rules
+
+## Updating Hooks
+
+```bash
+# Update hook versions
+pre-commit autoupdate
+
+# Review changes
+git diff .pre-commit-config.yaml
+```
+
+## Exclusions
+
+Hooks exclude common directories:
+- venv/, .venv/ (Python virtual environments)
+- build/, dist/ (Build artifacts)
+- docs/archive/ (Archived documentation)
+
+## Troubleshooting
+
+### Hook fails but you think it's wrong
+Check the specific hook documentation and configuration. Some rules may need adjustment for the project.
+
+### Pre-commit not running
+Ensure hooks are installed:
+```bash
+pre-commit install
+```
+
+### Slow execution
+Run hooks on specific files only:
+```bash
+pre-commit run <hook-name> <files...>
+```

scripts/ci/analyze-ci-metrics.sh

@@ -0,0 +1,158 @@
+#!/bin/bash
+# CI Reliability Metrics Analyzer
+# Analyzes CI logs from /opt/gitea-runner/logs/index.tsv to track success rates and failure patterns
+
+# Set locale for consistent decimal formatting
+export LC_ALL=C
+
+LOG_INDEX="/opt/gitea-runner/logs/index.tsv"
+METRICS_DIR="/opt/gitea-runner/metrics"
+REPORT_FILE="$METRICS_DIR/ci-reliability-report-$(date +%Y%m%d).txt"
+JSON_FILE="$METRICS_DIR/ci-reliability-$(date +%Y%m%d).json"
+
+mkdir -p "$METRICS_DIR"
+
+echo "=== CI Reliability Metrics Analysis ==="
+echo "Analysis date: $(date)"
+echo ""
+
+# Initialize counters
+declare -A workflow_total
+declare -A workflow_success
+declare -A workflow_failure
+declare -A job_total
+declare -A job_success
+declare -A job_failure
+declare -A failure_patterns
+
+total_runs=0
+total_success=0
+total_failure=0
+
+# Parse index.tsv and analyze each log file
+echo "Analyzing CI logs..."
+while IFS=$'\t' read -r timestamp run_id1 run_id2 attempt workflow job logfile; do
+  if [[ -z "$logfile" ]]; then
+    continue
+  fi
+
+  total_runs=$((total_runs + 1))
+  workflow_total["$workflow"]=$((${workflow_total[$workflow]:-0} + 1))
+  job_total["$job"]=$((${job_total[$job]:-0} + 1))
+
+  # Check if log file exists
+  if [[ ! -f "$logfile" ]]; then
+    echo "Warning: Log file not found: $logfile"
+    workflow_failure["$workflow"]=$((${workflow_failure[$workflow]:-0} + 1))
+    job_failure["$job"]=$((${job_failure[$job]:-0} + 1))
+    total_failure=$((total_failure + 1))
+    continue
+  fi
+
+  # Determine success/failure based on log content
+  if grep -q "exit status 0\|✅\|SUCCESS\|completed successfully" "$logfile" 2>/dev/null; then
+    workflow_success["$workflow"]=$((${workflow_success[$workflow]:-0} + 1))
+    job_success["$job"]=$((${job_success[$job]:-0} + 1))
+    total_success=$((total_success + 1))
+  else
+    workflow_failure["$workflow"]=$((${workflow_failure[$workflow]:-0} + 1))
+    job_failure["$job"]=$((${job_failure[$job]:-0} + 1))
+    total_failure=$((total_failure + 1))
+
+    # Extract failure patterns
+    if grep -q "Error:" "$logfile" 2>/dev/null; then
+      error_msg=$(grep "Error:" "$logfile" | head -1 | sed 's/.*Error: //;s/\[.*\].*//' | cut -d' ' -f1-5)
+      failure_patterns["$error_msg"]=$((${failure_patterns[$error_msg]:-0} + 1))
+    fi
+  fi
+done < <(tail -100 "$LOG_INDEX")
+
+# Generate report
+echo "Generating report..."
+{
+  echo "=== CI Reliability Metrics Report ==="
+  echo "Generated: $(date)"
+  echo "Data source: $LOG_INDEX (last 100 runs)"
+  echo ""
+  echo "=== Summary ==="
+  echo "Total runs analyzed: $total_runs"
+  echo "Total successful: $total_success"
+  echo "Total failed: $total_failure"
+  if [[ $total_runs -gt 0 ]]; then
+    success_rate=$(echo "scale=2; ($total_success * 100) / $total_runs" | bc)
+    echo "Overall success rate: ${success_rate}%"
+  fi
+  echo ""
+
+  echo "=== Workflow Success Rates ==="
+  for workflow in "${!workflow_total[@]}"; do
+    total=${workflow_total[$workflow]}
+    success=${workflow_success[$workflow]:-0}
+    failure=${workflow_failure[$workflow]:-0}
+    if [[ $total -gt 0 ]]; then
+      rate=$(LC_ALL=C echo "scale=2; ($success * 100) / $total" | bc)
+      printf "%-30s: %3d/%3d runs (%5.1f%%) | Success: %3d | Failed: %3d\n" \
+        "$workflow" "$success" "$total" "$rate" "$success" "$failure"
+    fi
+  done | sort -k6 -nr
+  echo ""
+
+  echo "=== Job Success Rates ==="
+  for job in "${!job_total[@]}"; do
+    total=${job_total[$job]}
+    success=${job_success[$job]:-0}
+    failure=${job_failure[$job]:-0}
+    if [[ $total -gt 0 ]]; then
+      rate=$(LC_ALL=C echo "scale=2; ($success * 100) / $total" | bc)
+      printf "%-35s: %3d/%3d runs (%5.1f%%) | Success: %3d | Failed: %3d\n" \
+        "$job" "$success" "$total" "$rate" "$success" "$failure"
+    fi
+  done | sort -k6 -nr
+  echo ""
+
+  if [[ ${#failure_patterns[@]} -gt 0 ]]; then
+    echo "=== Common Failure Patterns ==="
+    for pattern in "${!failure_patterns[@]}"; do
+      count=${failure_patterns[$pattern]}
+      printf "%-60s: %3d occurrences\n" "$pattern" "$count"
+    done | sort -k3 -nr
+    echo ""
+  fi
+} > "$REPORT_FILE"
+
+# Generate JSON output
+{
+  echo "{"
+  echo "  \"generated\": \"$(date -Iseconds)\","
+  echo "  \"total_runs\": $total_runs,"
+  echo "  \"total_success\": $total_success,"
+  echo "  "total_failure": $total_failure,"
+  if [[ $total_runs -gt 0 ]]; then
+    success_rate=$(echo "scale=2; ($total_success * 100) / $total_runs" | bc)
+    echo "  \"success_rate\": $success_rate,"
+  fi
+  echo "  \"workflows\": {"
+  first=true
+  for workflow in "${!workflow_total[@]}"; do
+    if [[ "$first" == "true" ]]; then
+      first=false
+    else
+      echo ","
+    fi
+    total=${workflow_total[$workflow]}
+    success=${workflow_success[$workflow]:-0}
+    failure=${workflow_failure[$workflow]:-0}
+    rate=$(echo "scale=2; ($success * 100) / $total" | bc)
+    printf "    \"%s\": {\"total\": %d, \"success\": %d, \"failure\": %d, \"rate\": %s}" \
+      "$workflow" "$total" "$success" "$failure" "$rate"
+  done
+  echo ""
+  echo "  }"
+  echo "}"
+} > "$JSON_FILE"
+
+echo "Report saved to: $REPORT_FILE"
+echo "JSON metrics saved to: $JSON_FILE"
+echo ""
+echo "=== Summary ==="
+cat "$REPORT_FILE" | head -20

scripts/wrappers/aitbc-agent-daemon-wrapper.py

@@ -29,7 +29,7 @@ exec_cmd = [
     "--address", "ait1d18e286fc0c12888aca94732b5507c8787af71a5",
     "--password-file", str(KEYSTORE_DIR / ".agent_daemon_password"),
     "--keystore-dir", str(KEYSTORE_DIR),
-    "--db-path", str(DATA_DIR / "chain.db"),
+    "--db-path", "/var/lib/aitbc/data/chain.db",
     "--rpc-url", "http://localhost:8006",
     "--poll-interval", "2",
     "--reply-message", "pong",

scripts/wrappers/aitbc-marketplace-wrapper.py

@@ -17,7 +17,7 @@ from aitbc import ENV_FILE, NODE_ENV_FILE, REPO_DIR, DATA_DIR, LOG_DIR
 # Set up environment using aitbc constants
 os.environ["AITBC_ENV_FILE"] = str(ENV_FILE)
 os.environ["AITBC_NODE_ENV_FILE"] = str(NODE_ENV_FILE)
-os.environ["PYTHONPATH"] = f"{REPO_DIR}/apps/marketplace/scripts:{REPO_DIR}/apps/marketplace/src:{REPO_DIR}/apps/coordinator-api/src"
+os.environ["PYTHONPATH"] = f"{REPO_DIR}:{REPO_DIR}/apps/marketplace/scripts:{REPO_DIR}/apps/marketplace/src:{REPO_DIR}/apps/coordinator-api/src:{REPO_DIR}/packages/py/aitbc-sdk/src:{REPO_DIR}/packages/py/aitbc-crypto/src"
 os.environ["DATA_DIR"] = str(DATA_DIR)
 os.environ["LOG_DIR"] = str(LOG_DIR)
 

systemd/aitbc-agent-daemon.service

@@ -18,6 +18,7 @@ Restart=always
 RestartSec=10
 StandardOutput=journal
 StandardError=journal
+SyslogIdentifier=AgentDaemon
 
 # Security settings
 NoNewPrivileges=true

systemd/aitbc-blockchain-event-bridge.service

@@ -14,6 +14,7 @@ Restart=always
 RestartSec=5
 StandardOutput=journal
 StandardError=journal
+SyslogIdentifier=EventBridge
 NoNewPrivileges=true
 PrivateTmp=true
 ProtectSystem=strict

systemd/aitbc-blockchain-rpc.service

@@ -16,6 +16,7 @@ Restart=always
 RestartSec=5
 StandardOutput=journal
 StandardError=journal
+SyslogIdentifier=BlockchainRPC
 
 [Install]
 WantedBy=multi-user.target

systemd/aitbc-coordinator-api.service

@@ -14,6 +14,7 @@ Restart=always
 RestartSec=5
 StandardOutput=journal
 StandardError=journal
+SyslogIdentifier=Coordinator
 
 # Allow database writes for SQLite WAL mode
 ProtectSystem=no

systemd/aitbc-exchange-api.service

@@ -10,6 +10,7 @@ Group=root
 WorkingDirectory=/opt/aitbc/apps/exchange
 EnvironmentFile=/etc/aitbc/.env
 EnvironmentFile=/etc/aitbc/node.env
+Environment="PYTHONPATH=/opt/aitbc"
 ExecStart=/opt/aitbc/venv/bin/python simple_exchange_api.py --port 8001
 ExecReload=/bin/kill -HUP $MAINPID
 Restart=always

systemd/aitbc-learning.service

@@ -8,7 +8,7 @@ Type=simple
 User=root
 Group=root
 WorkingDirectory=/opt/aitbc/apps/coordinator-api/
-Environment=PYTHONPATH=/opt/aitbc/apps/coordinator-api/src:/opt/aitbc/packages/py/aitbc-sdk/src:/opt/aitbc/packages/py/aitbc-crypto/src
+Environment=PYTHONPATH=/opt/aitbc:/opt/aitbc/apps/coordinator-api/src:/opt/aitbc/packages/py/aitbc-sdk/src:/opt/aitbc/packages/py/aitbc-crypto/src
 EnvironmentFile=/etc/aitbc/.env
 EnvironmentFile=/etc/aitbc/node.env
 ExecStart=/opt/aitbc/venv/bin/python -m uvicorn app.services.adaptive_learning_app:app --host 127.0.0.1 --port 8011

systemd/aitbc-marketplace.service

@@ -9,7 +9,7 @@ User=root
 Group=root
 WorkingDirectory=/opt/aitbc
 Environment=PATH=/usr/bin:/usr/local/bin:/usr/bin:/bin
-Environment=PYTHONPATH=/opt/aitbc/apps/marketplace/scripts:/opt/aitbc/apps/marketplace/src:/opt/aitbc/apps/coordinator-api/src
+Environment=PYTHONPATH=/opt/aitbc:/opt/aitbc/apps/marketplace/scripts:/opt/aitbc/apps/marketplace/src:/opt/aitbc/apps/coordinator-api/src:/opt/aitbc/packages/py/aitbc-sdk/src:/opt/aitbc/packages/py/aitbc-crypto/src
 EnvironmentFile=/etc/aitbc/.env
 EnvironmentFile=/etc/aitbc/node.env
 

systemd/aitbc-modality-optimization.service

@@ -5,11 +5,11 @@ Wants=aitbc-coordinator-api.service
 
 [Service]
 Type=simple
-User=debian
-Group=debian
+User=root
+Group=root
 WorkingDirectory=/opt/aitbc/apps/coordinator-api
 Environment=PATH=/opt/aitbc/venv/bin:/usr/bin
-Environment=PYTHONPATH=/opt/aitbc/apps/coordinator-api/src
+Environment=PYTHONPATH=/opt/aitbc:/opt/aitbc/apps/coordinator-api/src:/opt/aitbc/packages/py/aitbc-sdk/src:/opt/aitbc/packages/py/aitbc-crypto/src
 EnvironmentFile=/etc/aitbc/.env
 EnvironmentFile=/etc/aitbc/node.env
 ExecStart=/opt/aitbc/venv/bin/python -m uvicorn src.app.services.modality_optimization_app:app --host 127.0.0.1 --port 8021

systemd/aitbc-monitor.service

@@ -8,6 +8,7 @@ User=root
 Group=root
 WorkingDirectory=/opt/aitbc
 Environment=PATH=/usr/bin:/usr/local/bin:/usr/bin:/bin
+Environment=PYTHONPATH=/opt/aitbc:/opt/aitbc/packages/py/aitbc-sdk/src:/opt/aitbc/packages/py/aitbc-crypto/src
 EnvironmentFile=/etc/aitbc/.env
 EnvironmentFile=/etc/aitbc/node.env
 

systemd/aitbc-multimodal.service

@@ -5,11 +5,11 @@ Wants=aitbc-coordinator-api.service
 
 [Service]
 Type=simple
-User=debian
-Group=debian
+User=root
+Group=root
 WorkingDirectory=/opt/aitbc/apps/coordinator-api
 Environment=PATH=/opt/aitbc/venv/bin:/usr/bin
-Environment=PYTHONPATH=/opt/aitbc/apps/coordinator-api/src
+Environment=PYTHONPATH=/opt/aitbc:/opt/aitbc/apps/coordinator-api/src:/opt/aitbc/packages/py/aitbc-sdk/src:/opt/aitbc/packages/py/aitbc-crypto/src
 EnvironmentFile=/etc/aitbc/.env
 EnvironmentFile=/etc/aitbc/node.env
 ExecStart=/opt/aitbc/venv/bin/python -m uvicorn src.app.services.multimodal_app:app --host 127.0.0.1 --port 8020

systemd/aitbc-openclaw.service

@@ -5,11 +5,11 @@ Wants=aitbc-coordinator-api.service
 
 [Service]
 Type=simple
-User=debian
-Group=debian
+User=root
+Group=root
 WorkingDirectory=/opt/aitbc/apps/coordinator-api
 Environment=PATH=/opt/aitbc/venv/bin:/usr/bin
-Environment=PYTHONPATH=/opt/aitbc/apps/coordinator-api/src
+Environment=PYTHONPATH=/opt/aitbc:/opt/aitbc/apps/coordinator-api/src:/opt/aitbc/packages/py/aitbc-sdk/src:/opt/aitbc/packages/py/aitbc-crypto/src
 EnvironmentFile=/etc/aitbc/.env
 EnvironmentFile=/etc/aitbc/node.env
 ExecStart=/opt/aitbc/venv/bin/python -m uvicorn src.app.routers.openclaw_enhanced_app:app --host 127.0.0.1 --port 8014