diff --git a/scripts/SCRIPTS_ORGANIZATION.md b/scripts/SCRIPTS_ORGANIZATION.md new file mode 100644 index 00000000..83e059a9 --- /dev/null +++ b/scripts/SCRIPTS_ORGANIZATION.md @@ -0,0 +1,161 @@ +# Scripts Directory Organization + +## Overview + +The AITBC scripts directory has been reorganized into functional categories for better maintainability and navigation. + +## Directory Structure + +### 📁 github/ +GitHub and Git-related operations +- `all-prs-merged-summary.md` - Summary of merged pull requests +- `gitea-changes-review.md` - Gitea repository changes review +- `github-push-ready-summary.md` - GitHub push readiness summary +- `pr40-resolution-complete.md` - PR #40 resolution documentation +- `solve-github-prs.sh` - GitHub PR resolution script +- `solve-prs-with-poetry.sh` - PR resolution with Poetry dependency management + +### 📁 sync/ +Synchronization and data replication +- `bulk_sync.sh` - Bulk synchronization operations +- `fast_bulk_sync.sh` - Fast bulk synchronization +- `sync_detector.sh` - Synchronization detection and monitoring +- `sync-systemd.sh` - SystemD service synchronization + +### 📁 security/ +Security and audit operations +- `security_audit.py` - Comprehensive security audit script +- `security_monitor.sh` - Security monitoring and alerting + +### 📁 monitoring/ +System and service monitoring +- `health_check.sh` - System health checks +- `log_monitor.sh` - Log file monitoring +- `network_monitor.sh` - Network monitoring +- `monitor-prs.py` - Pull request monitoring +- `nightly_health_check.sh` - Nightly health check automation +- `production_monitoring.sh` - Production environment monitoring + +### 📁 maintenance/ +System maintenance and cleanup +- `cleanup-root-directory.sh` - Root directory cleanup +- `final-cleanup.sh` - Final cleanup operations +- `performance_tune.sh` - Performance tuning +- `weekly_maintenance.sh` - Weekly maintenance tasks + +### 📁 deployment/ +Deployment and provisioning +- `build-release.sh` - Release building automation +- `deploy_openclaw_dao.py` - OpenClaw DAO deployment +- `provision_node.sh` - Node provisioning +- `complete-agent-protocols.sh` - Complete agent protocols deployment +- `deploy.sh` - General deployment script +- `production-deploy.sh` - Production deployment +- `implement-agent-protocols.sh` - Agent protocols implementation +- `implement-ai-trading-analytics.sh` - AI trading analytics implementation + +### 📁 testing/ +Testing and quality assurance +- `comprehensive_e2e_test_fixed.py` - Comprehensive E2E testing +- `test_workflow.sh` - Workflow testing +- `debug-services.sh` - Service debugging +- `drain_test.py` - Drain testing +- `qa-cycle.py` - QA cycle automation +- `quick_test.py` - Quick testing +- `run_all_tests.sh` - All tests runner +- `run_test.py` - Test runner +- `scalability_validation.py` - Scalability validation +- `simple-test.sh` - Simple testing +- `test-all-services.sh` - All services testing +- `test-permissions.sh` - Permission testing + +### 📁 utils/ +Utility scripts and helpers +- `link-systemd.sh` - SystemD linking +- `manage-services.sh` - Service management +- `requirements_migrator.py` - Requirements migration +- `setup.sh` - System setup +- `workspace-manager.sh` - Workspace management +- `check-aitbc-services.sh` - AITBC services checking +- `check-documentation-requirements.sh` - Documentation requirements checking +- `claim-task.py` - Task claiming +- `clean-sudoers-fix.sh` - Sudoers cleanup +- `cleanup_fake_gpus_db.py` - Fake GPU database cleanup +- `cleanup_fake_gpus.py` - Fake GPU cleanup +- `complete-permission-fix.sh` - Complete permission fixes +- `create_structured_issue.py` - Structured issue creation +- `deploy_enhanced_genesis.py` - Enhanced genesis deployment +- `detect-aitbc-user.sh` - AITBC user detection +- `end_to_end_workflow.py` - End-to-end workflow +- `final-sudoers-fix.sh` - Final sudoers fixes +- `fix_database_persistence.py` - Database persistence fixes +- `fix_gpu_release.py` - GPU release fixes +- `fix-permissions.sh` - Permission fixes +- `fix-startup-issues.sh` - Startup issue fixes +- `fix-sudoers-syntax.sh` - Sudoers syntax fixes +- `generate-api-keys.py` - API key generation +- `git_helper.sh` - Git helper functions +- `git-pre-commit-hook.sh` - Git pre-commit hook +- `init_production_genesis.py` - Production genesis initialization +- `keystore.py` - Keystore management +- `organize-dev-logs.sh` - Development logs organization +- `pr-conflict-resolution-summary.md` - PR conflict resolution summary +- `quick-fix.sh` - Quick fixes +- `run_comprehensive_planning_cleanup.py` - Comprehensive planning cleanup +- `run_documentation_conversion.sh` - Documentation conversion +- `run_enhanced_planning_analysis.py` - Enhanced planning analysis +- `run_enhanced_planning_cleanup.py` - Enhanced planning cleanup +- `run_master_planning_cleanup.py` - Master planning cleanup +- `run_planning_cleanup.py` - Planning cleanup +- `run_production_node.py` - Production node runner +- `security_hardening.sh` - Security hardening +- `setup-dev-permissions.sh` - Development permissions setup +- `setup_production.py` - Production setup +- `sync.sh` - Synchronization +- `update-docs.sh` - Documentation updates +- `validate-requirements.sh` - Requirements validation +- `verify-codebase-update.sh` - Codebase update verification +- `verify-production-advanced.sh` - Advanced production verification + +## Usage + +### Quick Reference +```bash +# GitHub operations +./scripts/github/solve-github-prs.sh + +# System monitoring +./scripts/monitoring/health_check.sh + +# Deployment +./scripts/deployment/build-release.sh + +# Testing +./scripts/testing/comprehensive_e2e_test_fixed.py + +# Security +./scripts/security/security_audit.py + +# Maintenance +./scripts/maintenance/weekly_maintenance.sh +``` + +### Finding Scripts +Use this README to locate the appropriate script for your needs: +1. Identify the category (GitHub, sync, security, etc.) +2. Navigate to the relevant directory +3. Run the appropriate script + +## Benefits + +1. **Better Organization**: Scripts grouped by functionality +2. **Easier Navigation**: Clear directory structure +3. **Maintainability**: Related scripts grouped together +4. **Scalability**: Easy to add new scripts to appropriate categories +5. **Documentation**: Clear descriptions of each script's purpose + +## Maintenance + +- Add new scripts to appropriate functional directories +- Update this README when adding new categories or scripts +- Keep script descriptions current and accurate diff --git a/scripts/build-release.sh b/scripts/deployment/build-release.sh similarity index 100% rename from scripts/build-release.sh rename to scripts/deployment/build-release.sh diff --git a/scripts/deploy_openclaw_dao.py b/scripts/deployment/deploy_openclaw_dao.py similarity index 100% rename from scripts/deploy_openclaw_dao.py rename to scripts/deployment/deploy_openclaw_dao.py diff --git a/scripts/provision_node.sh b/scripts/deployment/provision_node.sh similarity index 100% rename from scripts/provision_node.sh rename to scripts/deployment/provision_node.sh diff --git a/scripts/all-prs-merged-summary.md b/scripts/github/all-prs-merged-summary.md similarity index 100% rename from scripts/all-prs-merged-summary.md rename to scripts/github/all-prs-merged-summary.md diff --git a/scripts/gitea-changes-review.md b/scripts/github/gitea-changes-review.md similarity index 100% rename from scripts/gitea-changes-review.md rename to scripts/github/gitea-changes-review.md diff --git a/scripts/github-push-ready-summary.md b/scripts/github/github-push-ready-summary.md similarity index 100% rename from scripts/github-push-ready-summary.md rename to scripts/github/github-push-ready-summary.md diff --git a/scripts/pr40-resolution-complete.md b/scripts/github/pr40-resolution-complete.md similarity index 100% rename from scripts/pr40-resolution-complete.md rename to scripts/github/pr40-resolution-complete.md diff --git a/scripts/solve-github-prs.sh b/scripts/github/solve-github-prs.sh similarity index 100% rename from scripts/solve-github-prs.sh rename to scripts/github/solve-github-prs.sh diff --git a/scripts/solve-prs-with-poetry.sh b/scripts/github/solve-prs-with-poetry.sh similarity index 100% rename from scripts/solve-prs-with-poetry.sh rename to scripts/github/solve-prs-with-poetry.sh diff --git a/scripts/cleanup-root-directory.sh b/scripts/maintenance/cleanup-root-directory.sh similarity index 100% rename from scripts/cleanup-root-directory.sh rename to scripts/maintenance/cleanup-root-directory.sh diff --git a/scripts/final-cleanup.sh b/scripts/maintenance/final-cleanup.sh similarity index 100% rename from scripts/final-cleanup.sh rename to scripts/maintenance/final-cleanup.sh diff --git a/scripts/performance_tune.sh b/scripts/maintenance/performance_tune.sh similarity index 100% rename from scripts/performance_tune.sh rename to scripts/maintenance/performance_tune.sh diff --git a/scripts/weekly_maintenance.sh b/scripts/maintenance/weekly_maintenance.sh similarity index 100% rename from scripts/weekly_maintenance.sh rename to scripts/maintenance/weekly_maintenance.sh diff --git a/scripts/health_check.sh b/scripts/monitoring/health_check.sh similarity index 100% rename from scripts/health_check.sh rename to scripts/monitoring/health_check.sh diff --git a/scripts/log_monitor.sh b/scripts/monitoring/log_monitor.sh similarity index 100% rename from scripts/log_monitor.sh rename to scripts/monitoring/log_monitor.sh diff --git a/scripts/network_monitor.sh b/scripts/monitoring/network_monitor.sh similarity index 100% rename from scripts/network_monitor.sh rename to scripts/monitoring/network_monitor.sh diff --git a/scripts/security_audit.py b/scripts/security/security_audit.py similarity index 100% rename from scripts/security_audit.py rename to scripts/security/security_audit.py diff --git a/scripts/security_monitor.sh b/scripts/security/security_monitor.sh similarity index 100% rename from scripts/security_monitor.sh rename to scripts/security/security_monitor.sh diff --git a/scripts/bulk_sync.sh b/scripts/sync/bulk_sync.sh similarity index 100% rename from scripts/bulk_sync.sh rename to scripts/sync/bulk_sync.sh diff --git a/scripts/fast_bulk_sync.sh b/scripts/sync/fast_bulk_sync.sh similarity index 100% rename from scripts/fast_bulk_sync.sh rename to scripts/sync/fast_bulk_sync.sh diff --git a/scripts/sync-systemd.sh b/scripts/sync/sync-systemd.sh similarity index 100% rename from scripts/sync-systemd.sh rename to scripts/sync/sync-systemd.sh diff --git a/scripts/sync_detector.sh b/scripts/sync/sync_detector.sh similarity index 100% rename from scripts/sync_detector.sh rename to scripts/sync/sync_detector.sh diff --git a/scripts/comprehensive_e2e_test_fixed.py b/scripts/testing/comprehensive_e2e_test_fixed.py similarity index 100% rename from scripts/comprehensive_e2e_test_fixed.py rename to scripts/testing/comprehensive_e2e_test_fixed.py diff --git a/scripts/test_workflow.sh b/scripts/testing/test_workflow.sh similarity index 100% rename from scripts/test_workflow.sh rename to scripts/testing/test_workflow.sh diff --git a/scripts/link-systemd.sh b/scripts/utils/link-systemd.sh similarity index 100% rename from scripts/link-systemd.sh rename to scripts/utils/link-systemd.sh diff --git a/scripts/manage-services.sh b/scripts/utils/manage-services.sh similarity index 100% rename from scripts/manage-services.sh rename to scripts/utils/manage-services.sh diff --git a/scripts/requirements_migrator.py b/scripts/utils/requirements_migrator.py similarity index 100% rename from scripts/requirements_migrator.py rename to scripts/utils/requirements_migrator.py diff --git a/scripts/utils/setup.sh b/scripts/utils/setup.sh new file mode 100755 index 00000000..586938ff --- /dev/null +++ b/scripts/utils/setup.sh @@ -0,0 +1,342 @@ +#!/bin/bash +# OpenClaw AITBC Integration Setup & Health Check +# Field-tested setup and management for OpenClaw + AITBC integration +# Version: 5.0 — Updated 2026-03-30 with AI operations and advanced coordination + +set -e + +AITBC_DIR="/opt/aitbc" +AITBC_CLI="$AITBC_DIR/aitbc-cli" +DATA_DIR="/var/lib/aitbc/data" +ENV_FILE="/etc/aitbc/.env" +GENESIS_RPC="http://localhost:8006" +FOLLOWER_RPC="http://10.1.223.40:8006" +WALLET_PASSWORD="123" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } +log_success() { echo -e "${GREEN}[OK]${NC} $1"; } +log_warning() { echo -e "${YELLOW}[WARN]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } + +# ── Prerequisites ────────────────────────────────────────────── +check_prerequisites() { + log_info "Checking prerequisites..." + local fail=0 + + command -v openclaw &>/dev/null && log_success "OpenClaw CLI found" || { log_error "OpenClaw not found"; fail=1; } + [ -x "$AITBC_CLI" ] && log_success "AITBC CLI found" || { log_error "AITBC CLI not found at $AITBC_CLI"; fail=1; } + + if curl -sf http://localhost:8006/health &>/dev/null; then + log_success "Genesis RPC (localhost:8006) healthy" + else + log_warning "Genesis RPC not responding — try: sudo systemctl start aitbc-blockchain-rpc.service" + fi + + if ssh aitbc1 'curl -sf http://localhost:8006/health' &>/dev/null; then + log_success "Follower RPC (aitbc1:8006) healthy" + else + log_warning "Follower RPC not responding — check aitbc1 services" + fi + + [ -d "$DATA_DIR/ait-mainnet" ] && log_success "Data dir $DATA_DIR/ait-mainnet exists" || log_warning "Data dir missing" + [ -f "$ENV_FILE" ] && log_success "Env file $ENV_FILE exists" || log_warning "Env file missing" + + [ $fail -eq 0 ] && log_success "Prerequisites satisfied" || { log_error "Prerequisites check failed"; exit 1; } +} + +# ── OpenClaw Agent Test ──────────────────────────────────────── +test_agent_communication() { + log_info "Testing OpenClaw agent communication..." + # IMPORTANT: use --message (long form), not -m + local SESSION_ID="health-$(date +%s)" + local GENESIS_HEIGHT + GENESIS_HEIGHT=$(curl -sf http://localhost:8006/rpc/head | jq -r '.height // "unknown"') + + openclaw agent --agent main --session-id "$SESSION_ID" \ + --message "AITBC integration health check. Genesis height: $GENESIS_HEIGHT. Report status." \ + --thinking low \ + && log_success "Agent communication working" \ + || log_warning "Agent communication failed (non-fatal)" +} + +# ── Blockchain Status ────────────────────────────────────────── +show_status() { + log_info "=== OpenClaw AITBC Integration Status ===" + + echo "" + echo "OpenClaw:" + openclaw --version 2>/dev/null || echo " (not available)" + + echo "" + echo "Genesis Node (aitbc):" + curl -sf http://localhost:8006/rpc/head | jq '{height, hash: .hash[0:18], timestamp}' 2>/dev/null \ + || echo " RPC not responding" + + echo "" + echo "Follower Node (aitbc1):" + ssh aitbc1 'curl -sf http://localhost:8006/rpc/head' 2>/dev/null | jq '{height, hash: .hash[0:18], timestamp}' \ + || echo " RPC not responding" + + echo "" + echo "Wallets (aitbc):" + cd "$AITBC_DIR" && source venv/bin/activate && ./aitbc-cli list 2>/dev/null || echo " CLI error" + + echo "" + echo "Wallets (aitbc1):" + ssh aitbc1 "cd $AITBC_DIR && source venv/bin/activate && ./aitbc-cli list" 2>/dev/null || echo " CLI error" + + echo "" + echo "Services (aitbc):" + systemctl is-active aitbc-blockchain-node.service 2>/dev/null | sed 's/^/ node: /' + systemctl is-active aitbc-blockchain-rpc.service 2>/dev/null | sed 's/^/ rpc: /' + + echo "" + echo "Services (aitbc1):" + ssh aitbc1 'systemctl is-active aitbc-blockchain-node.service' 2>/dev/null | sed 's/^/ node: /' + ssh aitbc1 'systemctl is-active aitbc-blockchain-rpc.service' 2>/dev/null | sed 's/^/ rpc: /' + + echo "" + echo "Data Directory:" + ls -lh "$DATA_DIR/ait-mainnet/" 2>/dev/null | head -5 || echo " not found" +} + +# ── Run Integration Test ─────────────────────────────────────── +run_integration_test() { + log_info "Running integration test..." + + local pass=0 total=0 + + # Test 1: RPC health + total=$((total+1)) + curl -sf http://localhost:8006/health &>/dev/null && { log_success "RPC health OK"; pass=$((pass+1)); } || log_error "RPC health FAIL" + + # Test 2: CLI works + total=$((total+1)) + cd "$AITBC_DIR" && source venv/bin/activate && ./aitbc-cli list &>/dev/null && { log_success "CLI OK"; pass=$((pass+1)); } || log_error "CLI FAIL" + + # Test 3: Cross-node SSH + total=$((total+1)) + ssh aitbc1 'echo ok' &>/dev/null && { log_success "SSH to aitbc1 OK"; pass=$((pass+1)); } || log_error "SSH FAIL" + + # Test 4: Agent communication + total=$((total+1)) + openclaw agent --agent main --message "ping" --thinking minimal &>/dev/null && { log_success "Agent OK"; pass=$((pass+1)); } || log_warning "Agent FAIL (non-fatal)" + + echo "" + log_info "Results: $pass/$total passed" +} + +# ── Main ─────────────────────────────────────────────────────── +main() { + case "${1:-status}" in + setup) + check_prerequisites + test_agent_communication + show_status + log_success "Setup verification complete" + ;; + test) + run_integration_test + ;; + status) + show_status + ;; + ai-setup) + setup_ai_operations + ;; + ai-test) + test_ai_operations + ;; + comprehensive) + show_comprehensive_status + ;; + help) + echo "Usage: $0 {setup|test|status|ai-setup|ai-test|comprehensive|help}" + echo " setup — Verify prerequisites and test agent communication" + echo " test — Run integration tests" + echo " status — Show current multi-node status" + echo " ai-setup — Setup AI operations and agents" + echo " ai-test — Test AI operations functionality" + echo " comprehensive — Show comprehensive status including AI operations" + echo " help — Show this help" + ;; + *) + log_error "Unknown command: $1" + main help + exit 1 + ;; + esac +} + +# ── AI Operations Setup ─────────────────────────────────────────── +setup_ai_operations() { + log_info "Setting up AI operations..." + + cd "$AITBC_DIR" + source venv/bin/activate + + # Create AI inference agent + log_info "Creating AI inference agent..." + if ./aitbc-cli agent create --name "ai-inference-worker" \ + --description "Specialized agent for AI inference tasks" \ + --verification full; then + log_success "AI inference agent created" + else + log_warning "AI inference agent creation failed" + fi + + # Allocate GPU resources + log_info "Allocating GPU resources..." + if ./aitbc-cli resource allocate --agent-id "ai-inference-worker" \ + --gpu 1 --memory 8192 --duration 3600; then + log_success "GPU resources allocated" + else + log_warning "GPU resource allocation failed" + fi + + # Create AI service marketplace listing + log_info "Creating AI marketplace listing..." + if ./aitbc-cli marketplace --action create \ + --name "AI Image Generation" \ + --type ai-inference \ + --price 50 \ + --wallet genesis-ops \ + --description "Generate high-quality images from text prompts"; then + log_success "AI marketplace listing created" + else + log_warning "AI marketplace listing creation failed" + fi + + # Setup follower AI operations + log_info "Setting up follower AI operations..." + if ssh aitbc1 "cd $AITBC_DIR && source venv/bin/activate && \ + ./aitbc-cli agent create --name 'ai-training-agent' \ + --description 'Specialized agent for AI model training' \ + --verification full && \ + ./aitbc-cli resource allocate --agent-id 'ai-training-agent' \ + --cpu 4 --memory 16384 --duration 7200"; then + log_success "Follower AI operations setup completed" + else + log_warning "Follower AI operations setup failed" + fi + + log_success "AI operations setup completed" +} + +# ── AI Operations Test ────────────────────────────────────────────── +test_ai_operations() { + log_info "Testing AI operations..." + + cd "$AITBC_DIR" + source venv/bin/activate + + # Test AI job submission + log_info "Testing AI job submission..." + if ./aitbc-cli ai-submit --wallet genesis-ops \ + --type inference \ + --prompt "Test image generation" \ + --payment 10; then + log_success "AI job submission test passed" + else + log_warning "AI job submission test failed" + fi + + # Test smart contract messaging + log_info "Testing smart contract messaging..." + TOPIC_ID=$(curl -s -X POST "$GENESIS_RPC/rpc/messaging/topics/create" \ + -H "Content-Type: application/json" \ + -d '{"agent_id": "test-agent", "agent_address": "ait158ec7a0713f30ccfb1aac6bfbab71f36271c5871", "title": "Test Topic", "description": "Test coordination"}' | \ + jq -r '.topic_id // "error"') + + if [ "$TOPIC_ID" != "error" ] && [ -n "$TOPIC_ID" ]; then + log_success "Smart contract messaging test passed - Topic: $TOPIC_ID" + else + log_warning "Smart contract messaging test failed" + fi + + log_success "AI operations testing completed" +} + +# ── Comprehensive Status ─────────────────────────────────────────── +show_comprehensive_status() { + log_info "Comprehensive AITBC + OpenClaw + AI Operations Status" + echo "" + + # Basic status + show_multi_node_status + echo "" + + # AI operations status + log_info "AI Operations Status:" + cd "$AITBC_DIR" + source venv/bin/activate + + # Check AI agents + AI_AGENTS=$(./aitbc-cli agent list 2>/dev/null | grep -c "agent_" || echo "0") + echo " AI Agents Created: $AI_AGENTS" + + # Check resource allocation + if ./aitbc-cli resource status &>/dev/null; then + echo " Resource Management: Operational" + else + echo " Resource Management: Not operational" + fi + + # Check marketplace + if ./aitbc-cli marketplace --action list &>/dev/null; then + echo " AI Marketplace: Operational" + else + echo " AI Marketplace: Not operational" + fi + + # Check smart contract messaging + if curl -s "$GENESIS_RPC/rpc/messaging/topics" &>/dev/null; then + TOPICS_COUNT=$(curl -s "$GENESIS_RPC/rpc/messaging/topics" | jq '.total_topics // 0' 2>/dev/null || echo "0") + echo " Smart Contract Messaging: Operational ($TOPICS_COUNT topics)" + else + echo " Smart Contract Messaging: Not operational" + fi + + echo "" + log_info "Health Check:" + if [ -f /tmp/aitbc1_heartbeat.py ]; then + python3 /tmp/aitbc1_heartbeat.py + else + log_warning "Heartbeat script not found" + fi +} + + ai-setup) + setup_ai_operations + ;; + ai-test) + test_ai_operations + ;; + comprehensive) + show_comprehensive_status + ;; + help) + echo "Usage: $0 {setup|test|status|ai-setup|ai-test|comprehensive|help}" + echo " setup — Verify prerequisites and test agent communication" + echo " test — Run integration tests" + echo " status — Show current multi-node status" + echo " ai-setup — Setup AI operations and agents" + echo " ai-test — Test AI operations functionality" + echo " comprehensive — Show comprehensive status including AI operations" + echo " help — Show this help" + ;; + *) + log_error "Unknown command: $1" + main help + exit 1 + ;; + esac +} + +main "$@" diff --git a/scripts/workspace-manager.sh b/scripts/utils/workspace-manager.sh similarity index 100% rename from scripts/workspace-manager.sh rename to scripts/utils/workspace-manager.sh