From 3f98f3f7bf19726c91b8816246eed39a65e1b934 Mon Sep 17 00:00:00 2001 From: aitbc Date: Sat, 18 Apr 2026 12:28:19 +0200 Subject: [PATCH] ci: add service host discovery and strict policy docs validation - Add multi-candidate host discovery (localhost, host.docker.internal, gateway) in api-endpoint-tests - Pass discovered service host via AITBC_API_HOST environment variable to test script - Update test_api_endpoints.py to use AITBC_API_HOST for all service URLs - Add validate-policies-strict job to docs-validation workflow for policy Markdown files - Add job names to package-tests matrix for better CI output clarity - Add --import --- .gitea/workflows/api-endpoint-tests.yml | 38 ++++++- .gitea/workflows/docs-validation.yml | 40 +++++++ .gitea/workflows/package-tests.yml | 2 + .gitea/workflows/production-tests.yml | 125 +++++++++++++++++++++ .gitea/workflows/python-tests.yml | 3 +- cli/utils/__init__.py | 5 + packages/py/aitbc-agent-sdk/pyproject.toml | 8 +- packages/py/aitbc-crypto/pyproject.toml | 8 +- packages/py/aitbc-sdk/pyproject.toml | 7 +- scripts/ci/test_api_endpoints.py | 19 ++-- 10 files changed, 239 insertions(+), 16 deletions(-) create mode 100644 .gitea/workflows/production-tests.yml diff --git a/.gitea/workflows/api-endpoint-tests.yml b/.gitea/workflows/api-endpoint-tests.yml index e61194a3..46675398 100644 --- a/.gitea/workflows/api-endpoint-tests.yml +++ b/.gitea/workflows/api-endpoint-tests.yml @@ -43,22 +43,51 @@ jobs: - name: Wait for services run: | echo "Waiting for AITBC services..." + gateway_host=$(ip route 2>/dev/null | awk '/default/ {print $3; exit}') + host_candidates=(localhost host.docker.internal) + if [[ -n "$gateway_host" ]]; then + host_candidates+=("$gateway_host") + fi + + service_host="" + for candidate in "${host_candidates[@]}"; do + code=$(curl -so /dev/null -w '%{http_code}' "http://$candidate:8000/health" 2>/dev/null) || code=0 + if [ "$code" -gt 0 ] && [ "$code" -lt 600 ]; then + service_host="$candidate" + break + fi + + code=$(curl -so /dev/null -w '%{http_code}' "http://$candidate:8000/v1/health" 2>/dev/null) || code=0 + if [ "$code" -gt 0 ] && [ "$code" -lt 600 ]; then + service_host="$candidate" + break + fi + done + + if [[ -z "$service_host" ]]; then + echo "❌ Could not find a reachable API host" + exit 1 + fi + + echo "$service_host" > /var/lib/aitbc-workspaces/api-tests/service_host + echo "Using service host: $service_host" + for port in 8000 8001 8003 8006; do port_ready=0 for i in $(seq 1 15); do - code=$(curl -so /dev/null -w '%{http_code}' "http://localhost:$port/health" 2>/dev/null) || code=0 + code=$(curl -so /dev/null -w '%{http_code}' "http://$service_host:$port/health" 2>/dev/null) || code=0 if [ "$code" -gt 0 ] && [ "$code" -lt 600 ]; then echo "✅ Port $port ready (HTTP $code)" port_ready=1 break fi - code=$(curl -so /dev/null -w '%{http_code}' "http://localhost:$port/api/health" 2>/dev/null) || code=0 + code=$(curl -so /dev/null -w '%{http_code}' "http://$service_host:$port/api/health" 2>/dev/null) || code=0 if [ "$code" -gt 0 ] && [ "$code" -lt 600 ]; then echo "✅ Port $port ready (HTTP $code)" port_ready=1 break fi - code=$(curl -so /dev/null -w '%{http_code}' "http://localhost:$port/" 2>/dev/null) || code=0 + code=$(curl -so /dev/null -w '%{http_code}' "http://$service_host:$port/" 2>/dev/null) || code=0 if [ "$code" -gt 0 ] && [ "$code" -lt 600 ]; then echo "✅ Port $port ready (HTTP $code)" port_ready=1 @@ -76,7 +105,8 @@ jobs: - name: Run API endpoint tests run: | cd /var/lib/aitbc-workspaces/api-tests/repo - venv/bin/python scripts/ci/test_api_endpoints.py + service_host=$(cat /var/lib/aitbc-workspaces/api-tests/service_host) + AITBC_API_HOST="$service_host" venv/bin/python scripts/ci/test_api_endpoints.py echo "✅ API endpoint tests completed" - name: Cleanup diff --git a/.gitea/workflows/docs-validation.yml b/.gitea/workflows/docs-validation.yml index 0ecf831b..a95579ae 100644 --- a/.gitea/workflows/docs-validation.yml +++ b/.gitea/workflows/docs-validation.yml @@ -101,3 +101,43 @@ jobs: - name: Cleanup if: always() run: rm -rf /var/lib/aitbc-workspaces/docs-validation + + validate-policies-strict: + runs-on: debian + timeout-minutes: 10 + + steps: + - name: Clone repository + run: | + WORKSPACE="/var/lib/aitbc-workspaces/docs-validation-policies" + rm -rf "$WORKSPACE" + mkdir -p "$WORKSPACE" + cd "$WORKSPACE" + git clone --depth 1 http://gitea.bubuit.net:3000/oib/aitbc.git repo + + - name: Install markdownlint + run: | + npm install -g markdownlint-cli + + - name: Strict lint policy docs + run: | + cd /var/lib/aitbc-workspaces/docs-validation-policies/repo + + # Ensure standard directories exist + mkdir -p /var/lib/aitbc/data /var/lib/aitbc/keystore /etc/aitbc /var/log/aitbc + + shopt -s globstar nullglob + mapfile -t targets < <(printf '%s\n' docs/policies/*.md docs/policies/**/*.md | awk '!seen[$0]++') + + if [[ ${#targets[@]} -eq 0 ]]; then + echo "❌ No policy Markdown files found" + exit 1 + fi + + echo "Strict docs scope: ${#targets[@]} policy Markdown files" + markdownlint "${targets[@]}" + echo "✅ Policy docs lint passed" + + - name: Cleanup + if: always() + run: rm -rf /var/lib/aitbc-workspaces/docs-validation-policies diff --git a/.gitea/workflows/package-tests.yml b/.gitea/workflows/package-tests.yml index 44726f43..349f48b0 100644 --- a/.gitea/workflows/package-tests.yml +++ b/.gitea/workflows/package-tests.yml @@ -17,6 +17,7 @@ concurrency: jobs: test-python-packages: + name: Python package - ${{ matrix.package.name }} runs-on: debian timeout-minutes: 15 @@ -101,6 +102,7 @@ jobs: run: rm -rf "/var/lib/aitbc-workspaces/pkg-${{ matrix.package.name }}" test-javascript-packages: + name: JavaScript package - ${{ matrix.package.name }} runs-on: debian timeout-minutes: 15 diff --git a/.gitea/workflows/production-tests.yml b/.gitea/workflows/production-tests.yml new file mode 100644 index 00000000..ef9f0523 --- /dev/null +++ b/.gitea/workflows/production-tests.yml @@ -0,0 +1,125 @@ +name: Production Tests + +on: + push: + branches: [main, develop] + paths: + - 'tests/production/**' + - 'apps/agent-coordinator/**' + - '.gitea/workflows/production-tests.yml' + pull_request: + branches: [main, develop] + workflow_dispatch: + +concurrency: + group: production-tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + test-production: + name: Production Integration Tests + runs-on: debian + timeout-minutes: 20 + + steps: + - name: Clone repository + run: | + WORKSPACE="/var/lib/aitbc-workspaces/production-tests" + rm -rf "$WORKSPACE" + mkdir -p "$WORKSPACE" + cd "$WORKSPACE" + git clone --depth 1 http://gitea.bubuit.net:3000/oib/aitbc.git repo + + - name: Setup test environment + run: | + cd /var/lib/aitbc-workspaces/production-tests/repo + python3 -m venv venv + venv/bin/pip install -q \ + pytest \ + pytest-asyncio \ + pytest-timeout \ + requests \ + pyjwt \ + fastapi \ + uvicorn[standard] \ + redis \ + bcrypt \ + websockets \ + numpy \ + psutil \ + prometheus-client + + # Ensure standard directories exist + mkdir -p /var/lib/aitbc/data /var/lib/aitbc/keystore /etc/aitbc /var/log/aitbc + + - name: Start Redis + run: | + redis-server --daemonize yes --port 6379 + sleep 2 + redis-cli ping || exit 1 + echo "✅ Redis started" + + - name: Start agent coordinator + run: | + cd /var/lib/aitbc-workspaces/production-tests/repo + source venv/bin/activate + export PYTHONPATH="apps/agent-coordinator/src:$PYTHONPATH" + + # Start agent coordinator in background + nohup uvicorn src.app.main:app \ + --host 0.0.0.0 \ + --port 9001 \ + --log-level info \ + > /tmp/agent-coordinator.log 2>&1 & + + echo $! > /tmp/agent-coordinator.pid + echo "✅ Agent coordinator started (PID: $(cat /tmp/agent-coordinator.pid))" + + - name: Wait for agent coordinator ready + run: | + echo "Waiting for agent coordinator on port 9001..." + for i in $(seq 1 30); do + code=$(curl -so /dev/null -w '%{http_code}' "http://localhost:9001/health" 2>/dev/null) || code=0 + if [ "$code" -ge 200 ] && [ "$code" -lt 600 ]; then + echo "✅ Agent coordinator ready (HTTP $code)" + exit 0 + fi + sleep 2 + done + echo "❌ Agent coordinator not ready" + cat /tmp/agent-coordinator.log + exit 1 + + - name: Run production tests + run: | + cd /var/lib/aitbc-workspaces/production-tests/repo + source venv/bin/activate + export PYTHONPATH="apps/agent-coordinator/src:$PYTHONPATH" + + pytest tests/production/ \ + -v \ + --tb=short \ + --timeout=30 \ + --import-mode=importlib \ + -k "not test_error_handling" + + echo "✅ Production tests completed" + + - name: Agent coordinator logs + if: always() + run: | + if [ -f /tmp/agent-coordinator.log ]; then + echo "=== Agent Coordinator Logs ===" + cat /tmp/agent-coordinator.log + fi + + - name: Cleanup + if: always() + run: | + if [ -f /tmp/agent-coordinator.pid ]; then + kill $(cat /tmp/agent-coordinator.pid) 2>/dev/null || true + rm -f /tmp/agent-coordinator.pid + fi + pkill -f "uvicorn src.app.main:app" 2>/dev/null || true + redis-cli shutdown 2>/dev/null || true + rm -rf /var/lib/aitbc-workspaces/production-tests diff --git a/.gitea/workflows/python-tests.yml b/.gitea/workflows/python-tests.yml index 6ed0b21e..f4912161 100644 --- a/.gitea/workflows/python-tests.yml +++ b/.gitea/workflows/python-tests.yml @@ -78,7 +78,8 @@ jobs: apps/wallet/tests/ \ packages/py/aitbc-crypto/tests/ \ packages/py/aitbc-sdk/tests/ \ - --tb=short -q --timeout=30 \ + --tb=short -q --timeout=30 --import-mode=importlib \ + --ignore=tests/production \ --ignore=apps/coordinator-api/tests/test_confidential*.py echo "✅ Python tests completed" diff --git a/cli/utils/__init__.py b/cli/utils/__init__.py index 0e93df1b..ddf877ae 100755 --- a/cli/utils/__init__.py +++ b/cli/utils/__init__.py @@ -242,6 +242,11 @@ def success(message: str): console.print(Panel(f"[green]{message}[/green]", title="✅")) +def info(message: str): + """Print informational message""" + console.print(Panel(f"[cyan]{message}[/cyan]", title="ℹ️")) + + def warning(message: str): """Print warning message""" console.print(Panel(f"[yellow]{message}[/yellow]", title="⚠️")) diff --git a/packages/py/aitbc-agent-sdk/pyproject.toml b/packages/py/aitbc-agent-sdk/pyproject.toml index 3f83315b..e55242e2 100644 --- a/packages/py/aitbc-agent-sdk/pyproject.toml +++ b/packages/py/aitbc-agent-sdk/pyproject.toml @@ -5,7 +5,8 @@ description = "AITBC Agent SDK" authors = [ {name = "AITBC Team", email = "team@aitbc.dev"} ] -requires-python = "^3.13" +readme = "README.md" +requires-python = ">=3.13" dependencies = [ "requests>=2.31.0", "pydantic>=2.5.0" @@ -14,3 +15,8 @@ dependencies = [ [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +[tool.poetry] +packages = [ + { include = "aitbc_agent", from = "src" } +] diff --git a/packages/py/aitbc-crypto/pyproject.toml b/packages/py/aitbc-crypto/pyproject.toml index fd4518c2..1bf42978 100644 --- a/packages/py/aitbc-crypto/pyproject.toml +++ b/packages/py/aitbc-crypto/pyproject.toml @@ -5,7 +5,8 @@ description = "AITBC Cryptography" authors = [ {name = "AITBC Team", email = "team@aitbc.dev"} ] -requires-python = "^3.13" +readme = "README.md" +requires-python = ">=3.13" dependencies = [ "cryptography>=41.0.0" ] @@ -13,3 +14,8 @@ dependencies = [ [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +[tool.poetry] +packages = [ + { include = "aitbc_crypto", from = "src" } +] diff --git a/packages/py/aitbc-sdk/pyproject.toml b/packages/py/aitbc-sdk/pyproject.toml index f929e64e..a50fb47e 100644 --- a/packages/py/aitbc-sdk/pyproject.toml +++ b/packages/py/aitbc-sdk/pyproject.toml @@ -6,7 +6,7 @@ authors = [ {name = "AITBC Team", email = "team@aitbc.dev"} ] readme = "README.md" -requires-python = "^3.13" +requires-python = ">=3.13" dependencies = [ "cryptography>=41.0.0", "requests>=2.31.0", @@ -16,3 +16,8 @@ dependencies = [ [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +[tool.poetry] +packages = [ + { include = "aitbc_sdk", from = "src" } +] diff --git a/scripts/ci/test_api_endpoints.py b/scripts/ci/test_api_endpoints.py index 96de2bf6..0fbd04f3 100755 --- a/scripts/ci/test_api_endpoints.py +++ b/scripts/ci/test_api_endpoints.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """CI test script for AITBC API endpoints.""" +import os import requests import json import time @@ -8,11 +9,13 @@ import statistics import sys # Service ports (must match systemd config) +API_HOST = os.environ.get("AITBC_API_HOST", "localhost") + SERVICES = { - "coordinator": {"url": "http://localhost:8000", "endpoints": ["/", "/health", "/info"]}, - "exchange": {"url": "http://localhost:8001", "endpoints": ["/", "/api/health", "/health", "/info"]}, - "wallet": {"url": "http://localhost:8003", "endpoints": ["/", "/health", "/wallets"]}, - "blockchain_rpc": {"url": "http://localhost:8006", "endpoints": ["/health", "/rpc/head", "/rpc/mempool"]}, + "coordinator": {"url": f"http://{API_HOST}:8000", "endpoints": ["/", "/health", "/info"]}, + "exchange": {"url": f"http://{API_HOST}:8001", "endpoints": ["/", "/api/health", "/health", "/info"]}, + "wallet": {"url": f"http://{API_HOST}:8003", "endpoints": ["/", "/health", "/wallets"]}, + "blockchain_rpc": {"url": f"http://{API_HOST}:8006", "endpoints": ["/health", "/rpc/head", "/rpc/mempool"]}, } @@ -76,10 +79,10 @@ def main(): print("\n⚡ Performance tests...") perf = test_performance([ - ("Coordinator", "http://localhost:8000/health"), - ("Exchange", "http://localhost:8001/api/health"), - ("Wallet", "http://localhost:8003/health"), - ("Blockchain RPC", "http://localhost:8006/health"), + ("Coordinator", f"http://{API_HOST}:8000/health"), + ("Exchange", f"http://{API_HOST}:8001/api/health"), + ("Wallet", f"http://{API_HOST}:8003/health"), + ("Blockchain RPC", f"http://{API_HOST}:8006/health"), ]) all_results["performance"] = perf