From fc803d80d078b0e3c526100f9addf43bc80df7ac Mon Sep 17 00:00:00 2001 From: aitbc Date: Mon, 20 Apr 2026 10:50:16 +0200 Subject: [PATCH] ci: standardize Python execution across workflows and improve venv caching robustness - Changed all Python/pip commands to use `venv/bin/python -m` pattern instead of direct tool invocation or source activation - package-tests.yml: pip, mypy, black, pytest now use `venv/bin/python -m` - python-tests.yml: ruff, pip, pytest now use `venv/bin/python -m` or `venv/bin/pytest` - staking-tests.yml: pytest now uses `venv/bin/pytest` instead of `python3 -m pytest` - Added missing dependencies to workflow --- .gitea/workflows/cli-level1-tests.yml | 4 ++- .gitea/workflows/package-tests.yml | 25 +++++++++++----- .gitea/workflows/python-tests.yml | 16 +++++----- .gitea/workflows/staking-tests.yml | 17 ++++++----- scripts/ci/setup-python-venv.sh | 43 ++++++++++++++++++--------- 5 files changed, 65 insertions(+), 40 deletions(-) diff --git a/.gitea/workflows/cli-level1-tests.yml b/.gitea/workflows/cli-level1-tests.yml index 5174547c..8e5b7d90 100644 --- a/.gitea/workflows/cli-level1-tests.yml +++ b/.gitea/workflows/cli-level1-tests.yml @@ -41,7 +41,9 @@ jobs: bash scripts/ci/setup-python-venv.sh \ --repo-dir "$PWD" \ - --venv-dir "$PWD/venv" + --venv-dir "$PWD/venv" \ + --requirements-file "$PWD/cli/requirements-cli.txt" \ + --extra-packages "PyYAML requests cryptography" echo "✅ Python environment ready" - name: Verify CLI imports diff --git a/.gitea/workflows/package-tests.yml b/.gitea/workflows/package-tests.yml index efffd1eb..b0ca6a36 100644 --- a/.gitea/workflows/package-tests.yml +++ b/.gitea/workflows/package-tests.yml @@ -63,28 +63,28 @@ jobs: --extra-packages "pytest mypy black" if [[ "${{ matrix.package.name }}" == "aitbc-sdk" ]]; then - venv/bin/pip install -q -e "$WORKSPACE/repo/packages/py/aitbc-crypto" + venv/bin/python -m pip install -q -e "$WORKSPACE/repo/packages/py/aitbc-crypto" fi # Install dependencies if [[ -f "pyproject.toml" ]]; then - venv/bin/pip install -q -e ".[dev]" 2>/dev/null || venv/bin/pip install -q -e . + venv/bin/python -m pip install -q -e ".[dev]" 2>/dev/null || venv/bin/python -m pip install -q -e . fi if [[ -f "requirements.txt" ]]; then - venv/bin/pip install -q -r requirements.txt + venv/bin/python -m pip install -q -r requirements.txt fi # Linting echo "=== Linting ===" if [[ -d "src" ]]; then - mypy src/ --ignore-missing-imports --no-error-summary 2>/dev/null || echo "⚠️ MyPy warnings" - black --check src/ 2>/dev/null || echo "⚠️ Black warnings" + venv/bin/python -m mypy src/ --ignore-missing-imports --no-error-summary 2>/dev/null || echo "⚠️ MyPy warnings" + venv/bin/python -m black --check src/ 2>/dev/null || echo "⚠️ Black warnings" fi # Tests echo "=== Tests ===" if [[ -d "tests" ]]; then - pytest tests/ -q --tb=short + venv/bin/python -m pytest tests/ -q --tb=short else echo "⚠️ No tests directory found" fi @@ -117,6 +117,7 @@ jobs: timeout-minutes: 30 strategy: + max-parallel: 1 matrix: package: - name: "aitbc-sdk-js" @@ -147,7 +148,11 @@ jobs: node --version npm --version - npm install --legacy-peer-deps + if [[ -f "package-lock.json" ]]; then + npm ci --legacy-peer-deps --no-audit --no-fund + else + npm install --legacy-peer-deps --no-audit --no-fund + fi # Build npm run build @@ -157,7 +162,11 @@ jobs: npm run lint 2>/dev/null && echo "✅ Lint passed" || echo "⚠️ Lint skipped" # Test - npm test + if [[ "${{ matrix.package.name }}" == "aitbc-token" ]]; then + npx hardhat test --no-compile + else + npm test + fi echo "✅ Tests passed" echo "✅ ${{ matrix.package.name }} completed" diff --git a/.gitea/workflows/python-tests.yml b/.gitea/workflows/python-tests.yml index 7a95d0b5..54719450 100644 --- a/.gitea/workflows/python-tests.yml +++ b/.gitea/workflows/python-tests.yml @@ -53,10 +53,9 @@ jobs: - name: Run linting run: | cd /var/lib/aitbc-workspaces/python-tests/repo - source venv/bin/activate - if command -v ruff >/dev/null 2>&1; then - ruff check apps/ packages/py/ --select E,F --ignore E501 -q || echo "⚠️ Ruff warnings" + if venv/bin/python -m ruff --version >/dev/null 2>&1; then + venv/bin/python -m ruff check apps/ packages/py/ --select E,F --ignore E501 -q || echo "⚠️ Ruff warnings" fi echo "✅ Linting completed" @@ -64,19 +63,18 @@ jobs: - name: Run tests run: | cd /var/lib/aitbc-workspaces/python-tests/repo - source venv/bin/activate # Install packages in development mode - pip install -e packages/py/aitbc-crypto/ - pip install -e packages/py/aitbc-sdk/ + venv/bin/python -m pip install -e packages/py/aitbc-crypto/ + venv/bin/python -m pip install -e packages/py/aitbc-sdk/ export PYTHONPATH="apps/coordinator-api/src:apps/blockchain-node/src:apps/wallet/src:packages/py/aitbc-crypto/src:packages/py/aitbc-sdk/src:." # Test if packages are importable - python3 -c "import aitbc_crypto; print('✅ aitbc_crypto imported')" - python3 -c "import aitbc_sdk; print('✅ aitbc_sdk imported')" + venv/bin/python -c "import aitbc_crypto; print('✅ aitbc_crypto imported')" + venv/bin/python -c "import aitbc_sdk; print('✅ aitbc_sdk imported')" - pytest tests/archived_phase_tests/ \ + venv/bin/python -m pytest tests/archived_phase_tests/ \ tests/cross_phase/ \ apps/wallet/tests/ \ packages/py/aitbc-crypto/tests/ \ diff --git a/.gitea/workflows/staking-tests.yml b/.gitea/workflows/staking-tests.yml index 8a76c633..066684c6 100644 --- a/.gitea/workflows/staking-tests.yml +++ b/.gitea/workflows/staking-tests.yml @@ -43,26 +43,24 @@ jobs: --repo-dir "$PWD" \ --venv-dir "$PWD/venv" \ --skip-requirements \ - --extra-packages "pytest pytest-asyncio" + --extra-packages "pytest pytest-asyncio sqlmodel" echo "✅ Python environment ready" - name: Run staking service tests run: | cd /var/lib/aitbc-workspaces/staking-tests/repo - source venv/bin/activate export PYTHONPATH="apps/coordinator-api/src:." echo "🧪 Running staking service tests..." - python3 -m pytest tests/services/test_staking_service.py -v --tb=short + venv/bin/pytest tests/services/test_staking_service.py -v --tb=short echo "✅ Service tests completed" - name: Generate test data run: | cd /var/lib/aitbc-workspaces/staking-tests/repo - source venv/bin/activate echo "🔧 Generating test data..." - python3 scripts/testing/generate_staking_test_data.py + venv/bin/python scripts/testing/generate_staking_test_data.py echo "✅ Test data generated" - name: Cleanup @@ -86,20 +84,22 @@ jobs: - name: Setup Python environment run: | cd /var/lib/aitbc-workspaces/staking-integration/repo + rm -rf venv bash scripts/ci/setup-python-venv.sh \ --repo-dir "$PWD" \ - --venv-dir "$PWD/venv" + --venv-dir "$PWD/venv" \ + --skip-requirements \ + --extra-packages "pytest pytest-asyncio sqlmodel" echo "✅ Python environment ready" - name: Run staking integration tests run: | cd /var/lib/aitbc-workspaces/staking-integration/repo - source venv/bin/activate export PYTHONPATH="apps/coordinator-api/src:." echo "🧪 Running staking integration tests..." - python3 -m pytest tests/integration/test_staking_lifecycle.py -v --tb=short + venv/bin/pytest tests/integration/test_staking_lifecycle.py -v --tb=short echo "✅ Integration tests completed" - name: Cleanup @@ -157,6 +157,7 @@ jobs: - name: Setup Python environment run: | cd /var/lib/aitbc-workspaces/staking-runner/repo + rm -rf venv bash scripts/ci/setup-python-venv.sh \ --repo-dir "$PWD" \ diff --git a/scripts/ci/setup-python-venv.sh b/scripts/ci/setup-python-venv.sh index ec0ddda5..0a85fb41 100644 --- a/scripts/ci/setup-python-venv.sh +++ b/scripts/ci/setup-python-venv.sh @@ -92,27 +92,37 @@ CACHE_KEY="py${PYTHON_VERSION}-req${REQUIREMENTS_HASH}-extra${EXTRA_HASH}" CACHE_VENV_DIR="$CACHE_ROOT/$CACHE_KEY" LOCK_FILE="$CACHE_ROOT/$CACHE_KEY.lock" +cached_environment_is_valid() { + [[ -x "$CACHE_VENV_DIR/bin/python" ]] || return 1 + [[ -x "$CACHE_VENV_DIR/bin/pip" ]] || return 1 + + "$CACHE_VENV_DIR/bin/python" -c 'import sys; print(sys.executable)' >/dev/null 2>&1 || return 1 + "$CACHE_VENV_DIR/bin/pip" --version >/dev/null 2>&1 || return 1 +} + build_cached_environment() { - local temp_dir - temp_dir="${CACHE_VENV_DIR}.tmp.$$" + rm -rf "$CACHE_VENV_DIR" + "$PYTHON_BIN" -m venv "$CACHE_VENV_DIR" - rm -rf "$temp_dir" - "$PYTHON_BIN" -m venv "$temp_dir" - source "$temp_dir/bin/activate" - - python -m pip install -q --upgrade pip setuptools wheel --no-cache-dir + if ! "$CACHE_VENV_DIR/bin/python" -m pip install -q --upgrade pip setuptools wheel --no-cache-dir; then + rm -rf "$CACHE_VENV_DIR" + return 1 + fi if [[ -n "$REQUIREMENTS_FILE" && -f "$REQUIREMENTS_FILE" ]]; then - python -m pip install -q -r "$REQUIREMENTS_FILE" --no-cache-dir + if ! "$CACHE_VENV_DIR/bin/python" -m pip install -q -r "$REQUIREMENTS_FILE" --no-cache-dir; then + rm -rf "$CACHE_VENV_DIR" + return 1 + fi fi if [[ -n "$EXTRA_PACKAGES" ]]; then read -r -a extra_array <<< "$EXTRA_PACKAGES" - python -m pip install -q "${extra_array[@]}" --no-cache-dir + if ! "$CACHE_VENV_DIR/bin/python" -m pip install -q "${extra_array[@]}" --no-cache-dir; then + rm -rf "$CACHE_VENV_DIR" + return 1 + fi fi - - deactivate || true - mv "$temp_dir" "$CACHE_VENV_DIR" } if command -v flock >/dev/null 2>&1; then @@ -120,10 +130,15 @@ if command -v flock >/dev/null 2>&1; then flock 9 fi -if [[ -x "$CACHE_VENV_DIR/bin/python" ]]; then +if cached_environment_is_valid; then echo "✅ Reusing cached Python environment: $CACHE_KEY" else - echo "📦 Building cached Python environment: $CACHE_KEY" + if [[ -e "$CACHE_VENV_DIR" ]]; then + echo "⚠️ Invalid cached Python environment detected, rebuilding: $CACHE_KEY" + rm -rf "$CACHE_VENV_DIR" + else + echo "📦 Building cached Python environment: $CACHE_KEY" + fi build_cached_environment fi