From a2f84648ab442a7e030e61b2f856215f3f011edc Mon Sep 17 00:00:00 2001 From: aitbc Date: Sun, 19 Apr 2026 21:05:26 +0200 Subject: [PATCH] ci: standardize Python venv setup across all workflows using setup-python-venv.sh script Replaced manual venv creation and pip install commands with calls to scripts/ci/setup-python-venv.sh across all CI workflows. The script provides consistent venv setup with configurable options for requirements installation, copy mode, and extra packages. - Changed from manual `python3 -m venv` + `pip install` to setup-python-venv.sh in all workflows - Added --skip-requirements flag where workflows don't need requirements --- .gitea/workflows/api-endpoint-tests.yml | 7 +- .gitea/workflows/cli-level1-tests.yml | 10 +- .gitea/workflows/integration-tests.yml | 7 +- .gitea/workflows/package-tests.yml | 18 +-- .gitea/workflows/production-tests.yml | 20 +--- .gitea/workflows/python-tests.yml | 12 +- .gitea/workflows/security-scanning.yml | 8 +- .gitea/workflows/staking-tests.yml | 23 ++-- scripts/ci/setup-python-venv.sh | 147 ++++++++++++++++++++++++ 9 files changed, 197 insertions(+), 55 deletions(-) create mode 100644 scripts/ci/setup-python-venv.sh diff --git a/.gitea/workflows/api-endpoint-tests.yml b/.gitea/workflows/api-endpoint-tests.yml index 45511ced..ddfe2e8b 100644 --- a/.gitea/workflows/api-endpoint-tests.yml +++ b/.gitea/workflows/api-endpoint-tests.yml @@ -34,8 +34,11 @@ jobs: - name: Setup test environment run: | cd /var/lib/aitbc-workspaces/api-tests/repo - python3 -m venv venv - venv/bin/pip install -q requests pytest httpx + bash scripts/ci/setup-python-venv.sh \ + --repo-dir "$PWD" \ + --venv-dir "$PWD/venv" \ + --skip-requirements \ + --extra-packages "requests pytest httpx" # Ensure standard directories exist mkdir -p /var/lib/aitbc/data /var/lib/aitbc/keystore /etc/aitbc /var/log/aitbc diff --git a/.gitea/workflows/cli-level1-tests.yml b/.gitea/workflows/cli-level1-tests.yml index 5bfb0345..4652f7b5 100644 --- a/.gitea/workflows/cli-level1-tests.yml +++ b/.gitea/workflows/cli-level1-tests.yml @@ -36,12 +36,10 @@ jobs: # Ensure standard directories exist mkdir -p /var/lib/aitbc/data /var/lib/aitbc/keystore /etc/aitbc /var/log/aitbc - python3 -m venv venv - source venv/bin/activate - pip install -q --upgrade pip setuptools wheel - pip install -q -r requirements.txt - pip install -q pytest - echo "✅ Python $(python3 --version) environment ready" + bash scripts/ci/setup-python-venv.sh \ + --repo-dir "$PWD" \ + --venv-dir "$PWD/venv" + echo "✅ Python environment ready" - name: Verify CLI imports run: | diff --git a/.gitea/workflows/integration-tests.yml b/.gitea/workflows/integration-tests.yml index 94959a0f..0ed7ae80 100644 --- a/.gitea/workflows/integration-tests.yml +++ b/.gitea/workflows/integration-tests.yml @@ -107,8 +107,11 @@ jobs: - name: Setup test environment run: | cd /var/lib/aitbc-workspaces/integration-tests/repo - python3 -m venv venv - venv/bin/pip install -q requests pytest httpx pytest-asyncio pytest-timeout click locust + bash scripts/ci/setup-python-venv.sh \ + --repo-dir "$PWD" \ + --venv-dir "$PWD/venv" \ + --skip-requirements \ + --extra-packages "requests pytest httpx pytest-asyncio pytest-timeout click locust" # Ensure standard directories exist mkdir -p /var/lib/aitbc/data /var/lib/aitbc/keystore /etc/aitbc /var/log/aitbc diff --git a/.gitea/workflows/package-tests.yml b/.gitea/workflows/package-tests.yml index 132bdb81..e86a372b 100644 --- a/.gitea/workflows/package-tests.yml +++ b/.gitea/workflows/package-tests.yml @@ -53,10 +53,12 @@ jobs: # Ensure standard directories exist mkdir -p /var/lib/aitbc/data /var/lib/aitbc/keystore /etc/aitbc /var/log/aitbc - # Create venv - python3 -m venv venv + bash "$WORKSPACE/repo/scripts/ci/setup-python-venv.sh" \ + --repo-dir "$PWD" \ + --venv-dir "$PWD/venv" \ + --mode copy \ + --extra-packages "pytest mypy black" source venv/bin/activate - pip install -q --upgrade pip setuptools wheel if [[ "${{ matrix.package.name }}" == "aitbc-sdk" ]]; then pip install -q -e "$WORKSPACE/repo/packages/py/aitbc-crypto" @@ -69,7 +71,6 @@ jobs: if [[ -f "requirements.txt" ]]; then pip install -q -r requirements.txt fi - pip install -q pytest mypy black # Linting echo "=== Linting ===" @@ -94,9 +95,12 @@ jobs: cd "$WORKSPACE/repo/${{ matrix.package.path }}" if [[ -f "pyproject.toml" ]]; then - python3 -m venv venv - source venv/bin/activate - pip install -q build + bash "$WORKSPACE/repo/scripts/ci/setup-python-venv.sh" \ + --repo-dir "$PWD" \ + --venv-dir "$PWD/venv-build" \ + --skip-requirements \ + --extra-packages "build" + source venv-build/bin/activate python -m build echo "✅ Package built" fi diff --git a/.gitea/workflows/production-tests.yml b/.gitea/workflows/production-tests.yml index 17b61558..a187fe57 100644 --- a/.gitea/workflows/production-tests.yml +++ b/.gitea/workflows/production-tests.yml @@ -33,21 +33,11 @@ jobs: - name: Setup test environment run: | cd /var/lib/aitbc-workspaces/production-tests/repo - python3 -m venv venv - venv/bin/pip install -q \ - pytest \ - pytest-asyncio \ - pytest-timeout \ - requests \ - pyjwt \ - fastapi \ - uvicorn[standard] \ - redis \ - bcrypt \ - websockets \ - numpy \ - psutil \ - prometheus-client + bash scripts/ci/setup-python-venv.sh \ + --repo-dir "$PWD" \ + --venv-dir "$PWD/venv" \ + --skip-requirements \ + --extra-packages "pytest pytest-asyncio pytest-timeout requests pyjwt fastapi uvicorn[standard] redis bcrypt websockets numpy psutil prometheus-client" # Ensure standard directories exist mkdir -p /var/lib/aitbc/data /var/lib/aitbc/keystore /etc/aitbc /var/log/aitbc diff --git a/.gitea/workflows/python-tests.yml b/.gitea/workflows/python-tests.yml index d88cba40..5ca6461e 100644 --- a/.gitea/workflows/python-tests.yml +++ b/.gitea/workflows/python-tests.yml @@ -39,12 +39,12 @@ jobs: # Ensure standard directories exist mkdir -p /var/lib/aitbc/data /var/lib/aitbc/keystore /etc/aitbc /var/log/aitbc - python3 -m venv venv - source venv/bin/activate - pip install -q --upgrade pip setuptools wheel - pip install -q -r requirements.txt - pip install -q pytest pytest-asyncio pytest-cov pytest-mock pytest-timeout click pynacl locust - echo "✅ Python $(python3 --version) environment ready" + bash scripts/ci/setup-python-venv.sh \ + --repo-dir "$PWD" \ + --venv-dir "$PWD/venv" \ + --mode copy \ + --extra-packages "pytest-cov pytest-mock pytest-timeout locust" + echo "✅ Python environment ready" - name: Run linting run: | diff --git a/.gitea/workflows/security-scanning.yml b/.gitea/workflows/security-scanning.yml index 33ae4b02..02946a36 100644 --- a/.gitea/workflows/security-scanning.yml +++ b/.gitea/workflows/security-scanning.yml @@ -42,9 +42,11 @@ jobs: # Ensure standard directories exist mkdir -p /var/lib/aitbc/data /var/lib/aitbc/keystore /etc/aitbc /var/log/aitbc - python3 -m venv venv - source venv/bin/activate - pip install -q bandit pip-audit + bash scripts/ci/setup-python-venv.sh \ + --repo-dir "$PWD" \ + --venv-dir "$PWD/venv" \ + --skip-requirements \ + --extra-packages "bandit pip-audit" echo "✅ Security tools installed" - name: Python dependency audit diff --git a/.gitea/workflows/staking-tests.yml b/.gitea/workflows/staking-tests.yml index 39e6652f..b9c3a0fc 100644 --- a/.gitea/workflows/staking-tests.yml +++ b/.gitea/workflows/staking-tests.yml @@ -36,11 +36,9 @@ jobs: run: | cd /var/lib/aitbc-workspaces/staking-tests/repo - python3 -m venv venv - source venv/bin/activate - pip install -q --upgrade pip setuptools wheel - pip install -q -r requirements.txt - pip install -q pytest pytest-asyncio + bash scripts/ci/setup-python-venv.sh \ + --repo-dir "$PWD" \ + --venv-dir "$PWD/venv" echo "✅ Python environment ready" - name: Run staking service tests @@ -84,11 +82,9 @@ jobs: run: | cd /var/lib/aitbc-workspaces/staking-integration/repo - python3 -m venv venv - source venv/bin/activate - pip install -q --upgrade pip setuptools wheel - pip install -q -r requirements.txt - pip install -q pytest pytest-asyncio + bash scripts/ci/setup-python-venv.sh \ + --repo-dir "$PWD" \ + --venv-dir "$PWD/venv" echo "✅ Python environment ready" - name: Run staking integration tests @@ -157,10 +153,9 @@ jobs: run: | cd /var/lib/aitbc-workspaces/staking-runner/repo - python3 -m venv venv - source venv/bin/activate - pip install -q --upgrade pip setuptools wheel - pip install -q -r requirements.txt + bash scripts/ci/setup-python-venv.sh \ + --repo-dir "$PWD" \ + --venv-dir "$PWD/venv" echo "✅ Python environment ready" - name: Run staking test runner diff --git a/scripts/ci/setup-python-venv.sh b/scripts/ci/setup-python-venv.sh new file mode 100644 index 00000000..837fc517 --- /dev/null +++ b/scripts/ci/setup-python-venv.sh @@ -0,0 +1,147 @@ +#!/bin/bash +set -euo pipefail + +REPO_DIR="$(pwd)" +VENV_DIR="" +REQUIREMENTS_FILE="" +SKIP_REQUIREMENTS="false" +MODE="symlink" +EXTRA_PACKAGES="" +CACHE_ROOT="/var/cache/aitbc/python-venvs" +PIP_CACHE_ROOT="/var/cache/aitbc/pip" +PYTHON_BIN="${PYTHON_BIN:-python3}" + +while [[ $# -gt 0 ]]; do + case "$1" in + --repo-dir) + REPO_DIR="$2" + shift 2 + ;; + --venv-dir) + VENV_DIR="$2" + shift 2 + ;; + --requirements-file) + REQUIREMENTS_FILE="$2" + shift 2 + ;; + --skip-requirements) + SKIP_REQUIREMENTS="true" + shift + ;; + --mode) + MODE="$2" + shift 2 + ;; + --extra-packages) + EXTRA_PACKAGES="$2" + shift 2 + ;; + --cache-root) + CACHE_ROOT="$2" + shift 2 + ;; + --python-bin) + PYTHON_BIN="$2" + shift 2 + ;; + *) + echo "Unknown argument: $1" >&2 + exit 1 + ;; + esac +done + +VENV_DIR="${VENV_DIR:-$REPO_DIR/venv}" +REQUIREMENTS_FILE="${REQUIREMENTS_FILE:-$REPO_DIR/requirements.txt}" + +if [[ "$SKIP_REQUIREMENTS" == "true" ]]; then + REQUIREMENTS_FILE="" +fi + +if [[ "$MODE" != "symlink" && "$MODE" != "copy" ]]; then + echo "Invalid mode: $MODE" >&2 + exit 1 +fi + +if [[ ! -d "$REPO_DIR" ]]; then + echo "Repository directory not found: $REPO_DIR" >&2 + exit 1 +fi + +if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + echo "Python binary not found: $PYTHON_BIN" >&2 + exit 1 +fi + +export PIP_DISABLE_PIP_VERSION_CHECK=1 +export PIP_NO_INPUT=1 +export PIP_PROGRESS_BAR=off +export PIP_CACHE_DIR="$PIP_CACHE_ROOT" + +mkdir -p "$CACHE_ROOT" "$PIP_CACHE_ROOT" + +PYTHON_VERSION="$($PYTHON_BIN -c 'import sys; print(".".join(map(str, sys.version_info[:3])))')" +if [[ -n "$REQUIREMENTS_FILE" && -f "$REQUIREMENTS_FILE" ]]; then + REQUIREMENTS_HASH="$(sha256sum "$REQUIREMENTS_FILE" | awk '{print $1}' | cut -c1-16)" +else + REQUIREMENTS_HASH="no-req" +fi +EXTRA_HASH="$(printf '%s' "$EXTRA_PACKAGES" | sha256sum | awk '{print $1}' | cut -c1-16)" +CACHE_KEY="py${PYTHON_VERSION}-req${REQUIREMENTS_HASH}-extra${EXTRA_HASH}" +CACHE_VENV_DIR="$CACHE_ROOT/$CACHE_KEY" +LOCK_FILE="$CACHE_ROOT/$CACHE_KEY.lock" + +build_cached_environment() { + local temp_dir + temp_dir="${CACHE_VENV_DIR}.tmp.$$" + + rm -rf "$temp_dir" + "$PYTHON_BIN" -m venv "$temp_dir" + source "$temp_dir/bin/activate" + + python -m pip install -q --upgrade pip setuptools wheel + + if [[ -n "$REQUIREMENTS_FILE" && -f "$REQUIREMENTS_FILE" ]]; then + python -m pip install -q -r "$REQUIREMENTS_FILE" + fi + + if [[ -n "$EXTRA_PACKAGES" ]]; then + read -r -a extra_array <<< "$EXTRA_PACKAGES" + python -m pip install -q "${extra_array[@]}" + fi + + deactivate || true + mv "$temp_dir" "$CACHE_VENV_DIR" +} + +if command -v flock >/dev/null 2>&1; then + exec 9>"$LOCK_FILE" + flock 9 +fi + +if [[ -x "$CACHE_VENV_DIR/bin/python" ]]; then + echo "✅ Reusing cached Python environment: $CACHE_KEY" +else + echo "📦 Building cached Python environment: $CACHE_KEY" + build_cached_environment +fi + +rm -rf "$VENV_DIR" + +case "$MODE" in + symlink) + ln -s "$CACHE_VENV_DIR" "$VENV_DIR" + ;; + copy) + mkdir -p "$VENV_DIR" + if command -v rsync >/dev/null 2>&1; then + rsync -a --delete "$CACHE_VENV_DIR/" "$VENV_DIR/" + else + cp -a "$CACHE_VENV_DIR/." "$VENV_DIR/" + fi + ;; +esac + +source "$VENV_DIR/bin/activate" +echo "✅ Python environment ready from cache: $CACHE_KEY"