ci: standardize Python venv setup across all workflows using setup-python-venv.sh script

Replaced manual venv creation and pip install commands with calls to scripts/ci/setup-python-venv.sh across all CI workflows. The script provides consistent venv setup with configurable options for requirements installation, copy mode, and extra packages.

- Changed from manual `python3 -m venv` + `pip install` to setup-python-venv.sh in all workflows
- Added --skip-requirements flag where workflows don't need requirements

.gitea/workflows/api-endpoint-tests.yml

@@ -34,8 +34,11 @@ jobs:
       - name: Setup test environment
         run: |
           cd /var/lib/aitbc-workspaces/api-tests/repo
-          python3 -m venv venv
-          venv/bin/pip install -q requests pytest httpx
+          bash scripts/ci/setup-python-venv.sh \
+            --repo-dir "$PWD" \
+            --venv-dir "$PWD/venv" \
+            --skip-requirements \
+            --extra-packages "requests pytest httpx"
           
           # Ensure standard directories exist
           mkdir -p /var/lib/aitbc/data /var/lib/aitbc/keystore /etc/aitbc /var/log/aitbc

.gitea/workflows/cli-level1-tests.yml

@@ -36,12 +36,10 @@ jobs:
           # Ensure standard directories exist
           mkdir -p /var/lib/aitbc/data /var/lib/aitbc/keystore /etc/aitbc /var/log/aitbc
           
-          python3 -m venv venv
-          source venv/bin/activate
-          pip install -q --upgrade pip setuptools wheel
-          pip install -q -r requirements.txt
-          pip install -q pytest
-          echo "✅ Python $(python3 --version) environment ready"
+          bash scripts/ci/setup-python-venv.sh \
+            --repo-dir "$PWD" \
+            --venv-dir "$PWD/venv"
+          echo "✅ Python environment ready"
 
       - name: Verify CLI imports
         run: |

.gitea/workflows/integration-tests.yml

@@ -107,8 +107,11 @@ jobs:
       - name: Setup test environment
         run: |
           cd /var/lib/aitbc-workspaces/integration-tests/repo
-          python3 -m venv venv
-          venv/bin/pip install -q requests pytest httpx pytest-asyncio pytest-timeout click locust
+          bash scripts/ci/setup-python-venv.sh \
+            --repo-dir "$PWD" \
+            --venv-dir "$PWD/venv" \
+            --skip-requirements \
+            --extra-packages "requests pytest httpx pytest-asyncio pytest-timeout click locust"
           
           # Ensure standard directories exist
           mkdir -p /var/lib/aitbc/data /var/lib/aitbc/keystore /etc/aitbc /var/log/aitbc

.gitea/workflows/package-tests.yml

@@ -53,10 +53,12 @@ jobs:
           # Ensure standard directories exist
           mkdir -p /var/lib/aitbc/data /var/lib/aitbc/keystore /etc/aitbc /var/log/aitbc
 
-          # Create venv
-          python3 -m venv venv
+          bash "$WORKSPACE/repo/scripts/ci/setup-python-venv.sh" \
+            --repo-dir "$PWD" \
+            --venv-dir "$PWD/venv" \
+            --mode copy \
+            --extra-packages "pytest mypy black"
           source venv/bin/activate
-          pip install -q --upgrade pip setuptools wheel
 
           if [[ "${{ matrix.package.name }}" == "aitbc-sdk" ]]; then
             pip install -q -e "$WORKSPACE/repo/packages/py/aitbc-crypto"
@@ -69,7 +71,6 @@ jobs:
           if [[ -f "requirements.txt" ]]; then
             pip install -q -r requirements.txt
           fi
-          pip install -q pytest mypy black
 
           # Linting
           echo "=== Linting ==="
@@ -94,9 +95,12 @@ jobs:
           cd "$WORKSPACE/repo/${{ matrix.package.path }}"
 
           if [[ -f "pyproject.toml" ]]; then
-            python3 -m venv venv
-            source venv/bin/activate
-            pip install -q build
+            bash "$WORKSPACE/repo/scripts/ci/setup-python-venv.sh" \
+              --repo-dir "$PWD" \
+              --venv-dir "$PWD/venv-build" \
+              --skip-requirements \
+              --extra-packages "build"
+            source venv-build/bin/activate
             python -m build
             echo "✅ Package built"
           fi

.gitea/workflows/production-tests.yml

@@ -33,21 +33,11 @@ jobs:
       - name: Setup test environment
         run: |
           cd /var/lib/aitbc-workspaces/production-tests/repo
-          python3 -m venv venv
-          venv/bin/pip install -q \
-            pytest \
-            pytest-asyncio \
-            pytest-timeout \
-            requests \
-            pyjwt \
-            fastapi \
-            uvicorn[standard] \
-            redis \
-            bcrypt \
-            websockets \
-            numpy \
-            psutil \
-            prometheus-client
+          bash scripts/ci/setup-python-venv.sh \
+            --repo-dir "$PWD" \
+            --venv-dir "$PWD/venv" \
+            --skip-requirements \
+            --extra-packages "pytest pytest-asyncio pytest-timeout requests pyjwt fastapi uvicorn[standard] redis bcrypt websockets numpy psutil prometheus-client"
 
           # Ensure standard directories exist
           mkdir -p /var/lib/aitbc/data /var/lib/aitbc/keystore /etc/aitbc /var/log/aitbc

.gitea/workflows/python-tests.yml

@@ -39,12 +39,12 @@ jobs:
           # Ensure standard directories exist
           mkdir -p /var/lib/aitbc/data /var/lib/aitbc/keystore /etc/aitbc /var/log/aitbc
           
-          python3 -m venv venv
-          source venv/bin/activate
-          pip install -q --upgrade pip setuptools wheel
-          pip install -q -r requirements.txt
-          pip install -q pytest pytest-asyncio pytest-cov pytest-mock pytest-timeout click pynacl locust
-          echo "✅ Python $(python3 --version) environment ready"
+          bash scripts/ci/setup-python-venv.sh \
+            --repo-dir "$PWD" \
+            --venv-dir "$PWD/venv" \
+            --mode copy \
+            --extra-packages "pytest-cov pytest-mock pytest-timeout locust"
+          echo "✅ Python environment ready"
 
       - name: Run linting
         run: |

.gitea/workflows/security-scanning.yml

@@ -42,9 +42,11 @@ jobs:
           # Ensure standard directories exist
           mkdir -p /var/lib/aitbc/data /var/lib/aitbc/keystore /etc/aitbc /var/log/aitbc
           
-          python3 -m venv venv
-          source venv/bin/activate
-          pip install -q bandit pip-audit
+          bash scripts/ci/setup-python-venv.sh \
+            --repo-dir "$PWD" \
+            --venv-dir "$PWD/venv" \
+            --skip-requirements \
+            --extra-packages "bandit pip-audit"
           echo "✅ Security tools installed"
 
       - name: Python dependency audit

.gitea/workflows/staking-tests.yml

@@ -36,11 +36,9 @@ jobs:
         run: |
           cd /var/lib/aitbc-workspaces/staking-tests/repo
           
-          python3 -m venv venv
-          source venv/bin/activate
-          pip install -q --upgrade pip setuptools wheel
-          pip install -q -r requirements.txt
-          pip install -q pytest pytest-asyncio
+          bash scripts/ci/setup-python-venv.sh \
+            --repo-dir "$PWD" \
+            --venv-dir "$PWD/venv"
           echo "✅ Python environment ready"
 
       - name: Run staking service tests
@@ -84,11 +82,9 @@ jobs:
         run: |
           cd /var/lib/aitbc-workspaces/staking-integration/repo
           
-          python3 -m venv venv
-          source venv/bin/activate
-          pip install -q --upgrade pip setuptools wheel
-          pip install -q -r requirements.txt
-          pip install -q pytest pytest-asyncio
+          bash scripts/ci/setup-python-venv.sh \
+            --repo-dir "$PWD" \
+            --venv-dir "$PWD/venv"
           echo "✅ Python environment ready"
 
       - name: Run staking integration tests
@@ -157,10 +153,9 @@ jobs:
         run: |
           cd /var/lib/aitbc-workspaces/staking-runner/repo
           
-          python3 -m venv venv
-          source venv/bin/activate
-          pip install -q --upgrade pip setuptools wheel
-          pip install -q -r requirements.txt
+          bash scripts/ci/setup-python-venv.sh \
+            --repo-dir "$PWD" \
+            --venv-dir "$PWD/venv"
           echo "✅ Python environment ready"
 
       - name: Run staking test runner

scripts/ci/setup-python-venv.sh

@@ -0,0 +1,147 @@
+#!/bin/bash
+set -euo pipefail
+
+REPO_DIR="$(pwd)"
+VENV_DIR=""
+REQUIREMENTS_FILE=""
+SKIP_REQUIREMENTS="false"
+MODE="symlink"
+EXTRA_PACKAGES=""
+CACHE_ROOT="/var/cache/aitbc/python-venvs"
+PIP_CACHE_ROOT="/var/cache/aitbc/pip"
+PYTHON_BIN="${PYTHON_BIN:-python3}"
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --repo-dir)
+            REPO_DIR="$2"
+            shift 2
+            ;;
+        --venv-dir)
+            VENV_DIR="$2"
+            shift 2
+            ;;
+        --requirements-file)
+            REQUIREMENTS_FILE="$2"
+            shift 2
+            ;;
+        --skip-requirements)
+            SKIP_REQUIREMENTS="true"
+            shift
+            ;;
+        --mode)
+            MODE="$2"
+            shift 2
+            ;;
+        --extra-packages)
+            EXTRA_PACKAGES="$2"
+            shift 2
+            ;;
+        --cache-root)
+            CACHE_ROOT="$2"
+            shift 2
+            ;;
+        --python-bin)
+            PYTHON_BIN="$2"
+            shift 2
+            ;;
+        *)
+            echo "Unknown argument: $1" >&2
+            exit 1
+            ;;
+    esac
+done
+
+VENV_DIR="${VENV_DIR:-$REPO_DIR/venv}"
+REQUIREMENTS_FILE="${REQUIREMENTS_FILE:-$REPO_DIR/requirements.txt}"
+
+if [[ "$SKIP_REQUIREMENTS" == "true" ]]; then
+    REQUIREMENTS_FILE=""
+fi
+
+if [[ "$MODE" != "symlink" && "$MODE" != "copy" ]]; then
+    echo "Invalid mode: $MODE" >&2
+    exit 1
+fi
+
+if [[ ! -d "$REPO_DIR" ]]; then
+    echo "Repository directory not found: $REPO_DIR" >&2
+    exit 1
+fi
+
+if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
+    echo "Python binary not found: $PYTHON_BIN" >&2
+    exit 1
+fi
+
+export PIP_DISABLE_PIP_VERSION_CHECK=1
+export PIP_NO_INPUT=1
+export PIP_PROGRESS_BAR=off
+export PIP_CACHE_DIR="$PIP_CACHE_ROOT"
+
+mkdir -p "$CACHE_ROOT" "$PIP_CACHE_ROOT"
+
+PYTHON_VERSION="$($PYTHON_BIN -c 'import sys; print(".".join(map(str, sys.version_info[:3])))')"
+if [[ -n "$REQUIREMENTS_FILE" && -f "$REQUIREMENTS_FILE" ]]; then
+    REQUIREMENTS_HASH="$(sha256sum "$REQUIREMENTS_FILE" | awk '{print $1}' | cut -c1-16)"
+else
+    REQUIREMENTS_HASH="no-req"
+fi
+EXTRA_HASH="$(printf '%s' "$EXTRA_PACKAGES" | sha256sum | awk '{print $1}' | cut -c1-16)"
+CACHE_KEY="py${PYTHON_VERSION}-req${REQUIREMENTS_HASH}-extra${EXTRA_HASH}"
+CACHE_VENV_DIR="$CACHE_ROOT/$CACHE_KEY"
+LOCK_FILE="$CACHE_ROOT/$CACHE_KEY.lock"
+
+build_cached_environment() {
+    local temp_dir
+    temp_dir="${CACHE_VENV_DIR}.tmp.$$"
+
+    rm -rf "$temp_dir"
+    "$PYTHON_BIN" -m venv "$temp_dir"
+    source "$temp_dir/bin/activate"
+
+    python -m pip install -q --upgrade pip setuptools wheel
+
+    if [[ -n "$REQUIREMENTS_FILE" && -f "$REQUIREMENTS_FILE" ]]; then
+        python -m pip install -q -r "$REQUIREMENTS_FILE"
+    fi
+
+    if [[ -n "$EXTRA_PACKAGES" ]]; then
+        read -r -a extra_array <<< "$EXTRA_PACKAGES"
+        python -m pip install -q "${extra_array[@]}"
+    fi
+
+    deactivate || true
+    mv "$temp_dir" "$CACHE_VENV_DIR"
+}
+
+if command -v flock >/dev/null 2>&1; then
+    exec 9>"$LOCK_FILE"
+    flock 9
+fi
+
+if [[ -x "$CACHE_VENV_DIR/bin/python" ]]; then
+    echo "✅ Reusing cached Python environment: $CACHE_KEY"
+else
+    echo "📦 Building cached Python environment: $CACHE_KEY"
+    build_cached_environment
+fi
+
+rm -rf "$VENV_DIR"
+
+case "$MODE" in
+    symlink)
+        ln -s "$CACHE_VENV_DIR" "$VENV_DIR"
+        ;;
+    copy)
+        mkdir -p "$VENV_DIR"
+        if command -v rsync >/dev/null 2>&1; then
+            rsync -a --delete "$CACHE_VENV_DIR/" "$VENV_DIR/"
+        else
+            cp -a "$CACHE_VENV_DIR/." "$VENV_DIR/"
+        fi
+        ;;
+esac
+
+source "$VENV_DIR/bin/activate"
+echo "✅ Python environment ready from cache: $CACHE_KEY"