diff --git a/.gitea/workflows/package-tests.yml b/.gitea/workflows/package-tests.yml index 7ae4ce27..fa9c3859 100644 --- a/.gitea/workflows/package-tests.yml +++ b/.gitea/workflows/package-tests.yml @@ -107,24 +107,15 @@ jobs: echo "Cloning repository..." # Ensure we're in the workspace directory - cd "$WORKSPACE_DIR" 2>/dev/null || cd /opt/aitbc/python-packages-workspace 2>/dev/null || { - echo "❌ Cannot access workspace, trying to recover..." - cd / || exit 1 - mkdir -p /opt/aitbc/python-packages-workspace - cd /opt/aitbc/python-packages-workspace - } + # Use external workspace strategy + WORKSPACE_BASE="/var/lib/aitbc-workspaces/python-packages" + rm -rf "$WORKSPACE_BASE" 2>/dev/null || true + mkdir -p "$WORKSPACE_BASE" + cd "$WORKSPACE_BASE" - # Set git configuration and check Git status - echo "Git configuration and status:" - git --version - git config --global http.sslVerify false - git config --global http.postBuffer 1048576000 - - # Clone with multiple fallback strategies - REPO_URL="http://10.0.3.107:3000/oib/aitbc.git" - - # Clean any existing repo directory first - rm -rf repo + # Clone repository + git clone http://10.0.3.107:3000/oib/aitbc.git repo + cd repo # Check filesystem space and permissions echo "Filesystem check:" @@ -723,14 +714,10 @@ jobs: # Clean and create isolated workspace cd / || true - rm -rf /opt/aitbc/javascript-packages-workspace 2>/dev/null || true - if [[ -d "/opt/aitbc/javascript-packages-workspace" ]]; then - find /opt/aitbc/javascript-packages-workspace -type f -delete 2>/dev/null || true - find /opt/aitbc/javascript-packages-workspace -type d -delete 2>/dev/null || true - rm -rf /opt/aitbc/javascript-packages-workspace 2>/dev/null || true - fi - mkdir -p /opt/aitbc/javascript-packages-workspace - cd /opt/aitbc/javascript-packages-workspace + WORKSPACE_BASE="/var/lib/aitbc-workspaces/javascript-packages" + rm -rf "$WORKSPACE_BASE" 2>/dev/null || true + mkdir -p "$WORKSPACE_BASE" + cd "$WORKSPACE_BASE" # Ensure no git lock files exist find . -name "*.lock" -delete 2>/dev/null || true @@ -954,10 +941,11 @@ jobs: - name: Setup workspace run: | echo "=== PACKAGE INTEGRATION TESTS ===" - cd / || true - rm -rf /opt/aitbc/integration-workspace - mkdir -p /opt/aitbc/integration-workspace - cd /opt/aitbc/integration-workspace + # Use external workspace for security tests + WORKSPACE_BASE="/var/lib/aitbc-workspaces/security-tests" + rm -rf "$WORKSPACE_BASE" 2>/dev/null || true + mkdir -p "$WORKSPACE_BASE" + cd "$WORKSPACE_BASE" # Ensure no git lock files exist find . -name "*.lock" -delete 2>/dev/null || true diff --git a/docs/WORKSPACE_STRATEGY.md b/docs/WORKSPACE_STRATEGY.md new file mode 100644 index 00000000..e749a49e --- /dev/null +++ b/docs/WORKSPACE_STRATEGY.md @@ -0,0 +1,170 @@ +# AITBC Workspace Strategy Guide + +## 🎯 Current Workspace Usage + +### In-Repo Workspaces (Current) +``` +/opt/aitbc/ +├── python-packages-workspace/ # Created inside repo +├── javascript-packages-workspace/ # Created inside repo +├── security-workspace/ # Created inside repo +└── compatibility-workspace/ # Created inside repo +``` + +## ✅ Benefits of Outside-Repo Workspaces + +### 1. **Clean Repository** +- No workspace directories in git status +- Cleaner commits and PRs +- No .gitignore conflicts + +### 2. **Better Isolation** +- Each workspace has isolated environment +- No cross-contamination between tests +- Easier cleanup and reset + +### 3. **Resource Management** +- Workspaces can use different base directories +- Better disk space management +- Parallel test execution possible + +### 4. **CI/CD Best Practices** +- Standard industry practice +- GitHub Actions, GitLab CI use this pattern +- Container-friendly approach + +## 🚀 Recommended Workspace Structure + +### Outside-Repo Strategy +```bash +# Base workspace directory +/var/lib/aitbc-workspaces/ +├── python-packages/ +├── javascript-packages/ +├── security-tests/ +├── integration-tests/ +├── compatibility-tests/ +└── temp/ +``` + +### Alternative: /opt Structure +```bash +/opt/aitbc-workspaces/ +├── python-packages/ +├── javascript-packages/ +├── security/ +├── integration/ +└── build-artifacts/ +``` + +## 📝 Implementation Examples + +### Current (Inside Repo) +```yaml +- name: Setup Python Workspace + run: | + cd /opt/aitbc + rm -rf python-packages-workspace + mkdir -p python-packages-workspace + cd python-packages-workspace + git clone http://10.0.3.107:3000/oib/aitbc.git repo +``` + +### Improved (Outside Repo) +```yaml +- name: Setup Python Workspace + run: | + # Use dedicated workspace directory + WORKSPACE_BASE="/var/lib/aitbc-workspaces/python-packages" + rm -rf "$WORKSPACE_BASE" + mkdir -p "$WORKSPACE_BASE" + cd "$WORKSPACE_BASE" + git clone http://10.0.3.107:3000/oib/aitbc.git repo + cd repo +``` + +### Even Better (With Cleanup) +```yaml +- name: Setup Workspace + run: | + # Clean workspace function + cleanup_workspace() { + local workspace="$1" + rm -rf "$workspace" 2>/dev/null || true + mkdir -p "$workspace" + } + + # Setup different workspaces + cleanup_workspace "/var/lib/aitbc-workspaces/python" + cleanup_workspace "/var/lib/aitbc-workspaces/javascript" + cleanup_workspace "/var/lib/aitbc-workspaces/security" +``` + +## 🔧 Workspace Management Functions + +### Reusable Setup Script +```bash +#!/bin/bash +# /opt/aitbc/scripts/setup-workspace.sh + +setup_workspace() { + local workspace_type="$1" + local workspace_base="/var/lib/aitbc-workspaces" + local workspace_dir="$workspace_base/$workspace_type" + local repo_url="http://10.0.3.107:3000/oib/aitbc.git" + + echo "=== Setting up $workspace_type workspace ===" + + # Cleanup and create + rm -rf "$workspace_dir" + mkdir -p "$workspace_dir" + cd "$workspace_dir" + + # Clone repository + git clone "$repo_url" repo + cd repo + + echo "✅ $workspace_type workspace ready at $workspace_dir/repo" +} + +# Usage examples +setup_workspace "python-packages" +setup_workspace "javascript-packages" +setup_workspace "security-tests" +``` + +## 📊 Comparison + +| Aspect | Inside Repo | Outside Repo | +|--------|-------------|--------------| +| **Clean Git History** | ❌ Workspaces in status | ✅ Clean status | +| **Isolation** | ❌ Shared space | ✅ Isolated | +| **Cleanup** | ❌ Complex | ✅ Simple | +| **Parallel Tests** | ❌ Conflicts | ✅ Possible | +| **Disk Usage** | ❌ In repo size | ✅ Separate | +| **Industry Standard** | ❌ Non-standard | ✅ Standard | + +## 🎯 Recommendation + +**Switch to outside-repo workspaces for:** + +1. **All CI/CD workflows** +2. **Package testing** (Python, JavaScript, Solidity) +3. **Security scanning** +4. **Integration testing** +5. **Build artifacts** + +**Keep inside-repo only for:** +1. **Development scripts** +2. **Documentation** +3. **Configuration files** + +## 🚀 Next Steps + +1. Create `/var/lib/aitbc-workspaces/` directory +2. Update CI workflows to use outside-repo workspaces +3. Add workspace management scripts +4. Update .gitignore to exclude any workspace directories +5. Test with existing workflows + +This will provide cleaner, more maintainable CI/CD pipelines! diff --git a/scripts/workspace-manager.sh b/scripts/workspace-manager.sh new file mode 100755 index 00000000..a5f205bc --- /dev/null +++ b/scripts/workspace-manager.sh @@ -0,0 +1,205 @@ +#!/bin/bash +# AITBC Workspace Management Script +# Handles setup, cleanup, and management of external workspaces + +set -euo pipefail + +# Configuration +WORKSPACE_BASE="/var/lib/aitbc-workspaces" +REPO_URL="http://10.0.3.107:3000/oib/aitbc.git" +GITEA_TOKEN="${GITEA_TOKEN:-b8fbb3e7e6cecf3a01f8a242fc652631c6dfd010}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Create workspace base directory +ensure_workspace_base() { + if [[ ! -d "$WORKSPACE_BASE" ]]; then + log_info "Creating workspace base directory: $WORKSPACE_BASE" + sudo mkdir -p "$WORKSPACE_BASE" + sudo chmod 755 "$WORKSPACE_BASE" + log_success "Workspace base directory created" + fi +} + +# Setup a specific workspace +setup_workspace() { + local workspace_type="$1" + local workspace_dir="$WORKSPACE_BASE/$workspace_type" + + log_info "=== Setting up $workspace_type workspace ===" + + # Cleanup existing workspace + if [[ -d "$workspace_dir" ]]; then + log_warning "Removing existing workspace: $workspace_dir" + rm -rf "$workspace_dir" + fi + + # Create new workspace + mkdir -p "$workspace_dir" + cd "$workspace_dir" + + # Clone repository + log_info "Cloning repository to: $workspace_dir/repo" + if ! git clone "$REPO_URL" repo; then + log_error "Failed to clone repository" + return 1 + fi + + cd repo + log_success "✅ $workspace_type workspace ready at $workspace_dir/repo" + log_info "Current directory: $(pwd)" + log_info "Repository contents:" + ls -la | head -10 + + # Set git config for CI + git config --global http.sslVerify false + git config --global http.postBuffer 1048576000 + + return 0 +} + +# Cleanup all workspaces +cleanup_all_workspaces() { + log_info "=== Cleaning up all workspaces ===" + + if [[ -d "$WORKSPACE_BASE" ]]; then + log_warning "Removing workspace base directory: $WORKSPACE_BASE" + rm -rf "$WORKSPACE_BASE" + log_success "All workspaces cleaned up" + else + log_info "No workspaces to clean up" + fi +} + +# List all workspaces +list_workspaces() { + log_info "=== AITBC Workspaces ===" + + if [[ ! -d "$WORKSPACE_BASE" ]]; then + log_info "No workspace base directory found" + return 0 + fi + + log_info "Workspace base: $WORKSPACE_BASE" + echo + + for workspace in "$WORKSPACE_BASE"/*; do + if [[ -d "$workspace" ]]; then + local workspace_name=$(basename "$workspace") + local size=$(du -sh "$workspace" 2>/dev/null | cut -f1) + local files=$(find "$workspace" -type f 2>/dev/null | wc -l) + + echo "📁 $workspace_name" + echo " Size: $size" + echo " Files: $files" + echo " Path: $workspace" + echo + fi + done +} + +# Check workspace status +check_workspace() { + local workspace_type="$1" + local workspace_dir="$WORKSPACE_BASE/$workspace_type" + + log_info "=== Checking $workspace_type workspace ===" + + if [[ ! -d "$workspace_dir" ]]; then + log_warning "Workspace does not exist: $workspace_dir" + return 1 + fi + + if [[ ! -d "$workspace_dir/repo" ]]; then + log_warning "Repository not found in workspace: $workspace_dir/repo" + return 1 + fi + + cd "$workspace_dir/repo" + + local git_status=$(git status --porcelain 2>/dev/null | wc -l) + local current_branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") + local last_commit=$(git log -1 --format="%h %s" 2>/dev/null || echo "unknown") + local workspace_size=$(du -sh "$workspace_dir" 2>/dev/null | cut -f1) + + echo "📊 Workspace Status:" + echo " Path: $workspace_dir/repo" + echo " Size: $workspace_size" + echo " Branch: $current_branch" + echo " Modified files: $git_status" + echo " Last commit: $last_commit" + + return 0 +} + +# Main function +main() { + local command="${1:-help}" + + case "$command" in + "setup") + ensure_workspace_base + setup_workspace "${2:-python-packages}" + ;; + "cleanup") + cleanup_all_workspaces + ;; + "list") + list_workspaces + ;; + "check") + check_workspace "${2:-python-packages}" + ;; + "help"|*) + echo "AITBC Workspace Management Script" + echo + echo "Usage: $0 [args]" + echo + echo "Commands:" + echo " setup [workspace] Setup a workspace (default: python-packages)" + echo " cleanup Clean up all workspaces" + echo " list List all workspaces" + echo " check [workspace] Check workspace status" + echo " help Show this help" + echo + echo "Available workspaces:" + echo " python-packages" + echo " javascript-packages" + echo " security-tests" + echo " integration-tests" + echo " compatibility-tests" + echo + echo "Examples:" + echo " $0 setup python-packages" + echo " $0 setup javascript-packages" + echo " $0 check python-packages" + echo " $0 list" + echo " $0 cleanup" + ;; + esac +} + +# Run main function +main "$@"