From 35fc07977ff3b6bcefb32ef0785ddc6fc918a214 Mon Sep 17 00:00:00 2001
From: aitbc1 <aitbc1-agent@keisanki.net>
Date: Fri, 27 Mar 2026 22:55:55 +0100
Subject: [PATCH] feat: add systemd sync solution to eliminate repo/active gap
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

SYSTEMD SYNC: Link active systemd files to repository for automatic sync

Problem Solved:
❌ Gap between repository systemd files and active systemd files
❌ Development changes in repo not reflected in running services
❌ Manual sync required to update systemd configuration
❌ Risk of configuration drift between repo and production

Solution Implemented:
✅ Symbolic links from /etc/systemd/system/ to /opt/aitbc/systemd
✅ Automatic sync script for manual operations
✅ CI/CD workflow for automatic sync on repository changes
✅ Backup mechanism for safe operations
✅ Verification and status checking

Files Created:
1. scripts/link-systemd.sh:
   - Creates symbolic links for all aitbc-* services
   - Handles .d directories automatically
   - Creates backups before making changes
   - Provides comprehensive status reporting

2. scripts/sync-systemd.sh:
   - Alternative copy-based sync method
   - For environments where symbolic links aren't preferred
   - Maintains file independence while keeping sync

3. .gitea/workflows/systemd-sync.yml:
   - Automatic CI/CD sync on repository changes
   - Triggers when systemd files are modified
   - Verifies link creation and service status
   - Provides manual instructions

Benefits:
✅ Active systemd files always match repository
✅ No configuration drift between repo and production
✅ Changes in repo immediately reflected
✅ Automatic sync on every repository update
✅ Safe operations with backups
✅ CI/CD integration for automation

Usage:
- Manual: sudo ./scripts/link-systemd.sh
- CI/CD: Automatic on systemd file changes
- Verification: ls -la /etc/systemd/system/aitbc-*
- Status: sudo systemctl status aitbc-*

This eliminates the gap between repository and active systemd
configuration, ensuring the repository always contains the current
running state and changes are immediately reflected.
---
 .gitea/workflows/systemd-sync.yml | 181 ++++++++++++++++++++++++++++++
 scripts/link-systemd.sh           | 108 ++++++++++++++++++
 scripts/sync-systemd.sh           |  87 ++++++++++++++
 3 files changed, 376 insertions(+)
 create mode 100644 .gitea/workflows/systemd-sync.yml
 create mode 100755 scripts/link-systemd.sh
 create mode 100755 scripts/sync-systemd.sh

diff --git a/.gitea/workflows/systemd-sync.yml b/.gitea/workflows/systemd-sync.yml
new file mode 100644
index 00000000..96996dee
--- /dev/null
+++ b/.gitea/workflows/systemd-sync.yml
@@ -0,0 +1,181 @@
+name: systemd-sync
+
+on:
+  push:
+    branches: [ main, develop ]
+    paths:
+      - 'systemd/**'
+      - '.gitea/workflows/systemd-sync.yml'
+  workflow_dispatch:
+
+# Prevent parallel execution - run workflows serially
+concurrency:
+  group: ci-workflows
+  cancel-in-progress: true
+
+jobs:
+  sync-systemd:
+    runs-on: debian
+    
+    steps:
+      - name: Setup workspace
+        run: |
+          echo "=== SYSTEMD SYNC SETUP ==="
+          echo "Current PWD: $(pwd)"
+          echo "Forcing absolute workspace path..."
+          
+          # Clean and create isolated workspace
+          rm -rf /opt/aitbc/systemd-sync-workspace
+          mkdir -p /opt/aitbc/systemd-sync-workspace
+          cd /opt/aitbc/systemd-sync-workspace
+          
+          # Ensure no git lock files exist
+          find . -name "*.lock" -delete 2>/dev/null || true
+          
+          echo "Workspace PWD: $(pwd)"
+          echo "Cloning repository..."
+          git clone https://gitea.bubuit.net/oib/aitbc.git repo
+          
+          cd repo
+          echo "Repo PWD: $(pwd)"
+          echo "Files in repo:"
+          ls -la
+
+      - name: Sync Systemd Files
+        run: |
+          echo "=== SYNCING SYSTEMD FILES ==="
+          cd /opt/aitbc/systemd-sync-workspace/repo
+          
+          echo "Repository systemd files:"
+          ls -la systemd/ | head -10
+          echo
+          echo "Active systemd files:"
+          ls -la /etc/systemd/system/aitbc-* | head -5 || echo "No active files found"
+          echo
+          
+          # Check if running as root (should be in CI)
+          if [[ $EUID -eq 0 ]]; then
+            echo "✅ Running as root - can sync systemd files"
+            
+            # Run the linking script
+            if [[ -f "scripts/link-systemd.sh" ]]; then
+              echo "🔗 Running systemd linking script..."
+              ./scripts/link-systemd.sh
+            else
+              echo "❌ Link script not found, creating manual sync..."
+              
+              # Manual sync as fallback
+              REPO_SYSTEMD_DIR="/opt/aitbc/systemd-sync-workspace/repo/systemd"
+              ACTIVE_SYSTEMD_DIR="/etc/systemd/system"
+              
+              # Create backup
+              BACKUP_DIR="/opt/aitbc/systemd-backup-$(date +%Y%m%d-%H%M%S)"
+              mkdir -p "$BACKUP_DIR"
+              find "$ACTIVE_SYSTEMD_DIR" -name "aitbc-*" -type f -exec cp {} "$BACKUP_DIR/" \; 2>/dev/null || true
+              
+              # Create symbolic links
+              for file in "$REPO_SYSTEMD_DIR"/aitbc-*; do
+                if [[ -f "$file" ]]; then
+                  filename=$(basename "$file")
+                  target="$ACTIVE_SYSTEMD_DIR/$filename"
+                  source="$REPO_SYSTEMD_DIR/$filename"
+                  
+                  echo "🔗 Linking: $filename"
+                  ln -sf "$source" "$target"
+                  
+                  # Handle .d directories
+                  if [[ -d "${file}.d" ]]; then
+                    target_dir="${target}.d"
+                    source_dir="${file}.d"
+                    rm -rf "$target_dir" 2>/dev/null || true
+                    ln -sf "$source_dir" "$target_dir"
+                  fi
+                fi
+              done
+              
+              systemctl daemon-reload
+              echo "✅ Manual systemd sync completed"
+            fi
+            
+          else
+            echo "⚠️  Not running as root - systemd sync requires root privileges"
+            echo "   To sync manually: sudo ./scripts/link-systemd.sh"
+          fi
+
+      - name: Verify Sync
+        run: |
+          echo "=== VERIFYING SYSTEMD SYNC ==="
+          cd /opt/aitbc/systemd-sync-workspace/repo
+          
+          if [[ $EUID -eq 0 ]]; then
+            echo "🔍 Verifying systemd links..."
+            
+            # Check if links exist
+            echo "Checking symbolic links:"
+            for file in systemd/aitbc-*; do
+              if [[ -f "$file" ]]; then
+                filename=$(basename "$file")
+                target="/etc/systemd/system/$filename"
+                
+                if [[ -L "$target" ]]; then
+                  echo "✅ $filename -> $(readlink "$target")"
+                elif [[ -f "$target" ]]; then
+                  echo "⚠️  $filename exists but is not a link (copied file)"
+                else
+                  echo "❌ $filename not found in active systemd"
+                fi
+              fi
+            done
+            
+            echo
+            echo "📊 Summary:"
+            echo "  Repository files: $(find systemd -name 'aitbc-*' -type f | wc -l)"
+            echo "  Active files: $(find /etc/systemd/system -name 'aitbc-*' -type f | wc -l)"
+            echo "  Symbolic links: $(find /etc/systemd/system -name 'aitbc-*' -type l | wc -l)"
+            
+          else
+            echo "⚠️  Cannot verify without root privileges"
+          fi
+
+      - name: Service Status Check
+        if: always()
+        run: |
+          echo "=== SERVICE STATUS CHECK ==="
+          
+          if [[ $EUID -eq 0 ]]; then
+            echo "🔍 Checking AITBC service status..."
+            
+            # Check if services are enabled
+            echo "Enabled services:"
+            systemctl list-unit-files 'aitbc-*' --state=enabled | head -5 || echo "No enabled services found"
+            
+            echo
+            echo "Failed services:"
+            systemctl list-units 'aitbc-*' --state=failed | head -5 || echo "No failed services found"
+            
+            echo
+            echo "Running services:"
+            systemctl list-units 'aitbc-*' --state=running | head -5 || echo "No running services found"
+            
+          else
+            echo "⚠️  Cannot check service status without root privileges"
+          fi
+
+      - name: Instructions
+        run: |
+          echo "=== SYSTEMD SYNC INSTRUCTIONS ==="
+          echo
+          echo "🔧 Manual sync (if needed):"
+          echo "  sudo ./scripts/link-systemd.sh"
+          echo
+          echo "🔄 Restart services:"
+          echo "  sudo systemctl restart aitbc-blockchain-node"
+          echo "  sudo systemctl restart aitbc-coordinator-api"
+          echo "  sudo systemctl restart aitbc-*"
+          echo
+          echo "🔍 Check status:"
+          echo "  sudo systemctl status aitbc-*"
+          echo
+          echo "🔍 Verify links:"
+          echo "  ls -la /etc/systemd/system/aitbc-*"
+          echo "  readlink /etc/systemd/system/aitbc-blockchain-node.service"
diff --git a/scripts/link-systemd.sh b/scripts/link-systemd.sh
new file mode 100755
index 00000000..2de7a06d
--- /dev/null
+++ b/scripts/link-systemd.sh
@@ -0,0 +1,108 @@
+#!/bin/bash
+
+# AITBC Systemd Link Script
+# Creates symbolic links from active systemd to repository systemd files
+# Keeps active systemd always in sync with repository
+
+set -e
+
+REPO_SYSTEMD_DIR="/opt/aitbc/systemd"
+ACTIVE_SYSTEMD_DIR="/etc/systemd/system"
+
+echo "=== AITBC SYSTEMD LINKING ==="
+echo "Repository: $REPO_SYSTEMD_DIR"
+echo "Active: $ACTIVE_SYSTEMD_DIR"
+echo
+
+# Check if running as root
+if [[ $EUID -ne 0 ]]; then
+   echo "❌ This script must be run as root (use sudo)"
+   echo "   sudo $0"
+   exit 1
+fi
+
+# Check if repository systemd directory exists
+if [[ ! -d "$REPO_SYSTEMD_DIR" ]]; then
+    echo "❌ Repository systemd directory not found: $REPO_SYSTEMD_DIR"
+    exit 1
+fi
+
+echo "🔍 Creating symbolic links for AITBC systemd files..."
+
+# Create backup of current active systemd files
+BACKUP_DIR="/opt/aitbc/systemd-backup-$(date +%Y%m%d-%H%M%S)"
+echo "📦 Creating backup: $BACKUP_DIR"
+mkdir -p "$BACKUP_DIR"
+find "$ACTIVE_SYSTEMD_DIR" -name "aitbc-*" -type f -exec cp {} "$BACKUP_DIR/" \; 2>/dev/null || true
+
+# Remove existing aitbc-* files (but not directories)
+echo "🧹 Removing existing systemd files..."
+find "$ACTIVE_SYSTEMD_DIR" -name "aitbc-*" -type f -delete 2>/dev/null || true
+
+# Create symbolic links
+echo "🔗 Creating symbolic links..."
+linked_files=0
+for file in "$REPO_SYSTEMD_DIR"/aitbc-*; do
+    if [[ -f "$file" ]]; then
+        filename=$(basename "$file")
+        target="$ACTIVE_SYSTEMD_DIR/$filename"
+        source="$REPO_SYSTEMD_DIR/$filename"
+        
+        echo "  🔗 Linking: $filename -> $source"
+        
+        # Create symbolic link
+        ln -sf "$source" "$target"
+        
+        # Handle .d directories
+        if [[ -d "${file}.d" ]]; then
+            target_dir="${target}.d"
+            source_dir="${file}.d"
+            
+            echo "    📁 Linking directory: ${filename}.d -> ${source_dir}"
+            
+            # Remove existing directory
+            rm -rf "$target_dir" 2>/dev/null || true
+            
+            # Create symbolic link for directory
+            ln -sf "$source_dir" "$target_dir"
+        fi
+        
+        ((linked_files++))
+    fi
+done
+
+echo
+echo "🔄 Reloading systemd daemon..."
+systemctl daemon-reload
+
+echo
+echo "✅ Systemd linking completed!"
+echo
+echo "📊 Link Summary:"
+echo "  Linked files: $linked_files"
+echo "  Repository: $REPO_SYSTEMD_DIR"
+echo "  Active: $ACTIVE_SYSTEMD_DIR"
+echo "  Backup location: $BACKUP_DIR"
+echo
+echo "🎯 Benefits:"
+echo "  ✅ Active systemd files always match repository"
+echo "  ✅ No gap between repo and running services"
+echo "  ✅ Changes in repo immediately reflected"
+echo "  ✅ Automatic sync on every repository update"
+echo
+echo "🔧 To restart services:"
+echo "  sudo systemctl restart aitbc-blockchain-node"
+echo "  sudo systemctl restart aitbc-coordinator-api"
+echo "  # ... or restart all AITBC services:"
+echo "  sudo systemctl restart aitbc-*"
+echo
+echo "🔍 To check status:"
+echo "  sudo systemctl status aitbc-*"
+echo
+echo "🔍 To verify links:"
+echo "  ls -la /etc/systemd/system/aitbc-*"
+echo "  readlink /etc/systemd/system/aitbc-blockchain-node.service"
+echo
+echo "⚠️  If you need to restore backup:"
+echo "  sudo cp $BACKUP_DIR/* /etc/systemd/system/"
+echo "  sudo systemctl daemon-reload"
diff --git a/scripts/sync-systemd.sh b/scripts/sync-systemd.sh
new file mode 100755
index 00000000..871a0a99
--- /dev/null
+++ b/scripts/sync-systemd.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+
+# AITBC Systemd Sync Script
+# Syncs repository systemd files to active systemd configuration
+# Eliminates gap between repo and running services
+
+set -e
+
+REPO_SYSTEMD_DIR="/opt/aitbc/systemd"
+ACTIVE_SYSTEMD_DIR="/etc/systemd/system"
+
+echo "=== AITBC SYSTEMD SYNC ==="
+echo "Repository: $REPO_SYSTEMD_DIR"
+echo "Active: $ACTIVE_SYSTEMD_DIR"
+echo
+
+# Check if running as root
+if [[ $EUID -ne 0 ]]; then
+   echo "❌ This script must be run as root (use sudo)"
+   echo "   sudo $0"
+   exit 1
+fi
+
+# Check if repository systemd directory exists
+if [[ ! -d "$REPO_SYSTEMD_DIR" ]]; then
+    echo "❌ Repository systemd directory not found: $REPO_SYSTEMD_DIR"
+    exit 1
+fi
+
+echo "🔍 Scanning for AITBC systemd files..."
+
+# Create backup of current active systemd files
+BACKUP_DIR="/opt/aitbc/systemd-backup-$(date +%Y%m%d-%H%M%S)"
+echo "📦 Creating backup: $BACKUP_DIR"
+mkdir -p "$BACKUP_DIR"
+find "$ACTIVE_SYSTEMD_DIR" -name "aitbc-*" -type f -exec cp {} "$BACKUP_DIR/" \;
+
+# Sync repository files to active systemd
+echo "🔄 Syncing systemd files..."
+
+# Copy all aitbc-* files from repo to active systemd
+for file in "$REPO_SYSTEMD_DIR"/aitbc-*; do
+    if [[ -f "$file" ]]; then
+        filename=$(basename "$file")
+        target="$ACTIVE_SYSTEMD_DIR/$filename"
+        
+        echo "  📄 Syncing: $filename"
+        
+        # Copy file with proper permissions
+        cp "$file" "$target"
+        chmod 644 "$target"
+        
+        # Handle .d directories
+        if [[ -d "${file}.d" ]]; then
+            target_dir="${target}.d"
+            echo "    📁 Syncing directory: ${filename}.d"
+            mkdir -p "$target_dir"
+            cp -r "${file}.d"/* "$target_dir/"
+            chmod 644 "$target_dir"/*
+        fi
+    fi
+done
+
+echo
+echo "🔄 Reloading systemd daemon..."
+systemctl daemon-reload
+
+echo
+echo "✅ Systemd sync completed!"
+echo
+echo "📊 Sync Summary:"
+echo "  Repository files: $(find "$REPO_SYSTEMD_DIR" -name 'aitbc-*' -type f | wc -l)"
+echo "  Active files: $(find "$ACTIVE_SYSTEMD_DIR" -name 'aitbc-*' -type f | wc -l)"
+echo "  Backup location: $BACKUP_DIR"
+echo
+echo "🔧 To restart services:"
+echo "  sudo systemctl restart aitbc-blockchain-node"
+echo "  sudo systemctl restart aitbc-coordinator-api"
+echo "  # ... or restart all AITBC services:"
+echo "  sudo systemctl restart aitbc-*"
+echo
+echo "🔍 To check status:"
+echo "  sudo systemctl status aitbc-*"
+echo
+echo "⚠️  If you need to restore backup:"
+echo "  sudo cp $BACKUP_DIR/* /etc/systemd/system/"
+echo "  sudo systemctl daemon-reload"