refactor: comprehensive scripts directory reorganization by functionality
Scripts Directory Reorganization - Complete: ✅ FUNCTIONAL ORGANIZATION: Scripts sorted into 8 logical categories - github/: GitHub and Git operations (6 files) - sync/: Synchronization and data replication (4 files) - security/: Security and audit operations (2 files) - monitoring/: System and service monitoring (6 files) - maintenance/: System maintenance and cleanup (4 files) - deployment/: Deployment and provisioning (11 files) - testing/: Testing and quality assurance (13 files) - utils/: Utility scripts and helpers (47 files) ✅ ROOT DIRECTORY CLEANED: Only README.md remains in scripts root - scripts/README.md: Main documentation - scripts/SCRIPTS_ORGANIZATION.md: Complete organization guide - All functional scripts moved to appropriate subdirectories ✅ SCRIPTS CATEGORIZATION: 📁 GitHub Operations: PR resolution, repository management, Git workflows 📁 Synchronization: Bulk sync, fast sync, sync detection, SystemD sync 📁 Security: Security audits, monitoring, vulnerability scanning 📁 Monitoring: Health checks, log monitoring, network monitoring, production monitoring 📁 Maintenance: Cleanup operations, performance tuning, weekly maintenance 📁 Deployment: Release building, node provisioning, DAO deployment, production deployment 📁 Testing: E2E testing, workflow testing, QA cycles, service testing 📁 Utilities: System management, setup scripts, helpers, tools ✅ ORGANIZATION BENEFITS: - Better Navigation: Scripts grouped by functionality - Easier Maintenance: Related scripts grouped together - Scalable Structure: Easy to add new scripts to appropriate categories - Clear Documentation: Comprehensive organization guide with descriptions - Improved Workflow: Quick access to relevant scripts by category ✅ DOCUMENTATION ENHANCED: - SCRIPTS_ORGANIZATION.md: Complete directory structure and usage guide - Quick Reference: Common script usage examples - Script Descriptions: Purpose and functionality for each script - Maintenance Guidelines: How to keep organization current DIRECTORY STRUCTURE: 📁 scripts/ ├── README.md (Main documentation) ├── SCRIPTS_ORGANIZATION.md (Organization guide) ├── github/ (6 files - GitHub operations) ├── sync/ (4 files - Synchronization) ├── security/ (2 files - Security) ├── monitoring/ (6 files - Monitoring) ├── maintenance/ (4 files - Maintenance) ├── deployment/ (11 files - Deployment) ├── testing/ (13 files - Testing) ├── utils/ (47 files - Utilities) ├── ci/ (existing - CI/CD) ├── deployment/ (existing - legacy deployment) ├── development/ (existing - Development tools) ├── monitoring/ (existing - Legacy monitoring) ├── services/ (existing - Service management) ├── testing/ (existing - Legacy testing) ├── utils/ (existing - Legacy utilities) ├── workflow/ (existing - Workflow automation) └── workflow-openclaw/ (existing - OpenClaw workflows) RESULT: Successfully reorganized 27 unorganized scripts into 8 functional categories, creating a clean, maintainable, and well-documented scripts directory structure with comprehensive organization guide.
This commit is contained in:
186
scripts/sync/bulk_sync.sh
Executable file
186
scripts/sync/bulk_sync.sh
Executable file
@@ -0,0 +1,186 @@
|
||||
#!/bin/bash
|
||||
|
||||
# AITBC Bulk Sync Script
|
||||
# Detects large sync differences and performs bulk synchronization
|
||||
|
||||
set -e
|
||||
|
||||
# Configuration
|
||||
GENESIS_NODE="10.1.223.40"
|
||||
GENESIS_PORT="8006"
|
||||
LOCAL_PORT="8006"
|
||||
MAX_SYNC_DIFF=100 # Trigger bulk sync if difference > 100 blocks
|
||||
BULK_BATCH_SIZE=500 # Process 500 blocks at a time
|
||||
|
||||
echo "=== 🔄 AITBC BULK SYNC DETECTOR ==="
|
||||
echo "Timestamp: $(date)"
|
||||
echo ""
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Function to get blockchain height
|
||||
get_height() {
|
||||
local url=$1
|
||||
curl -s "$url/rpc/head" | jq -r .height 2>/dev/null || echo "0"
|
||||
}
|
||||
|
||||
# Function to import a block
|
||||
import_block() {
|
||||
local block_data=$1
|
||||
curl -s -X POST "http://localhost:$LOCAL_PORT/rpc/importBlock" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$block_data" | jq -r .accepted 2>/dev/null || echo "false"
|
||||
}
|
||||
|
||||
# Function to get blocks range
|
||||
get_blocks_range() {
|
||||
local start=$1
|
||||
local end=$2
|
||||
curl -s "http://$GENESIS_NODE:$GENESIS_PORT/rpc/blocks-range?start=$start&end=$end" | jq -r '.blocks[]' 2>/dev/null
|
||||
}
|
||||
|
||||
echo "1. 🔍 DETECTING SYNC DIFFERENCE"
|
||||
echo "=============================="
|
||||
|
||||
# Get current heights
|
||||
local_height=$(get_height "http://localhost:$LOCAL_PORT")
|
||||
genesis_height=$(get_height "http://$GENESIS_NODE:$GENESIS_PORT")
|
||||
|
||||
echo "Local height: $local_height"
|
||||
echo "Genesis height: $genesis_height"
|
||||
|
||||
# Calculate difference
|
||||
if [ "$local_height" -eq 0 ] || [ "$genesis_height" -eq 0 ]; then
|
||||
echo -e "${RED}❌ ERROR: Cannot get blockchain heights${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
diff=$((genesis_height - local_height))
|
||||
echo "Sync difference: $diff blocks"
|
||||
|
||||
# Determine if bulk sync is needed
|
||||
if [ "$diff" -le $MAX_SYNC_DIFF ]; then
|
||||
echo -e "${GREEN}✅ Sync difference is within normal range ($diff <= $MAX_SYNC_DIFF)${NC}"
|
||||
echo "Normal sync should handle this difference."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo -e "${YELLOW}⚠️ LARGE SYNC DIFFERENCE DETECTED${NC}"
|
||||
echo "Difference ($diff) exceeds threshold ($MAX_SYNC_DIFF)"
|
||||
echo "Initiating bulk sync..."
|
||||
|
||||
echo ""
|
||||
echo "2. 🔄 INITIATING BULK SYNC"
|
||||
echo "=========================="
|
||||
|
||||
# Calculate sync range
|
||||
start_height=$((local_height + 1))
|
||||
end_height=$genesis_height
|
||||
|
||||
echo "Sync range: $start_height to $end_height"
|
||||
echo "Batch size: $BULK_BATCH_SIZE blocks"
|
||||
|
||||
# Process in batches
|
||||
current_start=$start_height
|
||||
total_imported=0
|
||||
total_failed=0
|
||||
|
||||
while [ "$current_start" -le "$end_height" ]; do
|
||||
current_end=$((current_start + BULK_BATCH_SIZE - 1))
|
||||
if [ "$current_end" -gt "$end_height" ]; then
|
||||
current_end=$end_height
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Processing batch: $current_start to $current_end"
|
||||
|
||||
# Get blocks from genesis node
|
||||
blocks_json=$(curl -s "http://$GENESIS_NODE:$GENESIS_PORT/rpc/blocks-range?start=$current_start&end=$current_end")
|
||||
|
||||
if [ $? -ne 0 ] || [ -z "$blocks_json" ]; then
|
||||
echo -e "${RED}❌ Failed to get blocks range${NC}"
|
||||
break
|
||||
fi
|
||||
|
||||
# Process each block in the batch
|
||||
batch_imported=0
|
||||
batch_failed=0
|
||||
|
||||
# Extract blocks and import them
|
||||
echo "$blocks_json" | jq -c '.blocks[]' 2>/dev/null | while read -r block; do
|
||||
if [ -n "$block" ] && [ "$block" != "null" ]; then
|
||||
# Extract block data for import
|
||||
block_height=$(echo "$block" | jq -r .height)
|
||||
block_hash=$(echo "$block" | jq -r .hash)
|
||||
parent_hash=$(echo "$block" | jq -r .parent_hash)
|
||||
proposer=$(echo "$block" | jq -r .proposer)
|
||||
timestamp=$(echo "$block" | jq -r .timestamp)
|
||||
tx_count=$(echo "$block" | jq -r .tx_count)
|
||||
|
||||
# Create import request
|
||||
import_request=$(cat << EOF
|
||||
{
|
||||
"height": $block_height,
|
||||
"hash": "$block_hash",
|
||||
"parent_hash": "$parent_hash",
|
||||
"proposer": "$proposer",
|
||||
"timestamp": "$timestamp",
|
||||
"tx_count": $tx_count
|
||||
}
|
||||
EOF
|
||||
)
|
||||
|
||||
# Import block
|
||||
result=$(import_block "$import_request")
|
||||
|
||||
if [ "$result" = "true" ]; then
|
||||
echo -e " ${GREEN}✅${NC} Imported block $block_height"
|
||||
((batch_imported++))
|
||||
else
|
||||
echo -e " ${RED}❌${NC} Failed to import block $block_height"
|
||||
((batch_failed++))
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Update counters
|
||||
total_imported=$((total_imported + batch_imported))
|
||||
total_failed=$((total_failed + batch_failed))
|
||||
|
||||
echo "Batch complete: $batch_imported imported, $batch_failed failed"
|
||||
|
||||
# Move to next batch
|
||||
current_start=$((current_end + 1))
|
||||
|
||||
# Brief pause to avoid overwhelming the system
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "3. 📊 SYNC RESULTS"
|
||||
echo "================"
|
||||
|
||||
# Final verification
|
||||
final_local_height=$(get_height "http://localhost:$LOCAL_PORT")
|
||||
final_diff=$((genesis_height - final_local_height))
|
||||
|
||||
echo "Initial difference: $diff blocks"
|
||||
echo "Final difference: $final_diff blocks"
|
||||
echo "Blocks imported: $total_imported"
|
||||
echo "Blocks failed: $total_failed"
|
||||
|
||||
# Determine success
|
||||
if [ "$final_diff" -le $MAX_SYNC_DIFF ]; then
|
||||
echo -e "${GREEN}✅ BULK SYNC SUCCESSFUL${NC}"
|
||||
echo "Sync difference is now within normal range."
|
||||
else
|
||||
echo -e "${YELLOW}⚠️ PARTIAL SYNC${NC}"
|
||||
echo "Some blocks may still need to sync normally."
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== 🔄 BULK SYNC COMPLETE ==="
|
||||
84
scripts/sync/fast_bulk_sync.sh
Executable file
84
scripts/sync/fast_bulk_sync.sh
Executable file
@@ -0,0 +1,84 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Fast AITBC Bulk Sync - Optimized for large sync differences
|
||||
|
||||
GENESIS_NODE="10.1.223.40"
|
||||
GENESIS_PORT="8006"
|
||||
LOCAL_PORT="8006"
|
||||
MAX_SYNC_DIFF=100
|
||||
BULK_BATCH_SIZE=1000
|
||||
|
||||
echo "=== 🚀 FAST AITBC BULK SYNC ==="
|
||||
echo "Timestamp: $(date)"
|
||||
|
||||
# Get current heights
|
||||
local_height=$(curl -s "http://localhost:$LOCAL_PORT/rpc/head" | jq -r .height 2>/dev/null || echo "0")
|
||||
genesis_height=$(curl -s "http://$GENESIS_NODE:$GENESIS_PORT/rpc/head" | jq -r .height 2>/dev/null || echo "0")
|
||||
|
||||
diff=$((genesis_height - local_height))
|
||||
echo "Current sync difference: $diff blocks"
|
||||
|
||||
if [ "$diff" -le $MAX_SYNC_DIFF ]; then
|
||||
echo "✅ Sync is within normal range"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "🔄 Starting fast bulk sync..."
|
||||
start_height=$((local_height + 1))
|
||||
end_height=$genesis_height
|
||||
|
||||
# Process in larger batches
|
||||
current_start=$start_height
|
||||
while [ "$current_start" -le "$end_height" ]; do
|
||||
current_end=$((current_start + BULK_BATCH_SIZE - 1))
|
||||
if [ "$current_end" -gt "$end_height" ]; then
|
||||
current_end=$end_height
|
||||
fi
|
||||
|
||||
echo "Processing batch: $current_start to $current_end"
|
||||
|
||||
# Get blocks and import them
|
||||
curl -s "http://$GENESIS_NODE:$GENESIS_PORT/rpc/blocks-range?start=$current_start&end=$current_end" | \
|
||||
jq -r '.blocks[] | @base64' | while read -r block_b64; do
|
||||
if [ -n "$block_b64" ] && [ "$block_b64" != "null" ]; then
|
||||
block=$(echo "$block_b64" | base64 -d)
|
||||
height=$(echo "$block" | jq -r .height)
|
||||
hash=$(echo "$block" | jq -r .hash)
|
||||
parent_hash=$(echo "$block" | jq -r .parent_hash)
|
||||
proposer=$(echo "$block" | jq -r .proposer)
|
||||
timestamp=$(echo "$block" | jq -r .timestamp)
|
||||
tx_count=$(echo "$block" | jq -r .tx_count)
|
||||
|
||||
# Create import request
|
||||
import_req="{\"height\":$height,\"hash\":\"$hash\",\"parent_hash\":\"$parent_hash\",\"proposer\":\"$proposer\",\"timestamp\":\"$timestamp\",\"tx_count\":$tx_count}"
|
||||
|
||||
# Import block
|
||||
result=$(curl -s -X POST "http://localhost:$LOCAL_PORT/rpc/importBlock" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$import_req" | jq -r .accepted 2>/dev/null || echo "false")
|
||||
|
||||
if [ "$result" = "true" ]; then
|
||||
echo "✅ Imported block $height"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
current_start=$((current_end + 1))
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
# Check final result
|
||||
final_height=$(curl -s "http://localhost:$LOCAL_PORT/rpc/head" | jq -r .height 2>/dev/null || echo "0")
|
||||
final_diff=$((genesis_height - final_height))
|
||||
|
||||
echo ""
|
||||
echo "📊 SYNC RESULTS:"
|
||||
echo "Initial difference: $diff blocks"
|
||||
echo "Final difference: $final_diff blocks"
|
||||
echo "Blocks synced: $((final_height - local_height))"
|
||||
|
||||
if [ "$final_diff" -le $MAX_SYNC_DIFF ]; then
|
||||
echo "✅ Fast bulk sync successful!"
|
||||
else
|
||||
echo "⚠️ Partial sync, may need additional runs"
|
||||
fi
|
||||
87
scripts/sync/sync-systemd.sh
Executable file
87
scripts/sync/sync-systemd.sh
Executable file
@@ -0,0 +1,87 @@
|
||||
#!/bin/bash
|
||||
|
||||
# AITBC Systemd Sync Script
|
||||
# Syncs repository systemd files to active systemd configuration
|
||||
# Eliminates gap between repo and running services
|
||||
|
||||
set -e
|
||||
|
||||
REPO_SYSTEMD_DIR="/opt/aitbc/systemd"
|
||||
ACTIVE_SYSTEMD_DIR="/etc/systemd/system"
|
||||
|
||||
echo "=== AITBC SYSTEMD SYNC ==="
|
||||
echo "Repository: $REPO_SYSTEMD_DIR"
|
||||
echo "Active: $ACTIVE_SYSTEMD_DIR"
|
||||
echo
|
||||
|
||||
# Check if running as root
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "❌ This script must be run as root (use sudo)"
|
||||
echo " sudo $0"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if repository systemd directory exists
|
||||
if [[ ! -d "$REPO_SYSTEMD_DIR" ]]; then
|
||||
echo "❌ Repository systemd directory not found: $REPO_SYSTEMD_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "🔍 Scanning for AITBC systemd files..."
|
||||
|
||||
# Create backup of current active systemd files
|
||||
BACKUP_DIR="/opt/aitbc/systemd-backup-$(date +%Y%m%d-%H%M%S)"
|
||||
echo "📦 Creating backup: $BACKUP_DIR"
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
find "$ACTIVE_SYSTEMD_DIR" -name "aitbc-*" -type f -exec cp {} "$BACKUP_DIR/" \;
|
||||
|
||||
# Sync repository files to active systemd
|
||||
echo "🔄 Syncing systemd files..."
|
||||
|
||||
# Copy all aitbc-* files from repo to active systemd
|
||||
for file in "$REPO_SYSTEMD_DIR"/aitbc-*; do
|
||||
if [[ -f "$file" ]]; then
|
||||
filename=$(basename "$file")
|
||||
target="$ACTIVE_SYSTEMD_DIR/$filename"
|
||||
|
||||
echo " 📄 Syncing: $filename"
|
||||
|
||||
# Copy file with proper permissions
|
||||
cp "$file" "$target"
|
||||
chmod 644 "$target"
|
||||
|
||||
# Handle .d directories
|
||||
if [[ -d "${file}.d" ]]; then
|
||||
target_dir="${target}.d"
|
||||
echo " 📁 Syncing directory: ${filename}.d"
|
||||
mkdir -p "$target_dir"
|
||||
cp -r "${file}.d"/* "$target_dir/"
|
||||
chmod 644 "$target_dir"/*
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "🔄 Reloading systemd daemon..."
|
||||
systemctl daemon-reload
|
||||
|
||||
echo
|
||||
echo "✅ Systemd sync completed!"
|
||||
echo
|
||||
echo "📊 Sync Summary:"
|
||||
echo " Repository files: $(find "$REPO_SYSTEMD_DIR" -name 'aitbc-*' -type f | wc -l)"
|
||||
echo " Active files: $(find "$ACTIVE_SYSTEMD_DIR" -name 'aitbc-*' -type f | wc -l)"
|
||||
echo " Backup location: $BACKUP_DIR"
|
||||
echo
|
||||
echo "🔧 To restart services:"
|
||||
echo " sudo systemctl restart aitbc-blockchain-node"
|
||||
echo " sudo systemctl restart aitbc-coordinator-api"
|
||||
echo " # ... or restart all AITBC services:"
|
||||
echo " sudo systemctl restart aitbc-*"
|
||||
echo
|
||||
echo "🔍 To check status:"
|
||||
echo " sudo systemctl status aitbc-*"
|
||||
echo
|
||||
echo "⚠️ If you need to restore backup:"
|
||||
echo " sudo cp $BACKUP_DIR/* /etc/systemd/system/"
|
||||
echo " sudo systemctl daemon-reload"
|
||||
52
scripts/sync/sync_detector.sh
Normal file
52
scripts/sync/sync_detector.sh
Normal file
@@ -0,0 +1,52 @@
|
||||
#!/bin/bash
|
||||
|
||||
GENESIS_NODE="10.1.223.40"
|
||||
GENESIS_PORT="8006"
|
||||
LOCAL_PORT="8006"
|
||||
MAX_SYNC_DIFF=100
|
||||
LOG_FILE="/var/log/aitbc/sync_detector.log"
|
||||
|
||||
log_sync() {
|
||||
echo "[$(date)] $1" >> "$LOG_FILE"
|
||||
}
|
||||
|
||||
check_sync_diff() {
|
||||
local_height=$(curl -s "http://localhost:$LOCAL_PORT/rpc/head" | jq -r .height 2>/dev/null || echo "0")
|
||||
genesis_height=$(curl -s "http://$GENESIS_NODE:$GENESIS_PORT/rpc/head" | jq -r .height 2>/dev/null || echo "0")
|
||||
|
||||
if [ "$local_height" -eq 0 ] || [ "$genesis_height" -eq 0 ]; then
|
||||
log_sync "ERROR: Cannot get blockchain heights"
|
||||
return 1
|
||||
fi
|
||||
|
||||
diff=$((genesis_height - local_height))
|
||||
echo "$diff"
|
||||
}
|
||||
|
||||
main() {
|
||||
log_sync "Starting sync check"
|
||||
|
||||
diff=$(check_sync_diff)
|
||||
log_sync "Sync difference: $diff blocks"
|
||||
|
||||
if [ "$diff" -gt "$MAX_SYNC_DIFF" ]; then
|
||||
log_sync "Large sync difference detected ($diff > $MAX_SYNC_DIFF), initiating bulk sync"
|
||||
/opt/aitbc/scripts/bulk_sync.sh >> "$LOG_FILE" 2>&1
|
||||
|
||||
new_diff=$(check_sync_diff)
|
||||
log_sync "Post-sync difference: $new_diff blocks"
|
||||
|
||||
if [ "$new_diff" -le "$MAX_SYNC_DIFF" ]; then
|
||||
log_sync "Bulk sync successful"
|
||||
else
|
||||
log_sync "Bulk sync partially successful, may need additional runs"
|
||||
fi
|
||||
else
|
||||
log_sync "Sync difference is normal ($diff <= $MAX_SYNC_DIFF)"
|
||||
fi
|
||||
|
||||
log_sync "Sync check completed"
|
||||
}
|
||||
|
||||
main
|
||||
EOF && chmod +x /opt/aitbc/scripts/sync_detector.sh && echo "✅ Sync detector script fixed and made executable"
|
||||
Reference in New Issue
Block a user