aitbc/scripts/deployment/production-deploy.sh

#!/bin/bash

# AITBC Production Deployment Script
# This script handles production deployment with zero-downtime

set -e

# Production Configuration
ENVIRONMENT="production"
VERSION=${1:-latest}
REGION=${2:-us-east-1}
NAMESPACE="aitbc-prod"
DOMAIN="aitbc.dev"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Logging
log() {
    echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"
}

error() {
    echo -e "${RED}[ERROR]${NC} $1"
    exit 1
}

success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1"
}

warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

# Pre-deployment checks
pre_deployment_checks() {
    log "Running pre-deployment checks..."
    
    # Check if we're on production branch
    current_branch=$(git branch --show-current)
    if [ "$current_branch" != "production" ]; then
        error "Must be on production branch to deploy to production"
    fi
    
    # Check if all tests pass
    log "Running tests..."
    pytest tests/unit/ -v --tb=short || error "Unit tests failed"
    pytest tests/integration/ -v --tb=short || error "Integration tests failed"
    pytest tests/security/ -v --tb=short || error "Security tests failed"
    pytest tests/performance/test_performance_lightweight.py::TestPerformance::test_cli_performance -v --tb=short || error "Performance tests failed"
    
    # Check if production infrastructure is ready
    log "Checking production infrastructure..."
    kubectl get nodes | grep -q "Ready" || error "Production nodes not ready"
    kubectl get namespace $NAMESPACE || kubectl create namespace $NAMESPACE
    
    success "Pre-deployment checks passed"
}

# Backup current deployment
backup_current_deployment() {
    log "Backing up current deployment..."
    
    # Create backup directory
    backup_dir="/opt/aitbc/backups/pre-deployment-$(date +%Y%m%d_%H%M%S)"
    mkdir -p $backup_dir
    
    # Backup current configuration
    kubectl get all -n $NAMESPACE -o yaml > $backup_dir/current-deployment.yaml
    
    # Backup database
    pg_dump $DATABASE_URL | gzip > $backup_dir/database_backup.sql.gz
    
    # Backup application data
    kubectl exec -n $NAMESPACE deployment/coordinator-api -- tar -czf /tmp/app_data_backup.tar.gz /app/data
    kubectl cp $NAMESPACE/deployment/coordinator-api:/tmp/app_data_backup.tar.gz $backup_dir/app_data_backup.tar.gz
    
    success "Backup completed: $backup_dir"
}

# Build production images
build_production_images() {
    log "Building production images..."
    
    # Build CLI image
    docker build -t aitbc/cli:$VERSION -f Dockerfile --target production . || error "Failed to build CLI image"
    
    # Build service images
    for service_dir in apps/*/; do
        if [ -f "$service_dir/Dockerfile" ]; then
            service_name=$(basename "$service_dir")
            log "Building $service_name image..."
            docker build -t aitbc/$service_name:$VERSION -f "$service_dir/Dockerfile" "$service_dir" || error "Failed to build $service_name image"
        fi
    done
    
    # Push images to registry
    log "Pushing images to registry..."
    docker push aitbc/cli:$VERSION
    
    for service_dir in apps/*/; do
        if [ -f "$service_dir/Dockerfile" ]; then
            service_name=$(basename "$service_dir")
            docker push aitbc/$service_name:$VERSION
        fi
    done
    
    success "Production images built and pushed"
}

# Deploy database
deploy_database() {
    log "Deploying database..."
    
    # Deploy PostgreSQL
    helm upgrade --install postgres bitnami/postgresql \
        --namespace $NAMESPACE \
        --set auth.postgresPassword=$POSTGRES_PASSWORD \
        --set auth.database=aitbc_prod \
        --set primary.persistence.size=100Gi \
        --set primary.resources.requests.memory=8Gi \
        --set primary.resources.requests.cpu=2000m \
        --set primary.resources.limits.memory=16Gi \
        --set primary.resources.limits.cpu=4000m \
        --set readReplicas.replicaCount=1 \
        --set readReplicas.persistence.size=50Gi \
        --wait \
        --timeout 10m || error "Failed to deploy PostgreSQL"
    
    # Deploy Redis
    helm upgrade --install redis bitnami/redis \
        --namespace $NAMESPACE \
        --set auth.password=$REDIS_PASSWORD \
        --set master.persistence.size=20Gi \
        --set master.resources.requests.memory=2Gi \
        --set master.resources.requests.cpu=1000m \
        --set master.resources.limits.memory=4Gi \
        --set master.resources.limits.cpu=2000m \
        --set replica.replicaCount=2 \
        --wait \
        --timeout 5m || error "Failed to deploy Redis"
    
    success "Database deployed successfully"
}

# Deploy core services
deploy_core_services() {
    log "Deploying core services..."
    
    # Deploy blockchain services
    for service in blockchain-node consensus-node network-node; do
        log "Deploying $service..."
        
        # Create deployment manifest
        cat > /tmp/$service-deployment.yaml << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: $service
  namespace: $NAMESPACE
spec:
  replicas: 2
  selector:
    matchLabels:
      app: $service
  template:
    metadata:
      labels:
        app: $service
    spec:
      containers:
      - name: $service
        image: aitbc/$service:$VERSION
        ports:
        - containerPort: 8007
          name: http
        env:
        - name: NODE_ENV
          value: "production"
        - name: DATABASE_URL
          valueFrom:
            secretKeyRef:
              name: aitbc-secrets
              key: database-url
        - name: REDIS_URL
          valueFrom:
            secretKeyRef:
              name: aitbc-secrets
              key: redis-url
        resources:
          requests:
            memory: "2Gi"
            cpu: "1000m"
          limits:
            memory: "4Gi"
            cpu: "2000m"
        livenessProbe:
          httpGet:
            path: /health
            port: 8007
          initialDelaySeconds: 30
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /health
            port: 8007
          initialDelaySeconds: 5
          periodSeconds: 5
---
apiVersion: v1
kind: Service
metadata:
  name: $service
  namespace: $NAMESPACE
spec:
  selector:
    app: $service
  ports:
  - port: 8007
    targetPort: 8007
  type: ClusterIP
EOF
        
        # Apply deployment
        kubectl apply -f /tmp/$service-deployment.yaml -n $NAMESPACE || error "Failed to deploy $service"
        
        # Wait for deployment
        kubectl rollout status deployment/$service -n $NAMESPACE --timeout=300s || error "Failed to rollout $service"
        
        rm /tmp/$service-deployment.yaml
    done
    
    success "Core services deployed successfully"
}

# Deploy application services
deploy_application_services() {
    log "Deploying application services..."
    
    services=("coordinator-api" "exchange-integration" "compliance-service" "trading-engine" "plugin-registry" "plugin-marketplace" "plugin-security" "plugin-analytics" "global-infrastructure" "global-ai-agents" "multi-region-load-balancer")
    
    for service in "${services[@]}"; do
        log "Deploying $service..."
        
        # Determine port
        case $service in
            "coordinator-api") port=8001 ;;
            "exchange-integration") port=8010 ;;
            "compliance-service") port=8011 ;;
            "trading-engine") port=8012 ;;
            "plugin-registry") port=8013 ;;
            "plugin-marketplace") port=8014 ;;
            "plugin-security") port=8015 ;;
            "plugin-analytics") port=8016 ;;
            "global-infrastructure") port=8017 ;;
            "global-ai-agents") port=8018 ;;
            "multi-region-load-balancer") port=8019 ;;
        esac
        
        # Create deployment manifest
        cat > /tmp/$service-deployment.yaml << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: $service
  namespace: $NAMESPACE
spec:
  replicas: 3
  selector:
    matchLabels:
      app: $service
  template:
    metadata:
      labels:
        app: $service
    spec:
      containers:
      - name: $service
        image: aitbc/$service:$VERSION
        ports:
        - containerPort: $port
          name: http
        env:
        - name: NODE_ENV
          value: "production"
        - name: DATABASE_URL
          valueFrom:
            secretKeyRef:
              name: aitbc-secrets
              key: database-url
        - name: REDIS_URL
          valueFrom:
            secretKeyRef:
              name: aitbc-secrets
              key: redis-url
        - name: JWT_SECRET
          valueFrom:
            secretKeyRef:
              name: aitbc-secrets
              key: jwt-secret
        - name: ENCRYPTION_KEY
          valueFrom:
            secretKeyRef:
              name: aitbc-secrets
              key: encryption-key
        resources:
          requests:
            memory: "1Gi"
            cpu: "500m"
          limits:
            memory: "2Gi"
            cpu: "1000m"
        livenessProbe:
          httpGet:
            path: /health
            port: $port
          initialDelaySeconds: 30
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /health
            port: $port
          initialDelaySeconds: 5
          periodSeconds: 5
---
apiVersion: v1
kind: Service
metadata:
  name: $service
  namespace: $NAMESPACE
spec:
  selector:
    app: $service
  ports:
  - port: $port
    targetPort: $port
  type: ClusterIP
EOF
        
        # Apply deployment
        kubectl apply -f /tmp/$service-deployment.yaml -n $NAMESPACE || error "Failed to deploy $service"
        
        # Wait for deployment
        kubectl rollout status deployment/$service -n $NAMESPACE --timeout=300s || error "Failed to rollout $service"
        
        rm /tmp/$service-deployment.yaml
    done
    
    success "Application services deployed successfully"
}

# Deploy ingress and load balancer
deploy_ingress() {
    log "Deploying ingress and load balancer..."
    
    # Create ingress manifest
    cat > /tmp/ingress.yaml << EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: aitbc-ingress
  namespace: $NAMESPACE
  annotations:
    kubernetes.io/ingress.class: "nginx"
    cert-manager.io/cluster-issuer: "letsencrypt-prod"
    nginx.ingress.kubernetes.io/rate-limit: "100"
    nginx.ingress.kubernetes.io/rate-limit-window: "1m"
spec:
  tls:
  - hosts:
    - api.$DOMAIN
    - marketplace.$DOMAIN
    - explorer.$DOMAIN
    secretName: aitbc-tls
  rules:
  - host: api.$DOMAIN
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: coordinator-api
            port:
              number: 8001
  - host: marketplace.$DOMAIN
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: plugin-marketplace
            port:
              number: 8014
  - host: explorer.$DOMAIN
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: explorer
            port:
              number: 8020
EOF
    
    # Apply ingress
    kubectl apply -f /tmp/ingress.yaml -n $NAMESPACE || error "Failed to deploy ingress"
    
    rm /tmp/ingress.yaml
    
    success "Ingress deployed successfully"
}

# Deploy monitoring
deploy_monitoring() {
    log "Deploying monitoring stack..."
    
    # Deploy Prometheus
    helm upgrade --install prometheus prometheus-community/kube-prometheus-stack \
        --namespace $NAMESPACE \
        --create-namespace \
        --set prometheus.prometheus.spec.retention=30d \
        --set prometheus.prometheus.spec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage=50Gi \
        --set grafana.adminPassword=$GRAFANA_PASSWORD \
        --set grafana.persistence.size=10Gi \
        --set defaultRules.create=true \
        --wait \
        --timeout 10m || error "Failed to deploy monitoring"
    
    # Import Grafana dashboards
    log "Importing Grafana dashboards..."
    
    # Create dashboard configmaps
    kubectl create configmap grafana-dashboards \
        --from-file=monitoring/grafana/dashboards/ \
        -n $NAMESPACE \
        --dry-run=client -o yaml | kubectl apply -f -
    
    success "Monitoring deployed successfully"
}

# Run post-deployment tests
post_deployment_tests() {
    log "Running post-deployment tests..."
    
    # Wait for all services to be ready
    kubectl wait --for=condition=ready pod -l app!=pod -n $NAMESPACE --timeout=600s
    
    # Test API endpoints
    endpoints=(
        "coordinator-api:8001"
        "exchange-integration:8010"
        "trading-engine:8012"
        "plugin-registry:8013"
        "plugin-marketplace:8014"
    )
    
    for service_port in "${endpoints[@]}"; do
        service=$(echo $service_port | cut -d: -f1)
        port=$(echo $service_port | cut -d: -f2)
        
        log "Testing $service..."
        
        # Port-forward and test
        kubectl port-forward -n $NAMESPACE deployment/$service $port:8007 &
        port_forward_pid=$!
        
        sleep 5
        
        if curl -f -s http://localhost:$port/health > /dev/null; then
            success "$service is healthy"
        else
            error "$service health check failed"
        fi
        
        # Kill port-forward
        kill $port_forward_pid 2>/dev/null || true
    done
    
    # Test external endpoints
    external_endpoints=(
        "https://api.$DOMAIN/health"
        "https://marketplace.$DOMAIN/api/v1/marketplace/featured"
    )
    
    for endpoint in "${external_endpoints[@]}"; do
        log "Testing $endpoint..."
        
        if curl -f -s $endpoint > /dev/null; then
            success "$endpoint is responding"
        else
            error "$endpoint is not responding"
        fi
    done
    
    success "Post-deployment tests passed"
}

# Create secrets
create_secrets() {
    log "Creating secrets..."
    
    # Create secret from environment variables
    kubectl create secret generic aitbc-secrets \
        --from-literal=database-url="$DATABASE_URL" \
        --from-literal=redis-url="$REDIS_URL" \
        --from-literal=jwt-secret="$JWT_SECRET" \
        --from-literal=encryption-key="$ENCRYPTION_KEY" \
        --from-literal=postgres-password="$POSTGRES_PASSWORD" \
        --from-literal=redis-password="$REDIS_PASSWORD" \
        --namespace $NAMESPACE \
        --dry-run=client -o yaml | kubectl apply -f -
    
    success "Secrets created"
}

# Main deployment function
main() {
    log "Starting AITBC production deployment..."
    log "Environment: $ENVIRONMENT"
    log "Version: $VERSION"
    log "Region: $REGION"
    log "Domain: $DOMAIN"
    
    # Check prerequisites
    command -v kubectl >/dev/null 2>&1 || error "kubectl is not installed"
    command -v helm >/dev/null 2>&1 || error "Helm is not installed"
    kubectl cluster-info >/dev/null 2>&1 || error "Cannot connect to Kubernetes cluster"
    
    # Run deployment steps
    pre_deployment_checks
    create_secrets
    backup_current_deployment
    build_production_images
    deploy_database
    deploy_core_services
    deploy_application_services
    deploy_ingress
    deploy_monitoring
    post_deployment_tests
    
    success "Production deployment completed successfully!"
    
    # Display deployment information
    log "Deployment Information:"
    log "Environment: $ENVIRONMENT"
    log "Version: $VERSION"
    log "Namespace: $NAMESPACE"
    log "Domain: $DOMAIN"
    log ""
    log "Services are available at:"
    log "  API: https://api.$DOMAIN"
    log "  Marketplace: https://marketplace.$DOMAIN"
    log "  Explorer: https://explorer.$DOMAIN"
    log "  Grafana: https://grafana.$DOMAIN"
    log ""
    log "To check deployment status:"
    log "  kubectl get pods -n $NAMESPACE"
    log "  kubectl get services -n $NAMESPACE"
    log ""
    log "To view logs:"
    log "  kubectl logs -f deployment/coordinator-api -n $NAMESPACE"
}

# Handle script interruption
trap 'error "Script interrupted"' INT TERM

# Export environment variables
export DATABASE_URL=${DATABASE_URL}
export REDIS_URL=${REDIS_URL}
export JWT_SECRET=${JWT_SECRET}
export ENCRYPTION_KEY=${ENCRYPTION_KEY}
export POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
export REDIS_PASSWORD=${REDIS_PASSWORD}
export GRAFANA_PASSWORD=${GRAFANA_PASSWORD}
export VERSION=${VERSION}
export NAMESPACE=${NAMESPACE}
export DOMAIN=${DOMAIN}

# Run main function
main "$@"