security: enforce required API_KEY_HASH_SECRET and migrate keystore password to credential system

Remove default fallback for API_KEY_HASH_SECRET in tenant context middleware and management service, requiring explicit environment variable configuration. Migrate keystore password handling from /etc/aitbc/keystore_password to /etc/aitbc/credentials/keystore_password with 600 permissions. Add load-keystore-secrets.sh pre-start hook and /run/aitbc/secrets/.env environment file to blockchain-node, blockchain

scripts/utils/encrypt_keystore_password.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""
+Encrypt keystore password file using AES-GCM encryption
+Uses the existing encryption suite from the wallet service
+"""
+
+import sys
+import os
+from pathlib import Path
+
+# Add wallet service to path
+sys.path.insert(0, '/opt/aitbc/apps/wallet/src')
+
+from app.crypto.encryption import EncryptionSuite
+from secrets import token_bytes
+
+def main():
+    keystore_dir = Path('/var/lib/aitbc/keystore')
+    password_file = keystore_dir / '.password'
+    encrypted_file = keystore_dir / 'passwords' / 'keystore_password.enc'
+
+    # Ensure passwords directory exists
+    encrypted_file.parent.mkdir(parents=True, exist_ok=True)
+
+    # Read existing password if it exists
+    if password_file.exists():
+        with open(password_file, 'r') as f:
+            password = f.read().strip()
+    else:
+        # Generate new secure password if none exists
+        password = token_bytes(32).hex()
+        with open(password_file, 'w') as f:
+            f.write(password)
+        os.chmod(password_file, 0o600)
+
+    # Get encryption password from environment or prompt
+    enc_password = os.environ.get('KEYSTORE_ENCRYPTION_PASSWORD')
+    if not enc_password:
+        print("Error: KEYSTORE_ENCRYPTION_PASSWORD environment variable not set")
+        print("Set it with: export KEYSTORE_ENCRYPTION_PASSWORD=<secure-password>")
+        sys.exit(1)
+
+    # Encrypt the password
+    encryption = EncryptionSuite()
+    salt = token_bytes(encryption.salt_bytes)
+    nonce = token_bytes(encryption.nonce_bytes)
+    ciphertext = encryption.encrypt(password=enc_password, plaintext=password.encode(), salt=salt, nonce=nonce)
+
+    # Write encrypted file with salt and nonce
+    with open(encrypted_file, 'wb') as f:
+        f.write(salt)
+        f.write(nonce)
+        f.write(ciphertext)
+
+    os.chmod(encrypted_file, 0o600)
+
+    print(f"Encrypted keystore password saved to: {encrypted_file}")
+    print(f"Original password file: {password_file} (will be removed)")
+    print(f"WARNING: Keep KEYSTORE_ENCRYPTION_PASSWORD secure - it's needed to decrypt")
+
+    # Remove clear text password file
+    if password_file.exists():
+        password_file.unlink()
+        print(f"Removed clear text password file: {password_file}")
+
+if __name__ == '__main__':
+    main()

scripts/utils/load-keystore-secrets.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Load AITBC secrets from credentials directory
+# This script is called by systemd services before main process starts
+
+set -e
+
+CREDENTIALS_DIR="/etc/aitbc/credentials"
+RUN_DIR="/run/aitbc/secrets"
+
+# Create runtime directory (tmpfs, cleared on reboot)
+mkdir -p "$RUN_DIR"
+chmod 700 "$RUN_DIR"
+
+# Create .env file from credentials
+ENV_FILE="$RUN_DIR/.env"
+
+if [ -f "$CREDENTIALS_DIR/api_hash_secret" ]; then
+    echo "API_KEY_HASH_SECRET=$(cat $CREDENTIALS_DIR/api_hash_secret)" >> "$ENV_FILE"
+fi
+
+if [ -f "$CREDENTIALS_DIR/proposer_id" ]; then
+    echo "proposer_id=$(cat $CREDENTIALS_DIR/proposer_id)" >> "$ENV_FILE"
+fi
+
+if [ -f "$CREDENTIALS_DIR/keystore_password" ]; then
+    echo "KEYSTORE_PASSWORD=$(cat $CREDENTIALS_DIR/keystore_password)" >> "$ENV_FILE"
+fi
+
+# Add non-sensitive config from main .env
+if [ -f "/etc/aitbc/.env" ]; then
+    # Skip lines that are comments or contain migrated secrets
+    grep -v '^#' /etc/aitbc/.env | grep -v 'API_KEY_HASH_SECRET' | grep -v 'proposer_id' >> "$ENV_FILE" || true
+fi
+
+chmod 600 "$ENV_FILE"
+
+echo "Secrets loaded to $ENV_FILE"

scripts/utils/migrate_secrets_to_keystore.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""
+Migrate secrets from .env to encrypted keystore storage
+"""
+
+import sys
+import os
+from pathlib import Path
+from secrets import token_bytes
+
+# Add wallet service to path
+sys.path.insert(0, '/opt/aitbc/apps/wallet/src')
+
+from app.crypto.encryption import EncryptionSuite
+
+def encrypt_secret(plaintext: str, encryption_password: str) -> bytes:
+    """Encrypt a secret using AES-GCM"""
+    encryption = EncryptionSuite()
+    salt = token_bytes(encryption.salt_bytes)
+    nonce = token_bytes(encryption.nonce_bytes)
+    ciphertext = encryption.encrypt(password=encryption_password, plaintext=plaintext.encode(), salt=salt, nonce=nonce)
+    return salt + nonce + ciphertext
+
+def main():
+    env_file = Path('/etc/aitbc/.env')
+    keystore_config_dir = Path('/var/lib/aitbc/keystore/config')
+    keystore_passwords_dir = Path('/var/lib/aitbc/keystore/passwords')
+
+    # Get encryption password from environment
+    enc_password = os.environ.get('KEYSTORE_ENCRYPTION_PASSWORD')
+    if not enc_password:
+        print("Error: KEYSTORE_ENCRYPTION_PASSWORD environment variable not set")
+        print("Set it with: export KEYSTORE_ENCRYPTION_PASSWORD=<secure-password>")
+        sys.exit(1)
+
+    # Read .env file
+    env_vars = {}
+    if env_file.exists():
+        with open(env_file, 'r') as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith('#') and '=' in line:
+                    key, value = line.split('=', 1)
+                    env_vars[key.strip()] = value.strip()
+
+    # Migrate API_KEY_HASH_SECRET
+    if 'API_KEY_HASH_SECRET' in env_vars:
+        api_secret = env_vars['API_KEY_HASH_SECRET']
+        encrypted = encrypt_secret(api_secret, enc_password)
+        output_file = keystore_config_dir / 'api_hash_secret.enc'
+        with open(output_file, 'wb') as f:
+            f.write(encrypted)
+        os.chmod(output_file, 0o600)
+        print(f"Migrated API_KEY_HASH_SECRET to: {output_file}")
+    else:
+        print("API_KEY_HASH_SECRET not found in .env")
+
+    # Migrate proposer_id
+    if 'proposer_id' in env_vars:
+        proposer_id = env_vars['proposer_id']
+        encrypted = encrypt_secret(proposer_id, enc_password)
+        output_file = keystore_config_dir / 'proposer_key.enc'
+        with open(output_file, 'wb') as f:
+            f.write(encrypted)
+        os.chmod(output_file, 0o600)
+        print(f"Migrated proposer_id to: {output_file}")
+    else:
+        print("proposer_id not found in .env")
+
+    # Update .env to remove migrated secrets
+    new_env_lines = []
+    with open(env_file, 'r') as f:
+        for line in f:
+            if line.strip().startswith('API_KEY_HASH_SECRET='):
+                new_env_lines.append('# API_KEY_HASH_SECRET migrated to keystore/config/api_hash_secret.enc\n')
+            elif line.strip().startswith('proposer_id='):
+                new_env_lines.append('# proposer_id migrated to keystore/config/proposer_key.enc\n')
+            else:
+                new_env_lines.append(line)
+
+    with open(env_file, 'w') as f:
+        f.writelines(new_env_lines)
+
+    print(f"Updated {env_file} to remove migrated secrets")
+    print("\nWARNING: Keep KEYSTORE_ENCRYPTION_PASSWORD secure - it's needed to decrypt")
+
+if __name__ == '__main__':
+    main()

scripts/utils/setup-credentials.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""
+Setup systemd credentials for AITBC services
+Stores secrets in /etc/aitbc/credentials with proper permissions
+"""
+
+import sys
+import os
+from pathlib import Path
+from secrets import token_hex
+
+def main():
+    credentials_dir = Path('/etc/aitbc/credentials')
+    credentials_dir.mkdir(parents=True, exist_ok=True)
+    os.chmod(credentials_dir, 0o700)
+
+    env_file = Path('/etc/aitbc/.env')
+
+    # Read current .env values
+    env_vars = {}
+    if env_file.exists():
+        with open(env_file, 'r') as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith('#') and '=' in line:
+                    key, value = line.split('=', 1)
+                    env_vars[key.strip()] = value.strip()
+
+    # Create credential files for sensitive values
+    credentials = {
+        'api_hash_secret': env_vars.get('API_KEY_HASH_SECRET', token_hex(32)),
+        'proposer_id': env_vars.get('proposer_id', ''),
+        'keystore_password': env_vars.get('KEYSTORE_PASSWORD', token_hex(32)),
+    }
+
+    for name, value in credentials.items():
+        if value:
+            cred_file = credentials_dir / name
+            with open(cred_file, 'w') as f:
+                f.write(value)
+            os.chmod(cred_file, 0o600)
+            print(f"Created credential: {cred_file}")
+
+    print(f"\nCredentials stored in: {credentials_dir}")
+    print("All files have 600 permissions (root read/write only)")
+
+if __name__ == '__main__':
+    main()

scripts/utils/setup_production.py

@@ -42,14 +42,21 @@ def main():
     run(f"mkdir -p {KEYS_DIR}")
     run(f"chown -R root:root {KEYS_DIR}")
     
-    # SECURITY FIX: Use environment variable instead of hardcoded password
-    # Avoid writing password to disk - pass directly to keystore script
+    # SECURITY FIX: Use credential system instead of writing password to disk
+    # Password is stored in /etc/aitbc/credentials/keystore_password with 600 permissions
     password = os.environ.get("AITBC_KEYSTORE_PASSWORD")
     if not password:
-        # Generate secure random password if not provided
-        password = secrets.token_hex(32)
-        PASSWORD_FILE.write_text(password)
-        run(f"chmod 600 {PASSWORD_FILE}")
+        # Read from credential file if not provided
+        cred_file = Path("/etc/aitbc/credentials/keystore_password")
+        if cred_file.exists():
+            password = cred_file.read_text().strip()
+        else:
+            # Generate secure random password and store in credentials
+            password = secrets.token_hex(32)
+            cred_file.parent.mkdir(parents=True, exist_ok=True)
+            cred_file.write_text(password)
+            os.chmod(cred_file, 0o600)
+            print(f"[INFO] Password stored in {cred_file} with 600 permissions")
     else:
         # Use provided password from environment without writing to disk
         # Clear password from environment variable for security