security: enforce required API_KEY_HASH_SECRET and migrate keystore password to credential system

Remove default fallback for API_KEY_HASH_SECRET in tenant context middleware and management service, requiring explicit environment variable configuration. Migrate keystore password handling from /etc/aitbc/keystore_password to /etc/aitbc/credentials/keystore_password with 600 permissions. Add load-keystore-secrets.sh pre-start hook and /run/aitbc/secrets/.env environment file to blockchain-node, blockchain

apps/blockchain-node/src/aitbc_chain/consensus/poa.py

@@ -195,7 +195,7 @@ class PoAProposer:
             if block_generation_mode == "mempool-only":
                 # Strict mempool-only mode: skip if empty
                 if mempool_size == 0:
-                    self._logger.info(f"[PROPOSE] Skipping block proposal: mempool is empty (chain={self._config.chain_id}, mode=mempool-only)")
+                    self._logger.debug(f"[PROPOSE] Skipping block proposal: mempool is empty (chain={self._config.chain_id}, mode=mempool-only)")
                     metrics_registry.increment("sync_empty_blocks_skipped_total")
                     return False
             elif block_generation_mode == "hybrid":
@@ -203,7 +203,7 @@ class PoAProposer:
                 if self._last_block_timestamp:
                     time_since_last_block = (datetime.utcnow() - self._last_block_timestamp).total_seconds()
                     if mempool_size == 0 and time_since_last_block < max_empty_block_interval:
-                        self._logger.info(f"[PROPOSE] Skipping block proposal: mempool empty, heartbeat not yet due (chain={self._config.chain_id}, mode=hybrid, idle_time={time_since_last_block:.1f}s)")
+                        self._logger.debug(f"[PROPOSE] Skipping block proposal: mempool empty, heartbeat not yet due (chain={self._config.chain_id}, mode=hybrid, idle_time={time_since_last_block:.1f}s)")
                         metrics_registry.increment("sync_empty_blocks_skipped_total")
                         return False
                     elif mempool_size == 0 and time_since_last_block >= max_empty_block_interval:
@@ -212,7 +212,7 @@ class PoAProposer:
                         metrics_registry.observe("sync_time_since_last_block_seconds", time_since_last_block)
                 elif mempool_size == 0:
                     # No previous block timestamp, skip (will be set after genesis)
-                    self._logger.info(f"[PROPOSE] Skipping block proposal: no previous block timestamp (chain={self._config.chain_id}, mode=hybrid)")
+                    self._logger.debug(f"[PROPOSE] Skipping block proposal: no previous block timestamp (chain={self._config.chain_id}, mode=hybrid)")
                     metrics_registry.increment("sync_empty_blocks_skipped_total")
                     return False
 

apps/blockchain-node/src/aitbc_chain/p2p_network.py

@@ -696,7 +696,6 @@ def main():
     )
     
     try:
-        from .config import settings
         from .mempool import init_mempool
         import pathlib
         

apps/coordinator-api/src/app/middleware/tenant_context.py

@@ -152,7 +152,12 @@ class TenantContextMiddleware(BaseHTTPMiddleware):
         # SECURITY FIX: Use HMAC with a secret key instead of plain sha256 for API key hashing
         # This prevents rainbow table attacks and provides better security
         import hmac
-        secret_key = os.environ.get("API_KEY_HASH_SECRET", "default-secret-change-in-production")
+        secret_key = os.environ.get("API_KEY_HASH_SECRET")
+        if not secret_key:
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail="API_KEY_HASH_SECRET environment variable not set"
+            )
         key_hash = hmac.new(secret_key.encode(), api_key.encode(), hashlib.sha256).hexdigest()
 
         db = next(get_db())

apps/coordinator-api/src/app/services/tenant_management.py

@@ -363,7 +363,9 @@ class TenantManagementService:
         api_key = f"ask_{secrets.token_urlsafe(32)}"
         # SECURITY FIX: Use HMAC with secret key instead of plain sha256 for API key hashing
         import hmac
-        secret_key = os.environ.get("API_KEY_HASH_SECRET", "default-secret-change-in-production")
+        secret_key = os.environ.get("API_KEY_HASH_SECRET")
+        if not secret_key:
+            raise ValueError("API_KEY_HASH_SECRET environment variable not set")
         key_hash = hmac.new(secret_key.encode(), api_key.encode(), hashlib.sha256).hexdigest()
         key_prefix = api_key[:8]
 

scripts/utils/encrypt_keystore_password.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""
+Encrypt keystore password file using AES-GCM encryption
+Uses the existing encryption suite from the wallet service
+"""
+
+import sys
+import os
+from pathlib import Path
+
+# Add wallet service to path
+sys.path.insert(0, '/opt/aitbc/apps/wallet/src')
+
+from app.crypto.encryption import EncryptionSuite
+from secrets import token_bytes
+
+def main():
+    keystore_dir = Path('/var/lib/aitbc/keystore')
+    password_file = keystore_dir / '.password'
+    encrypted_file = keystore_dir / 'passwords' / 'keystore_password.enc'
+
+    # Ensure passwords directory exists
+    encrypted_file.parent.mkdir(parents=True, exist_ok=True)
+
+    # Read existing password if it exists
+    if password_file.exists():
+        with open(password_file, 'r') as f:
+            password = f.read().strip()
+    else:
+        # Generate new secure password if none exists
+        password = token_bytes(32).hex()
+        with open(password_file, 'w') as f:
+            f.write(password)
+        os.chmod(password_file, 0o600)
+
+    # Get encryption password from environment or prompt
+    enc_password = os.environ.get('KEYSTORE_ENCRYPTION_PASSWORD')
+    if not enc_password:
+        print("Error: KEYSTORE_ENCRYPTION_PASSWORD environment variable not set")
+        print("Set it with: export KEYSTORE_ENCRYPTION_PASSWORD=<secure-password>")
+        sys.exit(1)
+
+    # Encrypt the password
+    encryption = EncryptionSuite()
+    salt = token_bytes(encryption.salt_bytes)
+    nonce = token_bytes(encryption.nonce_bytes)
+    ciphertext = encryption.encrypt(password=enc_password, plaintext=password.encode(), salt=salt, nonce=nonce)
+
+    # Write encrypted file with salt and nonce
+    with open(encrypted_file, 'wb') as f:
+        f.write(salt)
+        f.write(nonce)
+        f.write(ciphertext)
+
+    os.chmod(encrypted_file, 0o600)
+
+    print(f"Encrypted keystore password saved to: {encrypted_file}")
+    print(f"Original password file: {password_file} (will be removed)")
+    print(f"WARNING: Keep KEYSTORE_ENCRYPTION_PASSWORD secure - it's needed to decrypt")
+
+    # Remove clear text password file
+    if password_file.exists():
+        password_file.unlink()
+        print(f"Removed clear text password file: {password_file}")
+
+if __name__ == '__main__':
+    main()

scripts/utils/load-keystore-secrets.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Load AITBC secrets from credentials directory
+# This script is called by systemd services before main process starts
+
+set -e
+
+CREDENTIALS_DIR="/etc/aitbc/credentials"
+RUN_DIR="/run/aitbc/secrets"
+
+# Create runtime directory (tmpfs, cleared on reboot)
+mkdir -p "$RUN_DIR"
+chmod 700 "$RUN_DIR"
+
+# Create .env file from credentials
+ENV_FILE="$RUN_DIR/.env"
+
+if [ -f "$CREDENTIALS_DIR/api_hash_secret" ]; then
+    echo "API_KEY_HASH_SECRET=$(cat $CREDENTIALS_DIR/api_hash_secret)" >> "$ENV_FILE"
+fi
+
+if [ -f "$CREDENTIALS_DIR/proposer_id" ]; then
+    echo "proposer_id=$(cat $CREDENTIALS_DIR/proposer_id)" >> "$ENV_FILE"
+fi
+
+if [ -f "$CREDENTIALS_DIR/keystore_password" ]; then
+    echo "KEYSTORE_PASSWORD=$(cat $CREDENTIALS_DIR/keystore_password)" >> "$ENV_FILE"
+fi
+
+# Add non-sensitive config from main .env
+if [ -f "/etc/aitbc/.env" ]; then
+    # Skip lines that are comments or contain migrated secrets
+    grep -v '^#' /etc/aitbc/.env | grep -v 'API_KEY_HASH_SECRET' | grep -v 'proposer_id' >> "$ENV_FILE" || true
+fi
+
+chmod 600 "$ENV_FILE"
+
+echo "Secrets loaded to $ENV_FILE"

scripts/utils/migrate_secrets_to_keystore.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""
+Migrate secrets from .env to encrypted keystore storage
+"""
+
+import sys
+import os
+from pathlib import Path
+from secrets import token_bytes
+
+# Add wallet service to path
+sys.path.insert(0, '/opt/aitbc/apps/wallet/src')
+
+from app.crypto.encryption import EncryptionSuite
+
+def encrypt_secret(plaintext: str, encryption_password: str) -> bytes:
+    """Encrypt a secret using AES-GCM"""
+    encryption = EncryptionSuite()
+    salt = token_bytes(encryption.salt_bytes)
+    nonce = token_bytes(encryption.nonce_bytes)
+    ciphertext = encryption.encrypt(password=encryption_password, plaintext=plaintext.encode(), salt=salt, nonce=nonce)
+    return salt + nonce + ciphertext
+
+def main():
+    env_file = Path('/etc/aitbc/.env')
+    keystore_config_dir = Path('/var/lib/aitbc/keystore/config')
+    keystore_passwords_dir = Path('/var/lib/aitbc/keystore/passwords')
+
+    # Get encryption password from environment
+    enc_password = os.environ.get('KEYSTORE_ENCRYPTION_PASSWORD')
+    if not enc_password:
+        print("Error: KEYSTORE_ENCRYPTION_PASSWORD environment variable not set")
+        print("Set it with: export KEYSTORE_ENCRYPTION_PASSWORD=<secure-password>")
+        sys.exit(1)
+
+    # Read .env file
+    env_vars = {}
+    if env_file.exists():
+        with open(env_file, 'r') as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith('#') and '=' in line:
+                    key, value = line.split('=', 1)
+                    env_vars[key.strip()] = value.strip()
+
+    # Migrate API_KEY_HASH_SECRET
+    if 'API_KEY_HASH_SECRET' in env_vars:
+        api_secret = env_vars['API_KEY_HASH_SECRET']
+        encrypted = encrypt_secret(api_secret, enc_password)
+        output_file = keystore_config_dir / 'api_hash_secret.enc'
+        with open(output_file, 'wb') as f:
+            f.write(encrypted)
+        os.chmod(output_file, 0o600)
+        print(f"Migrated API_KEY_HASH_SECRET to: {output_file}")
+    else:
+        print("API_KEY_HASH_SECRET not found in .env")
+
+    # Migrate proposer_id
+    if 'proposer_id' in env_vars:
+        proposer_id = env_vars['proposer_id']
+        encrypted = encrypt_secret(proposer_id, enc_password)
+        output_file = keystore_config_dir / 'proposer_key.enc'
+        with open(output_file, 'wb') as f:
+            f.write(encrypted)
+        os.chmod(output_file, 0o600)
+        print(f"Migrated proposer_id to: {output_file}")
+    else:
+        print("proposer_id not found in .env")
+
+    # Update .env to remove migrated secrets
+    new_env_lines = []
+    with open(env_file, 'r') as f:
+        for line in f:
+            if line.strip().startswith('API_KEY_HASH_SECRET='):
+                new_env_lines.append('# API_KEY_HASH_SECRET migrated to keystore/config/api_hash_secret.enc\n')
+            elif line.strip().startswith('proposer_id='):
+                new_env_lines.append('# proposer_id migrated to keystore/config/proposer_key.enc\n')
+            else:
+                new_env_lines.append(line)
+
+    with open(env_file, 'w') as f:
+        f.writelines(new_env_lines)
+
+    print(f"Updated {env_file} to remove migrated secrets")
+    print("\nWARNING: Keep KEYSTORE_ENCRYPTION_PASSWORD secure - it's needed to decrypt")
+
+if __name__ == '__main__':
+    main()

scripts/utils/setup-credentials.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""
+Setup systemd credentials for AITBC services
+Stores secrets in /etc/aitbc/credentials with proper permissions
+"""
+
+import sys
+import os
+from pathlib import Path
+from secrets import token_hex
+
+def main():
+    credentials_dir = Path('/etc/aitbc/credentials')
+    credentials_dir.mkdir(parents=True, exist_ok=True)
+    os.chmod(credentials_dir, 0o700)
+
+    env_file = Path('/etc/aitbc/.env')
+
+    # Read current .env values
+    env_vars = {}
+    if env_file.exists():
+        with open(env_file, 'r') as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith('#') and '=' in line:
+                    key, value = line.split('=', 1)
+                    env_vars[key.strip()] = value.strip()
+
+    # Create credential files for sensitive values
+    credentials = {
+        'api_hash_secret': env_vars.get('API_KEY_HASH_SECRET', token_hex(32)),
+        'proposer_id': env_vars.get('proposer_id', ''),
+        'keystore_password': env_vars.get('KEYSTORE_PASSWORD', token_hex(32)),
+    }
+
+    for name, value in credentials.items():
+        if value:
+            cred_file = credentials_dir / name
+            with open(cred_file, 'w') as f:
+                f.write(value)
+            os.chmod(cred_file, 0o600)
+            print(f"Created credential: {cred_file}")
+
+    print(f"\nCredentials stored in: {credentials_dir}")
+    print("All files have 600 permissions (root read/write only)")
+
+if __name__ == '__main__':
+    main()

scripts/utils/setup_production.py

@@ -42,14 +42,21 @@ def main():
     run(f"mkdir -p {KEYS_DIR}")
     run(f"chown -R root:root {KEYS_DIR}")
     
-    # SECURITY FIX: Use environment variable instead of hardcoded password
-    # Avoid writing password to disk - pass directly to keystore script
+    # SECURITY FIX: Use credential system instead of writing password to disk
+    # Password is stored in /etc/aitbc/credentials/keystore_password with 600 permissions
     password = os.environ.get("AITBC_KEYSTORE_PASSWORD")
     if not password:
-        # Generate secure random password if not provided
+        # Read from credential file if not provided
+        cred_file = Path("/etc/aitbc/credentials/keystore_password")
+        if cred_file.exists():
+            password = cred_file.read_text().strip()
+        else:
+            # Generate secure random password and store in credentials
             password = secrets.token_hex(32)
-        PASSWORD_FILE.write_text(password)
-        run(f"chmod 600 {PASSWORD_FILE}")
+            cred_file.parent.mkdir(parents=True, exist_ok=True)
+            cred_file.write_text(password)
+            os.chmod(cred_file, 0o600)
+            print(f"[INFO] Password stored in {cred_file} with 600 permissions")
     else:
         # Use provided password from environment without writing to disk
         # Clear password from environment variable for security

systemd/aitbc-blockchain-node.service

@@ -9,6 +9,8 @@ User=root
 Group=root
 WorkingDirectory=/opt/aitbc
 Environment=PATH=/usr/bin:/usr/local/bin:/usr/bin:/bin
+ExecStartPre=/opt/aitbc/scripts/utils/load-keystore-secrets.sh
+EnvironmentFile=/run/aitbc/secrets/.env
 EnvironmentFile=/etc/aitbc/.env
 EnvironmentFile=/etc/aitbc/node.env
 ExecStart=/opt/aitbc/venv/bin/python /opt/aitbc/scripts/wrappers/aitbc-blockchain-node-wrapper.py

systemd/aitbc-blockchain-rpc.service

@@ -8,6 +8,8 @@ User=root
 Group=root
 WorkingDirectory=/opt/aitbc
 Environment=PATH=/usr/bin:/usr/local/bin:/usr/bin:/bin
+ExecStartPre=/opt/aitbc/scripts/utils/load-keystore-secrets.sh
+EnvironmentFile=/run/aitbc/secrets/.env
 EnvironmentFile=/etc/aitbc/.env
 EnvironmentFile=/etc/aitbc/node.env
 UnsetEnvironment=enable_block_production ENABLE_BLOCK_PRODUCTION

systemd/aitbc-coordinator-api.service

@@ -7,6 +7,8 @@ Type=simple
 User=root
 WorkingDirectory=/opt/aitbc/apps/coordinator-api/src
 Environment=PYTHONPATH=/opt/aitbc/apps/coordinator-api/src:/opt/aitbc/packages/py/aitbc-sdk/src:/opt/aitbc/packages/py/aitbc-crypto/src
+ExecStartPre=/opt/aitbc/scripts/utils/load-keystore-secrets.sh
+EnvironmentFile=/run/aitbc/secrets/.env
 EnvironmentFile=/etc/aitbc/.env
 EnvironmentFile=/etc/aitbc/node.env
 ExecStart=/opt/aitbc/venv/bin/python /opt/aitbc/scripts/wrappers/aitbc-coordinator-api-wrapper.py

systemd/aitbc-wallet.service

@@ -8,6 +8,8 @@ Type=simple
 User=root
 Group=root
 WorkingDirectory=/opt/aitbc
+ExecStartPre=/opt/aitbc/scripts/utils/load-keystore-secrets.sh
+EnvironmentFile=/run/aitbc/secrets/.env
 EnvironmentFile=/etc/aitbc/.env
 EnvironmentFile=/etc/aitbc/node.env
 ExecStart=/opt/aitbc/venv/bin/python /opt/aitbc/scripts/wrappers/aitbc-wallet-wrapper.py