feat: add production setup and infrastructure improvements

- Add production genesis initialization scripts
- Add keystore management for production
- Add production node runner
- Add setup production automation
- Add AI memory system for development tracking
- Add translation cache service
- Add development heartbeat monitoring
- Update blockchain RPC router
- Update coordinator API main configuration
- Update secure pickle service
- Update claim task script
- Update blockchain service configuration
- Update gitignore for production files

scripts/claim-task.py

@@ -6,7 +6,7 @@ Uses Git branch atomic creation as a distributed lock to prevent duplicate work.
 import os
 import json
 import subprocess
-from datetime import datetime
+from datetime import datetime, timedelta
 
 REPO_DIR = '/opt/aitbc'
 STATE_FILE = '/opt/aitbc/.claim-state.json'
@@ -16,6 +16,7 @@ MY_AGENT = os.getenv('AGENT_NAME', 'aitbc1')
 ISSUE_LABELS = ['security', 'bug', 'feature', 'refactor', 'task']  # priority order
 BONUS_LABELS = ['good-first-task-for-agent']
 AVOID_LABELS = ['needs-design', 'blocked', 'needs-reproduction']
+CLAIM_TTL = timedelta(hours=2)  # Stale claim timeout
 
 def query_api(path, method='GET', data=None):
     url = f"{API_BASE}/{path}"
@@ -105,15 +106,36 @@ def create_work_branch(issue_number, title):
     return branch_name
 
 def main():
-    now = datetime.utcnow().isoformat() + 'Z'
-    print(f"[{now}] Claim task cycle starting...")
+    now = datetime.utcnow()
+    print(f"[{now.isoformat()}Z] Claim task cycle starting...")
     
     state = load_state()
     current_claim = state.get('current_claim')
     
+    if current_claim:
+        claimed_at_str = state.get('claimed_at')
+        if claimed_at_str:
+            try:
+                # Convert 'Z' suffix to offset for fromisoformat
+                if claimed_at_str.endswith('Z'):
+                    claimed_at_str = claimed_at_str[:-1] + '+00:00'
+                claimed_at = datetime.fromisoformat(claimed_at_str)
+                age = now - claimed_at
+                if age > CLAIM_TTL:
+                    print(f"Claim for issue #{current_claim} is stale (age {age}). Releasing.")
+                    # Try to delete remote claim branch
+                    claim_branch = state.get('claim_branch', f'claim/{current_claim}')
+                    subprocess.run(['git', 'push', 'origin', '--delete', claim_branch], 
+                                   capture_output=True, cwd=REPO_DIR)
+                    # Clear state
+                    state = {'current_claim': None, 'claimed_at': None, 'work_branch': None}
+                    save_state(state)
+                    current_claim = None
+            except Exception as e:
+                print(f"Error checking claim age: {e}. Will attempt to proceed.")
+    
     if current_claim:
         print(f"Already working on issue #{current_claim} (branch {state.get('work_branch')})")
-        # Optional: could check if that PR has been merged/closed and release claim here
         return
     
     issues = get_open_unassigned_issues()

scripts/init_production_genesis.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+"""
+Initialize the production chain (ait-mainnet) with genesis allocations.
+This script:
+- Ensures the blockchain database is initialized
+- Creates the genesis block (if missing)
+- Populates account balances according to the production allocation
+- Outputs the addresses and their balances
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import yaml
+from datetime import datetime
+from pathlib import Path
+
+# Add the blockchain node src to path
+sys.path.insert(0, str(Path(__file__).parent.parent / "apps/blockchain-node/src"))
+
+from aitbc_chain.config import settings as cfg
+from aitbc_chain.database import init_db, session_scope
+from aitbc_chain.models import Block, Account
+from aitbc_chain.consensus.poa import PoAProposer, ProposerConfig
+from aitbc_chain.mempool import init_mempool
+import hashlib
+from sqlmodel import select
+
+# Production allocations (loaded from genesis_prod.yaml if available, else fallback)
+ALLOCATIONS = {}
+
+
+def load_allocations() -> dict[str, int]:
+    yaml_path = Path("/opt/aitbc/genesis_prod.yaml")
+    if yaml_path.exists():
+        import yaml
+        with yaml_path.open() as f:
+            data = yaml.safe_load(f)
+        allocations = {}
+        for acc in data.get("genesis", {}).get("accounts", []):
+            addr = acc["address"]
+            balance = int(acc["balance"])
+            allocations[addr] = balance
+        return allocations
+    else:
+        # Fallback hardcoded
+        return {
+            "aitbc1genesis": 10_000_000,
+            "aitbc1treasury": 5_000_000,
+            "aitbc1aiengine": 2_000_000,
+            "aitbc1surveillance": 1_500_000,
+            "aitbc1analytics": 1_000_000,
+            "aitbc1marketplace": 2_000_000,
+            "aitbc1enterprise": 3_000_000,
+            "aitbc1multimodal": 1_500_000,
+            "aitbc1zkproofs": 1_000_000,
+            "aitbc1crosschain": 2_000_000,
+            "aitbc1developer1": 500_000,
+            "aitbc1developer2": 300_000,
+            "aitbc1tester": 200_000,
+        }
+
+ALLOCATIONS = load_allocations()
+
+# Authorities (proposers) for PoA
+AUTHORITIES = ["aitbc1genesis"]
+
+
+def compute_genesis_hash(chain_id: str, timestamp: datetime) -> str:
+    payload = f"{chain_id}|0|0x00|{timestamp.isoformat()}".encode()
+    return "0x" + hashlib.sha256(payload).hexdigest()
+
+
+def ensure_genesis_block(chain_id: str) -> Block:
+    with session_scope() as session:
+        # Check if any block exists for this chain
+        head = session.exec(select(Block).where(Block.chain_id == chain_id).order_by(Block.height.desc()).limit(1)).first()
+        if head is not None:
+            print(f"[*] Chain already has block at height {head.height}")
+            return head
+
+        # Create deterministic genesis timestamp
+        timestamp = datetime(2025, 1, 1, 0, 0, 0)
+        block_hash = compute_genesis_hash(chain_id, timestamp)
+        genesis = Block(
+            chain_id=chain_id,
+            height=0,
+            hash=block_hash,
+            parent_hash="0x00",
+            proposer="genesis",
+            timestamp=timestamp,
+            tx_count=0,
+            state_root=None,
+        )
+        session.add(genesis)
+        session.commit()
+        print(f"[+] Created genesis block: height=0, hash={block_hash}")
+        return genesis
+
+
+def seed_accounts(chain_id: str) -> None:
+    with session_scope() as session:
+        for address, balance in ALLOCATIONS.items():
+            account = session.get(Account, (chain_id, address))
+            if account is None:
+                account = Account(chain_id=chain_id, address=address, balance=balance, nonce=0)
+                session.add(account)
+                print(f"[+] Created account {address} with balance {balance}")
+            else:
+                # Already exists; ensure balance matches if we want to enforce
+                if account.balance != balance:
+                    account.balance = balance
+                    print(f"[~] Updated account {address} balance to {balance}")
+        session.commit()
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--chain-id", default="ait-mainnet", help="Chain ID to initialize")
+    parser.add_argument("--db-path", type=Path, help="Path to SQLite database (overrides config)")
+    args = parser.parse_args()
+
+    # Override environment for config
+    os.environ["CHAIN_ID"] = args.chain_id
+    if args.db_path:
+        os.environ["DB_PATH"] = str(args.db_path)
+
+    from aitbc_chain.config import Settings
+    settings = Settings()
+
+    print(f"[*] Initializing database at {settings.db_path}")
+    init_db()
+    print("[*] Database initialized")
+
+    # Ensure mempool DB exists (though not needed for genesis)
+    mempool_path = settings.db_path.parent / "mempool.db"
+    init_mempool(backend="database", db_path=str(mempool_path), max_size=10000, min_fee=0)
+    print(f"[*] Mempool initialized at {mempool_path}")
+
+    # Create genesis block
+    ensure_genesis_block(args.chain_id)
+
+    # Seed accounts
+    seed_accounts(args.chain_id)
+
+    print("\n[+] Production genesis initialization complete.")
+    print(f"[!] Next steps:")
+    print(f"    1) Generate keystore for aitbc1genesis and aitbc1treasury using scripts/keystore.py")
+    print(f"    2) Update .env with CHAIN_ID={args.chain_id} and PROPOSER_KEY=<private key of aitbc1genesis>")
+    print(f"    3) Restart the blockchain node.")
+
+
+if __name__ == "__main__":
+    main()

scripts/keystore.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""
+Keystore management for AITBC production keys.
+Generates a random private key and encrypts it with a password using Fernet (AES-128).
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import hashlib
+import json
+import os
+import secrets
+from datetime import datetime
+from pathlib import Path
+
+from cryptography.fernet import Fernet
+
+
+def derive_key(password: str, salt: bytes = b"") -> bytes:
+    """Derive a 32-byte key from the password using SHA-256."""
+    if not salt:
+        salt = secrets.token_bytes(16)
+    # Simple KDF: hash(password + salt)
+    dk = hashlib.sha256(password.encode() + salt).digest()
+    return base64.urlsafe_b64encode(dk), salt
+
+
+def encrypt_private_key(private_key_hex: str, password: str) -> dict:
+    """Encrypt a hex-encoded private key with Fernet, returning a keystore dict."""
+    key, salt = derive_key(password)
+    f = Fernet(key)
+    token = f.encrypt(private_key_hex.encode())
+    return {
+        "cipher": "fernet",
+        "cipherparams": {"salt": base64.b64encode(salt).decode()},
+        "ciphertext": base64.b64encode(token).decode(),
+        "kdf": "sha256",
+        "kdfparams": {"dklen": 32, "salt": base64.b64encode(salt).decode()},
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Generate encrypted keystore for an account")
+    parser.add_argument("address", help="Account address (e.g., aitbc1treasury)")
+    parser.add_argument("--output-dir", type=Path, default=Path("/opt/aitbc/keystore"), help="Keystore directory")
+    parser.add_argument("--force", action="store_true", help="Overwrite existing keystore file")
+    parser.add_argument("--password", help="Encryption password (or read from KEYSTORE_PASSWORD / keystore/.password)")
+    args = parser.parse_args()
+
+    out_dir = args.output_dir
+    out_dir.mkdir(parents=True, exist_ok=True)
+    out_file = out_dir / f"{args.address}.json"
+
+    if out_file.exists() and not args.force:
+        print(f"Keystore file {out_file} exists. Use --force to overwrite.")
+        return
+
+    # Determine password: CLI > env var > password file
+    password = args.password
+    if not password:
+        password = os.getenv("KEYSTORE_PASSWORD")
+    if not password:
+        pw_file = Path("/opt/aitbc/keystore/.password")
+        if pw_file.exists():
+            password = pw_file.read_text().strip()
+    if not password:
+        print("No password provided. Set KEYSTORE_PASSWORD, pass --password, or create /opt/aitbc/keystore/.password")
+        sys.exit(1)
+
+    print(f"Generating keystore for {args.address}...")
+    private_key = secrets.token_hex(32)
+    print(f"Private key (hex): {private_key}")
+    print("** SAVE THIS KEY SECURELY ** (It cannot be recovered from the encrypted file without the password)")
+
+    encrypted = encrypt_private_key(private_key, password)
+    keystore = {
+        "address": args.address,
+        "crypto": encrypted,
+        "created_at": datetime.utcnow().isoformat() + "Z",
+    }
+
+    out_file.write_text(json.dumps(keystore, indent=2))
+    os.chmod(out_file, 0o600)
+    print(f"[+] Keystore written to {out_file}")
+    print(f"[!] Keep the password safe. Without it, the private key cannot be recovered.")
+
+
+if __name__ == "__main__":
+    main()

scripts/run_production_node.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""
+Production launcher for AITBC blockchain node.
+Sets up environment, initializes genesis if needed, and starts the node.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import subprocess
+from pathlib import Path
+
+# Configuration
+CHAIN_ID = "ait-mainnet"
+DATA_DIR = Path("/opt/aitbc/data/ait-mainnet")
+DB_PATH = DATA_DIR / "chain.db"
+KEYS_DIR = Path("/opt/aitbc/keystore")
+
+# Check for proposer key in keystore
+PROPOSER_KEY_FILE = KEYS_DIR / "aitbc1genesis.json"
+if not PROPOSER_KEY_FILE.exists():
+    print(f"[!] Proposer keystore not found at {PROPOSER_KEY_FILE}")
+    print("    Run scripts/keystore.py to generate it first.")
+    sys.exit(1)
+
+# Set environment variables
+os.environ["CHAIN_ID"] = CHAIN_ID
+os.environ["SUPPORTED_CHAINS"] = CHAIN_ID
+os.environ["DB_PATH"] = str(DB_PATH)
+os.environ["PROPOSER_ID"] = "aitbc1genesis"
+# PROPOSER_KEY will be read from keystore by the node? Currently .env expects hex directly.
+# We can read the keystore, decrypt, and set PROPOSER_KEY, but the node doesn't support that out of box.
+# So we require that PROPOSER_KEY is set in .env file manually after key generation.
+# This script will check for PROPOSER_KEY env var or fail with instructions.
+if not os.getenv("PROPOSER_KEY"):
+    print("[!] PROPOSER_KEY environment variable not set.")
+    print("    Please edit /opt/aitbc/apps/blockchain-node/.env and set PROPOSER_KEY to the hex private key of aitbc1genesis.")
+    sys.exit(1)
+
+# Ensure data directory
+DATA_DIR.mkdir(parents=True, exist_ok=True)
+
+# Optionally initialize genesis if DB doesn't exist
+if not DB_PATH.exists():
+    print("[*] Database not found. Initializing production genesis...")
+    result = subprocess.run([
+        sys.executable,
+        "/opt/aitbc/scripts/init_production_genesis.py",
+        "--chain-id", CHAIN_ID,
+        "--db-path", str(DB_PATH)
+    ], check=False)
+    if result.returncode != 0:
+        print("[!] Genesis initialization failed. Aborting.")
+        sys.exit(1)
+
+# Start the node
+print(f"[*] Starting blockchain node for chain {CHAIN_ID}...")
+# Change to the blockchain-node directory (since .env and uvicorn expect relative paths)
+os.chdir("/opt/aitbc/apps/blockchain-node")
+# Use the virtualenv Python
+venv_python = Path("/opt/aitbc/apps/blockchain-node/.venv/bin/python")
+if not venv_python.exists():
+    print(f"[!] Virtualenv not found at {venv_python}")
+    sys.exit(1)
+
+# Exec uvicorn
+os.execv(str(venv_python), [str(venv_python), "-m", "uvicorn", "aitbc_chain.app:app", "--host", "127.0.0.1", "--port", "8006"])

scripts/setup_production.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""
+Full production setup:
+- Generate keystore password file
+- Generate encrypted keystores for aitbc1genesis and aitbc1treasury
+- Initialize production database with allocations
+- Configure blockchain node .env for ait-mainnet
+- Restart services
+"""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+# Configuration
+CHAIN_ID = "ait-mainnet"
+DATA_DIR = Path("/opt/aitbc/data/ait-mainnet")
+DB_PATH = DATA_DIR / "chain.db"
+KEYS_DIR = Path("/opt/aitbc/keystore")
+PASSWORD_FILE = KEYS_DIR / ".password"
+NODE_VENV = Path("/opt/aitbc/apps/blockchain-node/.venv/bin/python")
+NODE_ENV = Path("/opt/aitbc/apps/blockchain-node/.env")
+SERVICE_NODE = "aitbc-blockchain-node"
+SERVICE_RPC = "aitbc-blockchain-rpc"
+
+def run(cmd, check=True, capture_output=False):
+    print(f"+ {cmd}")
+    if capture_output:
+        result = subprocess.run(cmd, shell=True, check=check, capture_output=True, text=True)
+    else:
+        result = subprocess.run(cmd, shell=True, check=check)
+    return result
+
+def main():
+    if os.geteuid() != 0:
+        print("Run as root (sudo)")
+        sys.exit(1)
+
+    # 1. Keystore directory and password
+    run(f"mkdir -p {KEYS_DIR}")
+    run(f"chown -R aitbc:aitbc {KEYS_DIR}")
+    if not PASSWORD_FILE.exists():
+        run(f"openssl rand -hex 32 > {PASSWORD_FILE}")
+        run(f"chmod 600 {PASSWORD_FILE}")
+    os.environ["KEYSTORE_PASSWORD"] = PASSWORD_FILE.read_text().strip()
+
+    # 2. Generate keystores
+    print("\n=== Generating keystore for aitbc1genesis ===")
+    result = run(
+        f"sudo -u aitbc {NODE_VENV} /opt/aitbc/scripts/keystore.py aitbc1genesis --output-dir {KEYS_DIR} --force",
+        capture_output=True
+    )
+    print(result.stdout)
+    genesis_priv = None
+    for line in result.stdout.splitlines():
+        if "Private key (hex):" in line:
+            genesis_priv = line.split(":",1)[1].strip()
+            break
+    if not genesis_priv:
+        print("ERROR: Could not extract genesis private key")
+        sys.exit(1)
+    (KEYS_DIR / "genesis_private_key.txt").write_text(genesis_priv)
+    os.chmod(KEYS_DIR / "genesis_private_key.txt", 0o600)
+
+    print("\n=== Generating keystore for aitbc1treasury ===")
+    result = run(
+        f"sudo -u aitbc {NODE_VENV} /opt/aitbc/scripts/keystore.py aitbc1treasury --output-dir {KEYS_DIR} --force",
+        capture_output=True
+    )
+    print(result.stdout)
+    treasury_priv = None
+    for line in result.stdout.splitlines():
+        if "Private key (hex):" in line:
+            treasury_priv = line.split(":",1)[1].strip()
+            break
+    if not treasury_priv:
+        print("ERROR: Could not extract treasury private key")
+        sys.exit(1)
+    (KEYS_DIR / "treasury_private_key.txt").write_text(treasury_priv)
+    os.chmod(KEYS_DIR / "treasury_private_key.txt", 0o600)
+
+    # 3. Data directory
+    run(f"mkdir -p {DATA_DIR}")
+    run(f"chown -R aitbc:aitbc {DATA_DIR}")
+
+    # 4. Initialize DB
+    os.environ["DB_PATH"] = str(DB_PATH)
+    os.environ["CHAIN_ID"] = CHAIN_ID
+    run(f"sudo -E -u aitbc {NODE_VENV} /opt/aitbc/scripts/init_production_genesis.py --chain-id {CHAIN_ID} --db-path {DB_PATH}")
+
+    # 5. Write .env for blockchain node
+    env_content = f"""CHAIN_ID={CHAIN_ID}
+SUPPORTED_CHAINS={CHAIN_ID}
+DB_PATH=./data/ait-mainnet/chain.db
+PROPOSER_ID=aitbc1genesis
+PROPOSER_KEY=0x{genesis_priv}
+PROPOSER_INTERVAL_SECONDS=5
+BLOCK_TIME_SECONDS=2
+
+RPC_BIND_HOST=127.0.0.1
+RPC_BIND_PORT=8006
+P2P_BIND_HOST=127.0.0.2
+P2P_BIND_PORT=8005
+
+MEMPOOL_BACKEND=database
+MIN_FEE=0
+GOSSIP_BACKEND=memory
+"""
+    NODE_ENV.write_text(env_content)
+    os.chmod(NODE_ENV, 0o644)
+    print(f"[+] Updated {NODE_ENV}")
+
+    # 6. Restart services
+    run("systemctl daemon-reload")
+    run(f"systemctl restart {SERVICE_NODE} {SERVICE_RPC}")
+
+    print("\n[+] Production setup complete!")
+    print(f"[+] Verify with: curl 'http://127.0.0.1:8006/head?chain_id={CHAIN_ID}' | jq")
+    print(f"[+] Keystore files in {KEYS_DIR} (encrypted, 600)")
+    print(f"[+] Private keys saved in {KEYS_DIR}/genesis_private_key.txt and treasury_private_key.txt (keep secure!)")
+
+if __name__ == "__main__":
+    main()