feat: implement AITBC mesh network deployment infrastructure

 Phase 0: Pre-implementation checklist completed
- Environment configurations (dev/staging/production)
- Directory structure setup (logs, backups, monitoring)
- Virtual environment with dependencies

 Master deployment script created
- Single command deployment with validation
- Progress tracking and rollback capability
- Health checks and deployment reporting

 Validation script created
- Module import validation
- Basic functionality testing
- Configuration and script verification

 Implementation fixes
- Fixed dataclass import in consensus keys
- Fixed async function syntax in tests
- Updated deployment script for virtual environment

🚀 Ready for deployment: ./scripts/deploy-mesh-network.sh dev
This commit is contained in:
aitbc
2026-04-02 12:08:15 +02:00
parent d68aa9a234
commit c876b0aa20
206 changed files with 47861 additions and 1 deletions

View File

@@ -0,0 +1,211 @@
"""
Validator Key Management
Handles cryptographic key operations for validators
"""
import os
import json
import time
from dataclasses import dataclass
from typing import Dict, Optional, Tuple
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.serialization import Encoding, PrivateFormat, NoEncryption
@dataclass
class ValidatorKeyPair:
address: str
private_key_pem: str
public_key_pem: str
created_at: float
last_rotated: float
class KeyManager:
"""Manages validator cryptographic keys"""
def __init__(self, keys_dir: str = "/opt/aitbc/keys"):
self.keys_dir = keys_dir
self.key_pairs: Dict[str, ValidatorKeyPair] = {}
self._ensure_keys_directory()
self._load_existing_keys()
def _ensure_keys_directory(self):
"""Ensure keys directory exists and has proper permissions"""
os.makedirs(self.keys_dir, mode=0o700, exist_ok=True)
def _load_existing_keys(self):
"""Load existing key pairs from disk"""
keys_file = os.path.join(self.keys_dir, "validator_keys.json")
if os.path.exists(keys_file):
try:
with open(keys_file, 'r') as f:
keys_data = json.load(f)
for address, key_data in keys_data.items():
self.key_pairs[address] = ValidatorKeyPair(
address=address,
private_key_pem=key_data['private_key_pem'],
public_key_pem=key_data['public_key_pem'],
created_at=key_data['created_at'],
last_rotated=key_data['last_rotated']
)
except Exception as e:
print(f"Error loading keys: {e}")
def generate_key_pair(self, address: str) -> ValidatorKeyPair:
"""Generate new RSA key pair for validator"""
# Generate private key
private_key = rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend()
)
# Serialize private key
private_key_pem = private_key.private_bytes(
encoding=Encoding.PEM,
format=PrivateFormat.PKCS8,
encryption_algorithm=NoEncryption()
).decode('utf-8')
# Get public key
public_key = private_key.public_key()
public_key_pem = public_key.public_bytes(
encoding=Encoding.PEM,
format=serialization.PublicFormat.SubjectPublicKeyInfo
).decode('utf-8')
# Create key pair object
current_time = time.time()
key_pair = ValidatorKeyPair(
address=address,
private_key_pem=private_key_pem,
public_key_pem=public_key_pem,
created_at=current_time,
last_rotated=current_time
)
# Store key pair
self.key_pairs[address] = key_pair
self._save_keys()
return key_pair
def get_key_pair(self, address: str) -> Optional[ValidatorKeyPair]:
"""Get key pair for validator"""
return self.key_pairs.get(address)
def rotate_key(self, address: str) -> Optional[ValidatorKeyPair]:
"""Rotate validator keys"""
if address not in self.key_pairs:
return None
# Generate new key pair
new_key_pair = self.generate_key_pair(address)
# Update rotation time
new_key_pair.created_at = self.key_pairs[address].created_at
new_key_pair.last_rotated = time.time()
self._save_keys()
return new_key_pair
def sign_message(self, address: str, message: str) -> Optional[str]:
"""Sign message with validator private key"""
key_pair = self.get_key_pair(address)
if not key_pair:
return None
try:
# Load private key from PEM
private_key = serialization.load_pem_private_key(
key_pair.private_key_pem.encode(),
password=None,
backend=default_backend()
)
# Sign message
signature = private_key.sign(
message.encode('utf-8'),
hashes.SHA256(),
default_backend()
)
return signature.hex()
except Exception as e:
print(f"Error signing message: {e}")
return None
def verify_signature(self, address: str, message: str, signature: str) -> bool:
"""Verify message signature"""
key_pair = self.get_key_pair(address)
if not key_pair:
return False
try:
# Load public key from PEM
public_key = serialization.load_pem_public_key(
key_pair.public_key_pem.encode(),
backend=default_backend()
)
# Verify signature
public_key.verify(
bytes.fromhex(signature),
message.encode('utf-8'),
hashes.SHA256(),
default_backend()
)
return True
except Exception as e:
print(f"Error verifying signature: {e}")
return False
def get_public_key_pem(self, address: str) -> Optional[str]:
"""Get public key PEM for validator"""
key_pair = self.get_key_pair(address)
return key_pair.public_key_pem if key_pair else None
def _save_keys(self):
"""Save key pairs to disk"""
keys_file = os.path.join(self.keys_dir, "validator_keys.json")
keys_data = {}
for address, key_pair in self.key_pairs.items():
keys_data[address] = {
'private_key_pem': key_pair.private_key_pem,
'public_key_pem': key_pair.public_key_pem,
'created_at': key_pair.created_at,
'last_rotated': key_pair.last_rotated
}
try:
with open(keys_file, 'w') as f:
json.dump(keys_data, f, indent=2)
# Set secure permissions
os.chmod(keys_file, 0o600)
except Exception as e:
print(f"Error saving keys: {e}")
def should_rotate_key(self, address: str, rotation_interval: int = 86400) -> bool:
"""Check if key should be rotated (default: 24 hours)"""
key_pair = self.get_key_pair(address)
if not key_pair:
return True
return (time.time() - key_pair.last_rotated) >= rotation_interval
def get_key_age(self, address: str) -> Optional[float]:
"""Get age of key in seconds"""
key_pair = self.get_key_pair(address)
if not key_pair:
return None
return time.time() - key_pair.created_at
# Global key manager
key_manager = KeyManager()

View File

@@ -0,0 +1,119 @@
"""
Multi-Validator Proof of Authority Consensus Implementation
Extends single validator PoA to support multiple validators with rotation
"""
import asyncio
import time
import hashlib
from typing import List, Dict, Optional, Set
from dataclasses import dataclass
from enum import Enum
from ..config import settings
from ..models import Block, Transaction
from ..database import session_scope
class ValidatorRole(Enum):
PROPOSER = "proposer"
VALIDATOR = "validator"
STANDBY = "standby"
@dataclass
class Validator:
address: str
stake: float
reputation: float
role: ValidatorRole
last_proposed: int
is_active: bool
class MultiValidatorPoA:
"""Multi-Validator Proof of Authority consensus mechanism"""
def __init__(self, chain_id: str):
self.chain_id = chain_id
self.validators: Dict[str, Validator] = {}
self.current_proposer_index = 0
self.round_robin_enabled = True
self.consensus_timeout = 30 # seconds
def add_validator(self, address: str, stake: float = 1000.0) -> bool:
"""Add a new validator to the consensus"""
if address in self.validators:
return False
self.validators[address] = Validator(
address=address,
stake=stake,
reputation=1.0,
role=ValidatorRole.STANDBY,
last_proposed=0,
is_active=True
)
return True
def remove_validator(self, address: str) -> bool:
"""Remove a validator from the consensus"""
if address not in self.validators:
return False
validator = self.validators[address]
validator.is_active = False
validator.role = ValidatorRole.STANDBY
return True
def select_proposer(self, block_height: int) -> Optional[str]:
"""Select proposer for the current block using round-robin"""
active_validators = [
v for v in self.validators.values()
if v.is_active and v.role in [ValidatorRole.PROPOSER, ValidatorRole.VALIDATOR]
]
if not active_validators:
return None
# Round-robin selection
proposer_index = block_height % len(active_validators)
return active_validators[proposer_index].address
def validate_block(self, block: Block, proposer: str) -> bool:
"""Validate a proposed block"""
if proposer not in self.validators:
return False
validator = self.validators[proposer]
if not validator.is_active:
return False
# Check if validator is allowed to propose
if validator.role not in [ValidatorRole.PROPOSER, ValidatorRole.VALIDATOR]:
return False
# Additional validation logic here
return True
def get_consensus_participants(self) -> List[str]:
"""Get list of active consensus participants"""
return [
v.address for v in self.validators.values()
if v.is_active and v.role in [ValidatorRole.PROPOSER, ValidatorRole.VALIDATOR]
]
def update_validator_reputation(self, address: str, delta: float) -> bool:
"""Update validator reputation"""
if address not in self.validators:
return False
validator = self.validators[address]
validator.reputation = max(0.0, min(1.0, validator.reputation + delta))
return True
# Global consensus instance
consensus_instances: Dict[str, MultiValidatorPoA] = {}
def get_consensus(chain_id: str) -> MultiValidatorPoA:
"""Get or create consensus instance for chain"""
if chain_id not in consensus_instances:
consensus_instances[chain_id] = MultiValidatorPoA(chain_id)
return consensus_instances[chain_id]

View File

@@ -0,0 +1,193 @@
"""
Practical Byzantine Fault Tolerance (PBFT) Consensus Implementation
Provides Byzantine fault tolerance for up to 1/3 faulty validators
"""
import asyncio
import time
import hashlib
from typing import List, Dict, Optional, Set, Tuple
from dataclasses import dataclass
from enum import Enum
from .multi_validator_poa import MultiValidatorPoA, Validator
class PBFTPhase(Enum):
PRE_PREPARE = "pre_prepare"
PREPARE = "prepare"
COMMIT = "commit"
EXECUTE = "execute"
class PBFTMessageType(Enum):
PRE_PREPARE = "pre_prepare"
PREPARE = "prepare"
COMMIT = "commit"
VIEW_CHANGE = "view_change"
@dataclass
class PBFTMessage:
message_type: PBFTMessageType
sender: str
view_number: int
sequence_number: int
digest: str
signature: str
timestamp: float
@dataclass
class PBFTState:
current_view: int
current_sequence: int
prepared_messages: Dict[str, List[PBFTMessage]]
committed_messages: Dict[str, List[PBFTMessage]]
pre_prepare_messages: Dict[str, PBFTMessage]
class PBFTConsensus:
"""PBFT consensus implementation"""
def __init__(self, consensus: MultiValidatorPoA):
self.consensus = consensus
self.state = PBFTState(
current_view=0,
current_sequence=0,
prepared_messages={},
committed_messages={},
pre_prepare_messages={}
)
self.fault_tolerance = max(1, len(consensus.get_consensus_participants()) // 3)
self.required_messages = 2 * self.fault_tolerance + 1
def get_message_digest(self, block_hash: str, sequence: int, view: int) -> str:
"""Generate message digest for PBFT"""
content = f"{block_hash}:{sequence}:{view}"
return hashlib.sha256(content.encode()).hexdigest()
async def pre_prepare_phase(self, proposer: str, block_hash: str) -> bool:
"""Phase 1: Pre-prepare"""
sequence = self.state.current_sequence + 1
view = self.state.current_view
digest = self.get_message_digest(block_hash, sequence, view)
message = PBFTMessage(
message_type=PBFTMessageType.PRE_PREPARE,
sender=proposer,
view_number=view,
sequence_number=sequence,
digest=digest,
signature="", # Would be signed in real implementation
timestamp=time.time()
)
# Store pre-prepare message
key = f"{sequence}:{view}"
self.state.pre_prepare_messages[key] = message
# Broadcast to all validators
await self._broadcast_message(message)
return True
async def prepare_phase(self, validator: str, pre_prepare_msg: PBFTMessage) -> bool:
"""Phase 2: Prepare"""
key = f"{pre_prepare_msg.sequence_number}:{pre_prepare_msg.view_number}"
if key not in self.state.pre_prepare_messages:
return False
# Create prepare message
prepare_msg = PBFTMessage(
message_type=PBFTMessageType.PREPARE,
sender=validator,
view_number=pre_prepare_msg.view_number,
sequence_number=pre_prepare_msg.sequence_number,
digest=pre_prepare_msg.digest,
signature="", # Would be signed
timestamp=time.time()
)
# Store prepare message
if key not in self.state.prepared_messages:
self.state.prepared_messages[key] = []
self.state.prepared_messages[key].append(prepare_msg)
# Broadcast prepare message
await self._broadcast_message(prepare_msg)
# Check if we have enough prepare messages
return len(self.state.prepared_messages[key]) >= self.required_messages
async def commit_phase(self, validator: str, prepare_msg: PBFTMessage) -> bool:
"""Phase 3: Commit"""
key = f"{prepare_msg.sequence_number}:{prepare_msg.view_number}"
# Create commit message
commit_msg = PBFTMessage(
message_type=PBFTMessageType.COMMIT,
sender=validator,
view_number=prepare_msg.view_number,
sequence_number=prepare_msg.sequence_number,
digest=prepare_msg.digest,
signature="", # Would be signed
timestamp=time.time()
)
# Store commit message
if key not in self.state.committed_messages:
self.state.committed_messages[key] = []
self.state.committed_messages[key].append(commit_msg)
# Broadcast commit message
await self._broadcast_message(commit_msg)
# Check if we have enough commit messages
if len(self.state.committed_messages[key]) >= self.required_messages:
return await self.execute_phase(key)
return False
async def execute_phase(self, key: str) -> bool:
"""Phase 4: Execute"""
# Extract sequence and view from key
sequence, view = map(int, key.split(':'))
# Update state
self.state.current_sequence = sequence
# Clean up old messages
self._cleanup_messages(sequence)
return True
async def _broadcast_message(self, message: PBFTMessage):
"""Broadcast message to all validators"""
validators = self.consensus.get_consensus_participants()
for validator in validators:
if validator != message.sender:
# In real implementation, this would send over network
await self._send_to_validator(validator, message)
async def _send_to_validator(self, validator: str, message: PBFTMessage):
"""Send message to specific validator"""
# Network communication would be implemented here
pass
def _cleanup_messages(self, sequence: int):
"""Clean up old messages to prevent memory leaks"""
old_keys = [
key for key in self.state.prepared_messages.keys()
if int(key.split(':')[0]) < sequence
]
for key in old_keys:
self.state.prepared_messages.pop(key, None)
self.state.committed_messages.pop(key, None)
self.state.pre_prepare_messages.pop(key, None)
def handle_view_change(self, new_view: int) -> bool:
"""Handle view change when proposer fails"""
self.state.current_view = new_view
# Reset state for new view
self.state.prepared_messages.clear()
self.state.committed_messages.clear()
self.state.pre_prepare_messages.clear()
return True

View File

@@ -0,0 +1,146 @@
"""
Validator Rotation Mechanism
Handles automatic rotation of validators based on performance and stake
"""
import asyncio
import time
from typing import List, Dict, Optional
from dataclasses import dataclass
from enum import Enum
from .multi_validator_poa import MultiValidatorPoA, Validator, ValidatorRole
class RotationStrategy(Enum):
ROUND_ROBIN = "round_robin"
STAKE_WEIGHTED = "stake_weighted"
REPUTATION_BASED = "reputation_based"
HYBRID = "hybrid"
@dataclass
class RotationConfig:
strategy: RotationStrategy
rotation_interval: int # blocks
min_stake: float
reputation_threshold: float
max_validators: int
class ValidatorRotation:
"""Manages validator rotation based on various strategies"""
def __init__(self, consensus: MultiValidatorPoA, config: RotationConfig):
self.consensus = consensus
self.config = config
self.last_rotation_height = 0
def should_rotate(self, current_height: int) -> bool:
"""Check if rotation should occur at current height"""
return (current_height - self.last_rotation_height) >= self.config.rotation_interval
def rotate_validators(self, current_height: int) -> bool:
"""Perform validator rotation based on configured strategy"""
if not self.should_rotate(current_height):
return False
if self.config.strategy == RotationStrategy.ROUND_ROBIN:
return self._rotate_round_robin()
elif self.config.strategy == RotationStrategy.STAKE_WEIGHTED:
return self._rotate_stake_weighted()
elif self.config.strategy == RotationStrategy.REPUTATION_BASED:
return self._rotate_reputation_based()
elif self.config.strategy == RotationStrategy.HYBRID:
return self._rotate_hybrid()
return False
def _rotate_round_robin(self) -> bool:
"""Round-robin rotation of validator roles"""
validators = list(self.consensus.validators.values())
active_validators = [v for v in validators if v.is_active]
# Rotate roles among active validators
for i, validator in enumerate(active_validators):
if i == 0:
validator.role = ValidatorRole.PROPOSER
elif i < 3: # Top 3 become validators
validator.role = ValidatorRole.VALIDATOR
else:
validator.role = ValidatorRole.STANDBY
self.last_rotation_height += self.config.rotation_interval
return True
def _rotate_stake_weighted(self) -> bool:
"""Stake-weighted rotation"""
validators = sorted(
[v for v in self.consensus.validators.values() if v.is_active],
key=lambda v: v.stake,
reverse=True
)
for i, validator in enumerate(validators[:self.config.max_validators]):
if i == 0:
validator.role = ValidatorRole.PROPOSER
elif i < 4:
validator.role = ValidatorRole.VALIDATOR
else:
validator.role = ValidatorRole.STANDBY
self.last_rotation_height += self.config.rotation_interval
return True
def _rotate_reputation_based(self) -> bool:
"""Reputation-based rotation"""
validators = sorted(
[v for v in self.consensus.validators.values() if v.is_active],
key=lambda v: v.reputation,
reverse=True
)
# Filter by reputation threshold
qualified_validators = [
v for v in validators
if v.reputation >= self.config.reputation_threshold
]
for i, validator in enumerate(qualified_validators[:self.config.max_validators]):
if i == 0:
validator.role = ValidatorRole.PROPOSER
elif i < 4:
validator.role = ValidatorRole.VALIDATOR
else:
validator.role = ValidatorRole.STANDBY
self.last_rotation_height += self.config.rotation_interval
return True
def _rotate_hybrid(self) -> bool:
"""Hybrid rotation considering both stake and reputation"""
validators = [v for v in self.consensus.validators.values() if v.is_active]
# Calculate hybrid score
for validator in validators:
validator.hybrid_score = validator.stake * validator.reputation
# Sort by hybrid score
validators.sort(key=lambda v: v.hybrid_score, reverse=True)
for i, validator in enumerate(validators[:self.config.max_validators]):
if i == 0:
validator.role = ValidatorRole.PROPOSER
elif i < 4:
validator.role = ValidatorRole.VALIDATOR
else:
validator.role = ValidatorRole.STANDBY
self.last_rotation_height += self.config.rotation_interval
return True
# Default rotation configuration
DEFAULT_ROTATION_CONFIG = RotationConfig(
strategy=RotationStrategy.HYBRID,
rotation_interval=100, # Rotate every 100 blocks
min_stake=1000.0,
reputation_threshold=0.7,
max_validators=10
)

View File

@@ -0,0 +1,138 @@
"""
Slashing Conditions Implementation
Handles detection and penalties for validator misbehavior
"""
import time
from typing import Dict, List, Optional, Set
from dataclasses import dataclass
from enum import Enum
from .multi_validator_poa import Validator, ValidatorRole
class SlashingCondition(Enum):
DOUBLE_SIGN = "double_sign"
UNAVAILABLE = "unavailable"
INVALID_BLOCK = "invalid_block"
SLOW_RESPONSE = "slow_response"
@dataclass
class SlashingEvent:
validator_address: str
condition: SlashingCondition
evidence: str
block_height: int
timestamp: float
slash_amount: float
class SlashingManager:
"""Manages validator slashing conditions and penalties"""
def __init__(self):
self.slashing_events: List[SlashingEvent] = []
self.slash_rates = {
SlashingCondition.DOUBLE_SIGN: 0.5, # 50% slash
SlashingCondition.UNAVAILABLE: 0.1, # 10% slash
SlashingCondition.INVALID_BLOCK: 0.3, # 30% slash
SlashingCondition.SLOW_RESPONSE: 0.05 # 5% slash
}
self.slash_thresholds = {
SlashingCondition.DOUBLE_SIGN: 1, # Immediate slash
SlashingCondition.UNAVAILABLE: 3, # After 3 offenses
SlashingCondition.INVALID_BLOCK: 1, # Immediate slash
SlashingCondition.SLOW_RESPONSE: 5 # After 5 offenses
}
def detect_double_sign(self, validator: str, block_hash1: str, block_hash2: str, height: int) -> Optional[SlashingEvent]:
"""Detect double signing (validator signed two different blocks at same height)"""
if block_hash1 == block_hash2:
return None
return SlashingEvent(
validator_address=validator,
condition=SlashingCondition.DOUBLE_SIGN,
evidence=f"Double sign detected: {block_hash1} vs {block_hash2} at height {height}",
block_height=height,
timestamp=time.time(),
slash_amount=self.slash_rates[SlashingCondition.DOUBLE_SIGN]
)
def detect_unavailability(self, validator: str, missed_blocks: int, height: int) -> Optional[SlashingEvent]:
"""Detect validator unavailability (missing consensus participation)"""
if missed_blocks < self.slash_thresholds[SlashingCondition.UNAVAILABLE]:
return None
return SlashingEvent(
validator_address=validator,
condition=SlashingCondition.UNAVAILABLE,
evidence=f"Missed {missed_blocks} consecutive blocks",
block_height=height,
timestamp=time.time(),
slash_amount=self.slash_rates[SlashingCondition.UNAVAILABLE]
)
def detect_invalid_block(self, validator: str, block_hash: str, reason: str, height: int) -> Optional[SlashingEvent]:
"""Detect invalid block proposal"""
return SlashingEvent(
validator_address=validator,
condition=SlashingCondition.INVALID_BLOCK,
evidence=f"Invalid block {block_hash}: {reason}",
block_height=height,
timestamp=time.time(),
slash_amount=self.slash_rates[SlashingCondition.INVALID_BLOCK]
)
def detect_slow_response(self, validator: str, response_time: float, threshold: float, height: int) -> Optional[SlashingEvent]:
"""Detect slow consensus participation"""
if response_time <= threshold:
return None
return SlashingEvent(
validator_address=validator,
condition=SlashingCondition.SLOW_RESPONSE,
evidence=f"Slow response: {response_time}s (threshold: {threshold}s)",
block_height=height,
timestamp=time.time(),
slash_amount=self.slash_rates[SlashingCondition.SLOW_RESPONSE]
)
def apply_slashing(self, validator: Validator, event: SlashingEvent) -> bool:
"""Apply slashing penalty to validator"""
slash_amount = validator.stake * event.slash_amount
validator.stake -= slash_amount
# Demote validator role if stake is too low
if validator.stake < 100: # Minimum stake threshold
validator.role = ValidatorRole.STANDBY
# Record slashing event
self.slashing_events.append(event)
return True
def get_validator_slash_count(self, validator_address: str, condition: SlashingCondition) -> int:
"""Get count of slashing events for validator and condition"""
return len([
event for event in self.slashing_events
if event.validator_address == validator_address and event.condition == condition
])
def should_slash(self, validator: str, condition: SlashingCondition) -> bool:
"""Check if validator should be slashed for condition"""
current_count = self.get_validator_slash_count(validator, condition)
threshold = self.slash_thresholds.get(condition, 1)
return current_count >= threshold
def get_slashing_history(self, validator_address: Optional[str] = None) -> List[SlashingEvent]:
"""Get slashing history for validator or all validators"""
if validator_address:
return [event for event in self.slashing_events if event.validator_address == validator_address]
return self.slashing_events.copy()
def calculate_total_slashed(self, validator_address: str) -> float:
"""Calculate total amount slashed for validator"""
events = self.get_slashing_history(validator_address)
return sum(event.slash_amount for event in events)
# Global slashing manager
slashing_manager = SlashingManager()