feat: add marketplace metrics, privacy features, and service registry endpoints

- Add Prometheus metrics for marketplace API throughput and error rates with new dashboard panels
- Implement confidential transaction models with encryption support and access control
- Add key management system with registration, rotation, and audit logging
- Create services and registry routers for service discovery and management
- Integrate ZK proof generation for privacy-preserving receipts
- Add metrics instru
This commit is contained in:
oib
2025-12-22 10:33:23 +01:00
parent d98b2c7772
commit c8be9d7414
260 changed files with 59033 additions and 351 deletions

View File

@ -0,0 +1,403 @@
# Cross-Chain Settlement Hooks Design
## Overview
This document outlines the architecture for cross-chain settlement hooks in AITBC, enabling job receipts and proofs to be settled across multiple blockchains using various bridge protocols.
## Architecture
### Core Components
```
┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐
│ AITBC Chain │ │ Settlement Hooks │ │ Target Chains │
│ │ │ │ │ │
│ - Job Receipts │───▶│ - Bridge Manager │───▶│ - Ethereum │
│ - Proofs │ │ - Adapters │ │ - Polygon │
│ - Payments │ │ - Router │ │ - BSC │
│ │ │ - Validator │ │ - Arbitrum │
└─────────────────┘ └──────────────────┘ └─────────────────┘
```
### Settlement Hook Interface
```python
from abc import ABC, abstractmethod
from typing import Dict, Any, List
from dataclasses import dataclass
@dataclass
class SettlementMessage:
"""Message to be settled across chains"""
source_chain_id: int
target_chain_id: int
job_id: str
receipt_hash: str
proof_data: Dict[str, Any]
payment_amount: int
payment_token: str
nonce: int
signature: str
class BridgeAdapter(ABC):
"""Abstract interface for bridge adapters"""
@abstractmethod
async def send_message(self, message: SettlementMessage) -> str:
"""Send message to target chain"""
pass
@abstractmethod
async def verify_delivery(self, message_id: str) -> bool:
"""Verify message was delivered"""
pass
@abstractmethod
async def estimate_cost(self, message: SettlementMessage) -> Dict[str, int]:
"""Estimate bridge fees"""
pass
@abstractmethod
def get_supported_chains(self) -> List[int]:
"""Get list of supported target chains"""
pass
@abstractmethod
def get_max_message_size(self) -> int:
"""Get maximum message size in bytes"""
pass
```
### Bridge Manager
```python
class BridgeManager:
"""Manages multiple bridge adapters"""
def __init__(self):
self.adapters: Dict[str, BridgeAdapter] = {}
self.default_adapter: str = None
def register_adapter(self, name: str, adapter: BridgeAdapter):
"""Register a bridge adapter"""
self.adapters[name] = adapter
async def settle_cross_chain(
self,
message: SettlementMessage,
bridge_name: str = None
) -> str:
"""Settle message across chains"""
adapter = self._get_adapter(bridge_name)
# Validate message
self._validate_message(message, adapter)
# Send message
message_id = await adapter.send_message(message)
# Store settlement record
await self._store_settlement(message_id, message)
return message_id
def _get_adapter(self, bridge_name: str = None) -> BridgeAdapter:
"""Get bridge adapter"""
if bridge_name:
return self.adapters[bridge_name]
return self.adapters[self.default_adapter]
```
## Bridge Implementations
### 1. LayerZero Adapter
```python
class LayerZeroAdapter(BridgeAdapter):
"""LayerZero bridge adapter"""
def __init__(self, endpoint_address: str, chain_id: int):
self.endpoint = endpoint_address
self.chain_id = chain_id
self.contract = self._load_contract()
async def send_message(self, message: SettlementMessage) -> str:
"""Send via LayerZero"""
# Encode settlement data
payload = self._encode_payload(message)
# Estimate fees
fees = await self._estimate_fees(message)
# Send transaction
tx = await self.contract.send(
message.target_chain_id,
self._get_target_address(message.target_chain_id),
payload,
message.payment_amount,
message.payment_token,
fees
)
return tx.hash
def _encode_payload(self, message: SettlementMessage) -> bytes:
"""Encode message for LayerZero"""
return abi.encode(
['uint256', 'bytes32', 'bytes', 'uint256', 'address'],
[
message.job_id,
message.receipt_hash,
json.dumps(message.proof_data),
message.payment_amount,
message.payment_token
]
)
```
### 2. Chainlink CCIP Adapter
```python
class ChainlinkCCIPAdapter(BridgeAdapter):
"""Chainlink CCIP bridge adapter"""
def __init__(self, router_address: str, chain_id: int):
self.router = router_address
self.chain_id = chain_id
self.contract = self._load_contract()
async def send_message(self, message: SettlementMessage) -> str:
"""Send via Chainlink CCIP"""
# Create CCIP message
ccip_message = {
'receiver': self._get_target_address(message.target_chain_id),
'data': self._encode_payload(message),
'tokenAmounts': [{
'token': message.payment_token,
'amount': message.payment_amount
}]
}
# Estimate fees
fees = await self.contract.getFee(ccip_message)
# Send transaction
tx = await self.contract.ccipSend(ccip_message, {'value': fees})
return tx.hash
```
### 3. Wormhole Adapter
```python
class WormholeAdapter(BridgeAdapter):
"""Wormhole bridge adapter"""
def __init__(self, bridge_address: str, chain_id: int):
self.bridge = bridge_address
self.chain_id = chain_id
self.contract = self._load_contract()
async def send_message(self, message: SettlementMessage) -> str:
"""Send via Wormhole"""
# Encode payload
payload = self._encode_payload(message)
# Send transaction
tx = await self.contract.publishMessage(
message.nonce,
payload,
message.payment_amount
)
return tx.hash
```
## Integration with Coordinator
### Settlement Hook in Coordinator
```python
class SettlementHook:
"""Settlement hook for coordinator"""
def __init__(self, bridge_manager: BridgeManager):
self.bridge_manager = bridge_manager
async def on_job_completed(self, job: Job) -> None:
"""Called when job completes"""
# Check if cross-chain settlement needed
if job.requires_cross_chain_settlement:
await self._settle_cross_chain(job)
async def _settle_cross_chain(self, job: Job) -> None:
"""Settle job across chains"""
# Create settlement message
message = SettlementMessage(
source_chain_id=await self._get_chain_id(),
target_chain_id=job.target_chain,
job_id=job.id,
receipt_hash=job.receipt.hash,
proof_data=job.receipt.proof,
payment_amount=job.payment_amount,
payment_token=job.payment_token,
nonce=await self._get_nonce(),
signature=await self._sign_message(job)
)
# Send via appropriate bridge
await self.bridge_manager.settle_cross_chain(
message,
bridge_name=job.preferred_bridge
)
```
### Coordinator API Endpoints
```python
@app.post("/v1/settlement/cross-chain")
async def initiate_cross_chain_settlement(
request: CrossChainSettlementRequest
):
"""Initiate cross-chain settlement"""
job = await get_job(request.job_id)
if not job.completed:
raise HTTPException(400, "Job not completed")
# Create settlement message
message = SettlementMessage(
source_chain_id=request.source_chain,
target_chain_id=request.target_chain,
job_id=job.id,
receipt_hash=job.receipt.hash,
proof_data=job.receipt.proof,
payment_amount=request.amount,
payment_token=request.token,
nonce=await generate_nonce(),
signature=await sign_settlement(job, request)
)
# Send settlement
message_id = await settlement_hook.settle_cross_chain(message)
return {"message_id": message_id, "status": "pending"}
@app.get("/v1/settlement/{message_id}/status")
async def get_settlement_status(message_id: str):
"""Get settlement status"""
status = await bridge_manager.get_settlement_status(message_id)
return status
```
## Configuration
### Bridge Configuration
```yaml
bridges:
layerzero:
enabled: true
endpoint_address: "0x..."
supported_chains: [1, 137, 56, 42161]
default_fee: "0.001"
chainlink_ccip:
enabled: true
router_address: "0x..."
supported_chains: [1, 137, 56, 42161]
default_fee: "0.002"
wormhole:
enabled: false
bridge_address: "0x..."
supported_chains: [1, 137, 56]
default_fee: "0.0015"
settlement:
default_bridge: "layerzero"
max_retries: 3
retry_delay: 30
timeout: 3600
```
## Security Considerations
### Message Validation
- Verify signatures on all settlement messages
- Validate chain IDs and addresses
- Check message size limits
- Prevent replay attacks with nonces
### Bridge Security
- Use reputable audited bridge contracts
- Implement bridge-specific security checks
- Monitor for bridge vulnerabilities
- Have fallback mechanisms
### Economic Security
- Validate payment amounts
- Check token allowances
- Implement fee limits
- Monitor for economic attacks
## Monitoring
### Metrics to Track
- Settlement success rate per bridge
- Average settlement time
- Cost per settlement
- Failed settlement reasons
- Bridge health status
### Alerts
- Settlement failures
- High settlement costs
- Bridge downtime
- Unusual settlement patterns
## Testing
### Test Scenarios
1. **Happy Path**: Successful settlement across chains
2. **Bridge Failure**: Handle bridge unavailability
3. **Message Too Large**: Handle size limits
4. **Insufficient Funds**: Handle payment failures
5. **Replay Attack**: Prevent duplicate settlements
### Test Networks
- Ethereum Sepolia
- Polygon Mumbai
- BSC Testnet
- Arbitrum Goerli
## Migration Path
### Phase 1: Single Bridge
- Implement LayerZero adapter
- Basic settlement functionality
- Test on testnets
### Phase 2: Multiple Bridges
- Add Chainlink CCIP
- Implement bridge selection logic
- Add cost optimization
### Phase 3: Advanced Features
- Add Wormhole support
- Implement atomic settlements
- Add settlement routing
## Future Enhancements
1. **Atomic Settlements**: Ensure all-or-nothing settlements
2. **Settlement Routing**: Automatically select optimal bridge
3. **Batch Settlements**: Settle multiple jobs together
4. **Cross-Chain Governance**: Governance across chains
5. **Privacy Features**: Confidential settlements
---
*Document Version: 1.0*
*Last Updated: 2025-01-10*
*Owner: Core Protocol Team*

View File

@ -0,0 +1,618 @@
# Python SDK Transport Abstraction Design
## Overview
This document outlines the design for a pluggable transport abstraction layer in the AITBC Python SDK, enabling support for multiple networks and cross-chain operations.
## Architecture
### Current SDK Structure
```
AITBCClient
├── Jobs API
├── Marketplace API
├── Wallet API
├── Receipts API
└── Direct HTTP calls to coordinator
```
### Proposed Transport-Based Structure
```
AITBCClient
├── Transport Layer (Pluggable)
│ ├── HTTPTransport
│ ├── WebSocketTransport
│ └── CrossChainTransport
├── Jobs API
├── Marketplace API
├── Wallet API
├── Receipts API
└── Settlement API (New)
```
## Transport Interface
### Base Transport Class
```python
from abc import ABC, abstractmethod
from typing import Dict, Any, Optional, Union
import asyncio
class Transport(ABC):
"""Abstract base class for all transports"""
def __init__(self, config: Dict[str, Any]):
self.config = config
self._connected = False
@abstractmethod
async def connect(self) -> None:
"""Establish connection"""
pass
@abstractmethod
async def disconnect(self) -> None:
"""Close connection"""
pass
@abstractmethod
async def request(
self,
method: str,
path: str,
data: Optional[Dict[str, Any]] = None,
params: Optional[Dict[str, Any]] = None,
headers: Optional[Dict[str, str]] = None
) -> Dict[str, Any]:
"""Make a request"""
pass
@abstractmethod
async def stream(
self,
method: str,
path: str,
data: Optional[Dict[str, Any]] = None
) -> AsyncIterator[Dict[str, Any]]:
"""Stream responses"""
pass
@property
def is_connected(self) -> bool:
"""Check if transport is connected"""
return self._connected
@property
def chain_id(self) -> Optional[int]:
"""Get the chain ID this transport is connected to"""
return self.config.get('chain_id')
```
### HTTP Transport Implementation
```python
import aiohttp
from typing import AsyncIterator
class HTTPTransport(Transport):
"""HTTP transport for REST API calls"""
def __init__(self, config: Dict[str, Any]):
super().__init__(config)
self.base_url = config['base_url']
self.session: Optional[aiohttp.ClientSession] = None
self.timeout = config.get('timeout', 30)
async def connect(self) -> None:
"""Create HTTP session"""
connector = aiohttp.TCPConnector(
limit=100,
limit_per_host=30,
ttl_dns_cache=300,
use_dns_cache=True,
)
timeout = aiohttp.ClientTimeout(total=self.timeout)
self.session = aiohttp.ClientSession(
connector=connector,
timeout=timeout,
headers=self.config.get('default_headers', {})
)
self._connected = True
async def disconnect(self) -> None:
"""Close HTTP session"""
if self.session:
await self.session.close()
self.session = None
self._connected = False
async def request(
self,
method: str,
path: str,
data: Optional[Dict[str, Any]] = None,
params: Optional[Dict[str, Any]] = None,
headers: Optional[Dict[str, str]] = None
) -> Dict[str, Any]:
"""Make HTTP request"""
if not self.session:
await self.connect()
url = f"{self.base_url}{path}"
async with self.session.request(
method=method,
url=url,
json=data,
params=params,
headers=headers
) as response:
if response.status >= 400:
error_data = await response.json()
raise APIError(error_data.get('error', 'Unknown error'))
return await response.json()
async def stream(
self,
method: str,
path: str,
data: Optional[Dict[str, Any]] = None
) -> AsyncIterator[Dict[str, Any]]:
"""Stream HTTP responses (not supported for basic HTTP)"""
raise NotImplementedError("HTTP transport does not support streaming")
```
### WebSocket Transport Implementation
```python
import websockets
import json
from typing import AsyncIterator
class WebSocketTransport(Transport):
"""WebSocket transport for real-time updates"""
def __init__(self, config: Dict[str, Any]):
super().__init__(config)
self.ws_url = config['ws_url']
self.websocket: Optional[websockets.WebSocketServerProtocol] = None
self._subscriptions: Dict[str, Any] = {}
async def connect(self) -> None:
"""Connect to WebSocket"""
self.websocket = await websockets.connect(
self.ws_url,
extra_headers=self.config.get('headers', {})
)
self._connected = True
# Start message handler
asyncio.create_task(self._handle_messages())
async def disconnect(self) -> None:
"""Disconnect WebSocket"""
if self.websocket:
await self.websocket.close()
self.websocket = None
self._connected = False
async def request(
self,
method: str,
path: str,
data: Optional[Dict[str, Any]] = None,
params: Optional[Dict[str, Any]] = None,
headers: Optional[Dict[str, str]] = None
) -> Dict[str, Any]:
"""Send request via WebSocket"""
if not self.websocket:
await self.connect()
message = {
'id': self._generate_id(),
'method': method,
'path': path,
'data': data,
'params': params
}
await self.websocket.send(json.dumps(message))
response = await self.websocket.recv()
return json.loads(response)
async def stream(
self,
method: str,
path: str,
data: Optional[Dict[str, Any]] = None
) -> AsyncIterator[Dict[str, Any]]:
"""Stream responses from WebSocket"""
if not self.websocket:
await self.connect()
# Subscribe to stream
subscription_id = self._generate_id()
message = {
'id': subscription_id,
'method': 'subscribe',
'path': path,
'data': data
}
await self.websocket.send(json.dumps(message))
# Yield messages as they come
async for message in self.websocket:
data = json.loads(message)
if data.get('subscription_id') == subscription_id:
yield data
async def _handle_messages(self):
"""Handle incoming WebSocket messages"""
async for message in self.websocket:
data = json.loads(message)
# Handle subscriptions and other messages
pass
```
### Cross-Chain Transport Implementation
```python
from ..settlement.manager import BridgeManager
from ..settlement.bridges.base import SettlementMessage, SettlementResult
class CrossChainTransport(Transport):
"""Transport for cross-chain settlements"""
def __init__(self, config: Dict[str, Any]):
super().__init__(config)
self.bridge_manager = BridgeManager(config.get('storage'))
self.base_transport = config.get('base_transport')
async def connect(self) -> None:
"""Initialize bridge manager"""
await self.bridge_manager.initialize(config.get('bridges', {}))
if self.base_transport:
await self.base_transport.connect()
self._connected = True
async def disconnect(self) -> None:
"""Disconnect all bridges"""
if self.base_transport:
await self.base_transport.disconnect()
self._connected = False
async def request(
self,
method: str,
path: str,
data: Optional[Dict[str, Any]] = None,
params: Optional[Dict[str, Any]] = None,
headers: Optional[Dict[str, str]] = None
) -> Dict[str, Any]:
"""Handle cross-chain requests"""
if path.startswith('/settlement/'):
return await self._handle_settlement_request(method, path, data)
# Forward to base transport for other requests
if self.base_transport:
return await self.base_transport.request(
method, path, data, params, headers
)
raise NotImplementedError(f"Path {path} not supported")
async def settle_cross_chain(
self,
message: SettlementMessage,
bridge_name: Optional[str] = None
) -> SettlementResult:
"""Settle message across chains"""
return await self.bridge_manager.settle_cross_chain(
message, bridge_name
)
async def estimate_settlement_cost(
self,
message: SettlementMessage,
bridge_name: Optional[str] = None
) -> Dict[str, Any]:
"""Estimate settlement cost"""
return await self.bridge_manager.estimate_settlement_cost(
message, bridge_name
)
async def _handle_settlement_request(
self,
method: str,
path: str,
data: Optional[Dict[str, Any]]
) -> Dict[str, Any]:
"""Handle settlement-specific requests"""
if method == 'POST' and path == '/settlement/cross-chain':
message = SettlementMessage(**data)
result = await self.settle_cross_chain(message)
return {
'message_id': result.message_id,
'status': result.status.value,
'transaction_hash': result.transaction_hash
}
elif method == 'GET' and path.startswith('/settlement/'):
message_id = path.split('/')[-1]
result = await self.bridge_manager.get_settlement_status(message_id)
return {
'message_id': message_id,
'status': result.status.value,
'error_message': result.error_message
}
else:
raise ValueError(f"Unsupported settlement request: {method} {path}")
```
## Multi-Network Client
### Network Configuration
```python
@dataclass
class NetworkConfig:
"""Configuration for a network"""
name: str
chain_id: int
transport: Transport
is_default: bool = False
bridges: List[str] = None
class MultiNetworkClient:
"""Client supporting multiple networks"""
def __init__(self):
self.networks: Dict[int, NetworkConfig] = {}
self.default_network: Optional[int] = None
def add_network(self, config: NetworkConfig) -> None:
"""Add a network configuration"""
self.networks[config.chain_id] = config
if config.is_default or self.default_network is None:
self.default_network = config.chain_id
def get_transport(self, chain_id: Optional[int] = None) -> Transport:
"""Get transport for a network"""
network_id = chain_id or self.default_network
if network_id not in self.networks:
raise ValueError(f"Network {network_id} not configured")
return self.networks[network_id].transport
async def connect_all(self) -> None:
"""Connect to all configured networks"""
for config in self.networks.values():
await config.transport.connect()
async def disconnect_all(self) -> None:
"""Disconnect from all networks"""
for config in self.networks.values():
await config.transport.disconnect()
```
## Updated SDK Client
### New Client Implementation
```python
class AITBCClient:
"""AITBC client with pluggable transports"""
def __init__(
self,
transport: Optional[Union[Transport, Dict[str, Any]]] = None,
multi_network: bool = False
):
if multi_network:
self._init_multi_network(transport or {})
else:
self._init_single_network(transport or {})
def _init_single_network(self, transport_config: Dict[str, Any]) -> None:
"""Initialize single network client"""
if isinstance(transport_config, Transport):
self.transport = transport_config
else:
# Default to HTTP transport
self.transport = HTTPTransport(transport_config)
self.multi_network = False
self._init_apis()
def _init_multi_network(self, configs: Dict[str, Any]) -> None:
"""Initialize multi-network client"""
self.multi_network_client = MultiNetworkClient()
# Configure networks
for name, config in configs.get('networks', {}).items():
transport = self._create_transport(config)
network_config = NetworkConfig(
name=name,
chain_id=config['chain_id'],
transport=transport,
is_default=config.get('default', False)
)
self.multi_network_client.add_network(network_config)
self.multi_network = True
self._init_apis()
def _create_transport(self, config: Dict[str, Any]) -> Transport:
"""Create transport from config"""
transport_type = config.get('type', 'http')
if transport_type == 'http':
return HTTPTransport(config)
elif transport_type == 'websocket':
return WebSocketTransport(config)
elif transport_type == 'crosschain':
return CrossChainTransport(config)
else:
raise ValueError(f"Unknown transport type: {transport_type}")
def _init_apis(self) -> None:
"""Initialize API clients"""
if self.multi_network:
self.jobs = MultiNetworkJobsAPI(self.multi_network_client)
self.settlement = MultiNetworkSettlementAPI(self.multi_network_client)
else:
self.jobs = JobsAPI(self.transport)
self.settlement = SettlementAPI(self.transport)
# Other APIs remain the same but use the transport
self.marketplace = MarketplaceAPI(self.transport)
self.wallet = WalletAPI(self.transport)
self.receipts = ReceiptsAPI(self.transport)
async def connect(self) -> None:
"""Connect to network(s)"""
if self.multi_network:
await self.multi_network_client.connect_all()
else:
await self.transport.connect()
async def disconnect(self) -> None:
"""Disconnect from network(s)"""
if self.multi_network:
await self.multi_network_client.disconnect_all()
else:
await self.transport.disconnect()
```
## Usage Examples
### Single Network with HTTP Transport
```python
from aitbc import AITBCClient, HTTPTransport
# Create client with HTTP transport
transport = HTTPTransport({
'base_url': 'https://api.aitbc.io',
'timeout': 30,
'default_headers': {'X-API-Key': 'your-key'}
})
client = AITBCClient(transport)
await client.connect()
# Use APIs normally
job = await client.jobs.create({...})
```
### Multi-Network Configuration
```python
from aitbc import AITBCClient
config = {
'networks': {
'ethereum': {
'type': 'http',
'chain_id': 1,
'base_url': 'https://api.aitbc.io',
'default': True
},
'polygon': {
'type': 'http',
'chain_id': 137,
'base_url': 'https://polygon-api.aitbc.io'
},
'arbitrum': {
'type': 'crosschain',
'chain_id': 42161,
'base_transport': HTTPTransport({
'base_url': 'https://arbitrum-api.aitbc.io'
}),
'bridges': {
'layerzero': {'enabled': True},
'chainlink': {'enabled': True}
}
}
}
}
client = AITBCClient(config, multi_network=True)
await client.connect()
# Create job on specific network
job = await client.jobs.create({...}, chain_id=137)
# Settle across chains
settlement = await client.settlement.settle_cross_chain(
job_id=job['id'],
target_chain_id=42161,
bridge_name='layerzero'
)
```
### Cross-Chain Settlement
```python
# Create job on Ethereum
job = await client.jobs.create({
'name': 'cross-chain-ai-job',
'target_chain': 42161, # Arbitrum
'requires_cross_chain_settlement': True
})
# Wait for completion
result = await client.jobs.wait_for_completion(job['id'])
# Settle to Arbitrum
settlement = await client.settlement.settle_cross_chain(
job_id=job['id'],
target_chain_id=42161,
bridge_name='layerzero'
)
# Monitor settlement
status = await client.settlement.get_status(settlement['message_id'])
```
## Migration Guide
### From Current SDK
```python
# Old way
client = AITBCClient(api_key='key', base_url='url')
# New way (backward compatible)
client = AITBCClient({
'base_url': 'url',
'default_headers': {'X-API-Key': 'key'}
})
# Or with explicit transport
transport = HTTPTransport({
'base_url': 'url',
'default_headers': {'X-API-Key': 'key'}
})
client = AITBCClient(transport)
```
## Benefits
1. **Flexibility**: Easy to add new transport types
2. **Multi-Network**: Support for multiple blockchains
3. **Cross-Chain**: Built-in support for cross-chain settlements
4. **Backward Compatible**: Existing code continues to work
5. **Testable**: Easy to mock transports for testing
6. **Extensible**: Plugin architecture for custom transports
---
*Document Version: 1.0*
*Last Updated: 2025-01-10*
*Owner: SDK Team*