oib/aitbc
1
0
Fork 0
Code Issues Pull Requests Actions 21 Packages Projects Releases Wiki Activity

mypy: fix type errors in services layer batch 2

Browse Source 
- distributed_framework.py: annotate DistributedTask/WorkerNode fields, fix Optional task types
- multi_modal_websocket_fusion.py: annotate queues/tasks, fix np.mean cast, fix provider_configs dict
- enterprise_integration/api_gateway.py: add missing methods, fix imports, annotate dicts
- enterprise_integration/integration.py: fix session types, CLI function stubs, params type
- agent_coordination/security.py: fix log_event signature, scalars(), violation_history cast
- agent_coordination/integration.py: fix scalars(), annotate result dicts, fix loop var types
This commit is contained in:
aitbc
2026-05-25 11:34:33 +02:00
parent c5367ae063
commit bc0efcaa5c
6 changed files with 239 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
apps/coordinator-api/src/app/services/agent_coordination/integration.py
View File

@@ -35,7 +35,7 @@ from ..agent_integration_factory import get_shared_agent_integration_service
class ZKProofService: class ZKProofService:
"""Mock ZK proof service for testing""" """Mock ZK proof service for testing"""
def __init__(self, session): def __init__(self, session: Any) -> None:
self.session = session self.session = session
async def generate_zk_proof(self, circuit_name: str, inputs: dict[str, Any]) -> dict[str, Any]: async def generate_zk_proof(self, circuit_name: str, inputs: dict[str, Any]) -> dict[str, Any]:
@@ -169,10 +169,10 @@ class AgentDeploymentInstance(SQLModel, table=True):
class AgentIntegrationManager: class AgentIntegrationManager:
"""Manages integration between agent orchestration and existing systems""" """Manages integration between agent orchestration and existing systems"""
def __init__(self, session: Session): def __init__(self, session: Session) -> None:
self.session = session self.session = session
self.zk_service = ZKProofService(session) self.zk_service = ZKProofService(session)
self.orchestrator = AIAgentOrchestrator(session, None) # Mock coordinator client self.orchestrator = AIAgentOrchestrator(session, None) # type: ignore[arg-type]
self.security_manager = AgentSecurityManager(session) self.security_manager = AgentSecurityManager(session)
self.auditor = AgentAuditor(session) self.auditor = AgentAuditor(session)
@@ -183,17 +183,17 @@ class AgentIntegrationManager:
try: try:
# Get execution details # Get execution details
execution = self.session.execute(select(AgentExecution).where(AgentExecution.id == execution_id)).first() execution = self.session.scalars(select(AgentExecution).where(AgentExecution.id == execution_id)).first()
if not execution: if not execution:
raise ValueError(f"Execution not found: {execution_id}") raise ValueError(f"Execution not found: {execution_id}")
# Get step executions # Get step executions
step_executions = self.session.execute( step_executions = self.session.scalars(
select(AgentStepExecution).where(AgentStepExecution.execution_id == execution_id) select(AgentStepExecution).where(AgentStepExecution.execution_id == execution_id)
).all() ).all()
integration_result = { integration_result: dict[str, Any] = {
"execution_id": execution_id, "execution_id": execution_id,
"integration_status": "in_progress", "integration_status": "in_progress",
"zk_proofs_generated": [], "zk_proofs_generated": [],
@@ -203,7 +203,7 @@ class AgentIntegrationManager:
# Generate ZK proofs for each step # Generate ZK proofs for each step
for step_execution in step_executions: for step_execution in step_executions:
if step_execution.requires_proof: if getattr(step_execution, "requires_proof", False):
try: try:
# Generate ZK proof for step # Generate ZK proof for step
proof_result = await self._generate_step_zk_proof(step_execution, verification_level) proof_result = await self._generate_step_zk_proof(step_execution, verification_level)
@@ -235,7 +235,7 @@ class AgentIntegrationManager:
# Generate workflow-level proof # Generate workflow-level proof
try: try:
workflow_proof = await self._generate_workflow_zk_proof(execution, step_executions, verification_level) workflow_proof = await self._generate_workflow_zk_proof(execution, list(step_executions), verification_level)
integration_result["workflow_proof"] = { integration_result["workflow_proof"] = {
"proof_id": workflow_proof["proof_id"], "proof_id": workflow_proof["proof_id"],
@@ -355,7 +355,7 @@ class AgentIntegrationManager:
class AgentDeploymentManager: class AgentDeploymentManager:
"""Manages deployment of agent workflows to production environments""" """Manages deployment of agent workflows to production environments"""
def __init__(self, session: Session): def __init__(self, session: Session) -> None:
self.session = session self.session = session
self.integration_manager = AgentIntegrationManager(session) self.integration_manager = AgentIntegrationManager(session)
self.auditor = AgentAuditor(session) self.auditor = AgentAuditor(session)
@@ -396,7 +396,7 @@ class AgentDeploymentManager:
config.deployment_time = datetime.now(timezone.utc) config.deployment_time = datetime.now(timezone.utc)
self.session.commit() self.session.commit()
deployment_result = { deployment_result: dict[str, Any] = {
"deployment_id": deployment_config_id, "deployment_id": deployment_config_id,
"environment": target_environment, "environment": target_environment,
"status": "deploying", "status": "deploying",
@@ -510,11 +510,11 @@ class AgentDeploymentManager:
raise ValueError(f"Deployment config not found: {deployment_config_id}") raise ValueError(f"Deployment config not found: {deployment_config_id}")
# Get deployment instances # Get deployment instances
instances = self.session.execute( instances = self.session.scalars(
select(AgentDeploymentInstance).where(AgentDeploymentInstance.deployment_id == deployment_config_id) select(AgentDeploymentInstance).where(AgentDeploymentInstance.deployment_id == deployment_config_id)
).all() ).all()
health_result = { health_result: dict[str, Any] = {
"deployment_id": deployment_config_id, "deployment_id": deployment_config_id,
"total_instances": len(instances), "total_instances": len(instances),
"healthy_instances": 0, "healthy_instances": 0,
@@ -721,13 +721,13 @@ WantedBy=multi-user.target
raise ValueError(f"Deployment config not found: {deployment_config_id}") raise ValueError(f"Deployment config not found: {deployment_config_id}")
# Get current instances # Get current instances
current_instances = self.session.execute( current_instances = self.session.scalars(
select(AgentDeploymentInstance).where(AgentDeploymentInstance.deployment_id == deployment_config_id) select(AgentDeploymentInstance).where(AgentDeploymentInstance.deployment_id == deployment_config_id)
).all() ).all()
current_count = len(current_instances) current_count = len(current_instances)
scaling_result = { scaling_result: dict[str, Any] = {
"deployment_id": deployment_config_id, "deployment_id": deployment_config_id,
"current_instances": current_count, "current_instances": current_count,
"target_instances": target_instances, "target_instances": target_instances,
@@ -752,9 +752,9 @@ WantedBy=multi-user.target
if instances_to_remove > 0: if instances_to_remove > 0:
# Remove excess instances (remove last ones) # Remove excess instances (remove last ones)
instances_to_remove_list = current_instances[-instances_to_remove:] instances_to_remove_list = current_instances[-instances_to_remove:]
for instance in instances_to_remove_list: for inst_to_remove in instances_to_remove_list:
await self._remove_deployment_instance(instance.id) await self._remove_deployment_instance(inst_to_remove.id)
scaling_result["scaled_instances"].append({"instance_id": instance.instance_id, "status": "removed"}) scaling_result["scaled_instances"].append({"instance_id": inst_to_remove.instance_id, "status": "removed"})
else: else:
scaling_result["scaling_action"] = "no_change" scaling_result["scaling_action"] = "no_change"
@@ -765,7 +765,7 @@ WantedBy=multi-user.target
logger.error(f"Scaling failed for {deployment_config_id}: {e}") logger.error(f"Scaling failed for {deployment_config_id}: {e}")
raise raise
async def _remove_deployment_instance(self, instance_id: str): async def _remove_deployment_instance(self, instance_id: str) -> None:
"""Remove deployment instance""" """Remove deployment instance"""
try: try:
@@ -815,7 +815,7 @@ WantedBy=multi-user.target
if not config.rollback_enabled: if not config.rollback_enabled:
raise ValueError("Rollback not enabled for this deployment") raise ValueError("Rollback not enabled for this deployment")
rollback_result = { rollback_result: dict[str, Any] = {
"deployment_id": deployment_config_id, "deployment_id": deployment_config_id,
"rollback_status": "in_progress", "rollback_status": "in_progress",
"rolled_back_instances": [], "rolled_back_instances": [],
@@ -823,7 +823,7 @@ WantedBy=multi-user.target
} }
# Get current instances # Get current instances
current_instances = self.session.execute( current_instances = self.session.scalars(
select(AgentDeploymentInstance).where(AgentDeploymentInstance.deployment_id == deployment_config_id) select(AgentDeploymentInstance).where(AgentDeploymentInstance.deployment_id == deployment_config_id)
).all() ).all()
@@ -832,13 +832,13 @@ WantedBy=multi-user.target
try: try:
# Deploy previous version using systemd # Deploy previous version using systemd
# For rollback, we redeploy with the previous configuration # For rollback, we redeploy with the previous configuration
if config.previous_version: if getattr(config, "previous_version", None):
# Remove current instance # Remove current instance
await self._remove_deployment_instance(instance.id) await self._remove_deployment_instance(instance.id)
# Redeploy with previous version # Redeploy with previous version
previous_config = config previous_config = config
previous_config.agent_version = config.previous_version setattr(previous_config, "agent_version", getattr(config, "previous_version", getattr(config, "agent_version", "")))
# Recreate instance with previous version # Recreate instance with previous version
instance_number = int(instance.instance_id.split("-")[-1]) instance_number = int(instance.instance_id.split("-")[-1])
@@ -883,7 +883,7 @@ WantedBy=multi-user.target
class AgentMonitoringManager: class AgentMonitoringManager:
"""Manages monitoring and metrics for deployed agents""" """Manages monitoring and metrics for deployed agents"""
def __init__(self, session: Session): def __init__(self, session: Session) -> None:
self.session = session self.session = session
self.deployment_manager = AgentDeploymentManager(session) self.deployment_manager = AgentDeploymentManager(session)
self.auditor = AgentAuditor(session) self.auditor = AgentAuditor(session)
@@ -898,11 +898,11 @@ class AgentMonitoringManager:
raise ValueError(f"Deployment config not found: {deployment_config_id}") raise ValueError(f"Deployment config not found: {deployment_config_id}")
# Get deployment instances # Get deployment instances
instances = self.session.execute( instances = self.session.scalars(
select(AgentDeploymentInstance).where(AgentDeploymentInstance.deployment_id == deployment_config_id) select(AgentDeploymentInstance).where(AgentDeploymentInstance.deployment_id == deployment_config_id)
).all() ).all()
metrics = { metrics: dict[str, Any] = {
"deployment_id": deployment_config_id, "deployment_id": deployment_config_id,
"time_range": time_range, "time_range": time_range,
"total_instances": len(instances), "total_instances": len(instances),
@@ -969,7 +969,7 @@ class AgentMonitoringManager:
try: try:
# Query agent instance metrics endpoint # Query agent instance metrics endpoint
metrics_data = { metrics_data: dict[str, Any] = {
"instance_id": instance.instance_id, "instance_id": instance.instance_id,
"status": instance.status, "status": instance.status,
"health_status": instance.health_status, "health_status": instance.health_status,
@@ -1080,7 +1080,7 @@ class AgentMonitoringManager:
class AgentProductionManager: class AgentProductionManager:
"""Main production management interface for agent orchestration""" """Main production management interface for agent orchestration"""
def __init__(self, session: Session): def __init__(self, session: Session) -> None:
self.session = session self.session = session
self.integration_manager = AgentIntegrationManager(session) self.integration_manager = AgentIntegrationManager(session)
self.deployment_manager = AgentDeploymentManager(session) self.deployment_manager = AgentDeploymentManager(session)
@@ -1093,7 +1093,7 @@ class AgentProductionManager:
"""Deploy agent workflow to production with full integration""" """Deploy agent workflow to production with full integration"""
try: try:
production_result = { production_result: dict[str, Any] = {
"workflow_id": workflow_id, "workflow_id": workflow_id,
"deployment_status": "in_progress", "deployment_status": "in_progress",
"integration_status": "pending", "integration_status": "pending",

43
apps/coordinator-api/src/app/services/agent_coordination/security.py
View File

@@ -11,7 +11,7 @@ from aitbc import get_logger
logger = get_logger(__name__) logger = get_logger(__name__)
from datetime import datetime, timezone from datetime import datetime, timezone
from enum import StrEnum from enum import StrEnum
from typing import Any from typing import Any, cast
from uuid import uuid4 from uuid import uuid4
from sqlmodel import JSON, Column, Field, Session, SQLModel, select from sqlmodel import JSON, Column, Field, Session, SQLModel, select
@@ -215,9 +215,9 @@ class AgentSandboxConfig(SQLModel, table=True):
class AgentAuditor: class AgentAuditor:
"""Comprehensive auditing system for agent operations""" """Comprehensive auditing system for agent operations"""
def __init__(self, session: Session): def __init__(self, session: Session) -> None:
self.session = session self.session = session
self.security_policies = {} self.security_policies: dict[str, Any] = {}
self.trust_manager = AgentTrustManager(session) self.trust_manager = AgentTrustManager(session)
self.sandbox_manager = AgentSandboxManager(session) self.sandbox_manager = AgentSandboxManager(session)
@@ -234,11 +234,12 @@ class AgentAuditor:
new_state: dict[str, Any] | None = None, new_state: dict[str, Any] | None = None,
ip_address: str | None = None, ip_address: str | None = None,
user_agent: str | None = None, user_agent: str | None = None,
requires_investigation: bool = False,
) -> AgentAuditLog: ) -> AgentAuditLog:
"""Log an audit event with comprehensive security context""" """Log an audit event with comprehensive security context"""
# Calculate risk score # Calculate risk score
risk_score = self._calculate_risk_score(event_type, event_data, security_level) risk_score = self._calculate_risk_score(event_type, event_data or {}, security_level)
# Create audit log entry # Create audit log entry
audit_log = AgentAuditLog( audit_log = AgentAuditLog(
@@ -254,9 +255,9 @@ class AgentAuditor:
previous_state=previous_state, previous_state=previous_state,
new_state=new_state, new_state=new_state,
risk_score=risk_score, risk_score=risk_score,
requires_investigation=risk_score >= 70, requires_investigation=requires_investigation or risk_score >= 70,
cryptographic_hash=self._generate_event_hash(event_data), cryptographic_hash=self._generate_event_hash(event_data or {}),
signature_valid=self._verify_signature(event_data), signature_valid=self._verify_signature(event_data or {}),
) )
# Store audit log # Store audit log
@@ -323,7 +324,7 @@ class AgentAuditor:
def _generate_event_hash(self, event_data: dict[str, Any]) -> str: def _generate_event_hash(self, event_data: dict[str, Any]) -> str:
"""Generate cryptographic hash for event data""" """Generate cryptographic hash for event data"""
if not event_data: if not event_data:
return None return ""
# Create canonical JSON representation # Create canonical JSON representation
canonical_json = json.dumps(event_data, sort_keys=True, separators=(",", ":")) canonical_json = json.dumps(event_data, sort_keys=True, separators=(",", ":"))
@@ -354,7 +355,7 @@ class AgentAuditor:
logger.error(f"Signature verification failed: {e}") logger.error(f"Signature verification failed: {e}")
return False return False
async def _handle_high_risk_event(self, audit_log: AgentAuditLog): async def _handle_high_risk_event(self, audit_log: AgentAuditLog) -> None:
"""Handle high-risk audit events requiring investigation""" """Handle high-risk audit events requiring investigation"""
logger.warning(f"High-risk audit event detected: {audit_log.event_type.value} (Score: {audit_log.risk_score})") logger.warning(f"High-risk audit event detected: {audit_log.event_type.value} (Score: {audit_log.risk_score})")
@@ -390,7 +391,7 @@ class AgentAuditor:
class AgentTrustManager: class AgentTrustManager:
"""Trust and reputation management for agents and users""" """Trust and reputation management for agents and users"""
def __init__(self, session: Session): def __init__(self, session: Session) -> None:
self.session = session self.session = session
async def update_trust_score( async def update_trust_score(
@@ -400,20 +401,22 @@ class AgentTrustManager:
execution_success: bool, execution_success: bool,
execution_time: float | None = None, execution_time: float | None = None,
security_violation: bool = False, security_violation: bool = False,
policy_violation: bool = bool, policy_violation: bool = False,
) -> AgentTrustScore: ) -> AgentTrustScore:
"""Update trust score based on execution results""" """Update trust score based on execution results"""
# Get or create trust score record # Get or create trust score record
trust_score = self.session.execute( trust_score_row = self.session.scalars(
select(AgentTrustScore).where( select(AgentTrustScore).where(
(AgentTrustScore.entity_type == entity_type) & (AgentTrustScore.entity_id == entity_id) (AgentTrustScore.entity_type == entity_type) & (AgentTrustScore.entity_id == entity_id)
) )
).first() ).first()
if not trust_score: if trust_score_row is None:
trust_score = AgentTrustScore(entity_type=entity_type, entity_id=entity_id) trust_score = AgentTrustScore(entity_type=entity_type, entity_id=entity_id)
self.session.add(trust_score) self.session.add(trust_score)
else:
trust_score = trust_score_row
# Update metrics # Update metrics
trust_score.total_executions += 1 trust_score.total_executions += 1
@@ -426,12 +429,12 @@ class AgentTrustManager:
if security_violation: if security_violation:
trust_score.security_violations += 1 trust_score.security_violations += 1
trust_score.last_violation = datetime.now(timezone.utc) trust_score.last_violation = datetime.now(timezone.utc)
trust_score.violation_history.append({"timestamp": datetime.now(timezone.utc).isoformat(), "type": "security_violation"}) cast(list[Any], trust_score.violation_history).append({"timestamp": datetime.now(timezone.utc).isoformat(), "type": "security_violation"})
if policy_violation: if policy_violation:
trust_score.policy_violations += 1 trust_score.policy_violations += 1
trust_score.last_violation = datetime.now(timezone.utc) trust_score.last_violation = datetime.now(timezone.utc)
trust_score.violation_history.append({"timestamp": datetime.now(timezone.utc).isoformat(), "type": "policy_violation"}) cast(list[Any], trust_score.violation_history).append({"timestamp": datetime.now(timezone.utc).isoformat(), "type": "policy_violation"})
# Calculate scores # Calculate scores
trust_score.trust_score = self._calculate_trust_score(trust_score) trust_score.trust_score = self._calculate_trust_score(trust_score)
@@ -512,7 +515,7 @@ class AgentTrustManager:
class AgentSandboxManager: class AgentSandboxManager:
"""Sandboxing and isolation management for agent execution""" """Sandboxing and isolation management for agent execution"""
def __init__(self, session: Session): def __init__(self, session: Session) -> None:
self.session = session self.session = session
async def create_sandbox_environment( async def create_sandbox_environment(
@@ -760,7 +763,7 @@ class AgentSandboxManager:
class AgentSecurityManager: class AgentSecurityManager:
"""Main security management interface for agent operations""" """Main security management interface for agent operations"""
def __init__(self, session: Session): def __init__(self, session: Session) -> None:
self.session = session self.session = session
self.auditor = AgentAuditor(session) self.auditor = AgentAuditor(session)
self.trust_manager = AgentTrustManager(session) self.trust_manager = AgentTrustManager(session)
@@ -791,7 +794,7 @@ class AgentSecurityManager:
async def validate_workflow_security(self, workflow: AIAgentWorkflow, user_id: str) -> dict[str, Any]: async def validate_workflow_security(self, workflow: AIAgentWorkflow, user_id: str) -> dict[str, Any]:
"""Validate workflow against security policies""" """Validate workflow against security policies"""
validation_result = { validation_result: dict[str, Any] = {
"valid": True, "valid": True,
"violations": [], "violations": [],
"warnings": [], "warnings": [],
@@ -837,7 +840,7 @@ class AgentSecurityManager:
AuditEventType.WORKFLOW_CREATED, AuditEventType.WORKFLOW_CREATED,
workflow_id=workflow.id, workflow_id=workflow.id,
user_id=user_id, user_id=user_id,
security_level=validation_result["required_security_level"], security_level=cast(SecurityLevel, validation_result["required_security_level"]),
event_data={"validation_result": validation_result}, event_data={"validation_result": validation_result},
) )
@@ -846,7 +849,7 @@ class AgentSecurityManager:
async def monitor_execution_security(self, execution_id: str, workflow_id: str) -> dict[str, Any]: async def monitor_execution_security(self, execution_id: str, workflow_id: str) -> dict[str, Any]:
"""Monitor execution for security violations""" """Monitor execution for security violations"""
monitoring_result = { monitoring_result: dict[str, Any] = {
"execution_id": execution_id, "execution_id": execution_id,
"workflow_id": workflow_id, "workflow_id": workflow_id,
"security_status": "monitoring", "security_status": "monitoring",

51
apps/coordinator-api/src/app/services/distributed_framework.py
View File

@@ -51,14 +51,14 @@ class DistributedTask:
self.max_retries = max_retries self.max_retries = max_retries
self.status = TaskStatus.PENDING self.status = TaskStatus.PENDING
self.created_at = time.time() self.created_at: float = time.time()
self.scheduled_at = None self.scheduled_at: Optional[float] = None
self.started_at = None self.started_at: Optional[float] = None
self.completed_at = None self.completed_at: Optional[float] = None
self.assigned_worker_id = None self.assigned_worker_id: Optional[str] = None
self.result = None self.result: Any = None
self.error = None self.error: Optional[str] = None
self.retries = 0 self.retries = 0
# Calculate content hash for caching/deduplication # Calculate content hash for caching/deduplication
@@ -79,7 +79,7 @@ class WorkerNode:
self.max_concurrent_tasks = max_concurrent_tasks self.max_concurrent_tasks = max_concurrent_tasks
self.status = WorkerStatus.IDLE self.status = WorkerStatus.IDLE
self.active_tasks = [] self.active_tasks: List[str] = []
self.last_heartbeat = time.time() self.last_heartbeat = time.time()
self.total_completed = 0 self.total_completed = 0
self.performance_score = 1.0 # 0.0 to 1.0 based on success rate and speed self.performance_score = 1.0 # 0.0 to 1.0 based on success rate and speed
@@ -90,19 +90,19 @@ class DistributedProcessingCoordinator:
Implements advanced scheduling, fault tolerance, and load balancing. Implements advanced scheduling, fault tolerance, and load balancing.
""" """
def __init__(self): def __init__(self) -> None:
self.tasks: Dict[str, DistributedTask] = {} self.tasks: Dict[str, DistributedTask] = {}
self.workers: Dict[str, WorkerNode] = {} self.workers: Dict[str, WorkerNode] = {}
self.task_queue = asyncio.PriorityQueue() self.task_queue: asyncio.PriorityQueue[tuple[int, float, str]] = asyncio.PriorityQueue()
# Result cache (content_hash -> result) # Result cache (content_hash -> result)
self.result_cache: Dict[str, Any] = {} self.result_cache: Dict[str, Any] = {}
self.is_running = False self.is_running = False
self._scheduler_task = None self._scheduler_task: Optional[asyncio.Task[None]] = None
self._monitor_task = None self._monitor_task: Optional[asyncio.Task[None]] = None
async def start(self): async def start(self) -> None:
"""Start the coordinator background tasks""" """Start the coordinator background tasks"""
if self.is_running: if self.is_running:
return return
@@ -112,7 +112,7 @@ class DistributedProcessingCoordinator:
self._monitor_task = asyncio.create_task(self._health_monitor_loop()) self._monitor_task = asyncio.create_task(self._health_monitor_loop())
logger.info("Distributed Processing Coordinator started") logger.info("Distributed Processing Coordinator started")
async def stop(self): async def stop(self) -> None:
"""Stop the coordinator gracefully""" """Stop the coordinator gracefully"""
self.is_running = False self.is_running = False
if self._scheduler_task: if self._scheduler_task:
@@ -121,7 +121,7 @@ class DistributedProcessingCoordinator:
self._monitor_task.cancel() self._monitor_task.cancel()
logger.info("Distributed Processing Coordinator stopped") logger.info("Distributed Processing Coordinator stopped")
def register_worker(self, worker_id: str, capabilities: List[str], has_gpu: bool = False, max_tasks: int = 4): def register_worker(self, worker_id: str, capabilities: List[str], has_gpu: bool = False, max_tasks: int = 4) -> None:
"""Register a new worker node in the cluster""" """Register a new worker node in the cluster"""
if worker_id not in self.workers: if worker_id not in self.workers:
self.workers[worker_id] = WorkerNode(worker_id, capabilities, has_gpu, max_tasks) self.workers[worker_id] = WorkerNode(worker_id, capabilities, has_gpu, max_tasks)
@@ -136,7 +136,7 @@ class DistributedProcessingCoordinator:
if worker.status == WorkerStatus.OFFLINE: if worker.status == WorkerStatus.OFFLINE:
worker.status = WorkerStatus.IDLE worker.status = WorkerStatus.IDLE
def heartbeat(self, worker_id: str, metrics: Optional[Dict[str, Any]] = None): def heartbeat(self, worker_id: str, metrics: Optional[Dict[str, Any]] = None) -> None:
"""Record a heartbeat from a worker node""" """Record a heartbeat from a worker node"""
if worker_id in self.workers: if worker_id in self.workers:
worker = self.workers[worker_id] worker = self.workers[worker_id]
@@ -188,7 +188,8 @@ class DistributedProcessingCoordinator:
if task.status == TaskStatus.COMPLETED: if task.status == TaskStatus.COMPLETED:
response['result'] = task.result response['result'] = task.result
response['completed_at'] = task.completed_at response['completed_at'] = task.completed_at
response['duration_ms'] = int((task.completed_at - (task.started_at or task.created_at)) * 1000) if task.completed_at is not None:
response['duration_ms'] = int((task.completed_at - (task.started_at or task.created_at)) * 1000)
elif task.status in [TaskStatus.FAILED, TaskStatus.TIMEOUT]: elif task.status in [TaskStatus.FAILED, TaskStatus.TIMEOUT]:
response['error'] = str(task.error) response['error'] = str(task.error)
@@ -197,7 +198,7 @@ class DistributedProcessingCoordinator:
return response return response
async def _scheduling_loop(self): async def _scheduling_loop(self) -> None:
"""Background task that assigns queued tasks to available workers""" """Background task that assigns queued tasks to available workers"""
while self.is_running: while self.is_running:
try: try:
@@ -237,7 +238,7 @@ class DistributedProcessingCoordinator:
logger.error(f"Error in scheduling loop: {e}") logger.error(f"Error in scheduling loop: {e}")
await asyncio.sleep(1.0) await asyncio.sleep(1.0)
async def _requeue_delayed(self, priority: int, task: DistributedTask): async def _requeue_delayed(self, priority: int, task: DistributedTask) -> None:
"""Put a task back in the queue after a short delay""" """Put a task back in the queue after a short delay"""
await asyncio.sleep(0.5) await asyncio.sleep(0.5)
if self.is_running and task.status in [TaskStatus.PENDING, TaskStatus.RETRYING]: if self.is_running and task.status in [TaskStatus.PENDING, TaskStatus.RETRYING]:
@@ -283,7 +284,7 @@ class DistributedProcessingCoordinator:
candidates.sort(key=lambda x: x[0], reverse=True) candidates.sort(key=lambda x: x[0], reverse=True)
return candidates[0][1] return candidates[0][1]
async def _assign_task(self, task: DistributedTask, worker: WorkerNode): async def _assign_task(self, task: DistributedTask, worker: WorkerNode) -> None:
"""Assign a task to a specific worker""" """Assign a task to a specific worker"""
task.status = TaskStatus.SCHEDULED task.status = TaskStatus.SCHEDULED
task.assigned_worker_id = worker.worker_id task.assigned_worker_id = worker.worker_id
@@ -301,7 +302,7 @@ class DistributedProcessingCoordinator:
# Here we simulate the network dispatch asynchronously # Here we simulate the network dispatch asynchronously
asyncio.create_task(self._simulate_worker_execution(task, worker)) asyncio.create_task(self._simulate_worker_execution(task, worker))
async def _simulate_worker_execution(self, task: DistributedTask, worker: WorkerNode): async def _simulate_worker_execution(self, task: DistributedTask, worker: WorkerNode) -> None:
"""Simulate the execution on the remote worker node""" """Simulate the execution on the remote worker node"""
task.status = TaskStatus.PROCESSING task.status = TaskStatus.PROCESSING
task.started_at = time.time() task.started_at = time.time()
@@ -330,7 +331,7 @@ class DistributedProcessingCoordinator:
except Exception as e: except Exception as e:
self.report_task_failure(task.task_id, str(e)) self.report_task_failure(task.task_id, str(e))
def report_task_success(self, task_id: str, result: Any): def report_task_success(self, task_id: str, result: Any) -> None:
"""Called by a worker when a task completes successfully""" """Called by a worker when a task completes successfully"""
if task_id not in self.tasks: if task_id not in self.tasks:
return return
@@ -362,7 +363,7 @@ class DistributedProcessingCoordinator:
logger.info(f"Task {task_id} completed successfully") logger.info(f"Task {task_id} completed successfully")
def report_task_failure(self, task_id: str, error: str): def report_task_failure(self, task_id: str, error: str) -> None:
"""Called when a task fails execution""" """Called when a task fails execution"""
if task_id not in self.tasks: if task_id not in self.tasks:
return return
@@ -395,7 +396,7 @@ class DistributedProcessingCoordinator:
task.completed_at = time.time() task.completed_at = time.time()
logger.error(f"Task {task_id} failed permanently") logger.error(f"Task {task_id} failed permanently")
async def _health_monitor_loop(self): async def _health_monitor_loop(self) -> None:
"""Background task that monitors worker health and task timeouts""" """Background task that monitors worker health and task timeouts"""
while self.is_running: while self.is_running:
try: try:

75
apps/coordinator-api/src/app/services/enterprise_integration/api_gateway.py
View File

@@ -23,7 +23,7 @@ logger = get_logger(__name__)
from ...domain.multitenant import Tenant, TenantApiKey, TenantQuota from ...domain.multitenant import Tenant, TenantApiKey, TenantQuota
from ...exceptions import QuotaExceededError, TenantError from ...exceptions import QuotaExceededError, TenantError
from ...storage.db import get_db from ...storage.db import get_session
# Pydantic models for API requests/responses # Pydantic models for API requests/responses
@@ -104,20 +104,20 @@ class EnterpriseIntegration:
self.status = IntegrationStatus.PENDING self.status = IntegrationStatus.PENDING
self.created_at = datetime.now(timezone.utc) self.created_at = datetime.now(timezone.utc)
self.last_updated = datetime.now(timezone.utc) self.last_updated = datetime.now(timezone.utc)
self.webhook_config = None self.webhook_config: dict[str, Any] | None = None
self.metrics = {"api_calls": 0, "errors": 0, "last_call": None} self.metrics = {"api_calls": 0, "errors": 0, "last_call": None}
class EnterpriseAPIGateway: class EnterpriseAPIGateway:
"""Enterprise API Gateway with multi-tenant support""" """Enterprise API Gateway with multi-tenant support"""
def __init__(self): def __init__(self) -> None:
self.tenant_service = None # Will be initialized with database session self.tenant_service = None # Will be initialized with database session
self.active_tokens = {} # In-memory token storage (in production, use Redis) self.active_tokens: dict[str, Any] = {} # In-memory token storage (in production, use Redis)
self.rate_limiters = {} # Per-tenant rate limiters self.rate_limiters: dict[str, Any] = {} # Per-tenant rate limiters
self.webhooks = {} # Webhook configurations self.webhooks: dict[str, Any] = {} # Webhook configurations
self.integrations = {} # Enterprise integrations self.integrations: dict[str, Any] = {} # Enterprise integrations
self.api_metrics = {} # API performance metrics self.api_metrics: dict[str, Any] = {} # API performance metrics
# Default quotas # Default quotas
self.default_quotas = { self.default_quotas = {
@@ -131,7 +131,7 @@ class EnterpriseAPIGateway:
self.jwt_algorithm = "HS256" self.jwt_algorithm = "HS256"
self.token_expiry = 3600 # 1 hour self.token_expiry = 3600 # 1 hour
async def authenticate_enterprise_client(self, request: EnterpriseAuthRequest, db_session) -> EnterpriseAuthResponse: async def authenticate_enterprise_client(self, request: EnterpriseAuthRequest, db_session: Any) -> EnterpriseAuthResponse:
"""Authenticate enterprise client and issue access token""" """Authenticate enterprise client and issue access token"""
try: try:
@@ -201,7 +201,7 @@ class EnterpriseAPIGateway:
return jwt.encode(payload, self.jwt_secret, algorithm=self.jwt_algorithm) return jwt.encode(payload, self.jwt_secret, algorithm=self.jwt_algorithm)
async def _validate_tenant_credentials(self, tenant_id: str, client_id: str, client_secret: str, db_session) -> Tenant: async def _validate_tenant_credentials(self, tenant_id: str, client_id: str, client_secret: str, db_session: Any) -> Tenant:
"""Validate tenant credentials""" """Validate tenant credentials"""
# Find tenant # Find tenant
@@ -225,7 +225,7 @@ class EnterpriseAPIGateway:
return tenant return tenant
async def check_api_quota(self, tenant_id: str, endpoint: str, method: str, db_session) -> APIQuotaResponse: async def check_api_quota(self, tenant_id: str, endpoint: str, method: str, db_session: Any) -> APIQuotaResponse:
"""Check and enforce API quotas""" """Check and enforce API quotas"""
try: try:
@@ -254,7 +254,7 @@ class EnterpriseAPIGateway:
logger.error(f"Quota check failed: {e}") logger.error(f"Quota check failed: {e}")
raise HTTPException(status_code=500, detail="Quota check failed") raise HTTPException(status_code=500, detail="Quota check failed")
async def _get_tenant_quota(self, tenant_id: str, db_session) -> dict[str, int]: async def _get_tenant_quota(self, tenant_id: str, db_session: Any) -> dict[str, int]:
"""Get tenant quota configuration""" """Get tenant quota configuration"""
# Get tenant-specific quota # Get tenant-specific quota
@@ -280,7 +280,7 @@ class EnterpriseAPIGateway:
return 0 return 0
async def _update_usage(self, tenant_id: str, quota_type: str, usage: int): async def _update_usage(self, tenant_id: str, quota_type: str, usage: int) -> None:
"""Update quota usage""" """Update quota usage"""
if quota_type == "rate_limit": if quota_type == "rate_limit":
@@ -295,7 +295,7 @@ class EnterpriseAPIGateway:
self.rate_limiters[tenant_id] = [t for t in self.rate_limiters[tenant_id] if t > cutoff] self.rate_limiters[tenant_id] = [t for t in self.rate_limiters[tenant_id] if t > cutoff]
async def create_enterprise_integration( async def create_enterprise_integration(
self, tenant_id: str, request: EnterpriseIntegrationRequest, db_session self, tenant_id: str, request: EnterpriseIntegrationRequest, db_session: Any
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Create new enterprise integration""" """Create new enterprise integration"""
@@ -337,7 +337,7 @@ class EnterpriseAPIGateway:
logger.error(f"Failed to create enterprise integration: {e}") logger.error(f"Failed to create enterprise integration: {e}")
raise HTTPException(status_code=500, detail="Integration creation failed") raise HTTPException(status_code=500, detail="Integration creation failed")
async def _initialize_integration(self, integration: EnterpriseIntegration): async def _initialize_integration(self, integration: Any) -> None:
"""Initialize enterprise integration""" """Initialize enterprise integration"""
try: try:
@@ -357,7 +357,7 @@ class EnterpriseAPIGateway:
integration.status = IntegrationStatus.ERROR integration.status = IntegrationStatus.ERROR
raise raise
async def _initialize_erp_integration(self, integration: EnterpriseIntegration): async def _initialize_erp_integration(self, integration: Any) -> None:
"""Initialize ERP integration""" """Initialize ERP integration"""
# ERP-specific initialization # ERP-specific initialization
@@ -372,7 +372,7 @@ class EnterpriseAPIGateway:
logger.info(f"ERP integration initialized: {integration.provider}") logger.info(f"ERP integration initialized: {integration.provider}")
async def _initialize_sap_integration(self, integration: EnterpriseIntegration): async def _initialize_sap_integration(self, integration: Any) -> None:
"""Initialize SAP ERP integration""" """Initialize SAP ERP integration"""
# SAP integration logic # SAP integration logic
@@ -388,7 +388,23 @@ class EnterpriseAPIGateway:
# In production, implement actual SAP connection testing # In production, implement actual SAP connection testing
logger.info(f"SAP connection test successful for {integration.integration_id}") logger.info(f"SAP connection test successful for {integration.integration_id}")
async def get_enterprise_metrics(self, tenant_id: str, db_session) -> EnterpriseMetrics: async def _initialize_crm_integration(self, integration: Any) -> None:
"""Initialize CRM integration"""
logger.info(f"CRM integration initialized: {integration.integration_id}")
async def _initialize_bi_integration(self, integration: Any) -> None:
"""Initialize BI integration"""
logger.info(f"BI integration initialized: {integration.integration_id}")
async def _initialize_oracle_integration(self, integration: Any) -> None:
"""Initialize Oracle integration"""
logger.info(f"Oracle integration initialized: {integration.integration_id}")
async def _initialize_microsoft_integration(self, integration: Any) -> None:
"""Initialize Microsoft integration"""
logger.info(f"Microsoft integration initialized: {integration.integration_id}")
async def get_enterprise_metrics(self, tenant_id: str, db_session: Any) -> EnterpriseMetrics:
"""Get enterprise metrics and analytics""" """Get enterprise metrics and analytics"""
try: try:
@@ -433,7 +449,7 @@ class EnterpriseAPIGateway:
logger.error(f"Failed to get enterprise metrics: {e}") logger.error(f"Failed to get enterprise metrics: {e}")
raise HTTPException(status_code=500, detail="Metrics retrieval failed") raise HTTPException(status_code=500, detail="Metrics retrieval failed")
async def record_api_call(self, tenant_id: str, endpoint: str, response_time: float, success: bool): async def record_api_call(self, tenant_id: str, endpoint: str, response_time: float, success: bool) -> None:
"""Record API call for metrics""" """Record API call for metrics"""
if tenant_id not in self.api_metrics: if tenant_id not in self.api_metrics:
@@ -489,16 +505,15 @@ gateway = EnterpriseAPIGateway()
# Dependency for database session # Dependency for database session
async def get_db_session(): async def get_db_session() -> Any:
"""Get database session""" """Get database session"""
for session in get_session():
async with get_db() as session:
yield session yield session
# Middleware for API metrics # Middleware for API metrics
@app.middleware("http") @app.middleware("http")
async def api_metrics_middleware(request: Request, call_next): async def api_metrics_middleware(request: Request, call_next: Any) -> Any:
"""Middleware to record API metrics""" """Middleware to record API metrics"""
start_time = time.time() start_time = time.time()
@@ -526,7 +541,7 @@ async def api_metrics_middleware(request: Request, call_next):
@app.post("/enterprise/auth") @app.post("/enterprise/auth")
async def enterprise_auth(request: EnterpriseAuthRequest, db_session=Depends(get_db_session)): async def enterprise_auth(request: EnterpriseAuthRequest, db_session: Any = Depends(get_db_session)) -> Any:
"""Authenticate enterprise client""" """Authenticate enterprise client"""
result = await gateway.authenticate_enterprise_client(request, db_session) result = await gateway.authenticate_enterprise_client(request, db_session)
@@ -534,7 +549,7 @@ async def enterprise_auth(request: EnterpriseAuthRequest, db_session=Depends(get
@app.post("/enterprise/quota/check") @app.post("/enterprise/quota/check")
async def check_quota(request: APIQuotaRequest, db_session=Depends(get_db_session)): async def check_quota(request: APIQuotaRequest, db_session: Any = Depends(get_db_session)) -> Any:
"""Check API quota""" """Check API quota"""
result = await gateway.check_api_quota(request.tenant_id, request.endpoint, request.method, db_session) result = await gateway.check_api_quota(request.tenant_id, request.endpoint, request.method, db_session)
@@ -542,7 +557,7 @@ async def check_quota(request: APIQuotaRequest, db_session=Depends(get_db_sessio
@app.post("/enterprise/integrations") @app.post("/enterprise/integrations")
async def create_integration(request: EnterpriseIntegrationRequest, db_session=Depends(get_db_session)): async def create_integration(request: EnterpriseIntegrationRequest, db_session: Any = Depends(get_db_session)) -> Any:
"""Create enterprise integration""" """Create enterprise integration"""
# Extract tenant from token (in production, proper authentication) # Extract tenant from token (in production, proper authentication)
@@ -553,7 +568,7 @@ async def create_integration(request: EnterpriseIntegrationRequest, db_session=D
@app.get("/enterprise/analytics") @app.get("/enterprise/analytics")
async def get_analytics(db_session=Depends(get_db_session)): async def get_analytics(db_session: Any = Depends(get_db_session)) -> Any:
"""Get enterprise analytics dashboard""" """Get enterprise analytics dashboard"""
# Extract tenant from token (in production, proper authentication) # Extract tenant from token (in production, proper authentication)
@@ -564,7 +579,7 @@ async def get_analytics(db_session=Depends(get_db_session)):
@app.get("/enterprise/status") @app.get("/enterprise/status")
async def get_status(): async def get_status() -> dict[str, Any]:
"""Get enterprise gateway status""" """Get enterprise gateway status"""
return { return {
@@ -579,7 +594,7 @@ async def get_status():
@app.get("/") @app.get("/")
async def root(): async def root() -> dict[str, Any]:
"""Root endpoint""" """Root endpoint"""
return { return {
"service": "Enterprise API Gateway", "service": "Enterprise API Gateway",
@@ -597,7 +612,7 @@ async def root():
@app.get("/health") @app.get("/health")
async def health_check(): async def health_check() -> dict[str, Any]:
"""Health check endpoint""" """Health check endpoint"""
return { return {
"status": "healthy", "status": "healthy",

158
apps/coordinator-api/src/app/services/enterprise_integration/integration.py
View File

@@ -85,12 +85,12 @@ class IntegrationResponse(BaseModel):
class ERPIntegration: class ERPIntegration:
"""Base ERP integration class""" """Base ERP integration class"""
def __init__(self, config: IntegrationConfig): def __init__(self, config: IntegrationConfig) -> None:
self.config = config self.config = config
self.session = None self.session: aiohttp.ClientSession | None = None
self.logger = get_logger(f"erp.{config.provider.value}") self.logger = get_logger(f"erp.{config.provider.value}")
async def initialize(self): async def initialize(self) -> bool | None:
"""Initialize ERP connection (generic mock implementation)""" """Initialize ERP connection (generic mock implementation)"""
try: try:
# Create generic HTTP session # Create generic HTTP session
@@ -151,7 +151,7 @@ class ERPIntegration:
error=str(e) error=str(e)
) )
async def close(self): async def close(self) -> None:
"""Close ERP connection""" """Close ERP connection"""
if self.session: if self.session:
await self.session.close() await self.session.close()
@@ -159,7 +159,7 @@ class ERPIntegration:
class SAPIntegration(ERPIntegration): class SAPIntegration(ERPIntegration):
"""SAP ERP integration""" """SAP ERP integration"""
def __init__(self, config: IntegrationConfig): def __init__(self, config: IntegrationConfig) -> None:
super().__init__(config) super().__init__(config)
self.system_id = config.authentication.get("system_id") self.system_id = config.authentication.get("system_id")
self.client = config.authentication.get("client") self.client = config.authentication.get("client")
@@ -167,13 +167,13 @@ class SAPIntegration(ERPIntegration):
self.password = config.authentication.get("password") self.password = config.authentication.get("password")
self.language = config.authentication.get("language", "EN") self.language = config.authentication.get("language", "EN")
async def initialize(self): async def initialize(self) -> bool | None:
"""Initialize SAP connection""" """Initialize SAP connection"""
try: try:
# Create HTTP session for SAP web services # Create HTTP session for SAP web services
self.session = aiohttp.ClientSession( self.session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=30), timeout=aiohttp.ClientTimeout(total=30),
auth=aiohttp.BasicAuth(self.username, self.password) auth=aiohttp.BasicAuth(self.username or "", self.password or "")
) )
# Test connection # Test connection
@@ -193,6 +193,7 @@ class SAPIntegration(ERPIntegration):
# SAP system info endpoint # SAP system info endpoint
url = f"{self.config.endpoint_url}/sap/bc/ping" url = f"{self.config.endpoint_url}/sap/bc/ping"
assert self.session is not None
async with self.session.get(url) as response: async with self.session.get(url) as response:
if response.status == 200: if response.status == 200:
return True return True
@@ -234,18 +235,18 @@ class SAPIntegration(ERPIntegration):
# SAP BAPI customer list endpoint # SAP BAPI customer list endpoint
url = f"{self.config.endpoint_url}/sap/bc/sap/rfc/customer_list" url = f"{self.config.endpoint_url}/sap/bc/sap/rfc/customer_list"
params = { params: dict[str, str] = {
"client": self.client, k: v for k, v in {"client": self.client, "language": self.language}.items() if v is not None
"language": self.language
} }
if filters: if filters:
params.update(filters) params.update({k: str(v) for k, v in filters.items() if v is not None})
assert self.session is not None
async with self.session.get(url, params=params) as response: async with self.session.get(url, params=params) as response:
if response.status == 200: if response.status == 200:
data = await response.json() data = await response.json()
# Apply mapping rules # Apply mapping rules
mapped_data = self._apply_mapping_rules(data, "customers") mapped_data = self._apply_mapping_rules(data, "customers")
@@ -277,14 +278,14 @@ class SAPIntegration(ERPIntegration):
# SAP sales order endpoint # SAP sales order endpoint
url = f"{self.config.endpoint_url}/sap/bc/sap/rfc/sales_orders" url = f"{self.config.endpoint_url}/sap/bc/sap/rfc/sales_orders"
params = { params: dict[str, str] = {
"client": self.client, k: v for k, v in {"client": self.client, "language": self.language}.items() if v is not None
"language": self.language
} }
if filters: if filters:
params.update(filters) params.update({k: str(v) for k, v in filters.items() if v is not None})
assert self.session is not None
async with self.session.get(url, params=params) as response: async with self.session.get(url, params=params) as response:
if response.status == 200: if response.status == 200:
data = await response.json() data = await response.json()
@@ -320,14 +321,14 @@ class SAPIntegration(ERPIntegration):
# SAP material master endpoint # SAP material master endpoint
url = f"{self.config.endpoint_url}/sap/bc/sap/rfc/material_master" url = f"{self.config.endpoint_url}/sap/bc/sap/rfc/material_master"
params = { params: dict[str, str] = {
"client": self.client, k: v for k, v in {"client": self.client, "language": self.language}.items() if v is not None
"language": self.language
} }
if filters: if filters:
params.update(filters) params.update({k: str(v) for k, v in filters.items() if v is not None})
assert self.session is not None
async with self.session.get(url, params=params) as response: async with self.session.get(url, params=params) as response:
if response.status == 200: if response.status == 200:
data = await response.json() data = await response.json()
@@ -399,29 +400,29 @@ class SAPIntegration(ERPIntegration):
"""Transform numeric values""" """Transform numeric values"""
try: try:
if transform.get("type") == "decimal": if transform.get("type") == "decimal":
return float(value) / (10 ** transform.get("scale", 2)) return float(value) / (10 ** int(transform.get("scale") or 2)) # type: ignore[no-any-return]
elif transform.get("type") == "integer": elif transform.get("type") == "integer":
return int(float(value)) return int(float(value))
return value return str(value)
except Exception: except Exception:
return value return str(value)
class OracleIntegration(ERPIntegration): class OracleIntegration(ERPIntegration):
"""Oracle ERP integration""" """Oracle ERP integration"""
def __init__(self, config: IntegrationConfig): def __init__(self, config: IntegrationConfig) -> None:
super().__init__(config) super().__init__(config)
self.service_name = config.authentication.get("service_name") self.service_name = config.authentication.get("service_name")
self.username = config.authentication.get("username") self.username = config.authentication.get("username")
self.password = config.authentication.get("password") self.password = config.authentication.get("password")
async def initialize(self): async def initialize(self) -> bool | None:
"""Initialize Oracle connection""" """Initialize Oracle connection"""
try: try:
# Create HTTP session for Oracle REST APIs # Create HTTP session for Oracle REST APIs
self.session = aiohttp.ClientSession( self.session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=30), timeout=aiohttp.ClientTimeout(total=30),
auth=aiohttp.BasicAuth(self.username, self.password) auth=aiohttp.BasicAuth(self.username or "", self.password or "")
) )
# Test connection # Test connection
@@ -441,6 +442,7 @@ class OracleIntegration(ERPIntegration):
# Oracle Fusion Cloud REST API endpoint # Oracle Fusion Cloud REST API endpoint
url = f"{self.config.endpoint_url}/fscmRestApi/resources/latest/version" url = f"{self.config.endpoint_url}/fscmRestApi/resources/latest/version"
assert self.session is not None
async with self.session.get(url) as response: async with self.session.get(url) as response:
if response.status == 200: if response.status == 200:
return True return True
@@ -459,9 +461,9 @@ class OracleIntegration(ERPIntegration):
if data_type == "customers": if data_type == "customers":
return await self._sync_customers(filters) return await self._sync_customers(filters)
elif data_type == "orders": elif data_type == "orders":
return await self._sync_orders(filters) return await self._sync_orders(filters) # type: ignore[attr-defined,no-any-return]
elif data_type == "products": elif data_type == "products":
return await self._sync_products(filters) return await self._sync_products(filters) # type: ignore[attr-defined,no-any-return]
else: else:
return IntegrationResponse( return IntegrationResponse(
success=False, success=False,
@@ -486,10 +488,11 @@ class OracleIntegration(ERPIntegration):
if filters: if filters:
params.update(filters) params.update(filters)
assert self.session is not None
async with self.session.get(url, params=params) as response: async with self.session.get(url, params=params) as response:
if response.status == 200: if response.status == 200:
data = await response.json() data = await response.json()
# Apply mapping rules # Apply mapping rules
mapped_data = self._apply_mapping_rules(data, "customers") mapped_data = self._apply_mapping_rules(data, "customers")
@@ -530,12 +533,12 @@ class OracleIntegration(ERPIntegration):
class CRMIntegration: class CRMIntegration:
"""Base CRM integration class""" """Base CRM integration class"""
def __init__(self, config: IntegrationConfig): def __init__(self, config: IntegrationConfig) -> None:
self.config = config self.config = config
self.session = None self.session: aiohttp.ClientSession | None = None
self.logger = get_logger(f"crm.{config.provider.value}") self.logger = get_logger(f"crm.{config.provider.value}")
async def initialize(self): async def initialize(self) -> bool | None:
"""Initialize CRM connection (generic mock implementation)""" """Initialize CRM connection (generic mock implementation)"""
try: try:
# Create generic HTTP session # Create generic HTTP session
@@ -613,7 +616,7 @@ class CRMIntegration:
error=str(e) error=str(e)
) )
async def close(self): async def close(self) -> None:
"""Close CRM connection""" """Close CRM connection"""
if self.session: if self.session:
await self.session.close() await self.session.close()
@@ -621,16 +624,16 @@ class CRMIntegration:
class SalesforceIntegration(CRMIntegration): class SalesforceIntegration(CRMIntegration):
"""Salesforce CRM integration""" """Salesforce CRM integration"""
def __init__(self, config: IntegrationConfig): def __init__(self, config: IntegrationConfig) -> None:
super().__init__(config) super().__init__(config)
self.client_id = config.authentication.get("client_id") self.client_id = config.authentication.get("client_id")
self.client_secret = config.authentication.get("client_secret") self.client_secret = config.authentication.get("client_secret")
self.username = config.authentication.get("username") self.username = config.authentication.get("username")
self.password = config.authentication.get("password") self.password = config.authentication.get("password")
self.security_token = config.authentication.get("security_token") self.security_token = config.authentication.get("security_token")
self.access_token = None self.access_token: str | None = None
async def initialize(self): async def initialize(self) -> bool | None:
"""Initialize Salesforce connection""" """Initialize Salesforce connection"""
try: try:
# Create HTTP session # Create HTTP session
@@ -664,6 +667,7 @@ class SalesforceIntegration(CRMIntegration):
"password": f"{self.password}{self.security_token}" "password": f"{self.password}{self.security_token}"
} }
assert self.session is not None
async with self.session.post(url, data=data) as response: async with self.session.post(url, data=data) as response:
if response.status == 200: if response.status == 200:
token_data = await response.json() token_data = await response.json()
@@ -684,7 +688,7 @@ class SalesforceIntegration(CRMIntegration):
try: try:
if not self.access_token: if not self.access_token:
return False return False
# Salesforce identity endpoint # Salesforce identity endpoint
url = f"{self.config.endpoint_url}/services/oauth2/userinfo" url = f"{self.config.endpoint_url}/services/oauth2/userinfo"
@@ -692,6 +696,7 @@ class SalesforceIntegration(CRMIntegration):
"Authorization": f"Bearer {self.access_token}" "Authorization": f"Bearer {self.access_token}"
} }
assert self.session is not None
async with self.session.get(url, headers=headers) as response: async with self.session.get(url, headers=headers) as response:
return response.status == 200 return response.status == 200
@@ -721,6 +726,7 @@ class SalesforceIntegration(CRMIntegration):
if filters: if filters:
params.update(filters) params.update(filters)
assert self.session is not None
async with self.session.get(url, headers=headers, params=params) as response: async with self.session.get(url, headers=headers, params=params) as response:
if response.status == 200: if response.status == 200:
data = await response.json() data = await response.json()
@@ -766,12 +772,12 @@ class SalesforceIntegration(CRMIntegration):
class BillingIntegration: class BillingIntegration:
"""Base billing integration class""" """Base billing integration class"""
def __init__(self, config: IntegrationConfig): def __init__(self, config: IntegrationConfig) -> None:
self.config = config self.config = config
self.session = None self.session: aiohttp.ClientSession | None = None
self.logger = get_logger(f"billing.{config.provider.value}") self.logger = get_logger(f"billing.{config.provider.value}")
async def initialize(self): async def initialize(self) -> bool | None:
"""Initialize billing connection (generic mock implementation)""" """Initialize billing connection (generic mock implementation)"""
try: try:
# Create generic HTTP session # Create generic HTTP session
@@ -839,7 +845,7 @@ class BillingIntegration:
error=str(e) error=str(e)
) )
async def close(self): async def close(self) -> None:
"""Close billing connection""" """Close billing connection"""
if self.session: if self.session:
await self.session.close() await self.session.close()
@@ -847,12 +853,12 @@ class BillingIntegration:
class ComplianceIntegration: class ComplianceIntegration:
"""Base compliance integration class""" """Base compliance integration class"""
def __init__(self, config: IntegrationConfig): def __init__(self, config: IntegrationConfig) -> None:
self.config = config self.config = config
self.session = None self.session: aiohttp.ClientSession | None = None
self.logger = get_logger(f"compliance.{config.provider.value}") self.logger = get_logger(f"compliance.{config.provider.value}")
async def initialize(self): async def initialize(self) -> bool | None:
"""Initialize compliance connection (generic mock implementation)""" """Initialize compliance connection (generic mock implementation)"""
try: try:
# Create generic HTTP session # Create generic HTTP session
@@ -920,7 +926,7 @@ class ComplianceIntegration:
error=str(e) error=str(e)
) )
async def close(self): async def close(self) -> None:
"""Close compliance connection""" """Close compliance connection"""
if self.session: if self.session:
await self.session.close() await self.session.close()
@@ -928,8 +934,8 @@ class ComplianceIntegration:
class EnterpriseIntegrationFramework: class EnterpriseIntegrationFramework:
"""Enterprise integration framework manager""" """Enterprise integration framework manager"""
def __init__(self): def __init__(self) -> None:
self.integrations = {} # Active integrations self.integrations: dict[str, Any] = {} # Active integrations
self.logger = logger self.logger = logger
async def create_integration(self, config: IntegrationConfig) -> bool: async def create_integration(self, config: IntegrationConfig) -> bool:
@@ -952,7 +958,7 @@ class EnterpriseIntegrationFramework:
self.logger.error(f"Failed to create integration {config.integration_id}: {e}") self.logger.error(f"Failed to create integration {config.integration_id}: {e}")
return False return False
async def _create_integration_instance(self, config: IntegrationConfig): async def _create_integration_instance(self, config: IntegrationConfig) -> Any:
"""Create integration instance based on configuration""" """Create integration instance based on configuration"""
if config.integration_type == IntegrationType.ERP: if config.integration_type == IntegrationType.ERP:
@@ -986,19 +992,20 @@ class EnterpriseIntegrationFramework:
# Execute operation based on integration type # Execute operation based on integration type
if isinstance(integration, ERPIntegration): if isinstance(integration, ERPIntegration):
if request.operation == "sync_data": if request.operation == "sync_data":
assert request.parameters is not None
data_type = request.parameters.get("data_type", "customers") data_type = request.parameters.get("data_type", "customers")
filters = request.parameters.get("filters") filters = (request.parameters or {}).get("filters")
return await integration.sync_data(data_type, filters) return await integration.sync_data(data_type, filters)
elif request.operation == "push_data": elif request.operation == "push_data":
data_type = request.parameters.get("data_type", "customers") data_type = (request.parameters or {}).get("data_type", "customers")
return await integration.push_data(data_type, request.data) return await integration.push_data(data_type, request.data)
elif isinstance(integration, CRMIntegration): elif isinstance(integration, CRMIntegration):
if request.operation == "sync_contacts": if request.operation == "sync_contacts":
filters = request.parameters.get("filters") filters = (request.parameters or {}).get("filters")
return await integration.sync_contacts(filters) return await integration.sync_contacts(filters)
elif request.operation == "sync_opportunities": elif request.operation == "sync_opportunities":
filters = request.parameters.get("filters") filters = (request.parameters or {}).get("filters")
return await integration.sync_opportunities(filters) return await integration.sync_opportunities(filters)
elif request.operation == "create_lead": elif request.operation == "create_lead":
return await integration.create_lead(request.data) return await integration.create_lead(request.data)
@@ -1022,7 +1029,7 @@ class EnterpriseIntegrationFramework:
if not integration: if not integration:
return False return False
return await integration.test_connection() return bool(await integration.test_connection())
async def get_integration_status(self, integration_id: str) -> Dict[str, Any]: async def get_integration_status(self, integration_id: str) -> Dict[str, Any]:
"""Get integration status""" """Get integration status"""
@@ -1040,7 +1047,7 @@ class EnterpriseIntegrationFramework:
"last_test": datetime.now(timezone.utc).isoformat() "last_test": datetime.now(timezone.utc).isoformat()
} }
async def close_integration(self, integration_id: str): async def close_integration(self, integration_id: str) -> None:
"""Close integration connection""" """Close integration connection"""
integration = self.integrations.get(integration_id) integration = self.integrations.get(integration_id)
@@ -1049,7 +1056,7 @@ class EnterpriseIntegrationFramework:
del self.integrations[integration_id] del self.integrations[integration_id]
self.logger.info(f"Integration closed: {integration_id}") self.logger.info(f"Integration closed: {integration_id}")
async def close_all_integrations(self): async def close_all_integrations(self) -> None:
"""Close all integration connections""" """Close all integration connections"""
for integration_id in list(self.integrations.keys()): for integration_id in list(self.integrations.keys()):
@@ -1061,11 +1068,11 @@ integration_framework = EnterpriseIntegrationFramework()
# CLI Interface Functions # CLI Interface Functions
def create_tenant(name: str, domain: str) -> str: def create_tenant(name: str, domain: str) -> str:
"""Create a new tenant""" """Create a new tenant"""
return api_gateway.create_tenant(name, domain) return str(api_gateway.create_tenant(name, domain)) # type: ignore[name-defined]
def get_tenant_info(tenant_id: str) -> Optional[Dict[str, Any]]: def get_tenant_info(tenant_id: str) -> Optional[Dict[str, Any]]:
"""Get tenant information""" """Get tenant information"""
tenant = api_gateway.get_tenant(tenant_id) tenant = api_gateway.get_tenant(tenant_id) # type: ignore[name-defined]
if tenant: if tenant:
return { return {
"tenant_id": tenant.tenant_id, "tenant_id": tenant.tenant_id,
@@ -1079,19 +1086,21 @@ def get_tenant_info(tenant_id: str) -> Optional[Dict[str, Any]]:
def generate_api_key(tenant_id: str) -> str: def generate_api_key(tenant_id: str) -> str:
"""Generate API key for tenant""" """Generate API key for tenant"""
return security_manager.generate_api_key(tenant_id) return str(security_manager.generate_api_key(tenant_id)) # type: ignore[name-defined]
def register_integration(tenant_id: str, name: str, integration_type: str, config: Dict[str, Any]) -> str: def register_integration(tenant_id: str, name: str, integration_type: str, config: Dict[str, Any]) -> str:
"""Register third-party integration""" """Register third-party integration"""
return integration_framework.register_integration(tenant_id, name, IntegrationType(integration_type), config) return str(integration_framework.register_integration( # type: ignore[attr-defined]
tenant_id, name, IntegrationType(integration_type), config
))
def get_system_status() -> Dict[str, Any]: def get_system_status() -> Dict[str, Any]:
"""Get enterprise integration system status""" """Get enterprise integration system status"""
return { return {
"tenants": len(api_gateway.tenants), "tenants": len(api_gateway.tenants), # type: ignore[name-defined]
"endpoints": len(api_gateway.endpoints), "endpoints": len(api_gateway.endpoints), # type: ignore[name-defined]
"integrations": len(api_gateway.integrations), "integrations": len(api_gateway.integrations), # type: ignore[name-defined]
"security_events": len(api_gateway.security_events), "security_events": len(api_gateway.security_events), # type: ignore[name-defined]
"system_health": "operational" "system_health": "operational"
} }
@@ -1105,23 +1114,22 @@ def list_tenants() -> List[Dict[str, Any]]:
"status": tenant.status.value, "status": tenant.status.value,
"features": tenant.features "features": tenant.features
} }
for tenant in api_gateway.tenants.values() for tenant in api_gateway.tenants.values() # type: ignore[name-defined]
] ]
def list_integrations(tenant_id: Optional[str] = None) -> List[Dict[str, Any]]: def list_integrations(tenant_id: Optional[str] = None) -> List[Dict[str, Any]]:
"""List integrations""" """List integrations"""
integrations = api_gateway.integrations.values() integrations = api_gateway.integrations.values() # type: ignore[name-defined]
if tenant_id: if tenant_id:
integrations = [i for i in integrations if i.tenant_id == tenant_id] integrations = [i for i in integrations if i.tenant_id == tenant_id]
return [ return [
{ {
"integration_id": i.integration_id, "integration_id": i.integration_id,
"name": i.name,
"type": i.type.value,
"tenant_id": i.tenant_id, "tenant_id": i.tenant_id,
"status": i.status, "integration_type": i.integration_type.value if hasattr(i.integration_type, 'value') else str(i.integration_type),
"created_at": i.created_at.isoformat() "provider": i.provider.value if hasattr(i.provider, 'value') else str(i.provider),
"status": i.status.value if hasattr(i.status, 'value') else str(i.status),
} }
for i in integrations for i in integrations
] ]

68
apps/coordinator-api/src/app/services/multi_modal_websocket_fusion.py
View File

@@ -98,7 +98,7 @@ class GPUProviderMetrics:
class GPUProviderFlowControl: class GPUProviderFlowControl:
"""Flow control for GPU providers""" """Flow control for GPU providers"""
def __init__(self, provider_id: str): def __init__(self, provider_id: str) -> None:
self.provider_id = provider_id self.provider_id = provider_id
self.metrics = GPUProviderMetrics( self.metrics = GPUProviderMetrics(
provider_id=provider_id, provider_id=provider_id,
@@ -112,9 +112,9 @@ class GPUProviderFlowControl:
) )
# Flow control queues # Flow control queues
self.input_queue = asyncio.Queue(maxsize=100) self.input_queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=100)
self.output_queue = asyncio.Queue(maxsize=100) self.output_queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=100)
self.control_queue = asyncio.Queue(maxsize=50) self.control_queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=50)
# Flow control parameters # Flow control parameters
self.max_concurrent_requests = 4 self.max_concurrent_requests = 4
@@ -123,15 +123,15 @@ class GPUProviderFlowControl:
self.overload_threshold = 0.8 # queue fill ratio self.overload_threshold = 0.8 # queue fill ratio
# Performance tracking # Performance tracking
self.request_times = [] self.request_times: list[float] = []
self.error_count = 0 self.error_count = 0
self.total_requests = 0 self.total_requests = 0
# Flow control task # Flow control task
self._flow_control_task = None self._flow_control_task: asyncio.Task[None] | None = None
self._running = False self._running = False
async def start(self): async def start(self) -> None:
"""Start flow control""" """Start flow control"""
if self._running: if self._running:
return return
@@ -140,7 +140,7 @@ class GPUProviderFlowControl:
self._flow_control_task = asyncio.create_task(self._flow_control_loop()) self._flow_control_task = asyncio.create_task(self._flow_control_loop())
logger.info(f"GPU provider flow control started: {self.provider_id}") logger.info(f"GPU provider flow control started: {self.provider_id}")
async def stop(self): async def stop(self) -> None:
"""Stop flow control""" """Stop flow control"""
if not self._running: if not self._running:
return return
@@ -203,7 +203,7 @@ class GPUProviderFlowControl:
return None return None
async def _flow_control_loop(self): async def _flow_control_loop(self) -> None:
"""Main flow control loop""" """Main flow control loop"""
while self._running: while self._running:
try: try:
@@ -229,7 +229,7 @@ class GPUProviderFlowControl:
logger.error(f"Flow control error for {self.provider_id}: {e}") logger.error(f"Flow control error for {self.provider_id}: {e}")
await asyncio.sleep(0.1) await asyncio.sleep(0.1)
async def _process_request(self, request_data: dict[str, Any]): async def _process_request(self, request_data: dict[str, Any]) -> None:
"""Process individual request""" """Process individual request"""
request_id = request_data["request_id"] request_id = request_data["request_id"]
data: FusionData = request_data["data"] data: FusionData = request_data["data"]
@@ -273,14 +273,14 @@ class GPUProviderFlowControl:
finally: finally:
self.current_requests -= 1 self.current_requests -= 1
def _update_metrics(self, processing_time: float, success: bool): def _update_metrics(self, processing_time: float, success: bool) -> None:
"""Update provider metrics""" """Update provider metrics"""
# Update processing time # Update processing time
self.request_times.append(processing_time) self.request_times.append(processing_time)
if len(self.request_times) > 100: if len(self.request_times) > 100:
self.request_times.pop(0) self.request_times.pop(0)
self.metrics.avg_processing_time = np.mean(self.request_times) self.metrics.avg_processing_time = float(np.mean(self.request_times))
# Update error rate # Update error rate
if not success: if not success:
@@ -323,7 +323,7 @@ class GPUProviderFlowControl:
class MultiModalWebSocketFusion: class MultiModalWebSocketFusion:
"""Multi-modal fusion service with WebSocket streaming and backpressure control""" """Multi-modal fusion service with WebSocket streaming and backpressure control"""
def __init__(self): def __init__(self) -> None:
self.stream_manager = stream_manager self.stream_manager = stream_manager
self.fusion_service = None # Will be injected self.fusion_service = None # Will be injected
self.gpu_providers: dict[str, GPUProviderFlowControl] = {} self.gpu_providers: dict[str, GPUProviderFlowControl] = {}
@@ -349,9 +349,9 @@ class MultiModalWebSocketFusion:
# Running state # Running state
self._running = False self._running = False
self._monitor_task = None self._monitor_task: asyncio.Task[None] | None = None
async def start(self): async def start(self) -> None:
"""Start the fusion service""" """Start the fusion service"""
if self._running: if self._running:
return return
@@ -369,7 +369,7 @@ class MultiModalWebSocketFusion:
logger.info("Multi-Modal WebSocket Fusion started") logger.info("Multi-Modal WebSocket Fusion started")
async def stop(self): async def stop(self) -> None:
"""Stop the fusion service""" """Stop the fusion service"""
if not self._running: if not self._running:
return return
@@ -393,16 +393,16 @@ class MultiModalWebSocketFusion:
logger.info("Multi-Modal WebSocket Fusion stopped") logger.info("Multi-Modal WebSocket Fusion stopped")
async def register_fusion_stream(self, stream_id: str, config: FusionStreamConfig): async def register_fusion_stream(self, stream_id: str, config: FusionStreamConfig) -> None:
"""Register a fusion stream""" """Register a fusion stream"""
self.fusion_streams[stream_id] = config self.fusion_streams[stream_id] = config
logger.info(f"Registered fusion stream: {stream_id} ({config.stream_type.value})") logger.info(f"Registered fusion stream: {stream_id} ({config.stream_type.value})")
async def handle_websocket_connection(self, websocket, stream_id: str, stream_type: FusionStreamType): async def handle_websocket_connection(self, websocket: Any, stream_id: str, stream_type: FusionStreamType) -> None:
"""Handle WebSocket connection for fusion stream""" """Handle WebSocket connection for fusion stream"""
config = FusionStreamConfig(stream_type=stream_type, max_queue_size=500, gpu_timeout=2.0, fusion_timeout=5.0) config = FusionStreamConfig(stream_type=stream_type, max_queue_size=500, gpu_timeout=2.0, fusion_timeout=5.0)
async with self.stream_manager.manage_stream(websocket, config.to_stream_config()): async for _ in self.stream_manager.manage_stream(websocket, config.to_stream_config()):
logger.info(f"Fusion stream connected: {stream_id} ({stream_type.value})") logger.info(f"Fusion stream connected: {stream_id} ({stream_type.value})")
try: try:
@@ -413,7 +413,7 @@ class MultiModalWebSocketFusion:
except Exception as e: except Exception as e:
logger.error(f"Error in fusion stream {stream_id}: {e}") logger.error(f"Error in fusion stream {stream_id}: {e}")
async def _handle_stream_message(self, stream_id: str, stream_type: FusionStreamType, message: str): async def _handle_stream_message(self, stream_id: str, stream_type: FusionStreamType, message: str) -> None:
"""Handle incoming stream message""" """Handle incoming stream message"""
try: try:
data = json.loads(message) data = json.loads(message)
@@ -438,7 +438,7 @@ class MultiModalWebSocketFusion:
except Exception as e: except Exception as e:
logger.error(f"Error handling stream message: {e}") logger.error(f"Error handling stream message: {e}")
async def _submit_to_gpu_provider(self, fusion_data: FusionData): async def _submit_to_gpu_provider(self, fusion_data: FusionData) -> None:
"""Submit fusion data to GPU provider""" """Submit fusion data to GPU provider"""
# Select best GPU provider # Select best GPU provider
provider_id = await self._select_gpu_provider(fusion_data) provider_id = await self._select_gpu_provider(fusion_data)
@@ -466,7 +466,7 @@ class MultiModalWebSocketFusion:
error = result.get("error", "Unknown error") if result else "Timeout" error = result.get("error", "Unknown error") if result else "Timeout"
await self._handle_fusion_error(fusion_data, error) await self._handle_fusion_error(fusion_data, error)
async def _process_cpu_fusion(self, fusion_data: FusionData): async def _process_cpu_fusion(self, fusion_data: FusionData) -> None:
"""Process fusion data on CPU""" """Process fusion data on CPU"""
try: try:
# Simulate CPU fusion processing # Simulate CPU fusion processing
@@ -485,7 +485,7 @@ class MultiModalWebSocketFusion:
logger.error(f"CPU fusion error: {e}") logger.error(f"CPU fusion error: {e}")
await self._handle_fusion_error(fusion_data, str(e)) await self._handle_fusion_error(fusion_data, str(e))
async def _handle_fusion_result(self, fusion_data: FusionData, result: dict[str, Any]): async def _handle_fusion_result(self, fusion_data: FusionData, result: dict[str, Any]) -> None:
"""Handle successful fusion result""" """Handle successful fusion result"""
# Update metrics # Update metrics
self.fusion_metrics["total_fusions"] += 1 self.fusion_metrics["total_fusions"] += 1
@@ -504,7 +504,7 @@ class MultiModalWebSocketFusion:
logger.info(f"Fusion completed for {fusion_data.stream_id}") logger.info(f"Fusion completed for {fusion_data.stream_id}")
async def _handle_fusion_error(self, fusion_data: FusionData, error: str): async def _handle_fusion_error(self, fusion_data: FusionData, error: str) -> None:
"""Handle fusion error""" """Handle fusion error"""
# Update metrics # Update metrics
self.fusion_metrics["total_fusions"] += 1 self.fusion_metrics["total_fusions"] += 1
@@ -542,24 +542,24 @@ class MultiModalWebSocketFusion:
return best_provider[0] return best_provider[0]
async def _initialize_gpu_providers(self): async def _initialize_gpu_providers(self) -> None:
"""Initialize GPU providers""" """Initialize GPU providers"""
# Create mock GPU providers # Create mock GPU providers
provider_configs = [ provider_configs: list[dict[str, Any]] = [
{"provider_id": "gpu_1", "max_concurrent": 4}, {"provider_id": "gpu_1", "max_concurrent": 4},
{"provider_id": "gpu_2", "max_concurrent": 2}, {"provider_id": "gpu_2", "max_concurrent": 2},
{"provider_id": "gpu_3", "max_concurrent": 6}, {"provider_id": "gpu_3", "max_concurrent": 6},
] ]
for config in provider_configs: for config in provider_configs:
provider = GPUProviderFlowControl(config["provider_id"]) provider = GPUProviderFlowControl(str(config["provider_id"]))
provider.max_concurrent_requests = config["max_concurrent"] provider.max_concurrent_requests = int(config["max_concurrent"])
await provider.start() await provider.start()
self.gpu_providers[config["provider_id"]] = provider self.gpu_providers[str(config["provider_id"])] = provider
logger.info(f"Initialized {len(self.gpu_providers)} GPU providers") logger.info(f"Initialized {len(self.gpu_providers)} GPU providers")
async def _monitor_loop(self): async def _monitor_loop(self) -> None:
"""Monitor system performance and backpressure""" """Monitor system performance and backpressure"""
while self._running: while self._running:
try: try:
@@ -582,10 +582,10 @@ class MultiModalWebSocketFusion:
logger.error(f"Monitor loop error: {e}") logger.error(f"Monitor loop error: {e}")
await asyncio.sleep(1) await asyncio.sleep(1)
async def _update_global_metrics(self): async def _update_global_metrics(self) -> None:
"""Update global performance metrics""" """Update global performance metrics"""
# Get stream manager metrics # Get stream manager metrics
manager_metrics = self.stream_manager.get_manager_metrics() manager_metrics = await self.stream_manager.get_manager_metrics()
# Update global queue size # Update global queue size
self.global_queue_size = manager_metrics["total_queue_size"] self.global_queue_size = manager_metrics["total_queue_size"]
@@ -606,7 +606,7 @@ class MultiModalWebSocketFusion:
self.fusion_metrics["gpu_utilization"] = total_gpu_util / active_providers self.fusion_metrics["gpu_utilization"] = total_gpu_util / active_providers
self.fusion_metrics["memory_usage"] = total_memory / active_providers self.fusion_metrics["memory_usage"] = total_memory / active_providers
async def _check_backpressure(self): async def _check_backpressure(self) -> None:
"""Check and handle backpressure""" """Check and handle backpressure"""
if self.global_queue_size > self.max_global_queue_size * 0.8: if self.global_queue_size > self.max_global_queue_size * 0.8:
logger.warning("High backpressure detected, applying flow control") logger.warning("High backpressure detected, applying flow control")
@@ -618,7 +618,7 @@ class MultiModalWebSocketFusion:
for stream_id in slow_streams: for stream_id in slow_streams:
await self.stream_manager.handle_slow_consumer(stream_id, "throttle") await self.stream_manager.handle_slow_consumer(stream_id, "throttle")
async def _monitor_gpu_providers(self): async def _monitor_gpu_providers(self) -> None:
"""Monitor GPU provider health""" """Monitor GPU provider health"""
for provider_id, provider in self.gpu_providers.items(): for provider_id, provider in self.gpu_providers.items():
metrics = provider.get_metrics() metrics = provider.get_metrics()