Files
aitbc/cli/aitbc_cli/commands/monitor.py
aitbc dc547a506a
Some checks failed
AITBC CI/CD Pipeline / lint-and-test (3.11) (pull_request) Has been cancelled
AITBC CI/CD Pipeline / lint-and-test (3.12) (pull_request) Has been cancelled
AITBC CI/CD Pipeline / lint-and-test (3.13) (pull_request) Has been cancelled
AITBC CLI Level 1 Commands Test / test-cli-level1 (3.11) (pull_request) Has been cancelled
AITBC CLI Level 1 Commands Test / test-cli-level1 (3.12) (pull_request) Has been cancelled
AITBC CLI Level 1 Commands Test / test-cli-level1 (3.13) (pull_request) Has been cancelled
Security Scanning / Bandit Security Scan (apps/coordinator-api/src) (pull_request) Has been cancelled
Security Scanning / Bandit Security Scan (cli/aitbc_cli) (pull_request) Has been cancelled
Security Scanning / Bandit Security Scan (packages/py/aitbc-core/src) (pull_request) Has been cancelled
Security Scanning / Bandit Security Scan (packages/py/aitbc-crypto/src) (pull_request) Has been cancelled
Security Scanning / Bandit Security Scan (packages/py/aitbc-sdk/src) (pull_request) Has been cancelled
Security Scanning / Bandit Security Scan (tests) (pull_request) Has been cancelled
Security Scanning / CodeQL Security Analysis (javascript) (pull_request) Has been cancelled
Security Scanning / CodeQL Security Analysis (python) (pull_request) Has been cancelled
Security Scanning / Dependency Security Scan (pull_request) Has been cancelled
Security Scanning / Container Security Scan (pull_request) Has been cancelled
Security Scanning / OSSF Scorecard (pull_request) Has been cancelled
AITBC CI/CD Pipeline / test-cli (pull_request) Has been cancelled
AITBC CI/CD Pipeline / test-services (pull_request) Has been cancelled
AITBC CI/CD Pipeline / test-production-services (pull_request) Has been cancelled
AITBC CI/CD Pipeline / security-scan (pull_request) Has been cancelled
AITBC CI/CD Pipeline / build (pull_request) Has been cancelled
AITBC CI/CD Pipeline / deploy-staging (pull_request) Has been cancelled
AITBC CI/CD Pipeline / deploy-production (pull_request) Has been cancelled
AITBC CI/CD Pipeline / performance-test (pull_request) Has been cancelled
AITBC CI/CD Pipeline / docs (pull_request) Has been cancelled
AITBC CI/CD Pipeline / release (pull_request) Has been cancelled
AITBC CI/CD Pipeline / notify (pull_request) Has been cancelled
AITBC CLI Level 1 Commands Test / test-summary (pull_request) Has been cancelled
Security Scanning / Security Summary Report (pull_request) Has been cancelled
chore: replace print() with proper logging; add jitter to automation
- CLI commands: replace print with click.echo (ensures proper stdout handling)
- Coordinator API services: add logging import and logger; replace print with logger.info
- Automation scripts: claim-task.py, monitor-prs.py, qa-cycle.py now use logging and have random jitter at startup
- Also includes fix for name shadowing in regulatory.py (aliased service imports) which was pending

This addresses issue #23 (print statements) and improves error handling.

Note: Many bare except clauses (issue #20) remain; will be addressed separately.
2026-03-15 19:01:56 +00:00

486 lines
18 KiB
Python
Executable File

"""Monitoring and dashboard commands for AITBC CLI"""
import click
import httpx
import json
import time
from pathlib import Path
from typing import Optional
from datetime import datetime, timedelta
from ..utils import output, error, success, console
@click.group()
def monitor():
"""Monitoring, metrics, and alerting commands"""
pass
@monitor.command()
@click.option("--refresh", type=int, default=5, help="Refresh interval in seconds")
@click.option("--duration", type=int, default=0, help="Duration in seconds (0 = indefinite)")
@click.pass_context
def dashboard(ctx, refresh: int, duration: int):
"""Real-time system dashboard"""
config = ctx.obj['config']
start_time = time.time()
try:
while True:
elapsed = time.time() - start_time
if duration > 0 and elapsed >= duration:
break
console.clear()
console.rule("[bold blue]AITBC Dashboard[/bold blue]")
console.click.echo(f"[dim]Refreshing every {refresh}s | Elapsed: {int(elapsed)}s[/dim]\n")
# Fetch system dashboard
try:
with httpx.Client(timeout=5) as client:
# Get dashboard data
try:
url = f"{config.coordinator_url}/api/v1/dashboard"
resp = client.get(
url,
headers={"X-Api-Key": config.api_key or ""}
)
if resp.status_code == 200:
dashboard = resp.json()
console.click.echo("[bold green]Dashboard Status:[/bold green] Online")
# Overall status
overall_status = dashboard.get("overall_status", "unknown")
console.click.echo(f" Overall Status: {overall_status}")
# Services summary
services = dashboard.get("services", {})
console.click.echo(f" Services: {len(services)}")
for service_name, service_data in services.items():
status = service_data.get("status", "unknown")
console.click.echo(f" {service_name}: {status}")
# Metrics summary
metrics = dashboard.get("metrics", {})
if metrics:
health_pct = metrics.get("health_percentage", 0)
console.click.echo(f" Health: {health_pct:.1f}%")
else:
console.click.echo(f"[bold yellow]Dashboard:[/bold yellow] HTTP {resp.status_code}")
except Exception as e:
console.click.echo(f"[bold red]Dashboard:[/bold red] Error - {e}")
except Exception as e:
console.click.echo(f"[red]Error fetching data: {e}[/red]")
console.click.echo(f"\n[dim]Press Ctrl+C to exit[/dim]")
time.sleep(refresh)
except KeyboardInterrupt:
console.click.echo("\n[bold]Dashboard stopped[/bold]")
@monitor.command()
@click.option("--period", default="24h", help="Time period (1h, 24h, 7d, 30d)")
@click.option("--export", "export_path", type=click.Path(), help="Export metrics to file")
@click.pass_context
def metrics(ctx, period: str, export_path: Optional[str]):
"""Collect and display system metrics"""
config = ctx.obj['config']
# Parse period
multipliers = {"h": 3600, "d": 86400}
unit = period[-1]
value = int(period[:-1])
seconds = value * multipliers.get(unit, 3600)
since = datetime.now() - timedelta(seconds=seconds)
metrics_data = {
"period": period,
"since": since.isoformat(),
"collected_at": datetime.now().isoformat(),
"coordinator": {},
"jobs": {},
"miners": {}
}
try:
with httpx.Client(timeout=10) as client:
# Coordinator metrics
try:
resp = client.get(
f"{config.coordinator_url}/status",
headers={"X-Api-Key": config.api_key or ""}
)
if resp.status_code == 200:
metrics_data["coordinator"] = resp.json()
metrics_data["coordinator"]["status"] = "online"
else:
metrics_data["coordinator"]["status"] = f"error_{resp.status_code}"
except Exception:
metrics_data["coordinator"]["status"] = "offline"
# Job metrics
try:
resp = client.get(
f"{config.coordinator_url}/jobs",
headers={"X-Api-Key": config.api_key or ""},
params={"limit": 100}
)
if resp.status_code == 200:
jobs = resp.json()
if isinstance(jobs, list):
metrics_data["jobs"] = {
"total": len(jobs),
"completed": sum(1 for j in jobs if j.get("status") == "completed"),
"pending": sum(1 for j in jobs if j.get("status") == "pending"),
"failed": sum(1 for j in jobs if j.get("status") == "failed"),
}
except Exception:
metrics_data["jobs"] = {"error": "unavailable"}
# Miner metrics
try:
resp = client.get(
f"{config.coordinator_url}/miners",
headers={"X-Api-Key": config.api_key or ""}
)
if resp.status_code == 200:
miners = resp.json()
if isinstance(miners, list):
metrics_data["miners"] = {
"total": len(miners),
"online": sum(1 for m in miners if m.get("status") == "ONLINE"),
"offline": sum(1 for m in miners if m.get("status") != "ONLINE"),
}
except Exception:
metrics_data["miners"] = {"error": "unavailable"}
except Exception as e:
error(f"Failed to collect metrics: {e}")
if export_path:
with open(export_path, "w") as f:
json.dump(metrics_data, f, indent=2)
success(f"Metrics exported to {export_path}")
output(metrics_data, ctx.obj['output_format'])
@monitor.command()
@click.argument("action", type=click.Choice(["add", "list", "remove", "test"]))
@click.option("--name", help="Alert name")
@click.option("--type", "alert_type", type=click.Choice(["coordinator_down", "miner_offline", "job_failed", "low_balance"]), help="Alert type")
@click.option("--threshold", type=float, help="Alert threshold value")
@click.option("--webhook", help="Webhook URL for notifications")
@click.pass_context
def alerts(ctx, action: str, name: Optional[str], alert_type: Optional[str],
threshold: Optional[float], webhook: Optional[str]):
"""Configure monitoring alerts"""
alerts_dir = Path.home() / ".aitbc" / "alerts"
alerts_dir.mkdir(parents=True, exist_ok=True)
alerts_file = alerts_dir / "alerts.json"
# Load existing alerts
existing = []
if alerts_file.exists():
with open(alerts_file) as f:
existing = json.load(f)
if action == "add":
if not name or not alert_type:
error("Alert name and type required (--name, --type)")
return
alert = {
"name": name,
"type": alert_type,
"threshold": threshold,
"webhook": webhook,
"created_at": datetime.now().isoformat(),
"enabled": True
}
existing.append(alert)
with open(alerts_file, "w") as f:
json.dump(existing, f, indent=2)
success(f"Alert '{name}' added")
output(alert, ctx.obj['output_format'])
elif action == "list":
if not existing:
output({"message": "No alerts configured"}, ctx.obj['output_format'])
else:
output(existing, ctx.obj['output_format'])
elif action == "remove":
if not name:
error("Alert name required (--name)")
return
existing = [a for a in existing if a["name"] != name]
with open(alerts_file, "w") as f:
json.dump(existing, f, indent=2)
success(f"Alert '{name}' removed")
elif action == "test":
if not name:
error("Alert name required (--name)")
return
alert = next((a for a in existing if a["name"] == name), None)
if not alert:
error(f"Alert '{name}' not found")
return
if alert.get("webhook"):
try:
with httpx.Client(timeout=10) as client:
resp = client.post(alert["webhook"], json={
"alert": name,
"type": alert["type"],
"message": f"Test alert from AITBC CLI",
"timestamp": datetime.now().isoformat()
})
output({"status": "sent", "response_code": resp.status_code}, ctx.obj['output_format'])
except Exception as e:
error(f"Webhook test failed: {e}")
else:
output({"status": "no_webhook", "alert": alert}, ctx.obj['output_format'])
@monitor.command()
@click.option("--period", default="7d", help="Analysis period (1d, 7d, 30d)")
@click.pass_context
def history(ctx, period: str):
"""Historical data analysis"""
config = ctx.obj['config']
multipliers = {"h": 3600, "d": 86400}
unit = period[-1]
value = int(period[:-1])
seconds = value * multipliers.get(unit, 3600)
since = datetime.now() - timedelta(seconds=seconds)
analysis = {
"period": period,
"since": since.isoformat(),
"analyzed_at": datetime.now().isoformat(),
"summary": {}
}
try:
with httpx.Client(timeout=10) as client:
try:
resp = client.get(
f"{config.coordinator_url}/jobs",
headers={"X-Api-Key": config.api_key or ""},
params={"limit": 500}
)
if resp.status_code == 200:
jobs = resp.json()
if isinstance(jobs, list):
completed = [j for j in jobs if j.get("status") == "completed"]
failed = [j for j in jobs if j.get("status") == "failed"]
analysis["summary"] = {
"total_jobs": len(jobs),
"completed": len(completed),
"failed": len(failed),
"success_rate": f"{len(completed) / max(1, len(jobs)) * 100:.1f}%",
}
except Exception:
analysis["summary"] = {"error": "Could not fetch job data"}
except Exception as e:
error(f"Analysis failed: {e}")
output(analysis, ctx.obj['output_format'])
@monitor.command()
@click.argument("action", type=click.Choice(["add", "list", "remove", "test"]))
@click.option("--name", help="Webhook name")
@click.option("--url", help="Webhook URL")
@click.option("--events", help="Comma-separated event types (job_completed,miner_offline,alert)")
@click.pass_context
def webhooks(ctx, action: str, name: Optional[str], url: Optional[str], events: Optional[str]):
"""Manage webhook notifications"""
webhooks_dir = Path.home() / ".aitbc" / "webhooks"
webhooks_dir.mkdir(parents=True, exist_ok=True)
webhooks_file = webhooks_dir / "webhooks.json"
existing = []
if webhooks_file.exists():
with open(webhooks_file) as f:
existing = json.load(f)
if action == "add":
if not name or not url:
error("Webhook name and URL required (--name, --url)")
return
webhook = {
"name": name,
"url": url,
"events": events.split(",") if events else ["all"],
"created_at": datetime.now().isoformat(),
"enabled": True
}
existing.append(webhook)
with open(webhooks_file, "w") as f:
json.dump(existing, f, indent=2)
success(f"Webhook '{name}' added")
output(webhook, ctx.obj['output_format'])
elif action == "list":
if not existing:
output({"message": "No webhooks configured"}, ctx.obj['output_format'])
else:
output(existing, ctx.obj['output_format'])
elif action == "remove":
if not name:
error("Webhook name required (--name)")
return
existing = [w for w in existing if w["name"] != name]
with open(webhooks_file, "w") as f:
json.dump(existing, f, indent=2)
success(f"Webhook '{name}' removed")
elif action == "test":
if not name:
error("Webhook name required (--name)")
return
wh = next((w for w in existing if w["name"] == name), None)
if not wh:
error(f"Webhook '{name}' not found")
return
try:
with httpx.Client(timeout=10) as client:
resp = client.post(wh["url"], json={
"event": "test",
"source": "aitbc-cli",
"message": "Test webhook notification",
"timestamp": datetime.now().isoformat()
})
output({"status": "sent", "response_code": resp.status_code}, ctx.obj['output_format'])
except Exception as e:
error(f"Webhook test failed: {e}")
CAMPAIGNS_DIR = Path.home() / ".aitbc" / "campaigns"
def _ensure_campaigns():
CAMPAIGNS_DIR.mkdir(parents=True, exist_ok=True)
campaigns_file = CAMPAIGNS_DIR / "campaigns.json"
if not campaigns_file.exists():
# Seed with default campaigns
default = {"campaigns": [
{
"id": "staking_launch",
"name": "Staking Launch Campaign",
"type": "staking",
"apy_boost": 2.0,
"start_date": "2026-02-01T00:00:00",
"end_date": "2026-04-01T00:00:00",
"status": "active",
"total_staked": 0,
"participants": 0,
"rewards_distributed": 0
},
{
"id": "liquidity_mining_q1",
"name": "Q1 Liquidity Mining",
"type": "liquidity",
"apy_boost": 3.0,
"start_date": "2026-01-15T00:00:00",
"end_date": "2026-03-15T00:00:00",
"status": "active",
"total_staked": 0,
"participants": 0,
"rewards_distributed": 0
}
]}
with open(campaigns_file, "w") as f:
json.dump(default, f, indent=2)
return campaigns_file
@monitor.command()
@click.option("--status", type=click.Choice(["active", "ended", "all"]), default="all", help="Filter by status")
@click.pass_context
def campaigns(ctx, status: str):
"""List active incentive campaigns"""
campaigns_file = _ensure_campaigns()
with open(campaigns_file) as f:
data = json.load(f)
campaign_list = data.get("campaigns", [])
# Auto-update status
now = datetime.now()
for c in campaign_list:
end = datetime.fromisoformat(c["end_date"])
if now > end and c["status"] == "active":
c["status"] = "ended"
with open(campaigns_file, "w") as f:
json.dump(data, f, indent=2)
if status != "all":
campaign_list = [c for c in campaign_list if c["status"] == status]
if not campaign_list:
output({"message": "No campaigns found"}, ctx.obj['output_format'])
return
output(campaign_list, ctx.obj['output_format'])
@monitor.command(name="campaign-stats")
@click.argument("campaign_id", required=False)
@click.pass_context
def campaign_stats(ctx, campaign_id: Optional[str]):
"""Campaign performance metrics (TVL, participants, rewards)"""
campaigns_file = _ensure_campaigns()
with open(campaigns_file) as f:
data = json.load(f)
campaign_list = data.get("campaigns", [])
if campaign_id:
campaign = next((c for c in campaign_list if c["id"] == campaign_id), None)
if not campaign:
error(f"Campaign '{campaign_id}' not found")
ctx.exit(1)
return
targets = [campaign]
else:
targets = campaign_list
stats = []
for c in targets:
start = datetime.fromisoformat(c["start_date"])
end = datetime.fromisoformat(c["end_date"])
now = datetime.now()
duration_days = (end - start).days
elapsed_days = min((now - start).days, duration_days)
progress_pct = round(elapsed_days / max(duration_days, 1) * 100, 1)
stats.append({
"campaign_id": c["id"],
"name": c["name"],
"type": c["type"],
"status": c["status"],
"apy_boost": c.get("apy_boost", 0),
"tvl": c.get("total_staked", 0),
"participants": c.get("participants", 0),
"rewards_distributed": c.get("rewards_distributed", 0),
"duration_days": duration_days,
"elapsed_days": elapsed_days,
"progress_pct": progress_pct,
"start_date": c["start_date"],
"end_date": c["end_date"]
})
if len(stats) == 1:
output(stats[0], ctx.obj['output_format'])
else:
output(stats, ctx.obj['output_format'])