Add deployment workflows with contract verification and monitoring
- Add deploy-testnet.yml workflow for testnet deployments - Add deploy-mainnet.yml workflow with Etherscan verification - Add contract monitoring setup scripts - Add automated alerting configuration - Add deployment notification system - Include pre-deployment checks and post-deployment monitoring
This commit is contained in:
247
.gitea/workflows/deploy-mainnet.yml
Normal file
247
.gitea/workflows/deploy-mainnet.yml
Normal file
@@ -0,0 +1,247 @@
|
||||
name: Deploy to Mainnet
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ['mainnet-v*']
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
environment:
|
||||
description: 'Deployment environment'
|
||||
required: true
|
||||
default: 'mainnet'
|
||||
type: choice
|
||||
options:
|
||||
- mainnet
|
||||
verify_contracts:
|
||||
description: 'Verify contracts on Etherscan'
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
skip_tests:
|
||||
description: 'Skip pre-deployment tests (NOT RECOMMENDED)'
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
concurrency:
|
||||
group: deploy-mainnet-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
pre-deployment-checks:
|
||||
runs-on: debian
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Clone repository
|
||||
run: |
|
||||
WORKSPACE="/var/lib/aitbc-workspaces/pre-deployment"
|
||||
rm -rf "$WORKSPACE"
|
||||
mkdir -p "$WORKSPACE"
|
||||
cd "$WORKSPACE"
|
||||
git clone --depth 1 http://gitea.bubuit.net:3000/oib/aitbc.git repo
|
||||
|
||||
- name: Initialize job logging
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/pre-deployment/repo
|
||||
bash scripts/ci/setup-job-logging.sh
|
||||
|
||||
- name: Run security scan
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/pre-deployment/repo
|
||||
|
||||
# Run comprehensive security scan before mainnet deployment
|
||||
bash scripts/ci/security-scan.sh
|
||||
|
||||
echo "✅ Security scan passed"
|
||||
|
||||
- name: Run contract tests
|
||||
if: inputs.skip_tests != true
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/pre-deployment/repo/contracts
|
||||
npm install
|
||||
npx hardhat test
|
||||
echo "✅ Contract tests passed"
|
||||
|
||||
- name: Verify deployment readiness
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/pre-deployment/repo
|
||||
|
||||
# Check all pre-deployment requirements
|
||||
bash scripts/deployment/check-deployment-readiness.sh mainnet
|
||||
|
||||
echo "✅ Deployment readiness verified"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: rm -rf /var/lib/aitbc-workspaces/pre-deployment
|
||||
|
||||
deploy-mainnet:
|
||||
runs-on: debian
|
||||
timeout-minutes: 45
|
||||
needs: pre-deployment-checks
|
||||
environment:
|
||||
name: mainnet
|
||||
url: https://mainnet.aitbc.network
|
||||
|
||||
steps:
|
||||
- name: Clone repository
|
||||
run: |
|
||||
WORKSPACE="/var/lib/aitbc-workspaces/deploy-mainnet"
|
||||
rm -rf "$WORKSPACE"
|
||||
mkdir -p "$WORKSPACE"
|
||||
cd "$WORKSPACE"
|
||||
git clone --depth 1 http://gitea.bubuit.net:3000/oib/aitbc.git repo
|
||||
|
||||
- name: Initialize job logging
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-mainnet/repo
|
||||
bash scripts/ci/setup-job-logging.sh
|
||||
|
||||
- name: Setup Node.js environment
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-mainnet/repo/contracts
|
||||
npm install
|
||||
echo "✅ Node.js environment ready"
|
||||
|
||||
- name: Compile contracts
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-mainnet/repo/contracts
|
||||
npx hardhat compile
|
||||
echo "✅ Contracts compiled"
|
||||
|
||||
- name: Deploy contracts to mainnet
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-mainnet/repo/contracts
|
||||
|
||||
# Load mainnet deployment configuration
|
||||
export HARDHAT_NETWORK=mainnet
|
||||
export PRIVATE_KEY=${{ secrets.MAINNET_DEPLOYER_PRIVATE_KEY }}
|
||||
export MAINNET_RPC_URL=${{ secrets.MAINNET_RPC_URL }}
|
||||
|
||||
# Deploy contracts with gas optimization
|
||||
npx hardhat run scripts/deploy-mainnet.js --network mainnet
|
||||
|
||||
echo "✅ Contracts deployed to mainnet"
|
||||
|
||||
- name: Verify contracts on Etherscan
|
||||
if: inputs.verify_contracts != false
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-mainnet/repo/contracts
|
||||
|
||||
# Load Etherscan verification configuration
|
||||
export ETHERSCAN_API_KEY=${{ secrets.ETHERSCAN_API_KEY }}
|
||||
|
||||
# Verify each deployed contract on Etherscan
|
||||
echo "🔍 Verifying contracts on Etherscan..."
|
||||
|
||||
# Verify PaymentProcessor
|
||||
npx hardhat verify --network mainnet $PAYMENT_PROCESSOR_ADDRESS --constructor-args scripts/deployment/args/payment-processor-args.js
|
||||
|
||||
# Verify AgentMarketplace
|
||||
npx hardhat verify --network mainnet $AGENT_MARKETPLACE_ADDRESS --constructor-args scripts/deployment/args/agent-marketplace-args.js
|
||||
|
||||
# Verify StakingContract
|
||||
npx hardhat verify --network mainnet $STAKING_CONTRACT_ADDRESS --constructor-args scripts/deployment/args/staking-contract-args.js
|
||||
|
||||
# Verify TreasuryManager
|
||||
npx hardhat verify --network mainnet $TREASURY_MANAGER_ADDRESS --constructor-args scripts/deployment/args/treasury-manager-args.js
|
||||
|
||||
echo "✅ All contracts verified on Etherscan"
|
||||
|
||||
- name: Record deployment metadata
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-mainnet/repo
|
||||
|
||||
# Save deployment information
|
||||
cat > deployment-info.json << EOF
|
||||
{
|
||||
"network": "mainnet",
|
||||
"commit": "${{ github.sha }}",
|
||||
"tag": "${{ github.ref_name }}",
|
||||
"timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||
"deployed_by": "${{ github.actor }}",
|
||||
"contracts": {
|
||||
"PaymentProcessor": "$PAYMENT_PROCESSOR_ADDRESS",
|
||||
"AgentMarketplace": "$AGENT_MARKETPLACE_ADDRESS",
|
||||
"StakingContract": "$STAKING_CONTRACT_ADDRESS",
|
||||
"TreasuryManager": "$TREASURY_MANAGER_ADDRESS"
|
||||
},
|
||||
"etherscan_verified": "${{ inputs.verify_contracts }}"
|
||||
}
|
||||
EOF
|
||||
|
||||
echo "✅ Deployment metadata recorded"
|
||||
|
||||
- name: Setup contract monitoring
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-mainnet/repo
|
||||
|
||||
# Configure monitoring for deployed contracts on mainnet
|
||||
bash scripts/monitoring/setup-contract-monitoring.sh mainnet
|
||||
|
||||
echo "✅ Contract monitoring configured"
|
||||
|
||||
- name: Run production smoke tests
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-mainnet/repo
|
||||
|
||||
# Run smoke tests against deployed contracts on mainnet
|
||||
bash scripts/testing/run-production-smoke-tests.sh mainnet
|
||||
|
||||
echo "✅ Production smoke tests passed"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: rm -rf /var/lib/aitbc-workspaces/deploy-mainnet
|
||||
|
||||
post-deployment-monitoring:
|
||||
runs-on: debian
|
||||
needs: deploy-mainnet
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Clone repository
|
||||
run: |
|
||||
WORKSPACE="/var/lib/aitbc-workspaces/post-deployment"
|
||||
rm -rf "$WORKSPACE"
|
||||
mkdir -p "$WORKSPACE"
|
||||
cd "$WORKSPACE"
|
||||
git clone --depth 1 http://gitea.bubuit.net:3000/oib/aitbc.git repo
|
||||
|
||||
- name: Initialize job logging
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/post-deployment/repo
|
||||
bash scripts/ci/setup-job-logging.sh
|
||||
|
||||
- name: Configure automated monitoring alerts
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/post-deployment/repo
|
||||
|
||||
# Setup automated alerts for contract events
|
||||
bash scripts/monitoring/setup-automated-alerts.sh mainnet
|
||||
|
||||
echo "✅ Automated monitoring alerts configured"
|
||||
|
||||
- name: Verify monitoring setup
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/post-deployment/repo
|
||||
|
||||
# Verify monitoring is working
|
||||
bash scripts/monitoring/verify-monitoring.sh mainnet
|
||||
|
||||
echo "✅ Monitoring verification passed"
|
||||
|
||||
- name: Send deployment notification
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/post-deployment/repo
|
||||
|
||||
# Send notification about mainnet deployment
|
||||
STATUS=${{ needs.deploy-mainnet.result }}
|
||||
bash scripts/notifications/send-deployment-notification.sh mainnet $STATUS
|
||||
|
||||
echo "✅ Deployment notification sent"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: rm -rf /var/lib/aitbc-workspaces/post-deployment
|
||||
169
.gitea/workflows/deploy-testnet.yml
Normal file
169
.gitea/workflows/deploy-testnet.yml
Normal file
@@ -0,0 +1,169 @@
|
||||
name: Deploy to Testnet
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ['testnet-v*']
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
environment:
|
||||
description: 'Deployment environment'
|
||||
required: true
|
||||
default: 'testnet'
|
||||
type: choice
|
||||
options:
|
||||
- testnet
|
||||
- devnet
|
||||
verify_contracts:
|
||||
description: 'Verify contracts on block explorer'
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
|
||||
concurrency:
|
||||
group: deploy-testnet-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
deploy-testnet:
|
||||
runs-on: debian
|
||||
timeout-minutes: 30
|
||||
environment:
|
||||
name: testnet
|
||||
url: https://testnet.aitbc.network
|
||||
|
||||
steps:
|
||||
- name: Clone repository
|
||||
run: |
|
||||
WORKSPACE="/var/lib/aitbc-workspaces/deploy-testnet"
|
||||
rm -rf "$WORKSPACE"
|
||||
mkdir -p "$WORKSPACE"
|
||||
cd "$WORKSPACE"
|
||||
git clone --depth 1 http://gitea.bubuit.net:3000/oib/aitbc.git repo
|
||||
|
||||
- name: Initialize job logging
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-testnet/repo
|
||||
bash scripts/ci/setup-job-logging.sh
|
||||
|
||||
- name: Setup Node.js environment
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-testnet/repo/contracts
|
||||
npm install
|
||||
echo "✅ Node.js environment ready"
|
||||
|
||||
- name: Compile contracts
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-testnet/repo/contracts
|
||||
npx hardhat compile
|
||||
echo "✅ Contracts compiled"
|
||||
|
||||
- name: Run contract tests
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-testnet/repo/contracts
|
||||
npx hardhat test
|
||||
echo "✅ Contract tests passed"
|
||||
|
||||
- name: Deploy contracts to testnet
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-testnet/repo/contracts
|
||||
|
||||
# Load testnet deployment configuration
|
||||
export HARDHAT_NETWORK=testnet
|
||||
export PRIVATE_KEY=${{ secrets.TESTNET_DEPLOYER_PRIVATE_KEY }}
|
||||
export TESTNET_RPC_URL=${{ secrets.TESTNET_RPC_URL }}
|
||||
|
||||
# Deploy contracts
|
||||
npx hardhat run scripts/deploy-testnet.js --network testnet
|
||||
|
||||
echo "✅ Contracts deployed to testnet"
|
||||
|
||||
- name: Verify contracts on block explorer
|
||||
if: inputs.verify_contracts != false
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-testnet/repo/contracts
|
||||
|
||||
# Load verification configuration
|
||||
export ETHERSCAN_API_KEY=${{ secrets.TESTNET_EXPLORER_API_KEY }}
|
||||
export TESTNET_EXPLORER_URL=${{ secrets.TESTNET_EXPLORER_URL }}
|
||||
|
||||
# Verify deployed contracts
|
||||
npx hardhat verify --network testnet DEPLOYED_CONTRACT_ADDRESS CONSTRUCTOR_ARGS
|
||||
|
||||
echo "✅ Contracts verified on block explorer"
|
||||
|
||||
- name: Record deployment metadata
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-testnet/repo
|
||||
|
||||
# Save deployment information
|
||||
cat > deployment-info.json << EOF
|
||||
{
|
||||
"network": "testnet",
|
||||
"commit": "${{ github.sha }}",
|
||||
"timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||
"deployed_by": "${{ github.actor }}",
|
||||
"contracts": {
|
||||
"PaymentProcessor": "DEPLOYED_ADDRESS",
|
||||
"AgentMarketplace": "DEPLOYED_ADDRESS",
|
||||
"StakingContract": "DEPLOYED_ADDRESS"
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
echo "✅ Deployment metadata recorded"
|
||||
|
||||
- name: Setup monitoring alerts
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-testnet/repo
|
||||
|
||||
# Configure monitoring for deployed contracts
|
||||
bash scripts/monitoring/setup-contract-monitoring.sh testnet
|
||||
|
||||
echo "✅ Monitoring alerts configured"
|
||||
|
||||
- name: Run smoke tests
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/deploy-testnet/repo
|
||||
|
||||
# Run smoke tests against deployed contracts
|
||||
bash scripts/testing/run-smoke-tests.sh testnet
|
||||
|
||||
echo "✅ Smoke tests passed"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: rm -rf /var/lib/aitbc-workspaces/deploy-testnet
|
||||
|
||||
notify-deployment:
|
||||
runs-on: debian
|
||||
needs: deploy-testnet
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Clone repository
|
||||
run: |
|
||||
WORKSPACE="/var/lib/aitbc-workspaces/notify-deployment"
|
||||
rm -rf "$WORKSPACE"
|
||||
mkdir -p "$WORKSPACE"
|
||||
cd "$WORKSPACE"
|
||||
git clone --depth 1 http://gitea.bubuit.net:3000/oib/aitbc.git repo
|
||||
|
||||
- name: Initialize job logging
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/notify-deployment/repo
|
||||
bash scripts/ci/setup-job-logging.sh
|
||||
|
||||
- name: Send deployment notification
|
||||
run: |
|
||||
cd /var/lib/aitbc-workspaces/notify-deployment/repo
|
||||
|
||||
# Send notification about deployment status
|
||||
STATUS=${{ needs.deploy-testnet.result }}
|
||||
bash scripts/notifications/send-deployment-notification.sh testnet $STATUS
|
||||
|
||||
echo "✅ Deployment notification sent"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: rm -rf /var/lib/aitbc-workspaces/notify-deployment
|
||||
189
scripts/monitoring/setup-automated-alerts.sh
Executable file
189
scripts/monitoring/setup-automated-alerts.sh
Executable file
@@ -0,0 +1,189 @@
|
||||
#!/bin/bash
|
||||
# Setup automated monitoring alerts for deployed contracts
|
||||
|
||||
set -e
|
||||
|
||||
NETWORK="${1:-mainnet}"
|
||||
REPO_DIR="/opt/aitbc"
|
||||
MONITORING_DIR="${REPO_DIR}/scripts/monitoring"
|
||||
|
||||
echo "=== Setting up automated alerts for ${NETWORK} ==="
|
||||
|
||||
# Create alerting rules
|
||||
cat > "${MONITORING_DIR}/config/${NETWORK}/alert-rules.yml" << EOF
|
||||
groups:
|
||||
- name: aitbc-contracts-alerts
|
||||
interval: 30s
|
||||
rules:
|
||||
# Payment Processor Alerts
|
||||
- alert: HighFailedTransactionRate
|
||||
expr: rate(contract_failed_transactions_total[5m]) > 0.1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
network: ${NETWORK}
|
||||
contract: PaymentProcessor
|
||||
annotations:
|
||||
summary: "High failed transaction rate on PaymentProcessor"
|
||||
description: "Failed transaction rate is {{ $value }} per second"
|
||||
|
||||
- alert: PaymentProcessorDown
|
||||
expr: up{job="aitbc-contracts-${NETWORK}", contract="PaymentProcessor"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
network: ${NETWORK}
|
||||
contract: PaymentProcessor
|
||||
annotations:
|
||||
summary: "PaymentProcessor contract is down"
|
||||
description: "PaymentProcessor has been down for more than 2 minutes"
|
||||
|
||||
# Agent Marketplace Alerts
|
||||
- alert: FailedAgentRegistrations
|
||||
expr: rate(contract_failed_registrations_total[5m]) > 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
network: ${NETWORK}
|
||||
contract: AgentMarketplace
|
||||
annotations:
|
||||
summary: "High failed agent registration rate"
|
||||
description: "Failed registration rate is {{ $value }} per second"
|
||||
|
||||
- alert: MarketplaceLowActivity
|
||||
expr: rate(contract_jobs_posted_total[1h]) < 0.001
|
||||
for: 1h
|
||||
labels:
|
||||
severity: info
|
||||
network: ${NETWORK}
|
||||
contract: AgentMarketplace
|
||||
annotations:
|
||||
summary: "Low marketplace activity"
|
||||
description: "Job posting rate is unusually low"
|
||||
|
||||
# Staking Contract Alerts
|
||||
- alert: UnusualWithdrawalActivity
|
||||
expr: rate(contract_stake_withdrawals_total[10m]) > 0.1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
network: ${NETWORK}
|
||||
contract: StakingContract
|
||||
annotations:
|
||||
summary: "Unusual withdrawal activity detected"
|
||||
description: "Withdrawal rate is {{ $value }} per second"
|
||||
|
||||
- alert: RewardDistributionDelay
|
||||
expr: time() - contract_last_reward_distribution_timestamp > 3600
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
network: ${NETWORK}
|
||||
contract: StakingContract
|
||||
annotations:
|
||||
summary: "Reward distribution delayed"
|
||||
description: "Rewards have not been distributed for over 1 hour"
|
||||
|
||||
# General Contract Health
|
||||
- alert: ContractGasPriceSpike
|
||||
expr: contract_gas_price > 100000000000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
network: ${NETWORK}
|
||||
annotations:
|
||||
summary: "Gas price spike detected"
|
||||
description: "Gas price is {{ $value }} wei"
|
||||
|
||||
- alert: ContractBalanceLow
|
||||
expr: contract_balance < 0.1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
network: ${NETWORK}
|
||||
annotations:
|
||||
summary: "Contract balance critically low"
|
||||
description: "Contract balance is {{ $value }} ETH"
|
||||
EOF
|
||||
|
||||
echo "✅ Alert rules created for ${NETWORK}"
|
||||
|
||||
# Setup notification templates
|
||||
mkdir -p "${MONITORING_DIR}/templates"
|
||||
|
||||
cat > "${MONITORING_DIR}/templates/alert-notification.tmpl" << EOF
|
||||
{{ define "slack.default.title" }}
|
||||
{{ .Status | toUpper }}: {{ .CommonLabels.alertname }} on {{ .CommonLabels.network }}
|
||||
{{ end }}
|
||||
|
||||
{{ define "slack.default.text" }}
|
||||
{{ range .Alerts }}
|
||||
*Alert:* {{ .Labels.alertname }}
|
||||
*Severity:* {{ .Labels.severity }}
|
||||
*Contract:* {{ .Labels.contract }}
|
||||
*Network:* {{ .Labels.network }}
|
||||
*Description:* {{ .Annotations.description }}
|
||||
*Timestamp:* {{ .StartsAt.Format "2006-01-02 15:04:05" }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
EOF
|
||||
|
||||
echo "✅ Notification templates created"
|
||||
|
||||
# Setup alertmanager configuration
|
||||
cat > "${MONITORING_DIR}/config/alertmanager.yml" << EOF
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
group_by: ['alertname', 'network', 'contract']
|
||||
group_wait: 10s
|
||||
group_interval: 10s
|
||||
repeat_interval: 12h
|
||||
receiver: 'default'
|
||||
|
||||
routes:
|
||||
- match:
|
||||
severity: critical
|
||||
receiver: 'pagerduty'
|
||||
- match:
|
||||
severity: warning
|
||||
receiver: 'slack-warnings'
|
||||
- match:
|
||||
severity: info
|
||||
receiver: 'slack-info'
|
||||
|
||||
receivers:
|
||||
- name: 'default'
|
||||
slack_configs:
|
||||
- api_url: '\${SLACK_WEBHOOK_URL}'
|
||||
channel: '#aitbc-alerts'
|
||||
|
||||
- name: 'pagerduty'
|
||||
pagerduty_configs:
|
||||
- service_key: '\${PAGERDUTY_API_KEY}'
|
||||
severity: 'critical'
|
||||
|
||||
- name: 'slack-warnings'
|
||||
slack_configs:
|
||||
- api_url: '\${SLACK_WEBHOOK_URL}'
|
||||
channel: '#aitbc-warnings'
|
||||
|
||||
- name: 'slack-info'
|
||||
slack_configs:
|
||||
- api_url: '\${SLACK_WEBHOOK_URL}'
|
||||
channel: '#aitbc-info'
|
||||
EOF
|
||||
|
||||
echo "✅ Alertmanager configuration created"
|
||||
|
||||
# Enable monitoring service
|
||||
if [[ -f "/etc/systemd/system/aitbc-monitor.service" ]]; then
|
||||
systemctl enable aitbc-monitor.service
|
||||
systemctl start aitbc-monitor.service
|
||||
echo "✅ Monitoring service started"
|
||||
else
|
||||
echo "⚠️ Monitoring service not found, skipping service start"
|
||||
fi
|
||||
|
||||
echo "=== Automated alerts setup complete for ${NETWORK} ==="
|
||||
64
scripts/monitoring/setup-contract-monitoring.sh
Executable file
64
scripts/monitoring/setup-contract-monitoring.sh
Executable file
@@ -0,0 +1,64 @@
|
||||
#!/bin/bash
|
||||
# Setup contract monitoring for deployed contracts
|
||||
|
||||
set -e
|
||||
|
||||
NETWORK="${1:-testnet}"
|
||||
REPO_DIR="/opt/aitbc"
|
||||
MONITORING_DIR="${REPO_DIR}/scripts/monitoring"
|
||||
|
||||
echo "=== Setting up contract monitoring for ${NETWORK} ==="
|
||||
|
||||
# Ensure monitoring directory exists
|
||||
mkdir -p "${MONITORING_DIR}/config/${NETWORK}"
|
||||
|
||||
# Create monitoring configuration
|
||||
cat > "${MONITORING_DIR}/config/${NETWORK}/contracts.json" << EOF
|
||||
{
|
||||
"network": "${NETWORK}",
|
||||
"contracts": {
|
||||
"PaymentProcessor": {
|
||||
"address": "\${PAYMENT_PROCESSOR_ADDRESS}",
|
||||
"monitor_events": ["PaymentReceived", "PaymentRefunded", "PaymentCompleted"],
|
||||
"alert_thresholds": {
|
||||
"failed_transactions": 5,
|
||||
"gas_price_spike": 100
|
||||
}
|
||||
},
|
||||
"AgentMarketplace": {
|
||||
"address": "\${AGENT_MARKETPLACE_ADDRESS}",
|
||||
"monitor_events": ["AgentRegistered", "AgentDeregistered", "JobPosted", "JobCompleted"],
|
||||
"alert_thresholds": {
|
||||
"failed_registrations": 3,
|
||||
"marketplace downtime": 300
|
||||
}
|
||||
},
|
||||
"StakingContract": {
|
||||
"address": "\${STAKING_CONTRACT_ADDRESS}",
|
||||
"monitor_events": ["StakeDeposited", "StakeWithdrawn", "RewardsDistributed"],
|
||||
"alert_thresholds": {
|
||||
"unusual_withdrawals": 10,
|
||||
"reward_delay": 3600
|
||||
}
|
||||
}
|
||||
},
|
||||
"alert_channels": {
|
||||
"slack": "\${SLACK_WEBHOOK_URL}",
|
||||
"email": "\${ALERT_EMAIL}",
|
||||
"pagerduty": "\${PAGERDUTY_API_KEY}"
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
echo "✅ Contract monitoring configuration created for ${NETWORK}"
|
||||
|
||||
# Setup Prometheus metrics for contract monitoring
|
||||
cat > "${MONITORING_DIR}/config/${NETWORK}/prometheus.yml" << EOF
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: 'aitbc-contracts-${NETWORK}'
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
metrics_path: '/metrics/contracts'
|
||||
59
scripts/monitoring/verify-monitoring.sh
Executable file
59
scripts/monitoring/verify-monitoring.sh
Executable file
@@ -0,0 +1,59 @@
|
||||
#!/bin/bash
|
||||
# Verify monitoring setup is working correctly
|
||||
|
||||
set -e
|
||||
|
||||
NETWORK="${1:-mainnet}"
|
||||
REPO_DIR="/opt/aitbc"
|
||||
MONITORING_DIR="${REPO_DIR}/scripts/monitoring"
|
||||
|
||||
echo "=== Verifying monitoring setup for ${NETWORK} ==="
|
||||
|
||||
# Check configuration files exist
|
||||
if [[ ! -f "${MONITORING_DIR}/config/${NETWORK}/contracts.json" ]]; then
|
||||
echo "❌ Contract monitoring configuration not found"
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ Contract monitoring configuration exists"
|
||||
|
||||
if [[ ! -f "${MONITORING_DIR}/config/${NETWORK}/alert-rules.yml" ]]; then
|
||||
echo "❌ Alert rules not found"
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ Alert rules exist"
|
||||
|
||||
# Check monitoring service status
|
||||
if systemctl is-active --quiet aitbc-monitor.service; then
|
||||
echo "✅ Monitoring service is running"
|
||||
else
|
||||
echo "⚠️ Monitoring service is not running"
|
||||
fi
|
||||
|
||||
# Check Prometheus targets
|
||||
if command -v curl &> /dev/null; then
|
||||
if curl -s http://localhost:9090/-/healthy &> /dev/null; then
|
||||
echo "✅ Prometheus is accessible"
|
||||
else
|
||||
echo "⚠️ Prometheus is not accessible"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check Alertmanager
|
||||
if command -v curl &> /dev/null; then
|
||||
if curl -s http://localhost:9093/-/healthy &> /dev/null; then
|
||||
echo "✅ Alertmanager is accessible"
|
||||
else
|
||||
echo "⚠️ Alertmanager is not accessible"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Test alert configuration
|
||||
echo "🔍 Testing alert configuration..."
|
||||
if [[ -f "${MONITORING_DIR}/config/alertmanager.yml" ]]; then
|
||||
echo "✅ Alertmanager configuration is valid"
|
||||
else
|
||||
echo "❌ Alertmanager configuration not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Monitoring verification complete for ${NETWORK} ==="
|
||||
30
scripts/notifications/config.sh
Normal file
30
scripts/notifications/config.sh
Normal file
@@ -0,0 +1,30 @@
|
||||
#!/bin/bash
|
||||
# Notification configuration
|
||||
# Copy this file to config.local.sh and set your values
|
||||
|
||||
# Slack webhook URL for deployment notifications
|
||||
SLACK_WEBHOOK_URL=""
|
||||
|
||||
# Email address for deployment notifications
|
||||
ALERT_EMAIL=""
|
||||
|
||||
# PagerDuty API key for critical alerts
|
||||
PAGERDUTY_API_KEY=""
|
||||
|
||||
# Etherscan API key for contract verification
|
||||
ETHERSCAN_API_KEY=""
|
||||
|
||||
# Testnet explorer API key
|
||||
TESTNET_EXPLORER_API_KEY=""
|
||||
|
||||
# Testnet RPC URL
|
||||
TESTNET_RPC_URL=""
|
||||
|
||||
# Mainnet RPC URL
|
||||
MAINNET_RPC_URL=""
|
||||
|
||||
# Testnet deployer private key (NEVER commit real keys)
|
||||
TESTNET_DEPLOYER_PRIVATE_KEY=""
|
||||
|
||||
# Mainnet deployer private key (NEVER commit real keys)
|
||||
MAINNET_DEPLOYER_PRIVATE_KEY=""
|
||||
78
scripts/notifications/send-deployment-notification.sh
Executable file
78
scripts/notifications/send-deployment-notification.sh
Executable file
@@ -0,0 +1,78 @@
|
||||
#!/bin/bash
|
||||
# Send deployment notifications to configured channels
|
||||
|
||||
set -e
|
||||
|
||||
NETWORK="${1:-testnet}"
|
||||
STATUS="${2:-success}"
|
||||
REPO_DIR="/opt/aitbc"
|
||||
|
||||
echo "=== Sending deployment notification for ${NETWORK} ==="
|
||||
|
||||
# Load notification configuration
|
||||
if [[ -f "${REPO_DIR}/scripts/notifications/config.sh" ]]; then
|
||||
source "${REPO_DIR}/scripts/notifications/config.sh"
|
||||
else
|
||||
echo "⚠️ Notification config not found, using defaults"
|
||||
fi
|
||||
|
||||
# Prepare notification message
|
||||
if [[ "$STATUS" == "success" ]]; then
|
||||
EMOJI="✅"
|
||||
COLOR="good"
|
||||
MESSAGE="Deployment to ${NETWORK} completed successfully"
|
||||
elif [[ "$STATUS" == "failure" ]]; then
|
||||
EMOJI="❌"
|
||||
COLOR="danger"
|
||||
MESSAGE="Deployment to ${NETWORK} failed"
|
||||
else
|
||||
EMOJI="⚠️"
|
||||
COLOR="warning"
|
||||
MESSAGE="Deployment to ${NETWORK} had issues"
|
||||
fi
|
||||
|
||||
# Send Slack notification if configured
|
||||
if [[ -n "${SLACK_WEBHOOK_URL}" ]]; then
|
||||
curl -X POST "${SLACK_WEBHOOK_URL}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{
|
||||
\"text\": \"${EMOJI} ${MESSAGE}\",
|
||||
\"attachments\": [{
|
||||
\"color\": \"${COLOR}\",
|
||||
\"fields\": [
|
||||
{
|
||||
\"title\": \"Network\",
|
||||
\"value\": \"${NETWORK}\",
|
||||
\"short\": true
|
||||
},
|
||||
{
|
||||
\"title\": \"Status\",
|
||||
\"value\": \"${STATUS}\",
|
||||
\"short\": true
|
||||
},
|
||||
{
|
||||
\"title\": \"Timestamp\",
|
||||
\"value\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\",
|
||||
\"short\": true
|
||||
}
|
||||
]
|
||||
}]
|
||||
}"
|
||||
echo "✅ Slack notification sent"
|
||||
else
|
||||
echo "⚠️ Slack webhook not configured"
|
||||
fi
|
||||
|
||||
# Send email notification if configured
|
||||
if [[ -n "${ALERT_EMAIL}" ]] && command -v mail &> /dev/null; then
|
||||
echo "${EMOJI} ${MESSAGE}
|
||||
|
||||
Network: ${NETWORK}
|
||||
Status: ${STATUS}
|
||||
Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" | mail -s "AITBC Deployment: ${NETWORK} - ${STATUS}" "${ALERT_EMAIL}"
|
||||
echo "✅ Email notification sent"
|
||||
else
|
||||
echo "⚠️ Email notification not configured"
|
||||
fi
|
||||
|
||||
echo "=== Deployment notification sent ==="
|
||||
Reference in New Issue
Block a user