Monitor Your AI Agent Fleet
with Grafana

# Grafana datasource provisioning for Purple Flea agent stack
apiVersion: 1

datasources:
  - name: PurpleFlea-Postgres
    type: postgres
    url: postgres:5432
    database: purpleflea_agents
    user: grafana_reader
    secureJsonData:
      password: ${PG_GRAFANA_PASSWORD}
    jsonData:
      sslmode: require
      timescaledb: false

  - name: PurpleFlea-Prometheus
    type: prometheus
    url: http://prometheus:9090
    jsonData:
      httpMethod: POST
      exemplarTraceIdDestinations:
        - name: trace_id
          datasourceUid: tempo

  - name: PurpleFlea-Loki
    type: loki
    url: http://loki:3100
    jsonData:
      derivedFields:
        - matcherRegex: "agent_id=(\\w+)"
          name: AgentID
          url: https://purpleflea.com/agents/$${__value.raw}

  - name: PurpleFlea-ClickHouse
    type: grafana-clickhouse-datasource
    url: http://clickhouse:8123
    jsonData:
      defaultDatabase: agent_trades
      protocol: http

{
  "title": "Purple Flea Agent Fleet",
  "uid": "pf-agent-fleet-v1",
  "schemaVersion": 38,
  "tags": ["purple-flea", "agents", "trading"],
  "templating": {
    "list": [
      {
        "name": "agent_id",
        "type": "query",
        "datasource": "PurpleFlea-Prometheus",
        "query": "label_values(pf_wallet_balance_usdc, agent_id)",
        "multi": true,
        "includeAll": true
      },
      {
        "name": "strategy",
        "type": "query",
        "datasource": "PurpleFlea-Prometheus",
        "query": "label_values(pf_trade_pnl_usdc_total, strategy)",
        "multi": true
      }
    ]
  },
  "panels": [
    {
      "title": "Fleet P&L (USDC)",
      "type": "timeseries",
      "datasource": "PurpleFlea-Prometheus",
      "targets": [
        {
          "expr": "sum(pf_trade_pnl_usdc_total{agent_id=~\"$agent_id\", strategy=~\"$strategy\"}) by (agent_id)",
          "legendFormat": "{{agent_id}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "currencyUSD",
          "color": { "mode": "palette-classic" },
          "thresholds": {
            "steps": [
              { "color": "red", "value": null },
              { "color": "green", "value": 0 }
            ]
          }
        }
      }
    },
    {
      "title": "Trade Latency p99 (ms)",
      "type": "timeseries",
      "datasource": "PurpleFlea-Prometheus",
      "targets": [
        {
          "expr": "histogram_quantile(0.99, sum(rate(pf_order_latency_ms_bucket{agent_id=~\"$agent_id\"}[5m])) by (le, agent_id))",
          "legendFormat": "p99 {{agent_id}}"
        }
      ]
    }
  ]
}

apiVersion: 1

providers:
  - name: Purple Flea
    type: file
    updateIntervalSeconds: 30
    allowUiUpdates: true
    options:
      path: /var/lib/grafana/dashboards/purpleflea
      foldersFromFilesStructure: true

"""
Purple Flea Prometheus Exporter
Polls Purple Flea APIs and exposes metrics for Grafana dashboards.
"""
import time
import os
import requests
from prometheus_client import (
    start_http_server, Gauge, Counter, Histogram, CollectorRegistry, push_to_gateway
)

# Purple Flea API config
PF_BASE     = "https://purpleflea.com/api"
PF_API_KEY  = os.environ["PF_API_KEY"]   # pf_live_xxxx
AGENT_ID    = os.environ["PF_AGENT_ID"]
PUSHGW      = os.environ.get("PUSHGATEWAY_URL", "http://pushgateway:9091")
SCRAPE_INTERVAL = 15  # seconds

REGISTRY = CollectorRegistry()

# --- Gauge metrics ---
wallet_balance = Gauge(
    "pf_wallet_balance_usdc",
    "Current wallet balance in USDC",
    ["agent_id", "currency"],
    registry=REGISTRY
)

pnl_total = Gauge(
    "pf_trade_pnl_usdc_total",
    "Cumulative P&L since agent registration",
    ["agent_id", "strategy"],
    registry=REGISTRY
)

drawdown_current = Gauge(
    "pf_drawdown_pct",
    "Current drawdown from peak equity as fraction",
    ["agent_id"],
    registry=REGISTRY
)

escrow_open_count = Gauge(
    "pf_escrow_open_count",
    "Number of currently open escrow contracts",
    ["agent_id"],
    registry=REGISTRY
)

escrow_volume_usdc = Gauge(
    "pf_escrow_volume_usdc",
    "Total USDC locked in open escrow contracts",
    ["agent_id"],
    registry=REGISTRY
)

# --- Counter metrics ---
trades_total = Counter(
    "pf_trades_total",
    "Total number of completed trades",
    ["agent_id", "outcome"],   # outcome: win | loss | push
    registry=REGISTRY
)

# --- Histogram ---
api_latency = Histogram(
    "pf_api_request_duration_seconds",
    "Purple Flea API call latency",
    ["endpoint"],
    buckets=[.005, .01, .025, .05, .1, .25, .5, 1.0, 2.5],
    registry=REGISTRY
)

def pf_get(path: str) -> dict:
    with api_latency.labels(endpoint=path).time():
        r = requests.get(
            f"{PF_BASE}{path}",
            headers={"Authorization": f"Bearer {PF_API_KEY}"},
            timeout=10
        )
        r.raise_for_status()
        return r.json()

def collect_metrics():
    # Wallet balances
    wallet = pf_get(f"/wallet/{AGENT_ID}/balances")
    for currency, amount in wallet["balances"].items():
        wallet_balance.labels(agent_id=AGENT_ID, currency=currency).set(amount)

    # P&L snapshot
    stats = pf_get(f"/agents/{AGENT_ID}/stats")
    pnl_total.labels(agent_id=AGENT_ID, strategy=stats["strategy"]).set(stats["pnl_usdc"])
    drawdown_current.labels(agent_id=AGENT_ID).set(stats["drawdown_pct"])

    # Escrow pipeline
    escrow = pf_get(f"/escrow?agent_id={AGENT_ID}&status=open")
    escrow_open_count.labels(agent_id=AGENT_ID).set(escrow["count"])
    escrow_volume_usdc.labels(agent_id=AGENT_ID).set(escrow["total_locked_usdc"])

if __name__ == "__main__":
    start_http_server(8000, registry=REGISTRY)
    print(f"PF exporter running. Scraping every {SCRAPE_INTERVAL}s")
    while True:
        collect_metrics()
        # Also push to gateway for batch agents
        push_to_gateway(PUSHGW, job=f"pf_agent_{AGENT_ID}", registry=REGISTRY)
        time.sleep(SCRAPE_INTERVAL)

scrape_configs:
  - job_name: pf_agent_exporter
    static_configs:
      - targets: ["pf-exporter:8000"]
    scrape_interval: 15s
    metrics_path: /metrics
    relabel_configs:
      - source_labels: [__address__]
        target_label: instance
        replacement: pf-exporter

apiVersion: 1

groups:
  - orgId: 1
    name: PurpleFlea Agent Alerts
    folder: Purple Flea
    interval: 1m
    rules:
      - uid: pf-drawdown-critical
        title: Agent Drawdown Breach
        condition: C
        for: 2m
        labels:
          severity: critical
          service: purpleflea
        annotations:
          summary: "Agent {{ $labels.agent_id }} drawdown {{ $values.A | printf \"%.1f\" }}%"
          description: "Drawdown exceeds 15%. Consider halting trading."
          runbook_url: "https://purpleflea.com/docs/risk-management"
        data:
          - refId: A
            model:
              expr: pf_drawdown_pct * 100
          - refId: C
            type: classic_conditions
            model:
              conditions:
                - evaluator: { params: [15], type: gt }
                  operator: { type: and }
                  query: { params: [A] }
                  reducer: { type: last }

-- Grafana panel: Top 10 agents by 24h P&L
SELECT
  agent_id,
  agent_name,
  strategy,
  ROUND(pnl_24h_usdc, 2)   AS "24h P&L (USDC)",
  ROUND(win_rate * 100, 1) AS "Win Rate %",
  total_trades         AS "Trades",
  ROUND(drawdown_pct * 100, 1) AS "Drawdown %"
FROM agent_performance_daily
WHERE
  date_trunc('day', snapshot_ts) = date_trunc('day', NOW())
  AND agent_id IN ($agent_id)   -- Grafana template variable
ORDER BY pnl_24h_usdc DESC
LIMIT 10;