Initial commit (with migration template)

monitoring/db/docker-compose.yml

@@ -0,0 +1,18 @@
+version: '3.8'
+
+services:
+  litellm-db:
+    image: postgres:16-alpine
+    container_name: litellm-db
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB}
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+    restart: always
+
+volumes:
+  pgdata:

monitoring/install_exporters.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# DGX Exporter (Node Exporter & DCGM Exporter) 자동 기동 스크립트
+
+echo "============================================="
+echo "1. 기존 Exporter 컨테이너가 실행 중이면 정리합니다."
+echo "============================================="
+docker rm -f node-exporter dcgm-exporter 2>/dev/null || true
+
+echo ""
+echo "============================================="
+echo "2. Node Exporter를 실행합니다. (포트: 9100)"
+echo "============================================="
+docker run -d \
+  --name=node-exporter \
+  --restart=always \
+  --net="host" \
+  --pid="host" \
+  -v "/:/host:ro,rslave" \
+  quay.io/prometheus/node-exporter:latest \
+  --path.rootfs=/host
+
+echo ""
+echo "============================================="
+echo "3. NVIDIA DCGM Exporter를 실행합니다. (포트: 9400)"
+echo "============================================="
+docker run -d \
+  --name=dcgm-exporter \
+  --restart=always \
+  --gpus all \
+  -p 9400:9400 \
+  nvcr.io/nvidia/k8s/dcgm-exporter:3.3.5-3.4.0-ubuntu22.04
+
+echo ""
+echo "============================================="
+echo "4. 실행된 Exporter 상태를 확인합니다."
+echo "============================================="
+docker ps -f name=node-exporter -f name=dcgm-exporter

monitoring/litellm_perf_dashboard.json

@@ -0,0 +1,409 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 3,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "title": "LLM Deployment Analytics nex-agi/Nex-N2-mini",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "dfqsz88ybzw1sa"
+      },
+      "targets": [
+        {
+          "expr": "sum(litellm_deployment_state{litellm_model_name=\"nex-agi/Nex-N2-mini\"})",
+          "legendFormat": "Status",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": { "text": "unhealthy", "color": "red" }
+              },
+              "type": "special"
+            },
+            {
+              "options": {
+                "0": { "text": "healthy", "color": "green" },
+                "1": { "text": "unhealthy", "color": "red" }
+              },
+              "type": "value"
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "textMode": "value"
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 3,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "title": "LLM Deployment Analytics nex-agi/Nex-N2-mini-fp8",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "dfqsz88ybzw1sa"
+      },
+      "targets": [
+        {
+          "expr": "sum(litellm_deployment_state{litellm_model_name=\"nex-agi/Nex-N2-mini-fp8\"})",
+          "legendFormat": "Status",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": { "text": "unhealthy", "color": "red" }
+              },
+              "type": "special"
+            },
+            {
+              "options": {
+                "0": { "text": "healthy", "color": "green" },
+                "1": { "text": "unhealthy", "color": "red" }
+              },
+              "type": "value"
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "textMode": "value"
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 0,
+        "y": 3
+      },
+      "id": 3,
+      "title": "Models Latency",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "dfqsz88ybzw1sa"
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.9, sum(rate(litellm_request_total_latency_metric_bucket[5m])) by (le, model))",
+          "legendFormat": "{{model}} - p90",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.5, sum(rate(litellm_request_total_latency_metric_bucket[5m])) by (le, model))",
+          "legendFormat": "{{model}} - p50",
+          "refId": "B"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth"
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        }
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 12,
+        "y": 3
+      },
+      "id": 4,
+      "title": "Models Time To First Byte Latency",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "dfqsz88ybzw1sa"
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.9, sum(rate(litellm_llm_api_time_to_first_token_metric_bucket[5m])) by (le, model))",
+          "legendFormat": "{{model}} - p90",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth"
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        }
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 5,
+        "w": 24,
+        "x": 0,
+        "y": 9
+      },
+      "id": 5,
+      "title": "Requests Rate",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "dfqsz88ybzw1sa"
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(litellm_proxy_total_requests_metric_total[1m])) by (api_key_alias, model)",
+          "legendFormat": "{{model}} ({{api_key_alias}})",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "bar"
+          },
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        }
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 5,
+        "w": 6,
+        "x": 0,
+        "y": 14
+      },
+      "id": 6,
+      "title": "Cumulative Total Tokens",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "dfqsz88ybzw1sa"
+      },
+      "targets": [
+        {
+          "expr": "sum(litellm_total_tokens_metric_total)",
+          "legendFormat": "Total Tokens",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth"
+          },
+          "unit": ""
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        }
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 5,
+        "w": 9,
+        "x": 6,
+        "y": 14
+      },
+      "id": 7,
+      "title": "Cumulative Tokens per Model",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "dfqsz88ybzw1sa"
+      },
+      "targets": [
+        {
+          "expr": "sum(litellm_total_tokens_metric_total) by (model)",
+          "legendFormat": "{{model}} total",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth"
+          },
+          "unit": ""
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        }
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 5,
+        "w": 9,
+        "x": 15,
+        "y": 14
+      },
+      "id": 8,
+      "title": "Real-time Tokens/sec (tps)",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "dfqsz88ybzw1sa"
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(litellm_total_tokens_metric_total[1m])) by (model)",
+          "legendFormat": "{{model}} tps",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth"
+          },
+          "unit": ""
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        }
+      }
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "value": "dfqsz88ybzw1sa",
+          "text": "prometheus"
+        },
+        "datasource": {
+          "type": "prometheus"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "LiteLLM Performance - Compact V8 (Auto-fixed)",
+  "uid": "litellm_perf_compact_v8",
+  "version": 1
+}

monitoring/monitoring-stack/dgx_live_monitor_dashboard.json

@@ -0,0 +1,594 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "title": "TOTAL POWER",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(DCGM_FI_DEV_POWER_USAGE)",
+          "legendFormat": "sum across all GPUs",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "continuous-GrYlRd"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 500 },
+              { "color": "red", "value": 1000 }
+            ]
+          },
+          "unit": "watt"
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "id": 2,
+      "title": "MEAN GPU UTIL",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "avg(DCGM_FI_DEV_GPU_UTIL)",
+          "legendFormat": "average of all devices",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 60 },
+              { "color": "red", "value": 85 }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "id": 3,
+      "title": "CLUSTER MEMORY",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(DCGM_FI_DEV_FB_USED)",
+          "legendFormat": "used / total (UMA)",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "megabytes"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Value"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "custom: / 479 GiB"
+              }
+            ]
+          }
+        ]
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "id": 4,
+      "title": "HOTTEST GPU",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "max(DCGM_FI_DEV_GPU_TEMP)",
+          "legendFormat": "hottest device",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "continuous-GrYlRd"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 65 },
+              { "color": "red", "value": 80 }
+            ]
+          },
+          "unit": "celsius"
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "id": 5,
+      "title": "• JOCODING1 (HEAD)",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus"
+      },
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_POWER_USAGE{instance=\"192.168.0.99:9400\"}",
+          "legendFormat": "POWER",
+          "refId": "A"
+        },
+        {
+          "expr": "DCGM_FI_DEV_GPU_UTIL{instance=\"192.168.0.99:9400\"}",
+          "legendFormat": "GPU",
+          "refId": "B"
+        },
+        {
+          "expr": "DCGM_FI_DEV_GPU_TEMP{instance=\"192.168.0.99:9400\"}",
+          "legendFormat": "TEMP",
+          "refId": "C"
+        },
+        {
+          "expr": "DCGM_FI_DEV_SM_CLOCK{instance=\"192.168.0.99:9400\"}",
+          "legendFormat": "SM CLK",
+          "refId": "D"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "POWER"
+            },
+            "properties": [
+              { "id": "unit", "value": "watt" }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "GPU"
+            },
+            "properties": [
+              { "id": "unit", "value": "percent" }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "TEMP"
+            },
+            "properties": [
+              { "id": "unit", "value": "celsius" }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "SM CLK"
+            },
+            "properties": [
+              { "id": "unit", "value": "megahertz" }
+            ]
+          }
+        ]
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "id": 6,
+      "title": "• JOCODING2",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus"
+      },
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_POWER_USAGE{instance=\"192.168.0.100:9400\"}",
+          "legendFormat": "POWER",
+          "refId": "A"
+        },
+        {
+          "expr": "DCGM_FI_DEV_GPU_UTIL{instance=\"192.168.0.100:9400\"}",
+          "legendFormat": "GPU",
+          "refId": "B"
+        },
+        {
+          "expr": "DCGM_FI_DEV_GPU_TEMP{instance=\"192.168.0.100:9400\"}",
+          "legendFormat": "TEMP",
+          "refId": "C"
+        },
+        {
+          "expr": "DCGM_FI_DEV_SM_CLOCK{instance=\"192.168.0.100:9400\"}",
+          "legendFormat": "SM CLK",
+          "refId": "D"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "POWER"
+            },
+            "properties": [
+              { "id": "unit", "value": "watt" }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "GPU"
+            },
+            "properties": [
+              { "id": "unit", "value": "percent" }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "TEMP"
+            },
+            "properties": [
+              { "id": "unit", "value": "celsius" }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "SM CLK"
+            },
+            "properties": [
+              { "id": "unit", "value": "megahertz" }
+            ]
+          }
+        ]
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 11
+      },
+      "id": 7,
+      "title": "• JOCODING3 (OFFLINE)",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus"
+      },
+      "targets": [],
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "dark-red"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value"
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 11
+      },
+      "id": 8,
+      "title": "• JOCODING4 (OFFLINE)",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus"
+      },
+      "targets": [],
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "dark-red"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value"
+      }
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "GB10 CLUSTER - LIVE MONITOR",
+  "uid": "gb10_live_monitor",
+  "version": 1
+}

monitoring/monitoring-stack/docker-compose.yml

@@ -0,0 +1,36 @@
+version: '3.8'
+
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: litellm-prometheus
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+    ports:
+      - "9090:9090"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--web.cors.origin=.*'
+    restart: always
+    networks:
+      - litellm-network
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: litellm-grafana
+    ports:
+      - "3000:3000"
+    volumes:
+      - grafana-storage:/var/lib/grafana
+    restart: always
+    networks:
+      - litellm-network
+
+volumes:
+  grafana-storage:
+
+networks:
+  litellm-network:
+    external: true

monitoring/monitoring-stack/install_exporters.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+docker rm -f node-exporter dcgm-exporter 2>/dev/null || true
+docker run -d --name=node-exporter --restart=always --net="host" --pid="host" -v "/:/host:ro,rslave" quay.io/prometheus/node-exporter:latest --path.rootfs=/host
+docker run -d --name=dcgm-exporter --restart=always --gpus all -p 9400:9400 nvcr.io/nvidia/k8s/dcgm-exporter:3.3.5-3.4.0-ubuntu22.04
+docker ps -f name=node-exporter -f name=dcgm-exporter

monitoring/monitoring-stack/prometheus/prometheus.yml

@@ -0,0 +1,21 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: 'litellm-proxy'
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: ['litellm:4000']
+
+  - job_name: 'dgx-node-metrics'
+    static_configs:
+      - targets:
+          - '192.168.0.99:9100'
+          - '192.168.0.100:9100'
+
+  - job_name: 'dgx-gpu-metrics'
+    static_configs:
+      - targets:
+          - '192.168.0.99:9400'
+          - '192.168.0.100:9400'

monitoring/test_load.py

@@ -0,0 +1,61 @@
+import urllib.request
+import json
+import time
+import random
+
+url = "http://localhost:8010/v1/chat/completions"
+api_key = "a3dde4205dacc4027f2d50f1afafb3b00de1514e0949b0187f248c940f58f120"
+
+models = ["Helios-LLM"]
+prompts = [
+    "Hello! How are you?",
+    "Tell me a short joke about AI.",
+    "What is the capital of France?",
+    "Explain quantum computing in one sentence."
+]
+
+print("🚀 Starting LiteLLM Load Test Client...")
+print("Every 3 seconds, a random request will be sent to LiteLLM Proxy.")
+print("Press Ctrl+C to stop.\n")
+
+count = 1
+try:
+    while True:
+        model = random.choice(models)
+        prompt = random.choice(prompts)
+        
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {api_key}"
+        }
+        
+        data = {
+            "model": model,
+            "messages": [{"role": "user", "content": prompt}],
+            "max_tokens": 40
+        }
+        
+        req = urllib.request.Request(
+            url, 
+            data=json.dumps(data).encode("utf-8"), 
+            headers=headers,
+            method="POST"
+        )
+        
+        start_time = time.time()
+        try:
+            with urllib.request.urlopen(req, timeout=10) as response:
+                res_data = json.loads(response.read().decode("utf-8"))
+                latency = time.time() - start_time
+                answer = res_data["choices"][0]["message"]["content"].replace("\n", " ").strip()
+                print(f"[{count:03d}] Send to -> {model} | Status: Success (Latency: {latency:.2f}s)")
+                print(f"      Q: \"{prompt}\"")
+                print(f"      A: \"{answer[:70]}...\"\n")
+        except Exception as e:
+            print(f"[{count:03d}] Send to -> {model} | Status: Failed ({e})\n")
+            
+        count += 1
+        time.sleep(3) # 3초 간격 호출
+        
+except KeyboardInterrupt:
+    print("\n👋 Load testing stopped by user.")