temp

.gitignore

@@ -0,0 +1 @@
+node_modules

docker/compose.yml

@@ -0,0 +1,50 @@
+services:
+  loki:
+    image: grafana/loki:latest
+    container_name: loki
+    ports:
+      - "3100:3100"
+    volumes:
+      - ./loki/loki-config.yaml:/etc/loki/local-config.yaml
+      - ./loki/data:/loki
+    command: -config.file=/etc/loki/local-config.yaml
+    restart: unless-stopped
+    networks:
+      - monitoring
+
+  promtail:
+    image: grafana/promtail:latest
+    container_name: promtail
+    volumes:
+      - ./promtail/promtail-config.yaml:/etc/promtail/config.yml
+      - ./logs:/var/log/ai-api  # 挂载你的 AI API 日志目录
+      - /var/run/docker.sock:/var/run/docker.sock  # 可选：用于 Docker 日志采集
+    command: -config.file=/etc/promtail/config.yml
+    restart: unless-stopped
+    depends_on:
+      - loki
+    networks:
+      - monitoring
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    ports:
+      - "3000:3000"
+    volumes:
+      - ./grafana/grafana.ini:/etc/grafana/grafana.ini
+      - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
+      - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
+      - ./grafana/data:/var/lib/grafana
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    restart: unless-stopped
+    depends_on:
+      - loki
+    networks:
+      - monitoring
+
+networks:
+  monitoring:
+    driver: bridge

docker/grafana/dashboards/ai-api-token-analysis.json

@@ -0,0 +1,74 @@
+{
+  "title": "AI API Token Analysis",
+  "panels": [
+    {
+      "title": "Top 10 Most Used Tokens (last 5m)",
+      "type": "barchart",
+      "datasource": "Loki",
+      "targets": [
+        {
+          "expr": "{job=\"ai-api\"} | logfmt | topk(10, count_over_time(5m)) by (token)",
+          "legendFormat": "{{token}}",
+          "refId": "A"
+        }
+      ],
+      "gridPos": { "x": 0, "y": 0, "w": 12, "h": 8 }
+    },
+    {
+      "title": "Requests per Second by Token (last 1m)",
+      "type": "graph",
+      "datasource": "Loki",
+      "targets": [
+        {
+          "expr": "sum by (token) (rate({job=\"ai-api\"}[1m]))",
+          "legendFormat": "{{token}}",
+          "refId": "A"
+        }
+      ],
+      "gridPos": { "x": 12, "y": 0, "w": 12, "h": 8 }
+    },
+    {
+      "title": "Error Rate (HTTP 4xx/5xx)",
+      "type": "stat",
+      "datasource": "Loki",
+      "targets": [
+        {
+          "expr": "sum by (job) (rate({job=\"ai-api\", response_code=~\"[45]..\"}[5m]))",
+          "legendFormat": "Error Rate",
+          "refId": "A"
+        }
+      ],
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "values": false
+        }
+      },
+      "gridPos": { "x": 0, "y": 8, "w": 6, "h": 4 }
+    },
+    {
+      "title": "Avg Latency by Token",
+      "type": "timeseries",
+      "datasource": "Loki",
+      "targets": [
+        {
+          "expr": "{job=\"ai-api\"} | logfmt | avg(latency_ms) by (token)",
+          "legendFormat": "{{token}}",
+          "refId": "A"
+        }
+      ],
+      "gridPos": { "x": 6, "y": 8, "w": 18, "h": 8 }
+    }
+  ],
+  "refresh": "5s",
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timezone": "browser",
+  "panels": [],
+  "schemaVersion": 26,
+  "version": 1,
+  "editable": true,
+  "uid": "ai-api-token-analysis"
+}

docker/grafana/provisioning/datasources/dashboard.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+  - name: 'default'
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletion: false
+    editable: true
+    options:
+      path: /etc/grafana/provisioning/dashboards

docker/grafana/provisioning/datasources/datasource.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    type: loki
+    url: http://loki:3100
+    access: proxy
+    isDefault: true
+    version: 1
+    editable: false

docker/log.sh

@@ -0,0 +1,3 @@
+echo '{"timestamp":"2024-06-01T10:00:01Z","method":"POST","endpoint":"/v1/ai/chat","token":"sk-abc123xyz","ip":"192.168.1.100","user_id":"user_789","response_code":200,"latency_ms":450,"model":"gpt-4"}' > logs/ai-api.json
+echo '{"timestamp":"2024-06-01T10:00:02Z","method":"POST","endpoint":"/v1/ai/chat","token":"sk-def456uvw","ip":"192.168.1.101","user_id":"user_123","response_code":401,"latency_ms":120,"model":"gpt-4"}' >> logs/ai-api.json
+echo '{"timestamp":"2024-06-01T10:00:03Z","method":"POST","endpoint":"/v1/ai/chat","token":"sk-abc123xyz","ip":"192.168.1.102","user_id":"user_789","response_code":200,"latency_ms":380,"model":"gpt-4"}' >> logs/ai-api.json

docker/loki/loki-config.yaml

@@ -0,0 +1,50 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  config:
+    from: 2020-10-24
+
+config_targets:
+  enabled: false
+
+limits_config:
+  # 保留30天日志
+  retention_period: 720h  # 30天 = 720小时
+  max_line_length: 1048576  # 支持长日志
+
+# 启用压缩和合并
+compactor:
+  working_directory: /loki/compactor
+  retention_period: 720h
+  compaction_interval: 1h
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+        http:
+
+ingester:
+  lifecycler:
+    address: 127.0.0.1
+    final_sleep: 0s
+    num_tokens: 512
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1

docker/promtail/promtail-config.yaml

@@ -0,0 +1,64 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  # 采集你的 AI API JSON 日志文件
+  - job_name: ai-api-json
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: ai-api
+          __path__: /var/log/ai-api/*.json  # 指向挂载的日志目录
+
+    pipeline_stages:
+      - json:
+          expressions:
+            token: token
+            endpoint: endpoint
+            user_id: user_id
+            response_code: response_code
+            latency_ms: latency_ms
+            model: model
+      - labels:
+          token:
+          endpoint:
+          user_id:
+          response_code:
+          model:
+
+      # 可选：过滤错误日志（只保留 response_code >= 400）
+      # - drop:
+      #     expression: "response_code < 400"
+
+      # 可选：添加时间戳（如果日志没有 timestamp 字段）
+      # - timestamp:
+      #     source: timestamp
+      #     format: RFC3339
+
+# 可选：如果你用 Docker 容器输出日志，也可以采集容器日志
+# - job_name: docker-containers
+#   docker_sd_configs:
+#     - host: unix:///var/run/docker.sock
+#       refresh_interval: 5s
+#   relabel_configs:
+#     - source_labels: [__meta_docker_container_name]
+#       regex: /(.+)
+#       target_label: container_name
+#     - source_labels: [__meta_docker_container_label_com_docker_compose_service]
+#       target_label: service
+#   pipeline_stages:
+#     - json:
+#         expressions:
+#           level: level
+#           msg: msg
+#     - labels:
+#         level:
+#         service: