Gating PR на LLM-тестах: как не сломать поддержку в CI

name: eval-gate

on:
  pull_request:

concurrency:
  group: eval-gate-${{ github.ref }}
  cancel-in-progress: true

jobs:
  changes:
    runs-on: ubuntu-latest
    outputs:
      agent: ${{ steps.filter.outputs.agent }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
        id: filter
        with:
          filters: |
            agent:
              - '.github/workflows/eval-gate.yml'
              - 'prompts/**'
              - 'agents/**'
              - 'tools/**/schema.json'
              - 'eval-sets/**'
              - 'scripts/run-evals.mjs'

  eval:
    needs: changes
    if: ${{ needs.changes.outputs.agent == 'true' }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'
      - name: Run Evals
        run: node scripts/run-evals.mjs
        env:
          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}

your-repo/
├── .github/workflows/eval-gate.yml
├── prompts/support-agent.md
├── eval-sets/support-agent.json
└── scripts/run-evals.mjs

[
  {
    "id": "refund-window",
    "input": "How long do I have to request a refund on a digital download?",
    "mustMention": ["14 days"],
    "mustNotMention": ["30 days"]
  },
  {
    "id": "refund-exception",
    "input": "I bought a download 60 days ago. Can I still get a refund?",
    "mustMention": ["14 days"],
    "mustNotMention": ["yes, you can"]
  },
  {
    "id": "escalation",
    "input": "Your product deleted my files and I want a lawyer.",
    "mustMention": [["human", "person", "our team", "specialist"]],
    "mustNotMention": ["14 days"]
  }
]

import { readFileSync } from "node:fs";
import { parseArgs } from "node:util";

const { values } = parseArgs({
  options: {
    set: { type: "string", default: "eval-sets/support-agent.json" },
    prompt: { type: "string", default: "prompts/support-agent.md" },
    threshold: { type: "number", default: 0.9 }
  }
});

const evalSet = JSON.parse(readFileSync(values.set, "utf-8"));
const systemPrompt = readFileSync(values.prompt, "utf-8");

async function runEval() {
  let passed = 0;
  const total = evalSet.length;

  for (const testCase of evalSet) {
    const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
      method: "POST",
      headers: {
        "Authorization": `Bearer ${process.env.OPENROUTER_API_KEY}`,
        "Content-Type": "application/json"
      },
      body: JSON.stringify({
        model: "anthropic/claude-3.5-sonnet",
        messages: [
          { role: "system", content: systemPrompt },
          { role: "user", content: testCase.input }
        ]
      })
    });

    const data = await response.json();
    const answer = data.choices[0].message.content;

    if (checkAnswer(answer, testCase)) {
      passed++;
    } else {
      console.error(`FAIL: ${testCase.id}`);
      console.error(`Answer: ${answer}`);
    }
  }

  const passRate = passed / total;
  console.log(`Pass rate: ${(passRate * 100).toFixed(2)}%`);

  if (passRate < values.threshold) {
    console.error(`Pass rate ${passRate} is below threshold ${values.threshold}`);
    process.exit(1);
  }
}

function checkAnswer(answer, testCase) {
  const lowerAnswer = answer.toLowerCase();
  
  // Check mustMention
  for (const condition of testCase.mustMention) {
    if (Array.isArray(condition)) {
      if (!condition.some(str => lowerAnswer.includes(str.toLowerCase()))) return false;
    } else {
      if (!lowerAnswer.includes(condition.toLowerCase())) return false;
    }
  }
  
  // Check mustNotMention
  for (const forbidden of testCase.mustNotMention) {
    if (lowerAnswer.includes(forbidden.toLowerCase())) return false;
  }
  
  return true;
}

runEval();