Logprob evaluator

import { GEPAOptimizer, LogprobEvaluator } from "@gaussia/sdk/prompt-optimizer";

const evaluator = new LogprobEvaluator({
  model,
  criteria: "The answer is one sentence AND addresses every part of the question using only the context.",
}).evaluate;

const result = await GEPAOptimizer.run(EvalSet, datasets, {
  model,
  seedPrompt: "Be brief.",
  objective: "Answer in one sentence that fully covers every part of the question.",
  evaluator,
  failureThreshold: 0.7,
});

const raw = await model.generateText({
  prompt: 'Answer with exactly one word, "Yes" or "No": is the sky blue on a clear day?',
  logprobs: 10, // top-10 alternatives per position
});

for (const pos of raw.logprobs ?? []) {
  const alts = (pos.topLogprobs ?? [])
    .slice(0, 4)
    .map((a) => `${JSON.stringify(a.token)}=${Math.exp(a.logprob).toFixed(3)}`)
    .join("  ");
  console.log(`token ${JSON.stringify(pos.token)} · top: ${alts}`);
}

import { LLMEvaluator, LogprobEvaluator } from "@gaussia/sdk/prompt-optimizer";

const criteria = "The answer FULLY and directly addresses every part of the user's question.";
const logprobJudge = new LogprobEvaluator({ model, criteria }).evaluate;
const structuredJudge = new LLMEvaluator({ model, criteria }).evaluate;

const query = "How do I reset my password and turn on two-factor authentication?";
const context = "Reset from the login page's 'Forgot password' link. Turn on 2FA under Settings → Security.";

const answers = [
  ["complete ", "Reset via the 'Forgot password' link, then enable 2FA under Settings → Security."],
  ["partial  ", "You can reset your password from the login page's 'Forgot password' link."], // ignores 2FA — borderline
  ["off-topic", "Our support team is available 9am–5pm on weekdays."],
];

for (const [label, answer] of answers) {
  const lp = await logprobJudge(answer, "", query, context);
  const st = await structuredJudge(answer, "", query, context);
  console.log(`${label} | logprob ${lp.toFixed(2)} | structured ${st.toFixed(2)}`);
}