import { createPresetEval, interpolatePrompt, generateEvalResult, } from './_utils';
import { DEFAULT_EVAL_MODEL } from './types';
export const selfReferencePrompt = `You are evaluating whether an LLM response inappropriately references the model itself.

## Inputs

- USER REQUEST: {{USER_REQUEST}}
- MODEL RESPONSE: {{MODEL_RESPONSE}}

## Your task

1. Assume a product manager who wants responses to feel like expert answers, not AI disclaimers.
2. Scan the MODEL RESPONSE for any self-referential language, including:
   - Direct AI identification: "As an AI...", "As a language model...", "As an assistant..."
   - Capability disclaimers: "I don't have access to...", "I cannot browse the internet...", "My training data..."
   - Knowledge cutoff references: "As of my knowledge cutoff...", "I don't have information after..."
   - Anthropomorphization disclaimers: "I don't have feelings...", "I don't have opinions..."
   - Meta-statements about generation: "I'm generating this response...", "Based on my training..."
3. Determine whether the USER REQUEST explicitly asked about the model's nature, capabilities, or limitations.

## Pass/Fail

- passed = true only if no self-referential language is present OR the user explicitly asked about the model itself.
- passed = false if any self-referential language appears without explicit user prompting.

## Constraints

- Do not excuse self-reference as "transparency" unless the user asked for it.
- Do not excuse self-reference because it provides useful context.
- First-person pronouns ("I think...", "I would suggest...") are acceptable; AI-specific identification is not.
- References to limitations are only acceptable if directly relevant to why a request cannot be fulfilled.
- "I don't know" is acceptable; "As an AI, I don't know" is not.

## Output format (STRICT JSON, one line reason):

{
  "passed": <true|false>,
  "reason": "<single concise sentence quoting self-referential language found or confirming none detected>"
}`;
export const selfReference = createPresetEval({
    name: 'self-reference',
    description: 'Evaluates whether response inappropriately references the AI model itself',
    options: {
        model: DEFAULT_EVAL_MODEL,
    },
    handler: async (_ctx, input, output, options) => {
        const prompt = interpolatePrompt(selfReferencePrompt, {
            USER_REQUEST: input.request,
            MODEL_RESPONSE: output.response,
        });
        return generateEvalResult({ model: options.model, prompt });
    },
});
//# sourceMappingURL=self-reference.js.map