Exact Match Evaluation

The simplest evaluation type is direct string comparison. LangChain has a prebuilt "exact match" evaluator you can use, or you can do the same with a custom evaluator.

You can check out the example results here.

# %pip install -U --quiet langchain langchain_openai

import os

# Update with your API URL if using a hosted instance of Langsmith.
os.environ["LANGCHAIN_ENDPOINT"] = "https://api.smith.langchain.com"
# Update with your API key
os.environ["LANGCHAIN_API_KEY"] = "YOUR API KEY"
os.environ["OPENAI_API_KEY"] = "Your openai api key"

Create Dataset

First you create a simple dataset of input and expected output pairs.

import langsmith

client = langsmith.Client()
dataset_name = "Oracle of Exactness"
if not client.has_dataset(dataset_name=dataset_name):
    ds = client.create_dataset(dataset_name)
    client.create_examples(
        inputs=[
            {
                "prompt_template": "State the year of the declaration of independence."
                "Respond with just the year in digits, nothign else"
            },
            {"prompt_template": "What's the average speed of an unladen swallow?"},
        ],
        outputs=[{"output": "1776"}, {"output": "5"}],
        dataset_id=ds.id,
    )

Evaluate

from langchain.smith import RunEvalConfig
from langchain_openai import ChatOpenAI
from langsmith.evaluation import EvaluationResult, run_evaluator

model = "gpt-3.5-turbo"


# This is your model/system that you want to evaluate
def predict_result(input_: dict) -> dict:
    response = ChatOpenAI(model=model).invoke(input_["prompt_template"])
    return {"output": response.content}


@run_evaluator
def compare_label(run, example) -> EvaluationResult:
    # Custom evaluators let you define how "exact" the match ought to be
    # It also lets you flexibly pick the fields to compare
    prediction = run.outputs.get("output") or ""
    target = example.outputs.get("output") or ""
    match = prediction and prediction == target
    return EvaluationResult(key="matches_label", score=match)


# This defines how you generate metrics about the model's performance
eval_config = RunEvalConfig(
    evaluators=["exact_match"],  # equivalent prebuilt evaluator
    custom_evaluators=[compare_label],
)

client.run_on_dataset(
    dataset_name=dataset_name,
    llm_or_chain_factory=predict_result,
    evaluation=eval_config,
    verbose=True,
    project_metadata={"version": "1.0.0", "model": model},
)

View the evaluation results for project 'impressionable-crew-29' at: https://smith.langchain.com/o/30239cd8-922f-4722-808d-897e1e722845/datasets/4f23ec54-3cf8-44fc-a729-ce08ad855bfd/compare?selectedSessions=a0672ba4-e513-4fef-84b8-bab439581721

View all tests for Dataset Oracle of Exactness at: https://smith.langchain.com/o/30239cd8-922f-4722-808d-897e1e722845/datasets/4f23ec54-3cf8-44fc-a729-ce08ad855bfd [------------------------------------------------->] 2/2

<h3>Experiment Results:</h3>

| feedback.exact_match | feedback.matches_label | error | execution_time | run_id | count | unique | top | freq | mean | std | min | 25% | 50% | 75% | max --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- 2.000000 | 2 | 0 | 2.000000 | 2 NaN | 2 | 0 | NaN | 2 NaN | False | NaN | NaN | 2b4532af-445e-46aa-8170-d34c3af724a8 NaN | 1 | NaN | NaN | 1 0.500000 | NaN | NaN | 0.545045 | NaN 0.707107 | NaN | NaN | 0.265404 | NaN 0.000000 | NaN | NaN | 0.357376 | NaN 0.250000 | NaN | NaN | 0.451211 | NaN 0.500000 | NaN | NaN | 0.545045 | NaN 0.750000 | NaN | NaN | 0.638880 | NaN 1.000000 | NaN | NaN | 0.732714 | NaN

{'project_name': 'impressionable-crew-29', 'results': {'893730f0-393d-4c40-92f9-16ce24aaec1f': {'input': {'prompt_template': "What's the average speed of an unladen swallow?"}, 'feedback': [EvaluationResult(key='exact_match', score=0, value=None, comment=None, correction=None, evaluator_info={'__run': RunInfo(run_id=UUID('089a016a-d847-4a26-850c-afc0e78879d5'))}, source_run_id=None, target_run_id=None), EvaluationResult(key='matches_label', score=False, value=None, comment=None, correction=None, evaluator_info={}, source_run_id=None, target_run_id=None)], 'execution_time': 0.732714, 'run_id': '2b4532af-445e-46aa-8170-d34c3af724a8', 'output': {'output': 'The average speed of an unladen European swallow is approximately 20.1 miles per hour (32.4 km/h).'}, 'reference': {'output': '5'}}, 'ec9d8754-d264-4cec-802e-0c33513843d8': {'input': {'prompt_template': 'State the year of the declaration of independence.Respond with just the year in digits, nothign else'}, 'feedback': [EvaluationResult(key='exact_match', score=1, value=None, comment=None, correction=None, evaluator_info={'__run': RunInfo(run_id=UUID('cd4c7ede-f367-4d9c-b424-577bf054bf21'))}, source_run_id=None, target_run_id=None), EvaluationResult(key='matches_label', score=True, value=None, comment=None, correction=None, evaluator_info={}, source_run_id=None, target_run_id=None)], 'execution_time': 0.357376, 'run_id': '82b65c5c-bfbf-4d2b-9c05-3bbd1cd4e711', 'output': {'output': '1776'}, 'reference': {'output': '1776'}}}}