LLMを活用した生成AIの評価(LLM as a Judge)

from langchain.chains import RetrievalQA
from langchain.prompts import PromptTemplate
from langchain.chat_models import ChatDatabricks

TEMPLATE = """あなたはITのシステムエンジニアです。ITインフラストラクチャの管理の質問に答えています。質問がこれらのトピックのいずれかに関連していない場合は、回答をご遠慮ください。答えがわからない場合は、答えをでっち上げようとせず、わからないと言ってください。箇条書きで答えるのは避けてください。

次の文脈を使用して、最後の質問に答えてください。
{context}
質問: {question}
"""

prompt = PromptTemplate(template=TEMPLATE, input_variables=["context", "question"])

chain = RetrievalQA.from_chain_type(
　　llm=chat_model,
　　chain_type="stuff",
　　retriever=get_retriever(),
　　chain_type_kwargs={"prompt": prompt},
)

from mlflow.models import infer_signature
import mlflow
import langchain
from mlflow import MlflowClient

mlflow.set_registry_uri("databricks-uc")
model_name = "nos_data_llm.rag.nogblog_langchain"

with mlflow.start_run(run_name=run_name) as run:
　signature = infer_signature(question, answer)

　model_info = mlflow.langchain.log_model(
　　chain,
　　loader_fn=get_retriever,
　　artifact_path="chain",
　　registered_model_name=model_name,
　　pip_requirements=[
　　　"mlflow==2.9.2",
　　　"langchain==" + langchain.__version__,
　　　"databricks-vectorsearch",
　　],
　　input_example=question,
　)

　evaluate_results = mlflow.evaluate(
　　model=model_info.model_uri,
　　data=eval_data.toPandas(),
　　targets="ground_truth",
　　model_type = "question-answering",
　　evaluators="default",
　　extra_metrics=[mlflow.metrics.latency(),mlflow.metrics.rouge1(),mlflow.metrics.rougeL(),answer_relevance_metric, answer_correctness_metric,answer_similarity_metric],
　　evaluator_config={
　　　"col_mapping": {
　　　"inputs": "query"
　　　}
　　},
)

custom_correctness = make_genai_metric(
　name="custom_correctness",
　definition=(
　　"Answer correctness is evaluated on the accuracy of the provided output based on the "
　　"provided targets, which is the ground truth. Scores can be assigned based on the degree "
　　"of semantic similarity and factual correctness of the provided output to the provided "
　　"targets, where a higher score indicates higher degree of accuracy."
　),
　grading_prompt=(
　　"Answer Correctness: Below are the details for different scores:\n"
　　"- Score 1: The output is completely incorrect. It is completely different from or "
　　"contradicts the provided targets.\n"
　　"- Score 2: The output demonstrates some degree of semantic similarity and includes "
　　"partially correct information. However, the output still has significant discrepancies "
　　"with the provided targets or inaccuracies.\n"
　　"- Score 3: The output addresses a couple of aspects of the input accurately, aligning "
　　"with the provided targets. However, there are still omissions or minor inaccuracies.\n"
　　"- Score 4: The output is mostly correct. It provides mostly accurate information, but "
　　"there may be one or more minor omissions or inaccuracies.\n"
　　"- Score 5: The output is correct. It demonstrates a high degree of accuracy and "
　　"semantic similarity to the targets."
　),

　examples=[custom_correctness_example_score_2, custom_correctness_example_score_4],
　model="openai:/gpt-4",
　grading_context_columns=["targets"],
　parameters={"max_tokens": 400},
　aggregations=["mean", "variance"],
　greater_is_better=True,
　max_workers=2,
)