eval pipelne WIP

agents/agent_decision.py

@@ -22,6 +22,7 @@ class AgentState(MessagesState):
     current_input: Optional[Union[str, Dict]]
     output: Optional[str]
     retrieval_confidence: float
+    sources: Optional[List[str]] # retrieved_chunks
 
 def create_agent_graph():
     message_filter = MessageFilter(config.rag.llm)
@@ -110,6 +111,7 @@ def create_agent_graph():
             "messages": updated_messages,
             "output": response_output,
             "retrieval_confidence": retrieval_confidence,
+            "sources": response.get("chunks", [])
         })
 
         print(f"new_state in run_rag_agent: {new_state}")
@@ -180,6 +182,7 @@ def init_agent_state() -> AgentState:
         "current_input": None,
         "output": None,
         "retrieval_confidence": 0.0,
+        "sources": [],
     }
 
 def process_query(query: Union[str, Dict], conversation_history: List[BaseMessage] = None) -> Dict[str, Any]:
@@ -215,7 +218,10 @@ def process_query(query: Union[str, Dict], conversation_history: List[BaseMessag
     # for m in result["messages"]:
     #     m.pretty_print()
 
+    print(f"state after: {state}")
+
     return {
         "response": response_text,
-        "messages": state.get("messages", [])
+        "messages": state.get("messages", []),
+        "chunks": state.get("sources", []),
     }
\ No newline at end of file

agents/rag_agent/__init__.py

@@ -158,10 +158,15 @@ class ResearchAssistantRAG:
                 chat_history=chat_history
             )
 
+            print(f"type(reranked_documents) = {type(reranked_documents)}")
+            print(f"reranked_documents = {reranked_documents}")
+
             # Add timing information
             processing_time = time.time() - start_time
             response["processing_time"] = processing_time
 
+            response["chunks"] = [doc["content"] for doc in reranked_documents]
+
             return response
 
         except Exception as e:

api/fastapi_server.py

@@ -7,6 +7,7 @@ import uvicorn
 from agents.agent_decision import process_query
 from typing import List, Union
 from langchain_core.messages import BaseMessage, HumanMessage, AIMessage, SystemMessage
+from typing import Dict, Optional, Union, List, TypedDict, Any, Literal
 
 UPLOAD_FOLDER = "uploads/backend"
 os.makedirs(UPLOAD_FOLDER, exist_ok=True)
@@ -48,11 +49,13 @@ def chat(request: QueryRequest,
         print(f"History: {history}")
         print("Incoming request:", request.model_dump())
 
-        response_data = process_query(
+        response_data: Dict[str, Any] = process_query(
             query=request.query,
             conversation_history=history
         )
 
+        print("response_data =", response_data)
+
         response_text = response_data['response']
 
         response.set_cookie(key="session_id", value=session_id)
@@ -62,9 +65,19 @@ def chat(request: QueryRequest,
             "agent": response_data.get("agent_name", "default_agent")
         }
 
+        if "chunks" in response_data:
+            print(f"Chunks: {response_data['chunks']}")
+            result["context"] = {
+                "data_points": {
+                    "text": response_data["chunks"]
+                }
+            }
+
         if "result_file" in response_data:
             result["result_file_url"] = f"/uploads/{response_data['result_file']}"
 
+        print(f"Result final: {result}")
+
         return result
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))

evals/evaluate.py

+import os
+import json
+import argparse
+import logging
+from pathlib import Path
+import requests
+from metrics import exact_match, fuzzy_match, partial_fuzzy_match, jaccard_similarity
+
+METRIC_FUNCS = {
+    "exact_match": exact_match,
+    "fuzzy_match": fuzzy_match,
+    "partial_fuzzy_match": partial_fuzzy_match,
+    "jaccard_similarity": jaccard_similarity,
+}
+
+def load_config(config_path):
+    with open(config_path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+def load_ground_truth(ground_truth_file):
+    gt = []
+    with open(ground_truth_file, "r", encoding="utf-8") as f:
+        for line in f:
+            gt.append(json.loads(line))
+    return gt
+
+def ask_chatbot(api_url, question):
+    try:
+        response = requests.post(api_url, json={"question": question})
+        if response.status_code == 200:
+            # Sửa nếu response format khác
+            return response.json().get("answer", "")
+        else:
+            return ""
+    except Exception as e:
+        return ""
+
+def evaluate(config):
+    ground_truths = load_ground_truth(config["ground_truth_file"])
+    metrics = config["metrics"]
+    api_url = config["target_url"]
+    results_dir = Path(config["results_dir"])
+    results_dir.mkdir(parents=True, exist_ok=True)
+    results_path = results_dir / "results.jsonl"
+
+    # Log
+    logging.basicConfig(level=logging.INFO)
+
+    # Lưu từng kết quả dòng
+    with open(results_path, "w", encoding="utf-8") as fout:
+        for idx, gt in enumerate(ground_truths):
+            q = gt["question"]
+            gt_ans = gt["ground_truth"]
+            pred = ask_chatbot(api_url, q)
+            result = {
+                "question": q,
+                "ground_truth": gt_ans,
+                "prediction": pred,
+            }
+            # Tính các metric
+            for metric in metrics:
+                func = METRIC_FUNCS.get(metric)
+                if func:
+                    result[metric] = func(pred, gt_ans)
+            fout.write(json.dumps(result, ensure_ascii=False) + "\n")
+            logging.info(f"Done: {idx+1}/{len(ground_truths)}")
+
+    logging.info(f"Evaluation completed. Results saved to {results_path}")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", type=str, default="evals/evaluate_config.json")
+    args = parser.parse_args()
+    config = load_config(args.config)
+    evaluate(config)

evals/evaluate_config.json

+{
+    "testdata_path": "ground_truth.jsonl",
+    "results_dir": "results",
+    "target_url": "http://localhost:8000/chat",
+    "target_parameters": {},
+    "requested_metrics": ["answer_length", "latency"],
+    "target_response_answer_jmespath": "response",
+    "target_response_context_jmespath": "context.data_points.text"
+}

evals/evaluate_metrics/__init__.py

+from .code_metrics import AnswerLengthMetric, LatencyMetric
+# from .prompt_metrics import CoherenceMetric, DontKnownessMetric, GroundednessMetric, RelevanceMetric
+
+metrics = [
+    # CoherenceMetric,
+    # RelevanceMetric,
+    # GroundednessMetric,
+    # DontKnownessMetric,
+    LatencyMetric,
+    AnswerLengthMetric,
+]
+
+metrics_by_name = {metric.METRIC_NAME: metric for metric in metrics}
\ No newline at end of file

evals/evaluate_metrics/base_metric.py

+import logging
+from abc import ABC, abstractmethod
+import pandas as pd
+
+logger = logging.getLogger("evaltools")
+
+class BaseMetric(ABC):
+    METRIC_NAME = "name_of_metric"
+
+    @classmethod
+    @abstractmethod
+    def get_aggregate_stats(cls, df):
+        """Returns a dictionary of aggregate statistics for the metric"""
+        pass
+
+    @classmethod
+    def get_aggregate_stats_for_numeric_rating(cls, df, rating_column_name):
+        # Narrow down dataframe to just the metric
+        df = df[[rating_column_name]]
+
+        # Drop invalid ratings - strings like "Failed"
+        rows_before = len(df)
+        df = df.apply(pd.to_numeric, errors="coerce")
+        df = df.dropna()
+        rows_after = len(df)
+        if rows_before != rows_after:
+            logger.warning(
+                "Dropped %d invalid ratings for metric %s",
+                rows_before - rows_after,
+                rating_column_name,
+            )
+
+        # Count how many ratings passed threshold of 4+
+        pass_count = int(df[rating_column_name].apply(lambda rating: rating >= 4).sum())
+
+        return {
+            "pass_count": pass_count,
+            "pass_rate": round(pass_count / rows_before, 2),
+            "mean_rating": round(df[rating_column_name].mean(), 2),
+        }

evals/evaluate_metrics/code_metrics.py

+import logging
+
+from .base_metric import BaseMetric
+
+logger = logging.getLogger("evaltools")
+
+
+class AnswerLengthMetric(BaseMetric):
+    METRIC_NAME = "answer_length"
+
+    @classmethod
+    def evaluator_fn(cls, **kwargs):
+        def answer_length(*, response, **kwargs):
+            if response is None:
+                logger.warning("Received response of None, can't compute answer_length metric. Setting to -1.")
+                return {cls.METRIC_NAME: -1}
+            return {cls.METRIC_NAME: len(response)}
+
+        return answer_length
+
+    @classmethod
+    def get_aggregate_stats(cls, df):
+        # remove -1 values from the mean calculation
+        df = df[df[cls.METRIC_NAME] != -1]
+        return {
+            "mean": round(df[cls.METRIC_NAME].mean(), 2),
+            "max": int(df[cls.METRIC_NAME].max()),
+            "min": int(df[cls.METRIC_NAME].min()),
+        }
+
+
+class LatencyMetric(BaseMetric):
+    METRIC_NAME = "latency"
+
+    @classmethod
+    def evaluator_fn(cls, **kwargs):
+        def latency(**kwargs):
+            # Return no additional data, since latency is already stored in the target response
+            return {}
+
+        return latency
+
+    @classmethod
+    def get_aggregate_stats(cls, df):
+        return {
+            "mean": round(df[cls.METRIC_NAME].mean(), 2),
+            "max": df[cls.METRIC_NAME].max(),
+            "min": df[cls.METRIC_NAME].min(),
+        }

evals/evaluate_metrics/prompt_metrics.py

+import logging
+import re
+from pathlib import Path
+
+import numpy as np
+from promptflow.client import load_flow
+
+from .base_metric import BaseMetric
+
+PROMPT_TEMPLATE_DIR = Path(__file__).resolve().parent / "prompts"
+
+logger = logging.getLogger("evaltools")
+
+
+class PromptBasedEvaluator:
+    def __init__(self, model_config, path, name):
+        prompty_model_config = {"configuration": model_config}
+        self._name = name
+        self._flow = load_flow(source=path, model=prompty_model_config)
+
+    def __call__(self, **kwargs) -> dict:
+        llm_output = self._flow(**kwargs)
+
+        score = np.nan
+        if llm_output:
+            match = re.search(r"\d", llm_output)
+            if match:
+                score = float(match.group())
+            else:
+                logging.warning(
+                    "No score found in answer: %s\nMake sure prompty file is correctly formatted.", llm_output
+                )
+
+        output = {}
+        output[self._name] = float(score)
+        return output
+
+
+class CustomRatingMetric(BaseMetric):
+    @classmethod
+    def evaluator_fn(cls, openai_config, **kwargs):
+        return PromptBasedEvaluator(
+            openai_config, path=PROMPT_TEMPLATE_DIR / f"{cls.METRIC_NAME}.prompty", name=cls.METRIC_NAME
+        )
+
+    @classmethod
+    def get_aggregate_stats(cls, df):
+        return cls.get_aggregate_stats_for_numeric_rating(df, cls.METRIC_NAME)
+
+
+class RelevanceMetric(CustomRatingMetric):
+    METRIC_NAME = "myrelevance"
+
+
+class CoherenceMetric(CustomRatingMetric):
+    METRIC_NAME = "mycoherence"
+
+
+class GroundednessMetric(CustomRatingMetric):
+    METRIC_NAME = "mygroundedness"
+
+
+class DontKnownessMetric(CustomRatingMetric):
+    METRIC_NAME = "dontknowness"

evals/evaluate_runner.py

+import argparse
+import logging
+from pathlib import Path
+from rich.logging import RichHandler
+
+from evaluation import run_evaluate_from_config
+
+
+# Giả sử bạn đã import run_evaluate_from_config ở file khác, hoặc đặt nó trong cùng file
+
+def get_openai_config():
+    import os
+    return {
+        # "api_key": os.environ["OPENAI_API_KEY"],
+        # "model": os.environ.get("OPENAI_MODEL", "gpt-4")
+        "api_key": "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",
+        "model": "gpt-4"
+    }
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=logging.WARNING, format="%(message)s", datefmt="[%X]", handlers=[RichHandler(rich_tracebacks=True)]
+    )
+    # Tùy chỉnh log level nếu muốn nhiều thông tin hơn:
+    # logger.setLevel(logging.INFO)
+    # logging.getLogger("evaltools").setLevel(logging.INFO)
+
+    parser = argparse.ArgumentParser(description="Run evaluation with OpenAI configuration.")
+    parser.add_argument("--targeturl", type=str, help="Specify the target URL.", required=True)
+    parser.add_argument("--resultsdir", type=Path, help="Specify the results directory.", required=True)
+    parser.add_argument("--numquestions", type=int, help="Specify the number of questions.", required=False)
+
+    args = parser.parse_args()
+
+    openai_config = get_openai_config()
+
+    run_evaluate_from_config(
+        working_dir=Path(__file__).parent,
+        config_path="evaluate_config.json",
+        num_questions=args.numquestions,
+        target_url=args.targeturl,
+        results_dir=args.resultsdir,
+        openai_config=openai_config,
+        model=openai_config["model"],     # Lấy model từ config
+        azure_credential=None,            # KHÔNG dùng azure_credential
+    )

evals/evaluation.py

+import json
+import logging
+import os
+import time
+from pathlib import Path
+
+import jmespath
+import pandas as pd
+import requests
+from rich.progress import track
+
+from evaluate_metrics import metrics_by_name
+
+logger = logging.getLogger("evaltools")
+
+
+def send_question_to_target(
+    question: str,
+    url: str,
+    parameters: dict = {},
+    raise_error=False,
+    response_answer_jmespath="message.content",
+    response_context_jmespath="context.data_points.text",
+):
+    headers = {"Content-Type": "application/json"}
+    # body = {
+    #     "messages": [{"content": question, "role": "user"}],
+    #     "context": parameters,
+    # }
+    body = {
+        "query": question,
+        "conversation_history": parameters.get("conversation_history", [])
+    }
+    try:
+        r = requests.post(url, headers=headers, json=body)
+        r.encoding = "utf-8"
+
+        latency = r.elapsed.total_seconds()
+
+        try:
+            response_dict = r.json()
+        except json.JSONDecodeError:
+            raise ValueError(
+                f"Response from target {url} is not valid JSON:\n\n{r.text} \n"
+                "Make sure that your configuration points at a chat endpoint that returns a single JSON object.\n"
+            )
+
+        try:
+            answer = jmespath.search(response_answer_jmespath, response_dict)
+            data_points = jmespath.search(response_context_jmespath, response_dict)
+            if isinstance(data_points, dict):
+                context = json.dumps(data_points, ensure_ascii=False)
+            elif isinstance(data_points, list):
+                context = "\n\n".join(data_points)
+            elif data_points is not None:
+                context = data_points
+            else:
+                raise ValueError("Context is missing")
+        except Exception:
+            raise ValueError(
+                "Response does not adhere to the expected schema. "
+                f"The answer should be accessible via the JMESPath expression '{response_answer_jmespath}' "
+                f"and the context should be accessible via the JMESPath expression '{response_context_jmespath}'. "
+                "Either adjust the app response or adjust send_question_to_target() in evaluate.py "
+                f"to match the actual schema.\nResponse: {response_dict}"
+            )
+
+        response_obj = {"answer": answer, "context": context, "latency": latency}
+        return response_obj
+    except Exception as e:
+        if raise_error:
+            raise e
+        return {
+            "answer": str(e),
+            "context": str(e),
+            "latency": -1,
+        }
+
+
+def truncate_for_log(s: str, max_length=50):
+    return s if len(s) < max_length else s[:max_length] + "..."
+
+
+def load_jsonl(path: Path) -> list[dict]:
+    with open(path, encoding="utf-8") as f:
+        return [json.loads(line) for line in f.readlines()]
+
+
+def run_evaluation(
+    openai_config: dict,
+    testdata_path: Path,
+    results_dir: Path,
+    target_url: str,
+    target_parameters=None,
+    requested_metrics=None,
+    num_questions=None,
+    target_response_answer_jmespath=None,
+    target_response_context_jmespath=None,
+    model=None,
+    azure_credential=None,
+):
+    logger.info("Running evaluation using data from %s", testdata_path)
+    testdata = load_jsonl(testdata_path)
+    if num_questions:
+        logger.info("Limiting evaluation to %s questions", num_questions)
+        testdata = testdata[:num_questions]
+
+    logger.info("Sending a test question to the target to ensure it is running...")
+    try:
+        question = "What information is in your knowledge base?"
+        target_data = send_question_to_target(
+            question,
+            target_url,
+            target_parameters,
+            raise_error=True,
+            response_answer_jmespath=target_response_answer_jmespath,
+            response_context_jmespath=target_response_context_jmespath,
+        )
+        logger.info(
+            'Successfully received response from target for question: "%s"\n"answer": "%s"\n"context": "%s"',
+            truncate_for_log(question),
+            truncate_for_log(target_data["answer"]),
+            truncate_for_log(target_data["context"]),
+        )
+    except Exception as e:
+        logger.error("Failed to send a test question to the target due to error: \n%s", e)
+        return False
+
+    logger.info("Sending a test chat completion to the GPT deployment to ensure it is running...")
+
+    # TODO: Tạo một hàm/tạo client cho openai hoặc pass nếu không dùng
+    gpt_response = None
+    try:
+        logger.info("(Skipped GPT deployment test -- implement if needed)")
+    except Exception as e:
+        logger.error("Failed to send test message to GPT deployment: %s", e)
+        return False
+
+    logger.info("Starting evaluation...")
+    for metric in requested_metrics:
+        if metric not in metrics_by_name:
+            logger.error(f"Requested metric {metric} is not available. Available metrics: {metrics_by_name.keys()}")
+            return False
+
+    requested_metrics = [
+        metrics_by_name[metric_name] for metric_name in requested_metrics if metric_name in metrics_by_name
+    ]
+
+    def evaluate_row(row):
+        output = {}
+        output["question"] = row["question"]
+        output["truth"] = row["truth"]
+        target_response = send_question_to_target(
+            question=row["question"],
+            url=target_url,
+            parameters=target_parameters,
+            response_answer_jmespath=target_response_answer_jmespath,
+            response_context_jmespath=target_response_context_jmespath,
+        )
+        output.update(target_response)
+        for metric in requested_metrics:
+            result = metric.evaluator_fn(openai_config=openai_config)(
+                query=row["question"],
+                response=output["answer"],
+                context=output["context"],
+                ground_truth=row["truth"],
+            )
+            output.update(result)
+
+        return output
+
+    questions_with_ratings = []
+    for row in track(testdata, description="Processing..."):
+        questions_with_ratings.append(evaluate_row(row))
+
+    logger.info("Evaluation calls have completed. Calculating overall metrics now...")
+    results_dir.mkdir(parents=True, exist_ok=True)
+    with open(results_dir / "eval_results.jsonl", "w", encoding="utf-8") as results_file:
+        for row in questions_with_ratings:
+            results_file.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+    df = pd.DataFrame(questions_with_ratings)
+    summary = {}
+    for metric in requested_metrics:
+        summary[metric.METRIC_NAME] = metric.get_aggregate_stats(df)
+    summary["num_questions"] = {"total": len(df)}
+
+    with open(results_dir / "summary.json", "w", encoding="utf-8") as summary_file:
+        summary_file.write(json.dumps(summary, indent=4))
+
+    with open(results_dir / "evaluate_parameters.json", "w", encoding="utf-8") as parameters_file:
+        parameters = {
+            "evaluation_gpt_model": model,
+            "evaluation_timestamp": int(time.time()),
+            "testdata_path": str(testdata_path),
+            "target_url": target_url,
+            "target_parameters": target_parameters,
+            "num_questions": num_questions,
+        }
+        parameters_file.write(json.dumps(parameters, indent=4))
+    logger.info("Evaluation results saved in %s", results_dir)
+    return True
+
+
+def process_config(obj: dict):
+    if isinstance(obj, dict):
+        for key in obj:
+            if isinstance(obj[key], dict):
+                process_config(obj[key])
+            elif isinstance(obj[key], str) and "<TIMESTAMP>" in obj[key]:
+                logger.info("Replaced %s in config with timestamp", key)
+                obj[key] = obj[key].replace("<TIMESTAMP>", str(int(time.time())))
+            elif isinstance(obj[key], str) and "<READFILE>" in obj[key]:
+                with open(obj[key].replace("<READFILE>", ""), encoding="utf-8") as f:
+                    logger.info("Replaced %s in config with contents of %s", key, f.name)
+                    obj[key] = f.read()
+
+
+def run_evaluate_from_config(
+    working_dir,
+    config_path,
+    num_questions=None,
+    target_url=None,
+    results_dir=None,
+    openai_config=None,
+    model=None,
+    azure_credential=None,
+):
+    config_path = working_dir / Path(config_path)
+    logger.info("Running evaluation from config %s", config_path)
+    with open(config_path, encoding="utf-8") as f:
+        config = json.load(f)
+        process_config(config)
+
+    if results_dir is None:
+        results_dir = working_dir / Path(config["results_dir"])
+
+    if openai_config is None:
+        # TODO: Lấy openai_config phù hợp môi trường của bạn
+        openai_config = {}
+
+    evaluation_run_complete = run_evaluation(
+        openai_config=openai_config,
+        testdata_path=working_dir / config["testdata_path"],
+        results_dir=results_dir,
+        target_url=target_url or config["target_url"],
+        target_parameters=config.get("target_parameters", {}),
+        num_questions=num_questions,
+        requested_metrics=config.get(
+            "requested_metrics",
+            ["gpt_groundedness", "gpt_relevance", "gpt_coherence", "answer_length", "latency"],
+        ),
+        target_response_answer_jmespath=config.get("target_response_answer_jmespath", "message.content"),
+        target_response_context_jmespath=config.get("target_response_context_jmespath", "context.data_points.text"),
+        model=model or os.environ.get("OPENAI_GPT_MODEL", None),
+        azure_credential=azure_credential,
+    )
+
+    if evaluation_run_complete:
+        results_config_path = results_dir / "config.json"
+        logger.info("Saving original config file back to to %s", results_config_path)
+        with open(config_path, encoding="utf-8") as input_config:
+            with open(results_config_path, "w", encoding="utf-8") as output_config:
+                output_config.write(input_config.read())
+    else:
+        logger.error("Evaluation was terminated early due to an error ⬆")

evals/ground_truth.jsonl

+{"question": "Was ist das Hauptziel der Bachelorarbeit 'Verladegeräusche in Speditionen'?", "truth": "Das Hauptziel der Arbeit ist die Analyse und Bewertung von Verladetätigkeiten in Speditionen mit verschiedenen Flurfördergeräten und Anhängern, um Emissionsdatenblätter zu erstellen, die als Grundlage für Schallimmissionsprognosen dienen können. Außerdem werden mögliche Lärmminderungsmaßnahmen vorgestellt."}
+{"question": "Welche Methoden wurden zur Messung und Analyse der Verladegeräusche verwendet?", "truth": "Zur Messung der Verladegeräusche wurde der Schalldruck im Fernfeld gemessen und das Hüllflächen-Verfahren zur Ermittlung der Schallleistung angewendet. Zusätzlich wurde die Richtcharakteristik entlang einer Kreisbahn im Nahfeld analysiert und eine Vergleichsanalyse zwischen simulierten und tatsächlichen Messungen durchgeführt."}
+{"question": "Welche Unterschiede wurden zwischen Gabelstapler und Gabelhubwagen bei den Verladegeräuschen festgestellt?", "truth": "Die Beladung mit Gabelstapler und Gabelhubwagen weist ähnliche Schallleistungspegel auf, jedoch gibt es Unterschiede in der Impulshaftigkeit der Geräuschentwicklung. Der Gabelhubwagen erzeugt mehr impulsartige Schallereignisse, während der Gabelstapler eine kontinuierlichere Geräuschabstrahlung aufweist."}
+{"question": "Welche Maßnahmen zur Lärmminderung werden in der Arbeit vorgeschlagen?", "truth": "Zu den vorgeschlagenen Maßnahmen zur Lärmminderung zählen die Verwendung von weicheren Oberflächen auf Überladebrücken, Dröhnfolien an deren Unterseite, speziell angepasste Stoßdämpfer in Anhängern sowie organisatorische Maßnahmen auf dem Gelände und das richtige Verhalten des Personals."}
+{"question": "Welche Erkenntnisse liefert die Arbeit in Bezug auf Verladegeräusche in verschiedenen Anhängertypen?", "truth": "Die Arbeit zeigt, dass es gewisse Variationen der Schallleistungspegel je nach Flurfördergerät und Anhängertyp gibt, aber keinen signifikanten Unterschied zwischen den Beladevorgängen der Flurfördergeräte in verschiedenen Anhängern. Die Schallabstrahlung erfolgt weitestgehend in alle Richtungen, ohne ausgeprägte Richtcharakteristik."}
+{"question": "Was versteht man unter 'Schallimmissionsprognose' laut der Arbeit?", "truth": "Eine Schallimmissionsprognose ist eine Prognose der Schallimmissionen, zum Beispiel von Industrie- und Gewerbelärm, basierend auf messtechnisch ermittelten Schalldruckpegeln. Für die Prognose werden schalltechnische Eingangsdaten benötigt, die für Simulationen verwendet werden."}
+{"question": "Welche Aufgaben übernehmen Logistikunternehmen im Zusammenhang mit dem Güterfluss?", "truth": "Logistikunternehmen, auch Speditionen genannt, sind verantwortlich für den Güterfluss sowie die Planung, Steuerung und Kontrolle aller damit verbundenen Prozesse. Sie kümmern sich um Menge, Art und Handhabungseigenschaften der Güter und organisieren deren Transport, Umschlag und Lagerung."}
+{"question": "Welche Arten von Flurfördergeräten werden im Text genannt?", "truth": "Im Text werden unter anderem Handgabelhubwagen und Gabelstapler als Beispiele für Flurfördergeräte genannt. Es gibt verschiedene Arten, die sich in Antriebsart, Bedienungsart und Bauart unterscheiden können."}
+{"question": "Welche Bedeutung hat die Logistikkette laut Arbeit?", "truth": "Die Logistikkette ist ein zentraler Bestandteil der Logistik. Sie beschreibt den Ablauf von der Güterbereitstellung über die Verwendung bis zur Verteilung und Zwischenlagerung der Güter. Unternehmen sind durch die Logistikkette miteinander verbunden, um den Austausch von Gütern zu ermöglichen."}
+{"question": "Worin besteht der Unterschied zwischen in der Arbeit genannten Emissionsdatenblättern und bisherigen Veröffentlichungen?", "truth": "Die in der Arbeit erstellten Emissionsdatenblätter bieten eine Grundlage für präzisere Schallimmissionsprognosen, insbesondere für verschiedene Flurfördergeräte und Anhängertypen. Bisherige Veröffentlichungen konzentrierten sich hauptsächlich auf Verbrauchermärkte und Verladungen mit Handhubwagen, was nur einen Teil der tatsächlichen Verladetätigkeiten abdeckt."}

evals/metrics.py

+# evals/metrics.py
+import difflib
+from rapidfuzz import fuzz
+
+def exact_match(response, ground_truth):
+    # True nếu giống hệt nhau, False nếu khác
+    return response.strip() == ground_truth.strip()
+
+def fuzzy_match(response, ground_truth):
+    # Fuzzy matching (tỷ lệ phần trăm tương đồng)
+    return fuzz.ratio(response, ground_truth) / 100.0
+
+def partial_fuzzy_match(response, ground_truth):
+    # Đánh giá theo độ trùng khớp một phần
+    return fuzz.partial_ratio(response, ground_truth) / 100.0
+
+def jaccard_similarity(response, ground_truth):
+    set1 = set(response.lower().split())
+    set2 = set(ground_truth.lower().split())
+    if not set1 or not set2:
+        return 0.0
+    return len(set1 & set2) / len(set1 | set2)

requirements.txt

@@ -33,4 +33,8 @@ pydantic~=2.11.3
 numpy~=1.26.4
 docling~=2.31.0
 python-dotenv~=1.1.0
-openai~=1.76.0
\ No newline at end of file
+openai~=1.76.0
+rapidfuzz~=3.13.0
+jmespath~=1.0.1
+rich~=14.0.0
+promptflow
\ No newline at end of file