wget "https://drive.google.com/uc?export=download&id=1TeXM_Z-F3-o6ouooigaEWT3axUJA65kv" -O hotpotqa.jsonl

pip install pandas

system_prompt = """I'd like for you to answer questions about a context text that will be provided. I'll give you a pair
with the form:
Context: "context text"
Question: "a question about the context"
Generate an explicit answer to the question that will be output. Make sure that the answer is the only output you provide,
and the analysis of the context should be kept to yourself. Answer directly and do not prefix the answer with anything such as
"Answer:" nor "The answer is:". The answer has to be the only output you explicitly provide.

The answer has to be as short, direct, and concise as possible. If the answer to the question can not be obtained from the provided
context paragraph, output "UNANSWERABLE". Here's the context and question for you to reason about and answer.
"""

prompt_template = """
Context: {context}

Question: {question}
"""

from typing import List, Dict, Any, Optional
import requests
import json
import pandas as pd

def request_prompt_adaptation(
    system_prompt: str, 
    prompt_template: str, 
    fields: List[str], 
    train_data: List[Dict[str, str | Dict[str, str]]], 
    origin_model: Dict[str, str],
    target_models: List[Dict[str, str]],
    notdiamond_api_key: str,
    evaluation_metric: Optional[str] = None,
    evaluation_config: Optional[Dict] = None,
    request_url: str = "https://api.notdiamond.ai/v2/prompt/adapt"
) -> str:
    """
    Helper method to call the prompt adaptation endpoint
    """
    
    if (evaluation_metric and evaluation_config) or (not evaluation_metric and not evaluation_config):
      raise ValueError("Either evaluation_metric or evaluation_config must be provided, but not both or neither.")
    
    request_body = {
        "system_prompt": system_prompt,
        "template": prompt_template,
        "fields": fields,
        "goldens": train_data,
        "origin_model": origin_model,
        "target_models": target_models,
    }
    
    if evaluation_metric:
        request_body["evaluation_metric"] = evaluation_metric
		
    if evaluation_config:	
        request_body["evaluation_config"] = json.dumps(evaluation_config)

    headers = {
        "Authorization": f"Bearer {notdiamond_api_key}",
        "content-type": "application/json"
    }
    
    resp = requests.post(
        request_url,
        headers=headers,
        json=request_body
    )
    
    if resp.status_code == 200:
        response = resp.json()
        return response["adaptation_run_id"]
    else:
        raise Exception(
            f"Request to adapt prompt failed with code {resp.status_code}: {resp.text}"
        )
        
def get_prompt_adaptation_results(
    adaptation_run_id: str,
    notdiamond_api_key: str,
    base_url: str = "https://api.notdiamond.ai"
) -> Dict[str, Any]:
    """
    Helper method to get the results of a prompt adaptation request.
    
    Calls: {base_url}/v2/prompt/adaptResults/{adaptation_run_id}
    Example: https://api.notdiamond.ai/v2/prompt/adaptResults/00000000-0000-0000-0000-000000000000
    """
    
    headers = {
        "Authorization": f"Bearer {notdiamond_api_key}",
        "content-type": "application/json"
    }
    
    # Construct the full endpoint URL
    endpoint_url = f"{base_url}/v2/prompt/adaptResults/{adaptation_run_id}"
    
    resp = requests.get(endpoint_url, headers=headers)
    
    if resp.status_code == 200:
        response = resp.json()
        return response
    else:
        raise Exception(
            f"Requesting prompt adaptation result failed with code {resp.status_code}: {resp.text}"
        )


def load_json_dataset(dataset_path: str, n_samples: int = 200) -> tuple[list[str], list[dict]]:
    df = pd.read_json(dataset_path, lines=True)
    n_samples = min(n_samples, len(df))
    if n_samples:
        df = df.iloc[[v for v in range(n_samples)]]

    fields: list[str] = ["question", "context"]

    golden_dataset = []
    for idx, row in df.iterrows():
        sample_fields = {"question": row["question"]}
        sample_fields["context"] = "\n\n".join(row["documents"])
        answer = row["response"]

        data_sample = dict(
            fields=sample_fields,
            answer=answer,
        )
        golden_dataset.append(data_sample)
    return fields, golden_dataset

Given the predicted answer and reference answer, compare them and check whether they mean the same.

Following are the given answers:

Predicted Answer: {predicted_answer}
Reference Answer: {gt_answer}

On a NEW LINE, give a score of 1 if the answers mean the same, and 0 if they differ, and nothing more.

evaluation_config = {
    "llm_judging_prompt": (
        "Does the assistant's answer properly answer the user's question? question: {question} answer: {answer} "
        "Score a 1 if the answer is correct, 0 otherwise. Do not output any other values or text - only the score."
        ),
    "llm_judge": "openai/gpt-4o-2024-11-20",
    "correctness_cutoff": 0,
}

fields, pa_ds = load_json_dataset("hotpotqa.jsonl", 25)

print(prompt_template.format(**pa_ds[0]['fields']))

origin_model = {"provider": "google", "model": "gemini-2.5-pro"}
target_models = [
    {"provider": "anthropic", "model": "claude-sonnet-4-5-20250929"},
]

prompt_adaptation_id = request_prompt_adaptation(
    system_prompt=system_prompt,
    prompt_template=prompt_template,
    fields=fields,
    train_data=pa_ds,
    origin_model=origin_model,
    target_models=target_models,
    evaluation_metric="LLMaaJ:Sem_Sim_1",
    notdiamond_api_key="YOUR_NOTDIAMOND_API_KEY"
)
print(prompt_adaptation_id)

prompt_adaptation_id = request_prompt_adaptation(
    system_prompt=system_prompt,
    prompt_template=prompt_template,
    fields=fields,
    train_data=pa_ds,
    origin_model=origin_model,
  	target_models=target_models,
		evaluation_config=evaluation_config,
    notdiamond_api_key="YOUR_NOTDIAMOND_API_KEY"
)

results = get_prompt_adaptation_results(prompt_adaptation_id, "YOUR_NOTDIAMOND_API_KEY")
print(results)

{
  "id": "uuid", // The prompt adaptation id
  "created_at": "datetime", // Timestamp
  "origin_model": {
    "model_name": "openai/gpt-4o-2024-11-20", // The original model the prompt was designed for
    "score": 0.8, // The original model's score on the dataset before optimization
    "evals": {"LLMaaJ:Sem_Sim_1": 0.8}, // The original model's evaluation results on the dataset
    "system_prompt": "...", // The baseline system prompt submitted
    "user_message_template": "...", // The baseline prompt template submitted
    "result_status": "completed" 
  },
  "target_models": [
    {
      "model_name": "anthropic/claude-sonnet-4-5", // The original model the prompt was designed for
      "pre_optimization_score": 0.64, // The target model's score on the dataset before optimization
      "pre_optimization_evals": {"LLMaaJ:Sem_Sim_1": 0.64}, // The target model's evaluation results on the dataset before optimization
      "post_optimization_score": 0.8, // The target model's score on the dataset after optimization
      "post_optimization_evals": {"LLMaaJ:Sem_Sim_1": 0.8}, // The targe model's evaluation results on the dataset after optimization
      "system_prompt": "...", // The baseline system prompt submitted
      "user_message_template": "...", // The baseline prompt template submitted
      "user_message_template_fields": ["..."], // Field arguments in the user_message_template
      "result_status": "completed"
    }
  ],
}