wget "https://drive.google.com/uc?export=download&id=1TeXM_Z-F3-o6ouooigaEWT3axUJA65kv" -O hotpotqa.jsonl

pip install notdiamond pandas

npm install notdiamond

export NOTDIAMOND_API_KEY=YOUR_NOTDIAMOND_API_KEY

system_prompt = """I'd like for you to answer questions about a context text that will be provided. I'll give you a pair
with the form:
Context: "context text"
Question: "a question about the context"
Generate an explicit answer to the question that will be output. Make sure that the answer is the only output you provide,
and the analysis of the context should be kept to yourself. Answer directly and do not prefix the answer with anything such as
"Answer:" nor "The answer is:". The answer has to be the only output you explicitly provide.

The answer has to be as short, direct, and concise as possible. If the answer to the question can not be obtained from the provided
context paragraph, output "UNANSWERABLE". Here's the context and question for you to reason about and answer.
"""

prompt_template = """
Context: {context}

Question: {question}
"""

fields = ["question", "context"]

const systemPrompt = `I'd like for you to answer questions about a context text that will be provided. I'll give you a pair
with the form:
Context: "context text"
Question: "a question about the context"
Generate an explicit answer to the question that will be output. Make sure that the answer is the only output you provide,
and the analysis of the context should be kept to yourself. Answer directly and do not prefix the answer with anything such as
"Answer:" nor "The answer is:". The answer has to be the only output you explicitly provide.

The answer has to be as short, direct, and concise as possible. If the answer to the question can not be obtained from the provided
context paragraph, output "UNANSWERABLE". Here's the context and question for you to reason about and answer.
`;

const promptTemplate = `
Context: {context}

Question: {question}
`;

const fields = ['question', 'context'];

import pandas as pd

def load_json_dataset(dataset_path: str, n_samples: int = 200) -> list[dict]:
    df = pd.read_json(dataset_path, lines=True)
    n_samples = min(n_samples, len(df))
    if n_samples:
        df = df.iloc[[v for v in range(n_samples)]]

    golden_dataset = []
    for idx, row in df.iterrows():
        sample_fields = {"question": row["question"]}
        sample_fields["context"] = "\n\n".join(row["documents"])
        answer = row["response"]

        data_sample = dict(
            fields=sample_fields,
            answer=answer,
        )
        golden_dataset.append(data_sample)
    return golden_dataset
  
# Define the train and test set sizes
all_goldens = load_json_dataset("hotpotqa.jsonl", 20)
train_goldens = all_goldens[:10]
test_goldens = all_goldens[10:]

print(f"Loaded {len(train_goldens)} train samples and {len(test_goldens)} test samples")
print(f"\nExample prompt:\n{prompt_template.format(**train_goldens[0]['fields'])[:500]}...")

import * as fs from 'fs';
import * as readline from 'readline';

interface DataSample {
  fields: { question: string; context: string };
  answer: string;
}

async function loadJsonDataset(datasetPath: string, nSamples: number = 200): Promise<DataSample[]> {
  const fileStream = fs.createReadStream(datasetPath);
  const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });

  const goldenDataset: DataSample[] = [];

  for await (const line of rl) {
    if (goldenDataset.length >= nSamples) break;
    
    const row = JSON.parse(line);
    goldenDataset.push({
      fields: {
        question: row.question,
        context: row.documents.join('\n\n')
      },
      answer: row.response
    });
  }

  return goldenDataset;
}

// Define the train and test set sizes
const allGoldens = await loadJsonDataset('hotpotqa.jsonl', 20);
const trainGoldens = allGoldens.slice(0, 10);
const testGoldens = allGoldens.slice(10);

console.log(`Loaded ${trainGoldens.length} train samples and ${testGoldens.length} test samples`);

import os

from notdiamond import NotDiamond

client = NotDiamond(api_key=os.environ.get("NOTDIAMOND_API_KEY"))

# Define target models to optimize for
target_models = [
    {"provider": "openai", "model": "gpt-5-mini-2025-08-07"},
    {"provider": "google", "model": "gemini-2.5-flash"},
]

# Request adaptation
response = client.prompt_adaptation.adapt(
    system_prompt=system_prompt,
    template=prompt_template,
    fields=fields,
    train_goldens=train_goldens,
    test_goldens=test_goldens,
    target_models=target_models,
    evaluation_metric="LLMaaJ:Sem_Sim_1",
    prototype_mode=True
)

adaptation_run_id = response.adaptation_run_id
print(f"Adaptation job started: {adaptation_run_id}")

import NotDiamond from 'notdiamond';

const client = new NotDiamond({api_key: process.env.NOTDIAMOND_API_KEY});

// Define target models to optimize for
const targetModels = [
  { provider: 'openai', model: 'gpt-5-mini-2025-08-07' },
  { provider: 'google', model: 'gemini-2.5-flash' }
];

// Request adaptation
const response = await client.promptAdaptation.adapt({
  system_prompt: systemPrompt,
  template: promptTemplate,
  fields: fields,
  train_goldens: trainGoldens,
  test_goldens: testGoldens,
  target_models: targetModels,
  evaluation_metric: 'LLMaaJ:Sem_Sim_1',
  prototype_mode: true
});

const adaptationRunId = response.adaptation_run_id;
console.log(`Adaptation job started: ${adaptationRunId}`);

# Check status
status = client.prompt_adaptation.get_adapt_status(adaptation_run_id)
print(f"Status: {status.status}")

# Get results when complete
results = client.prompt_adaptation.get_adapt_results(adaptation_run_id)

# View optimized prompts for each target model
for target in results.target_models:
    print(f"\nModel: {target.model_name}")
    print(f"Pre-optimization score: {target.pre_optimization_score}")
    print(f"Post-optimization score: {target.post_optimization_score}")
    print(f"\nOptimized system prompt:\n{target.system_prompt}")
    print(f"\nOptimized template:\n{target.user_message_template}")

// Check status
const status = await client.promptAdaptation.getAdaptStatus(adaptationRunId);
console.log(`Status: ${status.status}`);

// Get results when complete
const results = await client.promptAdaptation.getAdaptResults(adaptationRunId);

// View optimized prompts for each target model
for (const target of results.target_models) {
  console.log(`\nModel: ${target.model_name}`);
  console.log(`Pre-optimization score: ${target.pre_optimization_score}`);
  console.log(`\nOptimized system prompt:\n${target.system_prompt}`);
  console.log(`\nOptimized template:\n${target.user_message_template}`);
}