Prompt adaptation for classification tutorial
Prompt adaptation is in private beta
Please reach out to us if you want to test it out!
In the example below, we will show you how to use the prompt adaptation API on classification tasks. Specifically, we'll use the PolyAI/banking77 dataset for this tutorial.
Setup
First, we will install datasets library and download banking77 from Huggingface.
pip install -U datasets==3.6.0
from datasets import load_dataset
n_samples = 100
ds = load_dataset("PolyAI/banking77")["train"].select(list(range(n_samples)))
Next, we define the system prompt and prompt template for our current workflow.
system_prompt = """
You are a helpful assistant that categorizes banking-related questions provided by the user.
"""
prompt_template = """
Sharing a text from banking domain which needs to be classified into one of the 77 classes mentioned below
The text is customer support query.
Categories to classify the data :
{categories}
Text To classify : {question}
"""
Finally, we'll define some helper functions to help us call the adaptation APIs.
from typing import List, Dict, Any, Optional
import requests
import json
def request_prompt_adaptation(
system_prompt: str,
prompt_template: str,
fields: List[str],
train_data: List[Dict[str, str | Dict[str, str]]],
origin_model: Dict[str, str],
target_models: List[Dict[str, str]],
notdiamond_api_key: str,
evaluation_metric: Optional[str] = None,
evaluation_config: Optional[Dict] = None,
request_url: str = "https://api.notdiamond.ai/v2/prompt/adapt"
) -> str:
"""
Helper method to call the prompt adaptation endpoint
"""
if (evaluation_metric and evaluation_config) or (not evaluation_metric and not evaluation_config):
raise ValueError("Either evaluation_metric or evaluation_config must be provided, but not both or neither.")
request_body = {
"system_prompt": system_prompt,
"template": prompt_template,
"fields": fields,
"goldens": train_data,
"origin_model": origin_model,
"target_models": target_models,
}
if evaluation_metric:
request_body["evaluation_metric"] = evaluation_metric
if evaluation_config:
request_body["evaluation_config"] = json.dumps(evaluation_config)
headers = {
"Authorization": f"Bearer {notdiamond_api_key}",
"content-type": "application/json"
}
resp = requests.post(
request_url,
headers=headers,
json=request_body
)
if resp.status_code == 200:
response = resp.json()
return response["adaptation_run_id"]
else:
raise Exception(
f"Request to adapt prompt failed with code {resp.status_code}: {resp.text}"
)
def get_prompt_adaptation_results(
adaptation_run_id: str,
notdiamond_api_key: str,
request_url: str = "https://api.notdiamond.ai/v2/prompt/adaptResults"
) -> Dict[str, Any]:
"""
Helper method to get the results of a prompt adaptation request
"""
headers = {
"Authorization": f"Bearer {notdiamond_api_key}",
"content-type": "application/json"
}
resp = requests.get(
f"{request_url}/{adaptation_run_id}",
headers=headers
)
if resp.status_code == 200:
response = resp.json()
return response
else:
raise Exception(
f"Requesting prompt adaptation result failed with code {resp.status_code}: {resp.text}"
)
Request prompt adaptation
First we will format the dataset for prompt adaptation. Not Diamond expects a list of samples consisting of prompt template field arguments, so ensure that prompt_template.format(sample['fields']) returns a valid user prompt for each sample. For classification tasks, it is important that the "answer" is the ground truth class name of the sample.
categories = [ # pre-extracted for convenience
"activate_my_card",
"age_limit",
"apple_pay_or_google_pay",
"atm_support",
"automatic_top_up",
"balance_not_updated_after_bank_transfer",
"balance_not_updated_after_cheque_or_cash_deposit",
"beneficiary_not_allowed",
"cancel_transfer",
"card_about_to_expire",
"card_acceptance",
"card_arrival",
"card_delivery_estimate",
"card_linking",
"card_not_working",
"card_payment_fee_charged",
"card_payment_not_recognised",
"card_payment_wrong_exchange_rate",
"card_swallowed",
"cash_withdrawal_charge",
"cash_withdrawal_not_recognised",
"change_pin",
"compromised_card",
"contactless_not_working",
"country_support",
"declined_card_payment",
"declined_cash_withdrawal",
"declined_transfer",
"direct_debit_payment_not_recognised",
"disposable_card_limits",
"edit_personal_details",
"exchange_charge",
"exchange_rate",
"exchange_via_app",
"extra_charge_on_statement",
"failed_transfer",
"fiat_currency_support",
"get_disposable_virtual_card",
"get_physical_card",
"getting_spare_card",
"getting_virtual_card",
"lost_or_stolen_card",
"lost_or_stolen_phone",
"order_physical_card",
"passcode_forgotten",
"pending_card_payment",
"pending_cash_withdrawal",
"pending_top_up",
"pending_transfer",
"pin_blocked",
"receiving_money",
"refund_not_showing_up",
"request_refund",
"reverted_card_payment",
"supported_cards_and_currencies",
"terminate_account",
"top_up_by_bank_transfer_charge",
"top_up_by_card_charge",
"top_up_by_cash_or_cheque",
"top_up_failed",
"top_up_limits",
"top_up_reverted",
"topping_up_by_card",
"transaction_charged_twice",
"transfer_fee_charged",
"transfer_into_account",
"transfer_not_received_by_recipient",
"transfer_timing",
"unable_to_verify_identity",
"verify_my_identity",
"verify_source_of_funds",
"verify_top_up",
"virtual_card_not_working",
"visa_or_mastercard",
"why_verify_identity",
"wrong_amount_of_cash_received",
"wrong_exchange_rate_for_cash_withdrawal"
]
fields = ["question", "categories"]
pa_ds = [
{
"fields": {
"question": sample["text"],
"categories": json.dumps(categories)
},
"answer": json.dumps({
"intent": categories[sample["label"]], # The intent here should be the class name
})
}
for sample in ds
]
print(prompt_template.format(**pa_ds[0]['fields']))
Next, specify the origin_model which you query with the current system prompt and prompt template; and your target_models, which you would like to query with adapted prompts. You can list multiple target models.
origin_model = {"provider": "openai", "model": "gpt-4o-2024-08-06"}
target_models = [
{"provider": "anthropic", "model": "claude-3-5-sonnet-20241022"},
]
Use multiple target models for the best results
Our prompt adaptation API allows you to define up to 4 target models. If you're unsure about which model you should migrate to, defining multiple target models lets you see at a glance which model is best suited for your data.
Finally, call the API with your NOTDIAMOND_API_KEY and the adaptation request will be submitted to Not Diamond's servers. You will get back a prompt_adaptation_id.
Please only submit one job at a time
Prompt adaptation is still in beta and our endpoint is not designed for high-volume production traffic. Please submit only one job at a time and wait for it to complete before submitting another job. It is fine for a single job to include multiple target models.
prompt_adaptation_id = request_prompt_adaptation(
system_prompt=system_prompt,
prompt_template=prompt_template,
fields=fields,
train_data=pa_ds,
origin_model=origin_model,
target_models=target_models,
evaluation_metric="LLMaaJ:Sem_Sim_1",
notdiamond_api_key="YOUR_NOTDIAMOND_API_KEY"
)
Request status
Get adapted prompt and evaluation results
Once the prompt adaptation request is completed, you can request the results of the optimization using the same prompt_adaptation_id.
results = get_prompt_adaptation_results(prompt_adaptation_id, "YOUR_NOTDIAMOND_API_KEY")
print(results)
Updated about 7 hours ago