Validate Bulk IPA/Phonetic (bebras)
A regression benchmark for Bebras bulk pronunciation verification. Tests whether the model returns only words with wrong IPA/phonetic mappings from 20-word lists with English + Chinese disambiguation.
Questions
5
Leaderboard Entries
1
Best Score
20/100
Leaderboard
| Rank | Model | Score | Run Date | Actions |
|---|---|---|---|---|
| GPT-5.4 mini |
20
1670.0ms median
|
2026-04-13 20:24:24 | View Details |
Questions
Question
Bulk pronunciation QA case: case_01_perfectQuestion payload
{
"question_text": "Bulk pronunciation QA case: case_01_perfect",
"answer_type": "json",
"correct_answer": {
"inputs": {
"entries": [
{
"word": "apple",
"chinese_hint": "苹果（名词）",
"ipa": "ˈæpəl",
"phonetic": "AP-uhl",
"entry_id": "C1_001"
},
{
"word": "mountain",
"chinese_hint": "山（名词）",
"ipa": "ˈmaʊntən",
"phonetic": "MOWN-tuhn",
"entry_id": "C1_002"
},
{
"word": "river",
"chinese_hint": "河流（名词）",
"ipa": "ˈrɪvər",
"phonetic": "RIV-er",
"entry_id": "C1_003"
},
{
"word": "lantern",
"chinese_hint": "灯笼（名词）",
"ipa": "ˈlæntərn",
"phonetic": "LAN-tern",
"entry_id": "C1_004"
},
{
"word": "pencil",
"chinese_hint": "铅笔（名词）",
"ipa": "ˈpɛnsəl",
"phonetic": "PEN-suhl",
"entry_id": "C1_005"
},
{
"word": "kitchen",
"chinese_hint": "厨房（名词）",
"ipa": "ˈkɪtʃən",
"phonetic": "KITCH-uhn",
"entry_id": "C1_006"
},
{
"word": "window",
"chinese_hint": "窗户（名词）",
"ipa": "ˈwɪndoʊ",
"phonetic": "WIN-doh",
"entry_id": "C1_007"
},
{
"word": "garden",
"chinese_hint": "花园（名词）",
"ipa": "ˈɡɑrdən",
"phonetic": "GAR-den",
"entry_id": "C1_008"
},
{
"word": "bicycle",
"chinese_hint": "自行车（名词）",
"ipa": "ˈbaɪsɪkəl",
"phonetic": "BYE-si-kuhl",
"entry_id": "C1_009"
},
{
"word": "planet",
"chinese_hint": "行星（名词）",
"ipa": "ˈplænɪt",
"phonetic": "PLAN-it",
"entry_id": "C1_010"
},
{
"word": "silver",
"chinese_hint": "银（名词）",
"ipa": "ˈsɪlvər",
"phonetic": "SIL-ver",
"entry_id": "C1_011"
},
{
"word": "tiger",
"chinese_hint": "老虎（名词）",
"ipa": "ˈtaɪɡər",
"phonetic": "TYE-ger",
"entry_id": "C1_012"
},
{
"word": "blanket",
"chinese_hint": "毯子（名词）",
"ipa": "ˈblæŋkɪt",
"phonetic": "BLANG-kit",
"entry_id": "C1_013"
},
{
"word": "candle",
"chinese_hint": "蜡烛（名词）",
"ipa": "ˈkændəl",
"phonetic": "KAN-duhl",
"entry_id": "C1_014"
},
{
"word": "pocket",
"chinese_hint": "口袋（名词）",
"ipa": "ˈpɑkɪt",
"phonetic": "POCK-it",
"entry_id": "C1_015"
},
{
"word": "thunder",
"chinese_hint": "雷（名词）",
"ipa": "ˈθʌndər",
"phonetic": "THUN-der",
"entry_id": "C1_016"
},
{
"word": "mirror",
"chinese_hint": "镜子（名词）",
"ipa": "ˈmɪrər",
"phonetic": "MEER-er",
"entry_id": "C1_017"
},
{
"word": "castle",
"chinese_hint": "城堡（名词）",
"ipa": "ˈkæsəl",
"phonetic": "KAS-uhl",
"entry_id": "C1_018"
},
{
"word": "coffee",
"chinese_hint": "咖啡（名词）",
"ipa": "ˈkɔfi",
"phonetic": "KAW-fee",
"entry_id": "C1_019"
},
{
"word": "violin",
"chinese_hint": "小提琴（名词）",
"ipa": "ˌvaɪəˈlɪn",
"phonetic": "vy-uh-LIN",
"entry_id": "C1_020"
}
]
},
"expected": {
"wrong_ipa_entries": [],
"wrong_phonetic_entries": []
}
},
"category": "agent_regression_pronunciation",
"difficulty": "easy",
"tags": [
"agent_benchmark",
"bebras",
"pronunciation",
"ipa",
"case:case_01_perfect"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [
"wrong_ipa_entries",
"wrong_phonetic_entries"
],
"tolerance": 0.0,
"alternatives": []
}
}Question
Bulk pronunciation QA case: case_02_two_wrongQuestion payload
{
"question_text": "Bulk pronunciation QA case: case_02_two_wrong",
"answer_type": "json",
"correct_answer": {
"inputs": {
"entries": [
{
"word": "apple",
"chinese_hint": "苹果（名词）",
"ipa": "ˈæpəl",
"phonetic": "AP-uhl",
"entry_id": "C2_001"
},
{
"word": "mountain",
"chinese_hint": "山（名词）",
"ipa": "ˈmaʊntən",
"phonetic": "MOWN-tuhn",
"entry_id": "C2_002"
},
{
"word": "river",
"chinese_hint": "河流（名词）",
"ipa": "ˈrɪvər",
"phonetic": "RIV-er",
"entry_id": "C2_003"
},
{
"word": "lantern",
"chinese_hint": "灯笼（名词）",
"ipa": "ˈlæntərn",
"phonetic": "LAN-tern",
"entry_id": "C2_004"
},
{
"word": "pencil",
"chinese_hint": "铅笔（名词）",
"ipa": "pɛnˈsɪl",
"phonetic": "PEN-suhl",
"entry_id": "C2_005"
},
{
"word": "kitchen",
"chinese_hint": "厨房（名词）",
"ipa": "ˈkɪtʃən",
"phonetic": "KITCH-uhn",
"entry_id": "C2_006"
},
{
"word": "window",
"chinese_hint": "窗户（名词）",
"ipa": "ˈwɪndoʊ",
"phonetic": "WIN-doh",
"entry_id": "C2_007"
},
{
"word": "garden",
"chinese_hint": "花园（名词）",
"ipa": "ˈɡɑrdən",
"phonetic": "GAR-den",
"entry_id": "C2_008"
},
{
"word": "bicycle",
"chinese_hint": "自行车（名词）",
"ipa": "ˈbaɪsɪkəl",
"phonetic": "BYE-si-kuhl",
"entry_id": "C2_009"
},
{
"word": "planet",
"chinese_hint": "行星（名词）",
"ipa": "ˈplænɪt",
"phonetic": "PLAN-it",
"entry_id": "C2_010"
},
{
"word": "silver",
"chinese_hint": "银（名词）",
"ipa": "ˈsɪlvər",
"phonetic": "SIL-ver",
"entry_id": "C2_011"
},
{
"word": "tiger",
"chinese_hint": "老虎（名词）",
"ipa": "ˈtaɪɡər",
"phonetic": "TEE-ger",
"entry_id": "C2_012"
},
{
"word": "blanket",
"chinese_hint": "毯子（名词）",
"ipa": "ˈblæŋkɪt",
"phonetic": "BLANG-kit",
"entry_id": "C2_013"
},
{
"word": "candle",
"chinese_hint": "蜡烛（名词）",
"ipa": "ˈkændəl",
"phonetic": "KAN-duhl",
"entry_id": "C2_014"
},
{
"word": "pocket",
"chinese_hint": "口袋（名词）",
"ipa": "ˈpɑkɪt",
"phonetic": "POCK-it",
"entry_id": "C2_015"
},
{
"word": "thunder",
"chinese_hint": "雷（名词）",
"ipa": "ˈθʌndər",
"phonetic": "THUN-der",
"entry_id": "C2_016"
},
{
"word": "mirror",
"chinese_hint": "镜子（名词）",
"ipa": "ˈmɪrər",
"phonetic": "MEER-er",
"entry_id": "C2_017"
},
{
"word": "castle",
"chinese_hint": "城堡（名词）",
"ipa": "ˈkæsəl",
"phonetic": "KAS-uhl",
"entry_id": "C2_018"
},
{
"word": "coffee",
"chinese_hint": "咖啡（名词）",
"ipa": "ˈkɔfi",
"phonetic": "KAW-fee",
"entry_id": "C2_019"
},
{
"word": "violin",
"chinese_hint": "小提琴（名词）",
"ipa": "ˌvaɪəˈlɪn",
"phonetic": "vy-uh-LIN",
"entry_id": "C2_020"
}
]
},
"expected": {
"wrong_ipa_entries": [
{
"entry_id": "C2_005",
"word": "pencil",
"correct_ipa": "ˈpɛnsəl"
}
],
"wrong_phonetic_entries": [
{
"entry_id": "C2_012",
"word": "tiger",
"correct_phonetic": "TYE-ger"
}
]
}
},
"category": "agent_regression_pronunciation",
"difficulty": "medium",
"tags": [
"agent_benchmark",
"bebras",
"pronunciation",
"ipa",
"case:case_02_two_wrong"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [
"wrong_ipa_entries",
"wrong_phonetic_entries"
],
"tolerance": 0.0,
"alternatives": []
}
}Question
Bulk pronunciation QA case: case_03_three_wrongQuestion payload
{
"question_text": "Bulk pronunciation QA case: case_03_three_wrong",
"answer_type": "json",
"correct_answer": {
"inputs": {
"entries": [
{
"word": "apple",
"chinese_hint": "苹果（名词）",
"ipa": "ˈæpəl",
"phonetic": "uh-PULL",
"entry_id": "C3_001"
},
{
"word": "mountain",
"chinese_hint": "山（名词）",
"ipa": "ˈmaʊntən",
"phonetic": "MOWN-tuhn",
"entry_id": "C3_002"
},
{
"word": "river",
"chinese_hint": "河流（名词）",
"ipa": "ˈrɪvər",
"phonetic": "RIV-er",
"entry_id": "C3_003"
},
{
"word": "lantern",
"chinese_hint": "灯笼（名词）",
"ipa": "ˈlæntərn",
"phonetic": "LAN-tern",
"entry_id": "C3_004"
},
{
"word": "pencil",
"chinese_hint": "铅笔（名词）",
"ipa": "ˈpɛnsəl",
"phonetic": "PEN-suhl",
"entry_id": "C3_005"
},
{
"word": "kitchen",
"chinese_hint": "厨房（名词）",
"ipa": "ˈkɪtʃən",
"phonetic": "KITCH-uhn",
"entry_id": "C3_006"
},
{
"word": "window",
"chinese_hint": "窗户（名词）",
"ipa": "ˈwɪndoʊ",
"phonetic": "WIN-doh",
"entry_id": "C3_007"
},
{
"word": "garden",
"chinese_hint": "花园（名词）",
"ipa": "ˈɡɑrdeɪn",
"phonetic": "GAR-den",
"entry_id": "C3_008"
},
{
"word": "bicycle",
"chinese_hint": "自行车（名词）",
"ipa": "ˈbaɪsɪkəl",
"phonetic": "BYE-si-kuhl",
"entry_id": "C3_009"
},
{
"word": "planet",
"chinese_hint": "行星（名词）",
"ipa": "ˈplænɪt",
"phonetic": "PLAN-it",
"entry_id": "C3_010"
},
{
"word": "silver",
"chinese_hint": "银（名词）",
"ipa": "ˈsɪlvər",
"phonetic": "SIL-ver",
"entry_id": "C3_011"
},
{
"word": "tiger",
"chinese_hint": "老虎（名词）",
"ipa": "ˈtaɪɡər",
"phonetic": "TYE-ger",
"entry_id": "C3_012"
},
{
"word": "blanket",
"chinese_hint": "毯子（名词）",
"ipa": "ˈblæŋkɪt",
"phonetic": "BLANG-kit",
"entry_id": "C3_013"
},
{
"word": "candle",
"chinese_hint": "蜡烛（名词）",
"ipa": "ˈkændəl",
"phonetic": "CANE-duhl",
"entry_id": "C3_014"
},
{
"word": "pocket",
"chinese_hint": "口袋（名词）",
"ipa": "ˈpɑkɪt",
"phonetic": "POCK-it",
"entry_id": "C3_015"
},
{
"word": "thunder",
"chinese_hint": "雷（名词）",
"ipa": "ˈθʌndər",
"phonetic": "THUN-der",
"entry_id": "C3_016"
},
{
"word": "mirror",
"chinese_hint": "镜子（名词）",
"ipa": "ˈmɪrər",
"phonetic": "MEER-er",
"entry_id": "C3_017"
},
{
"word": "castle",
"chinese_hint": "城堡（名词）",
"ipa": "ˈkæsəl",
"phonetic": "KAS-uhl",
"entry_id": "C3_018"
},
{
"word": "coffee",
"chinese_hint": "咖啡（名词）",
"ipa": "ˈkɔfi",
"phonetic": "KAW-fee",
"entry_id": "C3_019"
},
{
"word": "violin",
"chinese_hint": "小提琴（名词）",
"ipa": "ˌvaɪəˈlɪn",
"phonetic": "vy-uh-LIN",
"entry_id": "C3_020"
}
]
},
"expected": {
"wrong_ipa_entries": [
{
"entry_id": "C3_008",
"word": "garden",
"correct_ipa": "ˈɡɑrdən"
}
],
"wrong_phonetic_entries": [
{
"entry_id": "C3_001",
"word": "apple",
"correct_phonetic": "AP-uhl"
},
{
"entry_id": "C3_014",
"word": "candle",
"correct_phonetic": "KAN-duhl"
}
]
}
},
"category": "agent_regression_pronunciation",
"difficulty": "medium",
"tags": [
"agent_benchmark",
"bebras",
"pronunciation",
"ipa",
"case:case_03_three_wrong"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [
"wrong_ipa_entries",
"wrong_phonetic_entries"
],
"tolerance": 0.0,
"alternatives": []
}
}Question
Bulk pronunciation QA case: case_04_four_wrongQuestion payload
{
"question_text": "Bulk pronunciation QA case: case_04_four_wrong",
"answer_type": "json",
"correct_answer": {
"inputs": {
"entries": [
{
"word": "apple",
"chinese_hint": "苹果（名词）",
"ipa": "ˈæpəl",
"phonetic": "AP-uhl",
"entry_id": "C4_001"
},
{
"word": "mountain",
"chinese_hint": "山（名词）",
"ipa": "ˈmaʊntən",
"phonetic": "MOWN-tuhn",
"entry_id": "C4_002"
},
{
"word": "river",
"chinese_hint": "河流（名词）",
"ipa": "ˈrɪvər",
"phonetic": "RIV-er",
"entry_id": "C4_003"
},
{
"word": "lantern",
"chinese_hint": "灯笼（名词）",
"ipa": "ˈlæntərn",
"phonetic": "LAN-tern",
"entry_id": "C4_004"
},
{
"word": "pencil",
"chinese_hint": "铅笔（名词）",
"ipa": "ˈpɛnsəl",
"phonetic": "PEN-suhl",
"entry_id": "C4_005"
},
{
"word": "kitchen",
"chinese_hint": "厨房（名词）",
"ipa": "ˈkɪtʃən",
"phonetic": "kee-CHEN",
"entry_id": "C4_006"
},
{
"word": "window",
"chinese_hint": "窗户（名词）",
"ipa": "ˈwɪndoʊ",
"phonetic": "WIN-doh",
"entry_id": "C4_007"
},
{
"word": "garden",
"chinese_hint": "花园（名词）",
"ipa": "ˈɡɑrdən",
"phonetic": "GAR-den",
"entry_id": "C4_008"
},
{
"word": "bicycle",
"chinese_hint": "自行车（名词）",
"ipa": "ˈbaɪsɪkəl",
"phonetic": "BYE-si-kuhl",
"entry_id": "C4_009"
},
{
"word": "planet",
"chinese_hint": "行星（名词）",
"ipa": "pləˈnɛt",
"phonetic": "PLAN-it",
"entry_id": "C4_010"
},
{
"word": "silver",
"chinese_hint": "银（名词）",
"ipa": "ˈsɪlvər",
"phonetic": "SIL-ver",
"entry_id": "C4_011"
},
{
"word": "tiger",
"chinese_hint": "老虎（名词）",
"ipa": "ˈtaɪɡər",
"phonetic": "TYE-ger",
"entry_id": "C4_012"
},
{
"word": "blanket",
"chinese_hint": "毯子（名词）",
"ipa": "ˈblæŋkɪt",
"phonetic": "BLANG-kit",
"entry_id": "C4_013"
},
{
"word": "candle",
"chinese_hint": "蜡烛（名词）",
"ipa": "ˈkændəl",
"phonetic": "KAN-duhl",
"entry_id": "C4_014"
},
{
"word": "pocket",
"chinese_hint": "口袋（名词）",
"ipa": "ˈpɑkɪt",
"phonetic": "POCK-it",
"entry_id": "C4_015"
},
{
"word": "thunder",
"chinese_hint": "雷（名词）",
"ipa": "ˈθuːndər",
"phonetic": "THUN-der",
"entry_id": "C4_016"
},
{
"word": "mirror",
"chinese_hint": "镜子（名词）",
"ipa": "ˈmɪrər",
"phonetic": "MEER-er",
"entry_id": "C4_017"
},
{
"word": "castle",
"chinese_hint": "城堡（名词）",
"ipa": "ˈkæsəl",
"phonetic": "KAS-uhl",
"entry_id": "C4_018"
},
{
"word": "coffee",
"chinese_hint": "咖啡（名词）",
"ipa": "ˈkɔfi",
"phonetic": "KAW-fee",
"entry_id": "C4_019"
},
{
"word": "violin",
"chinese_hint": "小提琴（名词）",
"ipa": "ˌvaɪəˈlɪn",
"phonetic": "VEE-oh-lin",
"entry_id": "C4_020"
}
]
},
"expected": {
"wrong_ipa_entries": [
{
"entry_id": "C4_010",
"word": "planet",
"correct_ipa": "ˈplænɪt"
},
{
"entry_id": "C4_016",
"word": "thunder",
"correct_ipa": "ˈθʌndər"
}
],
"wrong_phonetic_entries": [
{
"entry_id": "C4_006",
"word": "kitchen",
"correct_phonetic": "KITCH-uhn"
},
{
"entry_id": "C4_020",
"word": "violin",
"correct_phonetic": "vy-uh-LIN"
}
]
}
},
"category": "agent_regression_pronunciation",
"difficulty": "hard",
"tags": [
"agent_benchmark",
"bebras",
"pronunciation",
"ipa",
"case:case_04_four_wrong"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [
"wrong_ipa_entries",
"wrong_phonetic_entries"
],
"tolerance": 0.0,
"alternatives": []
}
}Question
Bulk pronunciation QA case: case_05_five_wrongQuestion payload
{
"question_text": "Bulk pronunciation QA case: case_05_five_wrong",
"answer_type": "json",
"correct_answer": {
"inputs": {
"entries": [
{
"word": "apple",
"chinese_hint": "苹果（名词）",
"ipa": "ˈæpəl",
"phonetic": "AP-uhl",
"entry_id": "C5_001"
},
{
"word": "mountain",
"chinese_hint": "山（名词）",
"ipa": "ˈmaʊntən",
"phonetic": "MOWN-tuhn",
"entry_id": "C5_002"
},
{
"word": "river",
"chinese_hint": "河流（名词）",
"ipa": "raɪvər",
"phonetic": "RIV-er",
"entry_id": "C5_003"
},
{
"word": "lantern",
"chinese_hint": "灯笼（名词）",
"ipa": "ˈlæntərn",
"phonetic": "LAN-tern",
"entry_id": "C5_004"
},
{
"word": "pencil",
"chinese_hint": "铅笔（名词）",
"ipa": "ˈpɛnsəl",
"phonetic": "PEN-suhl",
"entry_id": "C5_005"
},
{
"word": "kitchen",
"chinese_hint": "厨房（名词）",
"ipa": "ˈkɪtʃən",
"phonetic": "KITCH-uhn",
"entry_id": "C5_006"
},
{
"word": "window",
"chinese_hint": "窗户（名词）",
"ipa": "ˈwɪndoʊ",
"phonetic": "wine-doh",
"entry_id": "C5_007"
},
{
"word": "garden",
"chinese_hint": "花园（名词）",
"ipa": "ˈɡɑrdən",
"phonetic": "GAR-den",
"entry_id": "C5_008"
},
{
"word": "bicycle",
"chinese_hint": "自行车（名词）",
"ipa": "ˈbaɪsɪkəl",
"phonetic": "BYE-si-kuhl",
"entry_id": "C5_009"
},
{
"word": "planet",
"chinese_hint": "行星（名词）",
"ipa": "ˈplænɪt",
"phonetic": "PLAN-it",
"entry_id": "C5_010"
},
{
"word": "silver",
"chinese_hint": "银（名词）",
"ipa": "ˈsaɪlvər",
"phonetic": "SIL-ver",
"entry_id": "C5_011"
},
{
"word": "tiger",
"chinese_hint": "老虎（名词）",
"ipa": "ˈtaɪɡər",
"phonetic": "TYE-ger",
"entry_id": "C5_012"
},
{
"word": "blanket",
"chinese_hint": "毯子（名词）",
"ipa": "ˈblæŋkɪt",
"phonetic": "BLANG-kit",
"entry_id": "C5_013"
},
{
"word": "candle",
"chinese_hint": "蜡烛（名词）",
"ipa": "ˈkændəl",
"phonetic": "KAN-duhl",
"entry_id": "C5_014"
},
{
"word": "pocket",
"chinese_hint": "口袋（名词）",
"ipa": "ˈpɑkɪt",
"phonetic": "POCK-it",
"entry_id": "C5_015"
},
{
"word": "thunder",
"chinese_hint": "雷（名词）",
"ipa": "ˈθʌndər",
"phonetic": "THUN-der",
"entry_id": "C5_016"
},
{
"word": "mirror",
"chinese_hint": "镜子（名词）",
"ipa": "ˈmɪrər",
"phonetic": "my-ROAR",
"entry_id": "C5_017"
},
{
"word": "castle",
"chinese_hint": "城堡（名词）",
"ipa": "ˈkæsəl",
"phonetic": "KAS-uhl",
"entry_id": "C5_018"
},
{
"word": "coffee",
"chinese_hint": "咖啡（名词）",
"ipa": "koʊˈfi",
"phonetic": "KAW-fee",
"entry_id": "C5_019"
},
{
"word": "violin",
"chinese_hint": "小提琴（名词）",
"ipa": "ˌvaɪəˈlɪn",
"phonetic": "vy-uh-LIN",
"entry_id": "C5_020"
}
]
},
"expected": {
"wrong_ipa_entries": [
{
"entry_id": "C5_003",
"word": "river",
"correct_ipa": "ˈrɪvər"
},
{
"entry_id": "C5_011",
"word": "silver",
"correct_ipa": "ˈsɪlvər"
},
{
"entry_id": "C5_019",
"word": "coffee",
"correct_ipa": "ˈkɔfi"
}
],
"wrong_phonetic_entries": [
{
"entry_id": "C5_007",
"word": "window",
"correct_phonetic": "WIN-doh"
},
{
"entry_id": "C5_017",
"word": "mirror",
"correct_phonetic": "MEER-er"
}
]
}
},
"category": "agent_regression_pronunciation",
"difficulty": "hard",
"tags": [
"agent_benchmark",
"bebras",
"pronunciation",
"ipa",
"case:case_05_five_wrong"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [
"wrong_ipa_entries",
"wrong_phonetic_entries"
],
"tolerance": 0.0,
"alternatives": []
}
}