Letter Count
A benchmark to evaluate a model's ability to count how many times a specific letter appears in a word.
Questions
40
Total Runs
19
Best Score
77/100
Leaderboard
| Rank | Model | Score | Run Date | Actions |
|---|---|---|---|---|
| Claude Haiku 4.5 |
77
|
2026-02-27 19:43:16 | View Details | |
| 2 |
Qwen3 VL 8B (LMStudio)
5000 MB |
55
|
2026-02-28 18:22:11 | View Details |
| 3 |
Gemma 3 12B (LMStudio)
8100 MB |
52
|
2026-02-28 15:42:32 | View Details |
| 4 |
Granite 3.2 8B (LMStudio)
4900 MB |
52
|
2026-02-28 16:12:55 | View Details |
| 5 | GPT-5 mini |
47
|
2026-02-26 01:11:28 | View Details |
| 6 |
Qwen3 4B (LMStudio)
2800 MB |
47
|
2026-02-28 18:09:55 | View Details |
| 7 |
OLMo 3 7B (LMStudio)
4300 MB |
45
|
2026-02-28 17:32:44 | View Details |
| 8 | GPT-5 nano |
42
|
2026-02-25 18:31:23 | View Details |
| 9 |
Llama 3.2 1B (LMStudio)
1300 MB |
42
|
2026-02-28 17:10:59 | View Details |
| 10 |
Llama 3.1 8B (LMStudio)
4900 MB |
35
|
2026-02-28 16:56:55 | View Details |
| 11 |
Ministral 8B (LMStudio)
4900 MB |
32
|
2026-02-28 17:18:13 | View Details |
| 12 |
Gemma 2 9B (LMStudio)
5800 MB |
30
|
2026-02-28 14:54:35 | View Details |
| 13 |
Qwen3 1.7B (LMStudio)
1100 MB |
22
|
2026-02-28 18:03:14 | View Details |
| 14 |
Gemma 2B (LMStudio)
1500 MB |
20
|
2026-02-28 15:32:09 | View Details |
| 15 |
SmolLM2 1.7B (LMStudio)
1100 MB |
20
|
2026-02-28 19:17:14 | View Details |
| 16 |
Llama 2 7B (LMStudio)
4900 MB |
15
|
2026-02-28 03:00:10 | View Details |
| 17 |
Llama 3 8B (LMStudio)
4900 MB |
15
|
2026-02-28 16:41:37 | View Details |
| 18 |
Gemma 2 2B (LMStudio)
1500 MB |
12
|
2026-02-28 03:17:41 | View Details |
| 19 |
Phi-3.5 Mini (LMStudio)
2500 MB |
12
|
2026-02-28 17:45:59 | View Details |
Questions
Question
How many times does the letter 'p' appear in the word 'experience'?Question payload
{
"question_text": "How many times does the letter 'p' appear in the word 'experience'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'e' appear in the word 'possible'?Question payload
{
"question_text": "How many times does the letter 'e' appear in the word 'possible'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'n' appear in the word 'excitement'?Question payload
{
"question_text": "How many times does the letter 'n' appear in the word 'excitement'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'r' appear in the word 'information'?Question payload
{
"question_text": "How many times does the letter 'r' appear in the word 'information'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 't' appear in the word 'community'?Question payload
{
"question_text": "How many times does the letter 't' appear in the word 'community'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 's' appear in the word 'necessary'?Question payload
{
"question_text": "How many times does the letter 's' appear in the word 'necessary'?",
"answer_type": "numeric",
"correct_answer": 2,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'i' appear in the word 'application'?Question payload
{
"question_text": "How many times does the letter 'i' appear in the word 'application'?",
"answer_type": "numeric",
"correct_answer": 2,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 't' appear in the word 'together'?Question payload
{
"question_text": "How many times does the letter 't' appear in the word 'together'?",
"answer_type": "numeric",
"correct_answer": 2,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 't' appear in the word 'community'?Question payload
{
"question_text": "How many times does the letter 't' appear in the word 'community'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 's' appear in the word 'business'?Question payload
{
"question_text": "How many times does the letter 's' appear in the word 'business'?",
"answer_type": "numeric",
"correct_answer": 3,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'n' appear in the word 'different'?Question payload
{
"question_text": "How many times does the letter 'n' appear in the word 'different'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'a' appear in the word 'beautiful'?Question payload
{
"question_text": "How many times does the letter 'a' appear in the word 'beautiful'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'n' appear in the word 'necessary'?Question payload
{
"question_text": "How many times does the letter 'n' appear in the word 'necessary'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 't' appear in the word 'television'?Question payload
{
"question_text": "How many times does the letter 't' appear in the word 'television'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'e' appear in the word 'successful'?Question payload
{
"question_text": "How many times does the letter 'e' appear in the word 'successful'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'b' appear in the word 'breakfast'?Question payload
{
"question_text": "How many times does the letter 'b' appear in the word 'breakfast'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'c' appear in the word 'significant'?Question payload
{
"question_text": "How many times does the letter 'c' appear in the word 'significant'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'e' appear in the word 'television'?Question payload
{
"question_text": "How many times does the letter 'e' appear in the word 'television'?",
"answer_type": "numeric",
"correct_answer": 2,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'f' appear in the word 'breakfast'?Question payload
{
"question_text": "How many times does the letter 'f' appear in the word 'breakfast'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'f' appear in the word 'different'?Question payload
{
"question_text": "How many times does the letter 'f' appear in the word 'different'?",
"answer_type": "numeric",
"correct_answer": 2,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'n' appear in the word 'intelligence'?Question payload
{
"question_text": "How many times does the letter 'n' appear in the word 'intelligence'?",
"answer_type": "numeric",
"correct_answer": 2,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'm' appear in the word 'development'?Question payload
{
"question_text": "How many times does the letter 'm' appear in the word 'development'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'n' appear in the word 'engineering'?Question payload
{
"question_text": "How many times does the letter 'n' appear in the word 'engineering'?",
"answer_type": "numeric",
"correct_answer": 3,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'u' appear in the word 'community'?Question payload
{
"question_text": "How many times does the letter 'u' appear in the word 'community'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'e' appear in the word 'computer'?Question payload
{
"question_text": "How many times does the letter 'e' appear in the word 'computer'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'f' appear in the word 'successful'?Question payload
{
"question_text": "How many times does the letter 'f' appear in the word 'successful'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 't' appear in the word 'development'?Question payload
{
"question_text": "How many times does the letter 't' appear in the word 'development'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'a' appear in the word 'conversation'?Question payload
{
"question_text": "How many times does the letter 'a' appear in the word 'conversation'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'n' appear in the word 'excitement'?Question payload
{
"question_text": "How many times does the letter 'n' appear in the word 'excitement'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'y' appear in the word 'necessary'?Question payload
{
"question_text": "How many times does the letter 'y' appear in the word 'necessary'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'u' appear in the word 'community'?Question payload
{
"question_text": "How many times does the letter 'u' appear in the word 'community'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'f' appear in the word 'performance'?Question payload
{
"question_text": "How many times does the letter 'f' appear in the word 'performance'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'a' appear in the word 'atmosphere'?Question payload
{
"question_text": "How many times does the letter 'a' appear in the word 'atmosphere'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'a' appear in the word 'understanding'?Question payload
{
"question_text": "How many times does the letter 'a' appear in the word 'understanding'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'r' appear in the word 'president'?Question payload
{
"question_text": "How many times does the letter 'r' appear in the word 'president'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'l' appear in the word 'possible'?Question payload
{
"question_text": "How many times does the letter 'l' appear in the word 'possible'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 't' appear in the word 'excitement'?Question payload
{
"question_text": "How many times does the letter 't' appear in the word 'excitement'?",
"answer_type": "numeric",
"correct_answer": 2,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'd' appear in the word 'development'?Question payload
{
"question_text": "How many times does the letter 'd' appear in the word 'development'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'u' appear in the word 'university'?Question payload
{
"question_text": "How many times does the letter 'u' appear in the word 'university'?",
"answer_type": "numeric",
"correct_answer": 1,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}Question
How many times does the letter 'f' appear in the word 'different'?Question payload
{
"question_text": "How many times does the letter 'f' appear in the word 'different'?",
"answer_type": "numeric",
"correct_answer": 2,
"category": "Letter Counting",
"difficulty": "easy",
"tags": [
"letter_count",
"spelling",
"counting"
],
"evaluation_criteria": {
"exact_match": true,
"case_sensitive": false,
"contains": false,
"required_fields": [],
"tolerance": 0.0,
"alternatives": []
}
}