Data and schema

The support-ticket dataset and the triage schema used by the eval and fine-tuning lessons. The .jsonl files show their first rows.

data/make_tickets.py Python · 117 lines

1
"""
2
data/make_tickets.py — the one dataset lessons 07–11 share.
3

4
A support-ticket TRIAGE task, the kind an enterprise customer brings on day one:
5
    input : free-text ticket
6
    output: {"category": billing|outage|how-to|abuse, "severity": 1–5, "next_action": str}
7

8
Why synthetic? So everyone gets the same numbers and nothing private leaks.
9
Why this task? It has a checkable answer (so evals are honest), needs structured
10
output (lesson 07), is cheap to batch (08), and small models improve visibly
11
with a LoRA (10, 11).
12

13
Honest splits: tickets are de-duplicated, the test set never shares a ticket text with
14
training, and one phrasing per category appears ONLY in the test set, so evals measure
15
generalisation, not memory. (lessons/17-full-vs-peft-mlx/sft_data_check.py verifies this.)
16

17
Writes (deterministic, seed 7):
18
    data/tickets/train.jsonl   800 rows  chat format → fine-tuning (Fireworks SFT + MLX LoRA)
19
    data/tickets/valid.jsonl   100 rows  chat format → MLX validation loss
20
    data/tickets/test.jsonl    100 rows  {"ticket", "category", "severity"} → evals, NEVER trained on
21

22
    python data/make_tickets.py
23
"""
24
import json
25
import random
26
from pathlib import Path
27

28
OUT = Path(__file__).parent / "tickets"
29
SYSTEM = ("You triage customer support tickets. Reply with JSON only: "
30
          '{"category": "billing|outage|how-to|abuse", "severity": 1-5, "next_action": "<short>"}')
31

32
PRODUCTS = ["the API", "the dashboard", "SSO login", "webhooks", "the EU cluster", "batch jobs", "the CLI"]
33
REGIONS = ["us-east", "eu-west", "ap-south", "us-west"]
34
URGENT = ["", "", "", " This is urgent.", " Launch is tomorrow.", " It affects all users.", " We are in production."]
35
OPENERS = ["", "", "Hi team, ", "Hello, ", "Quick one: ", "Hey, ", "Good morning. "]
36
CLOSERS = ["", "", " Thanks.", " Please help.", " Account {acct}.", " Ref #{ref}.", " Cheers, {name}."]
37
NAMES = ["Priya", "Tom", "Wei", "Aisha", "Carlos", "Mei", "Olu", "Sara", "Ken", "Lena"]
38

39
TEMPLATES = {
40
    "billing": [
41
        "My card was declined when renewing the plan.", "We were charged twice on the last invoice.",
42
        "Can I get a refund for the unused seats (${amt})?", "The invoice shows the wrong company name.",
43
        "Why did the payment for ${amt} fail?", "Billing page shows a charge I don't recognise.",
44
    ],
45
    "outage": [
46
        "{prod} is down in {region}.", "Getting 500 errors from {prod} since 09:00.",
47
        "Requests to {prod} time out after 30s.", "Huge latency spike on {prod} in {region}.",
48
        "{prod} unreachable for our whole team.", "Error rate on {prod} jumped to 40%.",
49
    ],
50
    "abuse": [
51
        "Someone is sending spam from an account on your platform.", "We received a phishing email using your logo.",
52
        "I think my API key was stolen — unknown usage overnight.", "An account is scraping our public pages via {prod}.",
53
        "Report fraud: fake invoices sent in your name.", "Abuse report: bot signups from {region}.",
54
    ],
55
    "how-to": [
56
        "How do I rotate my API key?", "How to configure {prod} for a second region?",
57
        "Where can I find docs for {prod} rate limits?", "How do I set up SSO with Okta?",
58
        "How to export usage data to CSV?", "Where can I change the default model?",
59
    ],
60
}
61
BASE_SEV = {"outage": 4, "abuse": 4, "billing": 3, "how-to": 2}
62
ACTION = {"billing": "route to billing; verify charge", "outage": "page on-call; check status page",
63
          "abuse": "route to trust & safety; suspend key if confirmed", "how-to": "reply with docs link"}
64

65

66
def make(rng: random.Random, held_out: bool) -> dict:
67
    """held_out=True uses only each category's LAST template (never seen in training)."""
68
    cat = rng.choice(list(TEMPLATES))
69
    pool = TEMPLATES[cat][-1:] if held_out else TEMPLATES[cat][:-1]
70
    text = rng.choice(pool).format(prod=rng.choice(PRODUCTS), region=rng.choice(REGIONS),
71
                                   amt=rng.choice([49, 120, 480, 900, 2400]))
72
    urgency = rng.choice(URGENT)
73
    close = rng.choice(CLOSERS).format(acct=rng.randint(10000, 99999), ref=rng.randint(1000, 9999), name=rng.choice(NAMES))
74
    sev = min(5, BASE_SEV[cat] + (1 if urgency else 0))
75
    return {"ticket": rng.choice(OPENERS) + text + urgency + close, "category": cat, "severity": sev,
76
            "next_action": ACTION[cat]}
77

78

79
def as_chat(r: dict) -> dict:
80
    answer = {"category": r["category"], "severity": r["severity"], "next_action": r["next_action"]}
81
    return {"messages": [{"role": "system", "content": SYSTEM},
82
                         {"role": "user", "content": r["ticket"]},
83
                         {"role": "assistant", "content": json.dumps(answer)}]}
84

85

86
def main() -> None:
87
    rng = random.Random(7)
88
    seen: set[str] = set()
89

90
    def unique(n: int, held_out: bool) -> list[dict]:
91
        out = []
92
        while len(out) < n:
93
            r = make(rng, held_out)
94
            if r["ticket"] not in seen:          # de-duplicate across ALL splits
95
                seen.add(r["ticket"])
96
                out.append(r)
97
        return out
98

99
    train_pool = unique(970, held_out=False)
100
    splits = {
101
        "train": train_pool[:800],
102
        "valid": train_pool[800:900],
103
        # test = 70 unseen tickets in familiar phrasings + 30 in phrasings never seen in training
104
        "test": train_pool[900:970] + unique(30, held_out=True),
105
    }
106
    rng.shuffle(splits["test"])
107
    OUT.mkdir(exist_ok=True)
108
    for name, rs in splits.items():
109
        with (OUT / f"{name}.jsonl").open("w") as f:
110
            for r in rs:
111
                f.write(json.dumps(as_chat(r) if name != "test" else
112
                                   {k: r[k] for k in ("ticket", "category", "severity")}) + "\n")
113
        print(f"wrote {OUT / name}.jsonl  ({len(rs)} rows)")
114

115

116
if __name__ == "__main__":
117
    main()

data/tickets/test.jsonl JSON Lines · 100 lines · first 3 shown

1
{"ticket": "Good morning. How do I rotate my API key? This is urgent. Cheers, Carlos.", "category": "how-to", "severity": 3}
2
{"ticket": "Abuse report: bot signups from us-east. It affects all users. Ref #8731.", "category": "abuse", "severity": 5}
3
{"ticket": "Hello, I think my API key was stolen \u2014 unknown usage overnight. This is urgent. Thanks.", "category": "abuse", "severity": 5}

data/tickets/train.jsonl JSON Lines · 800 lines · first 3 shown

1
{"messages": [{"role": "system", "content": "You triage customer support tickets. Reply with JSON only: {\"category\": \"billing|outage|how-to|abuse\", \"severity\": 1-5, \"next_action\": \"<short>\"}"}, {"role": "user", "content": "We received a phishing email using your logo. We are in production. Account 22337."}, {"role": "assistant", "content": "{\"category\": \"abuse\", \"severity\": 5, \"next_action\": \"route to trust & safety; suspend key if confirmed\"}"}]}
2
{"messages": [{"role": "system", "content": "You triage customer support tickets. Reply with JSON only: {\"category\": \"billing|outage|how-to|abuse\", \"severity\": 1-5, \"next_action\": \"<short>\"}"}, {"role": "user", "content": "Good morning. the API is down in us-west."}, {"role": "assistant", "content": "{\"category\": \"outage\", \"severity\": 4, \"next_action\": \"page on-call; check status page\"}"}]}
3
{"messages": [{"role": "system", "content": "You triage customer support tickets. Reply with JSON only: {\"category\": \"billing|outage|how-to|abuse\", \"severity\": 1-5, \"next_action\": \"<short>\"}"}, {"role": "user", "content": "Quick one: We were charged twice on the last invoice. Launch is tomorrow. Please help."}, {"role": "assistant", "content": "{\"category\": \"billing\", \"severity\": 4, \"next_action\": \"route to billing; verify charge\"}"}]}

data/tickets/valid.jsonl JSON Lines · 100 lines · first 3 shown

1
{"messages": [{"role": "system", "content": "You triage customer support tickets. Reply with JSON only: {\"category\": \"billing|outage|how-to|abuse\", \"severity\": 1-5, \"next_action\": \"<short>\"}"}, {"role": "user", "content": "How to export usage data to CSV? We are in production. Account 48975."}, {"role": "assistant", "content": "{\"category\": \"how-to\", \"severity\": 3, \"next_action\": \"reply with docs link\"}"}]}
2
{"messages": [{"role": "system", "content": "You triage customer support tickets. Reply with JSON only: {\"category\": \"billing|outage|how-to|abuse\", \"severity\": 1-5, \"next_action\": \"<short>\"}"}, {"role": "user", "content": "Hi team, Where can I find docs for the API rate limits? We are in production. Cheers, Aisha."}, {"role": "assistant", "content": "{\"category\": \"how-to\", \"severity\": 3, \"next_action\": \"reply with docs link\"}"}]}
3
{"messages": [{"role": "system", "content": "You triage customer support tickets. Reply with JSON only: {\"category\": \"billing|outage|how-to|abuse\", \"severity\": 1-5, \"next_action\": \"<short>\"}"}, {"role": "user", "content": "Good morning. Getting 500 errors from the dashboard since 09:00. It affects all users. Ref #3327."}, {"role": "assistant", "content": "{\"category\": \"outage\", \"severity\": 5, \"next_action\": \"page on-call; check status page\"}"}]}

data/triage_schema.json JSON · 10 lines

1
{
2
  "type": "object",
3
  "properties": {
4
    "category": {"type": "string", "enum": ["billing", "outage", "how-to", "abuse"]},
5
    "severity": {"type": "integer", "minimum": 1, "maximum": 5},
6
    "next_action": {"type": "string", "maxLength": 120}
7
  },
8
  "required": ["category", "severity", "next_action"],
9
  "additionalProperties": false
10
}