diff --git a/.github/workflows/notify_failure.yml b/.github/workflows/notify_failure.yml index 7a62c49..76633cc 100644 --- a/.github/workflows/notify_failure.yml +++ b/.github/workflows/notify_failure.yml @@ -6,6 +6,7 @@ on: - "Check for Crowdin Updates" - "Test Failure Notification" - "Crowdin Multiple Translations Report" + - "Zendesk Ticket Triage" types: - completed diff --git a/.github/workflows/zendesk_triage.yml b/.github/workflows/zendesk_triage.yml new file mode 100644 index 0000000..eccd46b --- /dev/null +++ b/.github/workflows/zendesk_triage.yml @@ -0,0 +1,123 @@ +name: Zendesk Ticket Triage + +# Runs daily over a 48h window (not 24h, so a failed run doesn't silently drop a +# day of tickets). The overlap does not produce duplicate Discord posts: a dedup +# state file records each reported ticket's Zendesk updated_at, so an unchanged +# ticket is skipped entirely on the next run, and a changed one is re-reported +# and flagged with ๐Ÿ”„. +# +# State lives in the Actions cache, which is best-effort โ€” see the state note in +# the README. If it is ever missing the run degrades to re-reporting the window +# once, which is noisy but never wrong. +on: + schedule: + - cron: "0 7 * * *" + workflow_dispatch: + inputs: + query: + description: "Zendesk search query (overrides the rolling window entirely)" + required: false + window_hours: + description: "Analyze tickets created in the last N hours (default 48)" + required: false + max_tickets: + description: "Max tickets to analyze (default 1000, Zendesk's search cap)" + required: false + reset_state: + description: "Ignore saved state and re-report everything in the window" + type: boolean + default: false + +# Two overlapping runs would race on the same state file, and the loser's +# reported tickets would be forgotten. Queue instead of cancelling, so a +# manual run never discards a scheduled run's state write. +concurrency: + group: zendesk-triage + cancel-in-progress: false + +# The job only reads the repo; everything it writes goes to Zendesk/Discord over +# their own credentials. Nothing needs a writable GITHUB_TOKEN. +permissions: + contents: read + +jobs: + triage: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install -r zendesk_triage/requirements.txt + + # Unique key so every run writes a fresh entry; the restore-keys prefix pulls + # in the most recent previous one. run_attempt is in the key because cache + # entries are immutable: a re-run reuses run_id, so without it attempt 2's save + # would collide with attempt 1's and silently write nothing. Attempt 2 then + # restores attempt 1's entry via the prefix, so tickets already delivered + # aren't reposted. Restore and save are split (rather than the combined + # actions/cache) so the save can run with if: always() โ€” the script records + # partially-delivered tickets even when a later Discord POST fails, and the + # combined action would discard that on a failed job. + - name: Restore triage state + uses: actions/cache/restore@v4 + with: + path: .triage-state + key: zendesk-triage-state-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + zendesk-triage-state- + + # Inputs arrive via env, never interpolated straight into a shell command, + # and both outputs are stripped to digits so the run step below is safe. + - name: Resolve window and cap + id: cfg + env: + INPUT_WINDOW: ${{ github.event.inputs.window_hours }} + INPUT_MAX: ${{ github.event.inputs.max_tickets }} + RESET_STATE: ${{ github.event.inputs.reset_state }} + run: | + # The cap is a runaway guard, not a batch size: a 48h window is ~45 + # tickets, and anything over --batch-size is split across requests + # rather than truncated. 1000 is Zendesk's own search result limit โ€” + # asking for more just walks pagination into a 422, so don't. + window=$(printf '%s' "${INPUT_WINDOW:-48}" | tr -cd '0-9') + max=$(printf '%s' "${INPUT_MAX:-1000}" | tr -cd '0-9') + : "${window:=48}" + : "${max:=1000}" + echo "window=$window" >> "$GITHUB_OUTPUT" + echo "max=$max" >> "$GITHUB_OUTPUT" + if [ "$RESET_STATE" = "true" ]; then + rm -f .triage-state/seen.json + echo "State reset: every ticket in the window will be re-reported." + fi + echo "Window: ${window}h, max tickets: ${max}" + + - name: Run triage + env: + ZENDESK_SUBDOMAIN: ${{ secrets.ZENDESK_SUBDOMAIN }} + ZENDESK_EMAIL: ${{ secrets.ZENDESK_EMAIL }} + ZENDESK_API_TOKEN: ${{ secrets.ZENDESK_API_TOKEN }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} + ZENDESK_QUERY: ${{ github.event.inputs.query }} + ZENDESK_TRIAGE_MODEL: ${{ vars.ZENDESK_TRIAGE_MODEL }} + run: | + mkdir -p .triage-state + python zendesk_triage/triage.py \ + --window-hours "${{ steps.cfg.outputs.window }}" \ + --max-tickets "${{ steps.cfg.outputs.max }}" \ + --state .triage-state/seen.json + + # always(): the script writes state for tickets Discord accepted even when a + # later message fails, and that must survive the job's non-zero exit. + - name: Save triage state + if: always() + uses: actions/cache/save@v4 + with: + path: .triage-state + key: zendesk-triage-state-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/.gitignore b/.gitignore index 441708f..792c7cf 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,11 @@ do_not_commit.sh # Local Crowdin audit outputs (persist across sessions, not for commit) .crowdin_audit/ + +# Local Zendesk triage debugging artifacts โ€” these contain ticket content +zendesk_triage/*.json + +# Zendesk triage dedup state (ticket ids + timestamps; restored from CI cache) +.triage-state/ + +.claude/ diff --git a/README.md b/README.md index 1526090..3570f70 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,175 @@ Runs automatically every Monday at 00:00 UTC. > **Note:** Plural strings skip variable/tag comparison because languages have different plural forms (English: 2, Arabic: 6, Russian: 4). It would be nice to add suppot for plural validation in the future. +## Zendesk Ticket Triage + +Claude reviews recently-created unsolved Zendesk tickets via the API and posts a summary to Discord that links back to each original ticket and highlights the ones worth looking into. For each ticket it assigns a category, infers severity, guesses a likely root cause, identifies platform and app version, groups likely duplicates into clusters, and ranks by priority. + +### Categories + +`CATEGORY_SPECS` in [triage.py](zendesk_triage/triage.py) is the single source of truth โ€” the schema enum, the Discord labels, the urgency colours, and the prompt guidance are all derived from it, so adding a category is one edit. + +| Category | Notes | +| --- | --- | +| `abuse_report` | One user reporting another for illegal content. ~11% of non-review tickets | +| `security_report` | Vulnerability or exploit disclosure | +| `legal_or_data_request` | GDPR, subpoena, law enforcement | +| `bug_report` | Something is broken | +| `account_access` | Lost recovery phrase, locked out | +| `policy_question` | Law/regulation questions ("Chat Control", encryption backdoors) | +| `low_star_review` | โ‰ค3โ˜… app-store review โ€” these often hide a real bug | +| `positive_review` | 4-5โ˜… review, no actionable content | +| `feature_request`, `question`, `spam_or_solicitation`, `other` | | + +The first three are **urgent categories**: they are not bugs, so the model rates their severity `not_applicable`. Colouring by severity alone painted them the calmest blue and sorted them last, so category urgency wins โ€” they render dark red, sort ahead of everything else, and cannot be pushed out of the digest by the display cap. + +### App-store review filtering + +73% of tickets are AppFollow-imported app-store reviews, and 71% of those are 5โ˜… โ€” 59% of *all* tickets are 4-5โ˜… reviews that are never actionable. Those are counted, not classified, cutting the batch roughly 60% (a real run: 48 fetched โ†’ 20 classified). + +Detection uses the Zendesk `via.channel`, which identified reviews with no false positives in a 3,662-ticket sample (2,656/2,656). **Not** tags โ€” only 287 of those reviews carried the `app-store` tag. Reviews whose star rating can't be parsed are kept rather than dropped. Use `--include-positive-reviews` to disable, or `--review-star-floor` to move the threshold. + +### Content-free tickets + +Twitter DM tickets arrive with `description` identical to `subject` โ€” both just `"Conversation with "` โ€” which is 15% of non-review tickets and unclassifiable as fetched. For those only, `hydrate_descriptions` fetches a page of up to 10 comments and joins every body that differs from the subject into the description; later replies often carry the actual detail. Hydration is an enrichment, so an HTTP error or an unreachable endpoint leaves the ticket as-is rather than failing the run (`--no-hydrate` to skip it entirely). + +The script (`zendesk_triage/triage.py`) fetches the tickets in a rolling time window, sends the whole batch to Claude in one structured-output request, and posts Discord embeds: a summary embed plus one embed per highlighted ticket (linking to the ticket in Zendesk). + +The summary embed accounts for the batch in full, so nothing is dropped silently: + +``` +Analyzed **2** of **47** tickets in the window (created in the past 2 days). Skipped **45** already reported and unchanged. +Backlog: **5,609** unsolved tickets in total (not triaged). +**1** worth looking into. ๐Ÿ”„ **1** changed since last reported. +``` + +> **Scope:** the window covers tickets *created* recently, so the long tail of older unsolved tickets is counted in the backlog line but not triaged. That is deliberate โ€” the job is a new-ticket digest, not a backlog sweep. + +### Deduplication + +The daily window is 48h, so consecutive runs overlap. A state file (`--state`) records each reported ticket's Zendesk `updated_at`, giving three outcomes per ticket: + +| Ticket | Outcome | +| ------ | ------- | +| Not seen before | Analyzed and reported | +| Seen, `updated_at` unchanged | **Skipped before the model call** โ€” costs no tokens | +| Seen, `updated_at` moved | Re-analyzed, reported, and flagged ๐Ÿ”„ in the embed title | + +State is written only on a real run, and only for tickets covered by messages Discord **accepted**. Each message carries the ticket ids it accounts for, so a partial failure records exactly what landed: already-posted messages aren't repeated next run, and undelivered tickets stay eligible. The run then exits non-zero. `--dry-run` never writes state. + +Two caveats worth knowing: + +- **Any** agent action bumps `updated_at` (a reply, a tag, a status change), not just an end-user comment, so agent activity can trigger a re-report. Narrowing this to new end-user comments would need per-ticket comment fetches. +- Unchanged tickets are filtered out *before* the model call, which is what makes the dedup free. The trade-off is that duplicate-cluster detection only sees the new and changed tickets in a given run, not the whole window. + +> **Note:** This repo is public, so ticket content is never written to the run logs or the job summary โ€” ticket detail goes only to the Discord webhook (a private channel), and the links require Zendesk auth to open. The one exception is the local `--dump-batch` debugging flag, which writes ticket content to a file you name; `zendesk_triage/*.json` is gitignored to keep those out of the repo. + +### Required Secrets + +| Secret | Description | +| --------------------- | ------------------------------------------------------- | +| `ZENDESK_SUBDOMAIN` | Zendesk subdomain (`mycompany` โ†’ `mycompany.zendesk.com`) | +| `ZENDESK_EMAIL` | Agent email used for Zendesk API-token auth | +| `ZENDESK_API_TOKEN` | Zendesk API token | +| `ANTHROPIC_API_KEY` | Claude API key | +| `DISCORD_WEBHOOK_URL` | Discord webhook (reused from the failure-notification setup) | + +### Optional Configuration + +| Setting | Where | Default | Description | +| ---------------------- | ---------------- | ------------------------------------------------------- | ----------- | +| `--window-hours` | workflow input / flag | `48` | Analyze unsolved tickets created in the last N hours | +| `--state` | flag | *(unset)* | Dedup state file. The workflow points this at the cached `.triage-state/seen.json` | +| `--state-retention-days` | flag | `30` | Forget state entries older than N days | +| `ZENDESK_QUERY` | env / `--query` | *(unset)* | Explicit Zendesk search query. Overrides `--window-hours` entirely | +| `ZENDESK_TRIAGE_MODEL` | repo variable / `--model` | `claude-opus-4-8` | Set to a cheaper model (e.g. `claude-haiku-4-5`) to reduce cost on large batches | +| `--max-tickets` | workflow input / flag | `1000` (workflow) / `100` (flag) | Runaway guard on tickets analyzed per run, **not** a batch size. The workflow passes `1000`; a bare `python triage.py` uses the script's own `DEFAULT_MAX_TICKETS` of `100`. Zendesk's search API caps a query at 1000 results, so higher values don't fetch more | +| `--batch-size` | flag | `400` | Split batches larger than this across multiple requests | +| `--review-star-floor` | flag | `3` | Classify app-store reviews at or below N stars; count the rest | +| `--include-positive-reviews` | flag | off | Classify every review, including 4-5โ˜… ones | +| `--no-hydrate` | flag | off | Skip fetching comments for content-free tickets | +| `--effort` | flag | `medium` | Claude reasoning effort (`low`โ€“`max`) | + +#### Batch size vs. ticket cap + +These do different jobs, and conflating them is how you get a silently truncated digest: + +- **`--max-tickets`** bounds how much of the Zendesk result set is fetched. At the workflow's 1000 it never binds on a 48h window (~45 tickets); it exists so a spam flood or a wide `reset_state` backfill can't run away. 1000 is also [Zendesk's own search result limit](https://developer.zendesk.com/api-reference/ticketing/ticket-management/search/#results-limit) โ€” the API returns `422` for any page past it, so the fetch stops at 1000 regardless of what you pass, and reports the matched-vs-analyzed gap rather than failing. +- **`--batch-size`** bounds how many tickets go into a *single* model request. Anything larger is split across requests and the findings are concatenated. + +The split is necessary because output tokens, not context, are the binding constraint. Measured on real tickets: **~118 input tokens and ~102 output tokens per ticket**, with adaptive thinking drawing from the same `max_tokens` budget. + +| Batch | Input | Output needed | Fits in one request? | +| ----- | ----- | ------------- | -------------------- | +| 45 (typical daily) | ~5K | ~5K | Yes | +| 400 (`--batch-size`) | ~47K | ~41K | Yes, with room for thinking | +| 1000 (`--max-tickets`) | ~118K | ~102K | **No** โ€” leaves only ~26K of the 128K output ceiling for thinking | + +If a single request ever does hit the ceiling, the script exits with that explicit reason rather than failing on an incomplete-JSON parse error. + +> Chunking is per-request, so `cluster` labels and `priority_rank` are only meaningful within a chunk. Batches large enough to split are ones where completing at all matters more than cross-chunk cluster fidelity. + +### Schedule + +Runs daily at 07:00 UTC over a 48h window (~45 tickets). The window is 48h rather than 24h so a failed run doesn't silently drop a day of tickets; the resulting overlap doesn't produce duplicate posts because of the dedup state described above. + +Triggerable manually via **workflow_dispatch** (optional `query` / `window_hours` / `max_tickets` inputs, plus `reset_state` to re-report the whole window). Failures are reported through the Discord failure-notification workflow, which watches this workflow by name โ€” so renaming `Zendesk Ticket Triage` means updating the `workflows:` list in [`notify_failure.yml`](.github/workflows/notify_failure.yml) too. + +#### How state survives between runs + +State is kept in the **GitHub Actions cache**, not committed โ€” this repo is public, and ticket IDs plus timestamps would leak ticket volume and activity rates. The workflow writes a unique cache key per run attempt and restores the most recent one by prefix: + +```yaml +key: zendesk-triage-state-${{ github.run_id }}-${{ github.run_attempt }} +restore-keys: | + zendesk-triage-state- +``` + +`run_attempt` is in the key because cache entries are **immutable**: a re-run reuses `run_id`, so keying on that alone would make the second attempt's save collide with the first's and write nothing. With it, attempt 2 saves its own entry and restores attempt 1's through the prefix โ€” so tickets the first attempt already delivered aren't reposted. + +The cache is **best-effort**, and the script is written to tolerate that โ€” a missing, corrupt, or wrong-shaped state file degrades to "treat every ticket as new", which is noisy for one run but never wrong. Things that can lose state: + +- **7 days without a cache hit** evicts the entry. The daily run keeps it warm, so this only bites if the workflow is disabled for a week. +- **Repo cache eviction** under the 10GB limit (LRU). The state file is a few KB, so this is unlikely. +- **Branch scoping:** caches written on the default branch are readable everywhere; a run on a feature branch won't see them and vice versa. + +The save step is `actions/cache/save` with `if: always()`, deliberately split from the restore rather than using the combined `actions/cache`. The combined action skips its save when a job fails, which would discard the partial-delivery record described above โ€” so a Discord failure on message 3 of 3 would repost messages 1 and 2 on the next run. + +A `concurrency` group serialises runs, because two overlapping runs would race on the same state file and the loser's recorded tickets would be forgotten. + +If you outgrow the cache's guarantees, the next step up is a private store (a private gist, S3, or a private companion repo) โ€” **not** committing state to this public repo. + +### Tests + +``` +python -m unittest discover -s zendesk_triage -v +``` + +Offline tests covering the window arithmetic, dedup partitioning, state round-trip and pruning, corrupt-state degradation, Discord embed rendering and chunking, defensive JSON parsing, and the retry/pagination behaviour with a stub session. No secrets or network access needed. They run in CI on any push or PR touching `zendesk_triage/`. + +### Local Testing + +``` +pip install -r zendesk_triage/requirements.txt +export ZENDESK_SUBDOMAIN=... ZENDESK_EMAIL=... ZENDESK_API_TOKEN=... ANTHROPIC_API_KEY=... + +# fetch + analyze, print the Discord payload, post nothing +python zendesk_triage/triage.py --window-hours 48 --dry-run +``` + +No `ANTHROPIC_API_KEY`? Two debug backends skip the Anthropic API entirely: + +``` +# classify via the local `claude` CLI (authenticates as Claude Code) +python zendesk_triage/triage.py --backend claude-cli --window-hours 48 --dry-run + +# or dump the batch, classify it by hand, and feed the findings back +python zendesk_triage/triage.py --dump-batch /tmp/batch.json --window-hours 48 +python zendesk_triage/triage.py --backend file --findings /tmp/findings.json --dry-run +``` + +The `claude-cli` backend has no structured-output enforcement, so its field values are looser than the API path's (e.g. `"en"` where the schema asks for `"English"`), and each invocation carries ~25K tokens of Claude Code system-prompt overhead. Use it for debugging, not for scheduled runs. + ## Workflow Failure Notificaiton If a workflow fails and is in the list of workflows monitored by the failure notificaiton workflow, the failure notificaiton workflow will send a message to a discord webhook. diff --git a/zendesk_triage/requirements.txt b/zendesk_triage/requirements.txt new file mode 100644 index 0000000..00d0e6e --- /dev/null +++ b/zendesk_triage/requirements.txt @@ -0,0 +1,2 @@ +anthropic==0.116.0 +requests==2.32.3 diff --git a/zendesk_triage/test_triage.py b/zendesk_triage/test_triage.py new file mode 100644 index 0000000..e2f880e --- /dev/null +++ b/zendesk_triage/test_triage.py @@ -0,0 +1,1212 @@ +#!/usr/bin/env python3 +"""Tests for the Zendesk triage script. + +Stdlib unittest so the repo needs no test dependency. Run from anywhere: + + python -m unittest discover -s zendesk_triage -v + +Everything here is offline โ€” no Zendesk, Anthropic, or Discord calls. The fetch +tests drive fetch_tickets with a stub session instead. +""" +import json +import os +import re +import sys +import tempfile +import unittest +from datetime import datetime, timedelta, timezone + +import requests + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import triage # noqa: E402 (needs the path insert above) + + +STAMP = "%Y-%m-%dT%H:%M:%SZ" + + +def ticket(ticket_id, updated_at="2026-08-03T12:00:00Z", **extra): + """A Zendesk-shaped ticket, only the fields the script actually reads.""" + row = { + "id": ticket_id, + "result_type": "ticket", + "subject": f"Subject {ticket_id}", + "description": f"Description {ticket_id}", + "tags": [], + "priority": "normal", + "status": "open", + "updated_at": updated_at, + } + row.update(extra) + return row + + +def finding(ticket_id, **extra): + """A classification result, with every key build_*_embed touches.""" + row = { + "id": ticket_id, + "category": "bug_report", + "severity": "major", + "affected_component": "sync", + "summary": f"Summary {ticket_id}", + "likely_root_cause": "cause", + "language": "English", + "priority_rank": 1, + "worth_looking_into": True, + "cluster": "", + } + row.update(extra) + return row + + +def build_messages(*args, **kwargs): + """triage.build_messages returns (messages, coverage); most tests want messages.""" + return triage.build_messages(*args, **kwargs)[0] + + +class FakeResponse: + # retry-after: 0 keeps the retry tests instant instead of sleeping through + # the real backoff, and exercises the header-honoring path while it's at it. + def __init__(self, payload, status_code=200, retry_after="0"): + self._payload = payload + self.status_code = status_code + self.headers = {"retry-after": retry_after} + self.text = json.dumps(payload) + + def json(self): + return self._payload + + +class NonJsonResponse(FakeResponse): + """A 200 whose body isn't JSON โ€” a proxy error page, say.""" + + def __init__(self): + super().__init__({}) + self.text = "maintenance" + + def json(self): + raise requests.exceptions.JSONDecodeError("Expecting value", self.text, 0) + + +class FakeSession: + """Returns queued responses in order and records the requests made. + + A queued Exception is raised instead of returned, so transport failures can be + exercised alongside HTTP status codes. + """ + + def __init__(self, responses): + self._responses = list(responses) + self.calls = [] + + def request(self, method, url, **kwargs): + self.calls.append((method, url, kwargs)) + item = self._responses.pop(0) + if isinstance(item, Exception): + raise item + return item + + +class NoSleep: + """Patch out time.sleep so retry tests assert on delays without waiting.""" + + def __enter__(self): + self.slept = [] + self._real = triage.time.sleep + triage.time.sleep = self.slept.append + return self + + def __exit__(self, *exc): + triage.time.sleep = self._real + return False + + +# ---- Window construction --------------------------------------------------- + + +class TestWindowQuery(unittest.TestCase): + def test_cutoff_is_the_requested_number_of_hours_back(self): + query = triage.build_window_query(48) + cutoff = query.split("created>")[1].split(" ")[0] + parsed = datetime.strptime(cutoff, STAMP).replace(tzinfo=timezone.utc) + expected = datetime.now(timezone.utc) - timedelta(hours=48) + self.assertLess(abs((parsed - expected).total_seconds()), 120) + + def test_query_keeps_unsolved_filter_and_newest_first_ordering(self): + query = triage.build_window_query(48) + self.assertIn("type:ticket", query) + self.assertIn("status")[1].split(" ")[0] + long = triage.build_window_query(168).split("created>")[1].split(" ")[0] + self.assertLess(long, short) # ISO-8601 sorts chronologically + + def test_window_label_reads_naturally(self): + self.assertEqual(triage.window_label(24), "created in the past 1 day") + self.assertEqual(triage.window_label(48), "created in the past 2 days") + self.assertEqual(triage.window_label(168), "created in the past 7 days") + self.assertEqual(triage.window_label(36), "created in the past 36h") + + +# ---- Dedup state ----------------------------------------------------------- + + +class TestPartitionByState(unittest.TestCase): + def test_empty_state_makes_everything_new(self): + new, changed, unchanged = triage.partition_by_state( + [ticket(1), ticket(2)], triage.empty_state() + ) + self.assertEqual([t["id"] for t in new], [1, 2]) + self.assertEqual(changed, []) + self.assertEqual(unchanged, []) + + def test_same_updated_at_is_unchanged(self): + state = {"seen": {"1": {"updated_at": "2026-08-03T12:00:00Z"}}} + new, changed, unchanged = triage.partition_by_state( + [ticket(1, updated_at="2026-08-03T12:00:00Z")], state + ) + self.assertEqual((new, changed), ([], [])) + self.assertEqual([t["id"] for t in unchanged], [1]) + + def test_moved_updated_at_is_changed(self): + state = {"seen": {"1": {"updated_at": "2026-08-03T12:00:00Z"}}} + new, changed, unchanged = triage.partition_by_state( + [ticket(1, updated_at="2026-08-04T09:00:00Z")], state + ) + self.assertEqual((new, unchanged), ([], [])) + self.assertEqual([t["id"] for t in changed], [1]) + + def test_mixed_batch_splits_three_ways(self): + state = { + "seen": { + "1": {"updated_at": "2026-08-03T12:00:00Z"}, + "2": {"updated_at": "2026-08-01T00:00:00Z"}, + } + } + batch = [ + ticket(1, updated_at="2026-08-03T12:00:00Z"), # unchanged + ticket(2, updated_at="2026-08-04T09:00:00Z"), # changed + ticket(3), # new + ] + new, changed, unchanged = triage.partition_by_state(batch, state) + self.assertEqual([t["id"] for t in new], [3]) + self.assertEqual([t["id"] for t in changed], [2]) + self.assertEqual([t["id"] for t in unchanged], [1]) + + def test_ids_are_matched_as_strings_not_ints(self): + """State comes back from JSON, where keys are always strings.""" + state = {"seen": {"27564": {"updated_at": "2026-08-03T12:00:00Z"}}} + _, _, unchanged = triage.partition_by_state( + [ticket(27564, updated_at="2026-08-03T12:00:00Z")], state + ) + self.assertEqual(len(unchanged), 1) + + +class TestStateRoundTrip(unittest.TestCase): + def setUp(self): + self.dir = tempfile.TemporaryDirectory() + self.path = os.path.join(self.dir.name, "nested", "seen.json") + self.addCleanup(self.dir.cleanup) + + def test_save_then_load_recovers_reported_tickets(self): + triage.save_state(self.path, triage.empty_state(), [ticket(1), ticket(2)], 30) + state = triage.load_state(self.path) + self.assertEqual(sorted(state["seen"]), ["1", "2"]) + self.assertEqual(state["seen"]["1"]["updated_at"], "2026-08-03T12:00:00Z") + self.assertEqual(state["version"], triage.STATE_VERSION) + + def test_save_creates_missing_parent_directories(self): + triage.save_state(self.path, triage.empty_state(), [ticket(1)], 30) + self.assertTrue(os.path.exists(self.path)) + + def test_save_leaves_no_temp_file_behind(self): + triage.save_state(self.path, triage.empty_state(), [ticket(1)], 30) + siblings = os.listdir(os.path.dirname(self.path)) + self.assertEqual(siblings, ["seen.json"]) + + def test_resaving_updates_an_existing_entry(self): + triage.save_state(self.path, triage.empty_state(), [ticket(1, updated_at="A")], 30) + state = triage.load_state(self.path) + triage.save_state(self.path, state, [ticket(1, updated_at="B")], 30) + self.assertEqual(triage.load_state(self.path)["seen"]["1"]["updated_at"], "B") + + def test_entries_past_retention_are_pruned(self): + old = (datetime.now(timezone.utc) - timedelta(days=40)).strftime(STAMP) + recent = (datetime.now(timezone.utc) - timedelta(days=2)).strftime(STAMP) + state = { + "version": 1, + "seen": { + "1": {"updated_at": "A", "last_reported": old}, + "2": {"updated_at": "B", "last_reported": recent}, + }, + } + kept, pruned = triage.save_state(self.path, state, [], 30) + self.assertEqual((kept, pruned), (1, 1)) + self.assertEqual(list(triage.load_state(self.path)["seen"]), ["2"]) + + def test_entries_with_unparseable_timestamps_are_dropped(self): + state = {"version": 1, "seen": {"1": {"updated_at": "A", "last_reported": "nonsense"}}} + kept, pruned = triage.save_state(self.path, state, [], 30) + self.assertEqual((kept, pruned), (0, 1)) + + def test_a_ticket_reported_now_survives_pruning(self): + old = (datetime.now(timezone.utc) - timedelta(days=40)).strftime(STAMP) + state = {"version": 1, "seen": {"1": {"updated_at": "A", "last_reported": old}}} + kept, _ = triage.save_state(self.path, state, [ticket(1, updated_at="B")], 30) + self.assertEqual(kept, 1) + + +class TestStateDegradation(unittest.TestCase): + """A missing or damaged cache must never crash the run โ€” worst case it + re-reports the window once.""" + + def setUp(self): + self.dir = tempfile.TemporaryDirectory() + self.addCleanup(self.dir.cleanup) + + def _write(self, name, content): + path = os.path.join(self.dir.name, name) + with open(path, "w", encoding="utf-8") as fh: + fh.write(content) + return path + + def test_missing_file(self): + path = os.path.join(self.dir.name, "absent.json") + self.assertEqual(triage.load_state(path), triage.empty_state()) + + def test_unparseable_json(self): + self.assertEqual(triage.load_state(self._write("c.json", "{{{")), triage.empty_state()) + + def test_json_that_is_not_an_object(self): + self.assertEqual(triage.load_state(self._write("l.json", "[]")), triage.empty_state()) + + def test_object_without_a_seen_map(self): + self.assertEqual( + triage.load_state(self._write("n.json", '{"version": 1}')), triage.empty_state() + ) + + def test_seen_of_the_wrong_type(self): + self.assertEqual( + triage.load_state(self._write("w.json", '{"seen": []}')), triage.empty_state() + ) + + def test_absent_version_is_a_cache_miss(self): + """Without a version we can't know the fields mean what we think.""" + path = self._write("v.json", '{"seen": {"1": {"updated_at": "A"}}}') + self.assertEqual(triage.load_state(path), triage.empty_state()) + + def test_unknown_version_is_a_cache_miss(self): + path = self._write("v2.json", '{"version": 99, "seen": {"1": {"updated_at": "A"}}}') + self.assertEqual(triage.load_state(path), triage.empty_state()) + + def test_matching_version_loads_normally(self): + path = self._write( + "ok.json", + json.dumps({"version": triage.STATE_VERSION, "seen": {"1": {"updated_at": "A"}}}), + ) + self.assertEqual(list(triage.load_state(path)["seen"]), ["1"]) + + def test_state_written_by_save_state_round_trips_the_version(self): + """Guards against save_state and load_state disagreeing on the version.""" + path = os.path.join(self.dir.name, "rt.json") + triage.save_state(path, triage.empty_state(), [ticket(1)], 30) + self.assertEqual(list(triage.load_state(path)["seen"]), ["1"]) + + +# ---- Discord rendering ----------------------------------------------------- + + +class TestSummaryEmbed(unittest.TestCase): + def description(self, findings, stats): + highlights = [f for f in findings if f.get("worth_looking_into")] + return triage.build_summary_embed(findings, highlights, "acme", stats)["description"] + + def test_reports_analyzed_against_matched(self): + text = self.description([finding(1)], {"matched": 47}) + self.assertIn("Analyzed **1** of **47** tickets in the window", text) + + def test_names_the_window(self): + text = self.description([finding(1)], {"matched": 5, "scope": "created in the past 2 days"}) + self.assertIn("(created in the past 2 days)", text) + + def test_reports_skipped_unchanged_tickets(self): + text = self.description([finding(1)], {"matched": 47, "skipped_unchanged": 45}) + self.assertIn("Skipped **45** already reported and unchanged", text) + + def test_omits_the_skip_line_when_nothing_was_skipped(self): + self.assertNotIn("Skipped", self.description([finding(1)], {"skipped_unchanged": 0})) + + def test_reports_the_untriaged_backlog_with_thousands_separators(self): + text = self.description([finding(1)], {"total_unsolved": 5609}) + self.assertIn("Backlog: **5,609** unsolved tickets in total", text) + + def test_omits_the_backlog_line_when_the_count_is_unavailable(self): + self.assertNotIn("Backlog", self.description([finding(1)], {"total_unsolved": None})) + + def test_flags_how_many_were_re_reports(self): + text = self.description([finding(1)], {"updated_count": 3}) + self.assertIn("๐Ÿ”„ **3** changed since last reported", text) + + def test_counts_crash_and_data_loss_as_serious(self): + findings = [finding(1, severity="crash"), finding(2, severity="data_loss")] + self.assertIn("**2** crash/data-loss", self.description(findings, {})) + + def test_works_with_no_stats_at_all(self): + text = self.description([finding(1)], None) + self.assertIn("Analyzed **1**", text) + self.assertNotIn("of **", text) + + def test_groups_repeated_clusters(self): + findings = [finding(1, cluster="push"), finding(2, cluster="push"), finding(3, cluster="solo")] + embed = triage.build_summary_embed(findings, findings, "acme", {}) + names = [f["name"] for f in embed["fields"]] + self.assertIn("Likely duplicate clusters", names) + clusters = next(f for f in embed["fields"] if f["name"] == "Likely duplicate clusters") + self.assertIn("push", clusters["value"]) + self.assertNotIn("solo", clusters["value"]) # a single ticket is not a cluster + + +class TestHighlightEmbed(unittest.TestCase): + def test_update_marker_only_appears_for_re_reports(self): + fresh = triage.build_highlight_embed(finding(1), "acme", is_update=False) + repeat = triage.build_highlight_embed(finding(1), "acme", is_update=True) + self.assertFalse(fresh["title"].startswith("๐Ÿ”„")) + self.assertTrue(repeat["title"].startswith("๐Ÿ”„")) + + def test_links_back_to_the_ticket(self): + embed = triage.build_highlight_embed(finding(42), "acme") + self.assertEqual(embed["url"], "https://acme.zendesk.com/agent/tickets/42") + + def test_title_stays_within_the_discord_limit(self): + embed = triage.build_highlight_embed(finding(1, summary="x" * 500), "acme", is_update=True) + self.assertLessEqual(len(embed["title"]), 256) + + +class TestBuildMessages(unittest.TestCase): + def test_only_tickets_worth_looking_into_get_their_own_embed(self): + findings = [finding(1), finding(2, worth_looking_into=False)] + messages = build_messages(findings, "acme") + self.assertEqual(len(messages[0]["embeds"]), 2) # summary + one highlight + + def test_highlights_are_ordered_by_priority_rank(self): + findings = [finding(1, priority_rank=3), finding(2, priority_rank=1)] + embeds = build_messages(findings, "acme")[0]["embeds"] + self.assertIn("#2", embeds[1]["title"]) + self.assertIn("#1", embeds[2]["title"]) + + def test_updated_ids_reach_the_right_embed(self): + findings = [finding(1), finding(2)] + embeds = build_messages(findings, "acme", {}, updated_ids={2})[0]["embeds"] + titles = {e["title"].lstrip("๐Ÿ”„ ").split(" ")[0]: e["title"] for e in embeds[1:]} + self.assertFalse(titles["#1"].startswith("๐Ÿ”„")) + self.assertTrue(titles["#2"].startswith("๐Ÿ”„")) + + def test_embeds_are_chunked_to_the_discord_per_message_limit(self): + findings = [finding(i, priority_rank=i) for i in range(triage.MAX_HIGHLIGHTS)] + messages = build_messages(findings, "acme") + for message in messages: + self.assertLessEqual(len(message["embeds"]), triage.MAX_EMBEDS_PER_MESSAGE) + total = sum(len(m["embeds"]) for m in messages) + self.assertEqual(total, triage.MAX_HIGHLIGHTS + 1) # + the summary + + def test_highlights_beyond_the_cap_are_dropped_but_announced(self): + over = triage.MAX_HIGHLIGHTS + 5 + findings = [finding(i, priority_rank=i) for i in range(over)] + messages = build_messages(findings, "acme") + self.assertIn(f"top {triage.MAX_HIGHLIGHTS} of {over}", messages[0]["content"]) + + def test_no_content_line_when_nothing_was_dropped(self): + messages = build_messages([finding(1)], "acme") + self.assertNotIn("content", messages[0]) + + +# ---- Parsing helpers ------------------------------------------------------- + + +class TestTaxonomyIsDerived(unittest.TestCase): + """CATEGORY_SPECS is the single source of truth. These caught a real bug: a stale + hardcoded CATEGORY_LABEL further down the module was shadowing the derived one, + so seven new categories silently rendered as raw enum values.""" + + def test_every_category_has_a_discord_label(self): + missing = [c for c in triage.CATEGORIES if c not in triage.CATEGORY_LABEL] + self.assertEqual(missing, []) + + def test_every_category_is_explained_in_the_system_prompt(self): + missing = [c for c in triage.CATEGORIES if c not in triage.SYSTEM_PROMPT] + self.assertEqual(missing, []) + + def test_every_category_is_listed_in_the_cli_instructions(self): + missing = [c for c in triage.CATEGORIES if c not in triage.CLI_JSON_INSTRUCTIONS] + self.assertEqual(missing, []) + + def test_every_schema_field_is_listed_in_the_cli_instructions(self): + """The CLI backend has no structured-output enforcement, so a field absent + from these instructions comes back empty โ€” which is how platform, app_version + and reported_session_id silently went unpopulated.""" + fields = triage.SCHEMA["properties"]["tickets"]["items"]["properties"] + missing = [f for f in fields if f not in triage.CLI_JSON_INSTRUCTIONS] + self.assertEqual(missing, []) + + def test_every_enum_value_is_listed_in_the_cli_instructions(self): + for values in (triage.CATEGORIES, triage.SEVERITIES, triage.PLATFORMS): + for value in values: + self.assertIn(value, triage.CLI_JSON_INSTRUCTIONS) + + def test_schema_enum_matches_the_category_list(self): + item = triage.SCHEMA["properties"]["tickets"]["items"] + self.assertEqual(item["properties"]["category"]["enum"], triage.CATEGORIES) + + def test_schema_requires_every_property(self): + """Structured outputs reject a schema whose fields aren't all required.""" + item = triage.SCHEMA["properties"]["tickets"]["items"] + self.assertEqual(sorted(item["required"]), sorted(item["properties"])) + + def test_category_names_are_unique(self): + self.assertEqual(len(triage.CATEGORIES), len(set(triage.CATEGORIES))) + + def test_urgent_categories_are_real_categories(self): + self.assertTrue(triage.URGENT_CATEGORIES.issubset(set(triage.CATEGORIES))) + + def test_platform_enum_is_wired_into_the_schema(self): + item = triage.SCHEMA["properties"]["tickets"]["items"] + self.assertEqual(item["properties"]["platform"]["enum"], triage.PLATFORMS) + + +class TestUrgency(unittest.TestCase): + def test_urgent_category_beats_a_benign_severity(self): + """An abuse report is not a bug, so severity is not_applicable โ€” which used to + paint the most serious ticket in the digest the calmest colour.""" + abuse = finding(1, category="abuse_report", severity="not_applicable") + self.assertEqual(triage.embed_color(abuse), triage.CATEGORY_COLOR["abuse_report"]) + self.assertNotEqual(triage.embed_color(abuse), + triage.SEVERITY_COLOR["not_applicable"]) + + def test_non_urgent_category_still_uses_severity(self): + self.assertEqual(triage.embed_color(finding(1, category="bug_report", severity="crash")), + triage.SEVERITY_COLOR["crash"]) + + def test_unknown_severity_falls_back_to_grey(self): + self.assertEqual(triage.embed_color({"category": "other", "severity": "???"}), 0x95A5A6) + + def test_urgent_tickets_are_highlighted_even_if_not_flagged(self): + abuse = finding(1, category="abuse_report", worth_looking_into=False) + shown, _ = triage.select_highlights([abuse]) + self.assertEqual([f["id"] for f in shown], [1]) + + def test_urgent_tickets_sort_ahead_of_better_ranked_ordinary_ones(self): + ordinary = finding(1, category="bug_report", priority_rank=1) + abuse = finding(2, category="abuse_report", priority_rank=99) + shown, _ = triage.select_highlights([ordinary, abuse]) + self.assertEqual([f["id"] for f in shown], [2, 1]) + + def test_urgent_tickets_cannot_be_pushed_out_by_the_display_cap(self): + ordinary = [finding(i, priority_rank=i) for i in range(triage.MAX_HIGHLIGHTS + 5)] + abuse = finding(9999, category="abuse_report", priority_rank=9999) + shown, omitted = triage.select_highlights(ordinary + [abuse]) + self.assertIn(9999, [f["id"] for f in shown]) + self.assertNotIn(9999, [f["id"] for f in omitted]) + + +class TestReviewFiltering(unittest.TestCase): + def review(self, ticket_id, stars, channel="any_channel"): + return ticket(ticket_id, subject="โ˜…" * stars + "โ˜†" * (5 - stars) + " \n\tGreat app", + via={"channel": channel}) + + def test_star_count_is_read_from_the_subject(self): + self.assertEqual(triage.review_stars(self.review(1, 5)), 5) + self.assertEqual(triage.review_stars(self.review(2, 1)), 1) + + def test_non_review_subject_has_no_stars(self): + self.assertIsNone(triage.review_stars(ticket(1, subject="Notifications broken"))) + + def test_channel_identifies_a_review_without_stars_in_the_subject(self): + """The channel is the reliable signal: only 287 of 2,656 sampled reviews + carried the app-store tag, so tag-based filtering would miss most.""" + self.assertTrue(triage.is_store_review( + ticket(1, subject="no stars here", via={"channel": "any_channel"}))) + + def test_web_tickets_are_not_reviews(self): + self.assertFalse(triage.is_store_review(ticket(1, via={"channel": "web"}))) + + def test_positive_reviews_are_skipped(self): + keep, skipped = triage.partition_reviews( + [self.review(1, 5), self.review(2, 4)], star_floor=3) + self.assertEqual(keep, []) + self.assertEqual(len(skipped), 2) + + def test_low_star_reviews_are_kept_because_they_hide_bugs(self): + keep, skipped = triage.partition_reviews( + [self.review(1, 1), self.review(2, 2), self.review(3, 3)], star_floor=3) + self.assertEqual(len(keep), 3) + self.assertEqual(skipped, []) + + def test_real_support_tickets_are_never_skipped(self): + support = ticket(1, subject="Cannot send messages", via={"channel": "web"}) + keep, skipped = triage.partition_reviews([support], star_floor=3) + self.assertEqual(keep, [support]) + self.assertEqual(skipped, []) + + def test_a_review_with_an_unparseable_rating_is_kept(self): + """Better to spend a few tokens than silently drop a real complaint.""" + odd = ticket(1, subject="loved it", via={"channel": "any_channel"}) + keep, skipped = triage.partition_reviews([odd], star_floor=3) + self.assertEqual(keep, [odd]) + self.assertEqual(skipped, []) + + def test_the_floor_is_configurable(self): + keep, skipped = triage.partition_reviews([self.review(1, 4)], star_floor=4) + self.assertEqual(len(keep), 1) + self.assertEqual(skipped, []) + + +class TestContentFreeTickets(unittest.TestCase): + """Twitter DM tickets arrive with subject and body both 'Conversation with + ' โ€” 15% of non-review tickets, unclassifiable as fetched.""" + + def test_detects_a_description_that_repeats_the_subject(self): + self.assertTrue(triage.is_content_free( + ticket(1, subject="Conversation with x", description="Conversation with x"))) + + def test_tolerates_whitespace_differences(self): + self.assertTrue(triage.is_content_free( + ticket(1, subject="Conversation with x", description="Conversation with x\n"))) + + def test_a_real_description_is_not_content_free(self): + self.assertFalse(triage.is_content_free( + ticket(1, subject="Notifications", description="I get no notifications"))) + + def test_hydration_pulls_the_first_informative_comment(self): + row = ticket(1, subject="Conversation with x", description="Conversation with x") + session = FakeSession([FakeResponse({"comments": [ + {"body": "Conversation with x"}, + {"body": "My messages will not send since the update"}, + ]})]) + self.assertEqual(triage.hydrate_descriptions(session, "acme", [row]), 1) + self.assertIn("will not send", row["description"]) + + def test_hydration_skips_tickets_that_already_have_content(self): + row = ticket(1, subject="Notifications", description="No notifications at all") + session = FakeSession([]) + self.assertEqual(triage.hydrate_descriptions(session, "acme", [row]), 0) + self.assertEqual(session.calls, []) # no wasted API call + + def test_hydration_survives_an_api_failure(self): + row = ticket(1, subject="Conversation with x", description="Conversation with x") + session = FakeSession([FakeResponse({}, status_code=404)]) + self.assertEqual(triage.hydrate_descriptions(session, "acme", [row]), 0) + self.assertEqual(row["description"], "Conversation with x") # left as-is + + def test_hydration_survives_a_transport_failure(self): + """An unreachable comments endpoint must not abort the whole digest.""" + row = ticket(1, subject="Conversation with x", description="Conversation with x") + session = FakeSession([requests.ConnectionError("unreachable")] * 2) + with NoSleep(): + self.assertEqual(triage.hydrate_descriptions(session, "acme", [row]), 0) + self.assertEqual(row["description"], "Conversation with x") + + def test_hydration_survives_a_non_json_body(self): + """A 200 carrying an HTML error page is as harmless as an HTTP error here.""" + row = ticket(1, subject="Conversation with x", description="Conversation with x") + session = FakeSession([NonJsonResponse()]) + self.assertEqual(triage.hydrate_descriptions(session, "acme", [row]), 0) + self.assertEqual(row["description"], "Conversation with x") # left as-is + + def test_one_unreachable_ticket_does_not_block_the_next(self): + rows = [ + ticket(1, subject="Conversation with a", description="Conversation with a"), + ticket(2, subject="Conversation with b", description="Conversation with b"), + ] + session = FakeSession([ + requests.ConnectionError("unreachable"), requests.ConnectionError("unreachable"), + FakeResponse({"comments": [{"body": "Cannot log in since the update"}]}), + ]) + with NoSleep(): + self.assertEqual(triage.hydrate_descriptions(session, "acme", rows), 1) + self.assertIn("Cannot log in", rows[1]["description"]) + + def test_hydration_joins_every_informative_comment(self): + """It joins all bodies differing from the subject within the fetched page, + not just the first โ€” later replies often carry the detail.""" + row = ticket(1, subject="Conversation with x", description="Conversation with x") + session = FakeSession([FakeResponse({"comments": [ + {"body": "Conversation with x"}, + {"body": "first detail"}, + {"body": "second detail"}, + ]})]) + triage.hydrate_descriptions(session, "acme", [row]) + self.assertIn("first detail", row["description"]) + self.assertIn("second detail", row["description"]) + + def test_hydration_requests_a_bounded_page_of_comments(self): + row = ticket(1, subject="Conversation with x", description="Conversation with x") + session = FakeSession([FakeResponse({"comments": [{"body": "detail"}]})]) + triage.hydrate_descriptions(session, "acme", [row]) + _, _, kwargs = session.calls[0] + self.assertEqual(kwargs["params"], {"per_page": 10}) + + def test_hydration_leaves_the_ticket_alone_when_no_comment_adds_anything(self): + row = ticket(1, subject="Conversation with x", description="Conversation with x") + session = FakeSession([FakeResponse({"comments": [{"body": "Conversation with x"}]})]) + self.assertEqual(triage.hydrate_descriptions(session, "acme", [row]), 0) + + +class TestEmbedCharLimit(unittest.TestCase): + """Discord caps a message at 10 embeds *and* 6,000 chars across them; chunking on + count alone can build a payload Discord rejects.""" + + def fat(self, ticket_id): + # ~1,300 chars of field text: 10 of these would be ~13,000, over the limit. + return finding(ticket_id, summary="s" * 200, likely_root_cause="r" * 300, + affected_component="c" * 100, language="l" * 40) + + def test_every_message_respects_both_limits(self): + findings = [self.fat(i) for i in range(triage.MAX_HIGHLIGHTS)] + for message in build_messages(findings, "acme"): + self.assertLessEqual(len(message["embeds"]), triage.MAX_EMBEDS_PER_MESSAGE) + total = sum(triage.embed_char_count(e) for e in message["embeds"]) + self.assertLessEqual(total, triage.MAX_EMBED_CHARS_PER_MESSAGE) + + def test_char_limit_splits_where_the_count_limit_would_not(self): + """9 fat highlights + summary = 10 embeds: within the count limit, over 6,000 chars.""" + messages = build_messages([self.fat(i) for i in range(9)], "acme") + embeds = sum(len(m["embeds"]) for m in messages) + self.assertLessEqual(embeds, triage.MAX_EMBEDS_PER_MESSAGE) # count alone: 1 message + self.assertGreater(len(messages), 1) # chars forced the split + + def test_lean_embeds_are_not_split_early(self): + """The char limit must not fragment ordinary digests.""" + messages = build_messages([finding(i, priority_rank=i) for i in range(9)], "acme") + self.assertEqual(len(messages), 1) + + def test_no_embed_is_dropped_while_chunking(self): + findings = [self.fat(i) for i in range(15)] + messages = build_messages(findings, "acme") + self.assertEqual(sum(len(m["embeds"]) for m in messages), 16) # 15 + summary + + def test_char_count_covers_titles_descriptions_and_fields(self): + embed = {"title": "abc", "description": "de", + "fields": [{"name": "fg", "value": "hij"}]} + self.assertEqual(triage.embed_char_count(embed), 3 + 2 + 2 + 3) + + def test_char_count_tolerates_missing_keys(self): + self.assertEqual(triage.embed_char_count({}), 0) + + +class TestCoverage(unittest.TestCase): + """coverage[i] is what message i accounts for, so a partial post failure records + exactly the tickets that reached Discord.""" + + def test_summary_message_covers_non_highlighted_tickets(self): + findings = [finding(1), finding(2, worth_looking_into=False)] + _, coverage = triage.build_messages(findings, "acme") + self.assertIn(2, coverage[0]) # counted by the summary + self.assertIn(1, coverage[0]) # its own embed is in the same message + + def test_highlights_omitted_by_the_cap_are_covered_by_nothing(self): + over = triage.MAX_HIGHLIGHTS + 3 + findings = [finding(i, priority_rank=i) for i in range(over)] + _, coverage = triage.build_messages(findings, "acme") + covered = set().union(*coverage) + shown, omitted = triage.select_highlights(findings) + self.assertEqual(len(omitted), 3) + for f in omitted: + self.assertNotIn(f["id"], covered) + for f in shown: + self.assertIn(f["id"], covered) + + def test_coverage_has_one_entry_per_message(self): + findings = [finding(i, priority_rank=i) for i in range(triage.MAX_HIGHLIGHTS)] + messages, coverage = triage.build_messages(findings, "acme") + self.assertEqual(len(messages), len(coverage)) + + def test_no_ticket_is_covered_twice(self): + findings = [finding(i, priority_rank=i) for i in range(20)] + _, coverage = triage.build_messages(findings, "acme") + flat = [tid for ids in coverage for tid in ids] + self.assertEqual(len(flat), len(set(flat))) + + +class TestPostToDiscord(unittest.TestCase): + """Returns the accepted count rather than exiting, so main can record exactly the + tickets that landed before signalling the failure.""" + + def test_all_accepted(self): + session = FakeSession([FakeResponse({}, status_code=204)] * 3) + self.assertEqual(triage.post_to_discord(session, "https://hook", [{}, {}, {}]), 3) + self.assertEqual(len(session.calls), 3) + + def test_stops_at_the_first_failure_and_reports_the_prefix(self): + session = FakeSession([ + FakeResponse({}, status_code=204), + FakeResponse({}, status_code=400), + ]) + self.assertEqual(triage.post_to_discord(session, "https://hook", [{}, {}, {}]), 1) + + def test_does_not_post_after_a_failure(self): + session = FakeSession([FakeResponse({}, status_code=404)]) + triage.post_to_discord(session, "https://hook", [{}, {}, {}]) + self.assertEqual(len(session.calls), 1) + + def test_first_message_failing_reports_zero(self): + session = FakeSession([FakeResponse({}, status_code=500)] * 6) + with NoSleep(): + self.assertEqual(triage.post_to_discord(session, "https://hook", [{}]), 0) + + def test_no_messages_is_zero(self): + self.assertEqual(triage.post_to_discord(FakeSession([]), "https://hook", []), 0) + + +class TestSelectHighlights(unittest.TestCase): + def test_splits_at_the_display_cap(self): + findings = [finding(i, priority_rank=i) for i in range(triage.MAX_HIGHLIGHTS + 4)] + shown, omitted = triage.select_highlights(findings) + self.assertEqual(len(shown), triage.MAX_HIGHLIGHTS) + self.assertEqual(len(omitted), 4) + + def test_orders_by_priority_rank(self): + shown, _ = triage.select_highlights( + [finding(1, priority_rank=5), finding(2, priority_rank=1)] + ) + self.assertEqual([f["id"] for f in shown], [2, 1]) + + def test_ignores_tickets_not_worth_looking_into(self): + shown, omitted = triage.select_highlights([finding(1, worth_looking_into=False)]) + self.assertEqual((shown, omitted), ([], [])) + + +class TestTicketsFromPayload(unittest.TestCase): + def test_returns_the_list(self): + payload = {"tickets": [finding(1)]} + self.assertEqual(triage.tickets_from_payload(payload, "x"), [finding(1)]) + + def test_missing_key_exits_instead_of_raising_keyerror(self): + with self.assertRaises(SystemExit): + triage.tickets_from_payload({"results": []}, "x") + + def test_wrong_type_exits(self): + for payload in ({"tickets": {}}, [], "nope", None): + with self.assertRaises(SystemExit): + triage.tickets_from_payload(payload, "x") + + def test_an_empty_list_is_valid(self): + self.assertEqual(triage.tickets_from_payload({"tickets": []}, "x"), []) + + def test_a_finding_missing_renderer_keys_exits(self): + """The claude-cli backend has no structured-output enforcement, so an entry + without category/severity would otherwise KeyError inside build_summary_embed.""" + for entry in ({"id": 1}, {"id": 1, "category": "bug_report"}, + {"category": "bug_report", "severity": "major"}): + with self.assertRaises(SystemExit): + triage.tickets_from_payload({"tickets": [entry]}, "x") + + def test_a_non_object_entry_exits(self): + with self.assertRaises(SystemExit): + triage.tickets_from_payload({"tickets": [["not", "an", "object"]]}, "x") + + +class TestExtractJsonObject(unittest.TestCase): + def test_bare_object(self): + self.assertEqual(triage.extract_json_object('{"a": 1}'), {"a": 1}) + + def test_object_inside_a_markdown_fence(self): + self.assertEqual(triage.extract_json_object('```json\n{"a": 1}\n```'), {"a": 1}) + + def test_object_surrounded_by_prose(self): + self.assertEqual( + triage.extract_json_object('Sure! Here you go:\n{"a": 1}\nHope that helps.'), {"a": 1} + ) + + def test_nested_braces_survive(self): + self.assertEqual( + triage.extract_json_object('{"t": [{"id": 1}, {"id": 2}]}'), + {"t": [{"id": 1}, {"id": 2}]}, + ) + + def test_no_object_exits(self): + with self.assertRaises(SystemExit): + triage.extract_json_object("no json here") + + def test_malformed_object_exits(self): + with self.assertRaises(SystemExit): + triage.extract_json_object('{"a": }') + + +class TestAnalyzeInChunks(unittest.TestCase): + """A batch of 2000 would need ~204K output tokens, past the 128K ceiling, so + oversized batches must split rather than truncate.""" + + def setUp(self): + self.seen = [] + + def analyzer(self, chunk): + self.seen.append(len(chunk)) + return [finding(t["id"]) for t in chunk] + + def tickets(self, count): + return [{"id": i} for i in range(count)] + + def test_a_batch_within_the_limit_is_one_request(self): + result = triage.analyze_in_chunks(self.analyzer, self.tickets(45), 400) + self.assertEqual(self.seen, [45]) + self.assertEqual(len(result), 45) + + def test_a_batch_exactly_at_the_limit_is_not_split(self): + triage.analyze_in_chunks(self.analyzer, self.tickets(400), 400) + self.assertEqual(self.seen, [400]) + + def test_an_oversized_batch_is_split(self): + triage.analyze_in_chunks(self.analyzer, self.tickets(1000), 400) + self.assertEqual(self.seen, [400, 400, 200]) + + def test_every_ticket_appears_exactly_once_across_chunks(self): + result = triage.analyze_in_chunks(self.analyzer, self.tickets(2000), 400) + self.assertEqual(len(result), 2000) + self.assertEqual(sorted(f["id"] for f in result), list(range(2000))) + + def test_no_chunk_exceeds_the_batch_size(self): + triage.analyze_in_chunks(self.analyzer, self.tickets(2000), 400) + self.assertTrue(all(size <= 400 for size in self.seen)) + + def test_an_empty_batch_makes_a_single_no_op_request(self): + self.assertEqual(triage.analyze_in_chunks(self.analyzer, [], 400), []) + + def test_a_failing_chunk_propagates_rather_than_reporting_partial_results(self): + def exploding(chunk): + if len(self.seen) == 1: + raise RuntimeError("api error") + self.seen.append(len(chunk)) + return [] + + with self.assertRaises(RuntimeError): + triage.analyze_in_chunks(exploding, self.tickets(1000), 400) + + +class TestLoadFindings(unittest.TestCase): + def setUp(self): + self.dir = tempfile.TemporaryDirectory() + self.addCleanup(self.dir.cleanup) + + def _write(self, content): + path = os.path.join(self.dir.name, "f.json") + with open(path, "w", encoding="utf-8") as fh: + fh.write(content) + return path + + MINIMAL = {"id": 1, "category": "bug_report", "severity": "major"} + + def test_accepts_an_object_with_a_tickets_list(self): + payload = json.dumps({"tickets": [self.MINIMAL]}) + self.assertEqual(triage.load_findings(self._write(payload)), [self.MINIMAL]) + + def test_accepts_a_bare_list(self): + payload = json.dumps([self.MINIMAL]) + self.assertEqual(triage.load_findings(self._write(payload)), [self.MINIMAL]) + + def test_rejects_anything_else(self): + with self.assertRaises(SystemExit): + triage.load_findings(self._write('{"nope": 1}')) + + def test_rejects_a_finding_missing_keys_the_renderer_indexes(self): + """build_summary_embed does f["category"] / f["severity"] directly.""" + for payload in ('[{"id": 1}]', + '[{"id": 1, "category": "bug_report"}]', + '[{"category": "bug_report", "severity": "major"}]'): + with self.assertRaises(SystemExit): + triage.load_findings(self._write(payload)) + + def test_rejects_a_non_object_entry(self): + with self.assertRaises(SystemExit): + triage.load_findings(self._write('[["not", "an", "object"]]')) + + def test_an_empty_list_is_valid(self): + self.assertEqual(triage.load_findings(self._write("[]")), []) + + +class TestCompactTicket(unittest.TestCase): + def test_long_descriptions_are_truncated_and_marked(self): + compact = triage.compact_ticket(ticket(1, description="x" * 5000)) + self.assertIn("โ€ฆ[truncated]", compact["description"]) + self.assertLess(len(compact["description"]), 5000) + + def test_short_descriptions_are_left_alone(self): + self.assertEqual(triage.compact_ticket(ticket(1, description="hi"))["description"], "hi") + + def test_missing_description_becomes_empty_string(self): + self.assertEqual(triage.compact_ticket(ticket(1, description=None))["description"], "") + + def test_satisfaction_score_is_lifted_out_of_the_nested_object(self): + compact = triage.compact_ticket(ticket(1, satisfaction_rating={"score": "bad"})) + self.assertEqual(compact["satisfaction_rating"], "bad") + + def test_absent_satisfaction_rating_is_none(self): + self.assertIsNone(triage.compact_ticket(ticket(1))["satisfaction_rating"]) + + def test_updated_at_is_not_sent_to_the_model(self): + """It is only needed for dedup state, so it stays out of the prompt.""" + self.assertNotIn("updated_at", triage.compact_ticket(ticket(1))) + + +# ---- Fetching -------------------------------------------------------------- + + +class TestFetchTickets(unittest.TestCase): + def test_returns_the_total_match_count_alongside_the_batch(self): + session = FakeSession([FakeResponse({"count": 47, "results": [ticket(1), ticket(2)]})]) + tickets, total = triage.fetch_tickets(session, "acme", "q", 100) + self.assertEqual(len(tickets), 2) + self.assertEqual(total, 47) + + def test_follows_pagination(self): + session = FakeSession([ + FakeResponse({"count": 3, "results": [ticket(1)], "next_page": "https://n/2"}), + FakeResponse({"count": 3, "results": [ticket(2), ticket(3)]}), + ]) + tickets, total = triage.fetch_tickets(session, "acme", "q", 100) + self.assertEqual([t["id"] for t in tickets], [1, 2, 3]) + self.assertEqual(total, 3) + + def test_max_tickets_caps_the_batch_but_not_the_reported_total(self): + session = FakeSession([ + FakeResponse({"count": 500, "results": [ticket(i) for i in range(10)]}), + ]) + tickets, total = triage.fetch_tickets(session, "acme", "q", 4) + self.assertEqual(len(tickets), 4) + self.assertEqual(total, 500) # the gap is what the digest surfaces + + def test_non_ticket_search_results_are_ignored(self): + session = FakeSession([ + FakeResponse({"count": 2, "results": [ticket(1), {"result_type": "user", "id": 9}]}), + ]) + tickets, _ = triage.fetch_tickets(session, "acme", "q", 100) + self.assertEqual([t["id"] for t in tickets], [1]) + + def test_total_is_none_when_zendesk_omits_the_count(self): + session = FakeSession([FakeResponse({"results": [ticket(1)]})]) + _, total = triage.fetch_tickets(session, "acme", "q", 100) + self.assertIsNone(total) + + def test_forbidden_response_exits_with_a_hint(self): + session = FakeSession([FakeResponse({}, status_code=403)]) + with self.assertRaises(SystemExit): + triage.fetch_tickets(session, "acme", "q", 100) + + def test_pagination_stops_at_the_zendesk_result_limit(self): + """Past 1000 results the search API 422s, so we never ask for that page. + + A caller asking for more gets the limit, not an error: one page beyond the + cap is queued here and must go unrequested. + """ + limit = triage.SEARCH_RESULT_LIMIT + pages = [ + FakeResponse({ + "count": 5000, + "results": [ticket(i) for i in range(offset, offset + 100)], + "next_page": "https://n/next", + }) + for offset in range(0, limit + 100, 100) + ] + session = FakeSession(pages) + tickets, total = triage.fetch_tickets(session, "acme", "q", 5000) + self.assertEqual(len(tickets), limit) + self.assertEqual(total, 5000) # the digest still reports the real backlog + self.assertEqual(len(session.calls), limit // 100) + + def test_max_tickets_below_the_limit_still_wins(self): + session = FakeSession([ + FakeResponse({"count": 500, "results": [ticket(i) for i in range(100)]}), + ]) + tickets, _ = triage.fetch_tickets(session, "acme", "q", 7) + self.assertEqual(len(tickets), 7) + + def test_an_unexpected_422_keeps_the_tickets_already_fetched(self): + """Belt and braces: a lower-than-documented limit truncates, not crashes.""" + session = FakeSession([ + FakeResponse({"count": 900, "results": [ticket(1)], "next_page": "https://n/2"}), + FakeResponse({"error": "invalid"}, status_code=422), + ]) + tickets, total = triage.fetch_tickets(session, "acme", "q", 900) + self.assertEqual([t["id"] for t in tickets], [1]) + self.assertEqual(total, 900) + + def test_a_422_on_the_first_page_still_exits(self): + """Nothing fetched means nothing to salvage โ€” that's a real failure.""" + session = FakeSession([FakeResponse({"error": "invalid"}, status_code=422)]) + with self.assertRaises(SystemExit): + triage.fetch_tickets(session, "acme", "q", 100) + + +class TestFetchTotalUnsolved(unittest.TestCase): + def test_returns_the_count(self): + session = FakeSession([FakeResponse({"count": 5609})]) + self.assertEqual(triage.fetch_total_unsolved(session, "acme"), 5609) + + def test_failure_is_non_fatal(self): + """The backlog number is context, not a reason to abort the digest.""" + session = FakeSession([FakeResponse({}, status_code=500)] * 2) + self.assertIsNone(triage.fetch_total_unsolved(session, "acme")) + + def test_uses_a_short_retry_budget(self): + """A full 6-attempt backoff would stall the digest ~60s for optional data.""" + session = FakeSession([FakeResponse({}, status_code=500)] * 6) + triage.fetch_total_unsolved(session, "acme") + self.assertEqual(len(session.calls), 2) + + def test_a_transport_failure_is_non_fatal_too(self): + """request_with_retry re-raises once its budget is spent; None is documented.""" + session = FakeSession([requests.ConnectionError("no route")] * 2) + with NoSleep(): + self.assertIsNone(triage.fetch_total_unsolved(session, "acme")) + + def test_a_non_json_body_is_non_fatal(self): + """A 200 with an HTML error page (proxy, maintenance) must not abort the run.""" + session = FakeSession([NonJsonResponse()]) + self.assertIsNone(triage.fetch_total_unsolved(session, "acme")) + + +class TestRequestWithRetry(unittest.TestCase): + def test_returns_the_first_success_without_retrying(self): + session = FakeSession([FakeResponse({"ok": True})]) + resp = triage.request_with_retry(session, "GET", "https://x") + self.assertEqual(resp.json(), {"ok": True}) + self.assertEqual(len(session.calls), 1) + + def test_retries_a_server_error_then_succeeds(self): + session = FakeSession([ + FakeResponse({}, status_code=500), + FakeResponse({"ok": True}), + ]) + resp = triage.request_with_retry(session, "GET", "https://x", attempts=3) + self.assertEqual(resp.json(), {"ok": True}) + self.assertEqual(len(session.calls), 2) + + def test_does_not_retry_a_client_error(self): + session = FakeSession([FakeResponse({}, status_code=404)]) + resp = triage.request_with_retry(session, "GET", "https://x") + self.assertEqual(resp.status_code, 404) + self.assertEqual(len(session.calls), 1) + + def test_gives_up_after_the_attempt_budget(self): + session = FakeSession([FakeResponse({}, status_code=503)] * 3) + resp = triage.request_with_retry(session, "GET", "https://x", attempts=3) + self.assertEqual(resp.status_code, 503) + self.assertEqual(len(session.calls), 3) + + def test_transport_failures_retry_then_raise_when_exhausted(self): + session = FakeSession([requests.ConnectionError("boom")] * 3) + with NoSleep(): + with self.assertRaises(requests.ConnectionError): + triage.request_with_retry(session, "GET", "https://x", attempts=3) + self.assertEqual(len(session.calls), 3) + + def test_a_transport_failure_can_recover_on_a_later_attempt(self): + session = FakeSession([requests.Timeout("slow"), FakeResponse({"ok": True})]) + with NoSleep(): + resp = triage.request_with_retry(session, "GET", "https://x", attempts=3) + self.assertEqual(resp.json(), {"ok": True}) + self.assertEqual(len(session.calls), 2) + + def test_no_sleep_after_the_final_attempt(self): + """Sleeping after the last try only delays the caller โ€” nothing follows it.""" + session = FakeSession([FakeResponse({}, status_code=503)] * 3) + with NoSleep() as clock: + triage.request_with_retry(session, "GET", "https://x", attempts=3) + self.assertEqual(len(clock.slept), 2) # 3 attempts, 2 gaps + + def test_numeric_retry_after_is_honoured(self): + session = FakeSession([ + FakeResponse({}, status_code=429, retry_after="7"), + FakeResponse({"ok": True}), + ]) + with NoSleep() as clock: + triage.request_with_retry(session, "GET", "https://x", attempts=3) + self.assertEqual(clock.slept, [7.0]) + + def test_retry_after_is_capped(self): + session = FakeSession([ + FakeResponse({}, status_code=429, retry_after="9999"), + FakeResponse({"ok": True}), + ]) + with NoSleep() as clock: + triage.request_with_retry(session, "GET", "https://x", attempts=3) + self.assertEqual(clock.slept, [60]) + + def test_http_date_retry_after_falls_back_instead_of_crashing(self): + """RFC 9110 allows an HTTP-date here; float() on it used to raise ValueError.""" + session = FakeSession([ + FakeResponse({}, status_code=503, retry_after="Wed, 21 Oct 2026 07:28:00 GMT"), + FakeResponse({"ok": True}), + ]) + with NoSleep() as clock: + resp = triage.request_with_retry(session, "GET", "https://x", attempts=3) + self.assertEqual(resp.json(), {"ok": True}) + self.assertEqual(clock.slept, [1.0]) # fell back to the backoff delay + + def test_zero_attempts_is_rejected_rather_than_unbound(self): + with self.assertRaises(ValueError): + triage.request_with_retry(FakeSession([]), "GET", "https://x", attempts=0) + + +class TestRetryAfterSeconds(unittest.TestCase): + def test_missing_header_uses_the_default(self): + self.assertEqual(triage.retry_after_seconds(FakeResponse({}, retry_after=None), 4.0), 4.0) + + def test_numeric_header_wins(self): + self.assertEqual(triage.retry_after_seconds(FakeResponse({}, retry_after="12"), 4.0), 12.0) + + def test_unparseable_header_uses_the_default(self): + for raw in ("Wed, 21 Oct 2026 07:28:00 GMT", "", "soon", "12s"): + self.assertEqual(triage.retry_after_seconds(FakeResponse({}, retry_after=raw), 4.0), 4.0) + + def test_negative_and_non_finite_values_use_the_default(self): + """time.sleep() rejects a negative or NaN duration, so passing one through + would crash the run on a hostile or buggy Retry-After header.""" + for raw in ("-30", "-0.5", "nan", "inf", "-inf"): + self.assertEqual(triage.retry_after_seconds(FakeResponse({}, retry_after=raw), 4.0), + 4.0, msg=f"retry-after={raw!r}") + + def test_zero_is_honoured_rather_than_replaced(self): + """Zero is a valid instruction to retry immediately, not a missing value.""" + self.assertEqual(triage.retry_after_seconds(FakeResponse({}, retry_after="0"), 4.0), 0.0) + + def test_a_negative_retry_after_does_not_crash_a_real_retry_loop(self): + session = FakeSession([ + FakeResponse({}, status_code=503, retry_after="-30"), + FakeResponse({"ok": True}), + ]) + with NoSleep() as clock: + resp = triage.request_with_retry(session, "GET", "https://x", attempts=3) + self.assertEqual(resp.json(), {"ok": True}) + self.assertTrue(all(s >= 0 for s in clock.slept), clock.slept) + + +# ---- Workflow wiring ------------------------------------------------------- + + +class TestFailureNotificationWiring(unittest.TestCase): + """The failure notifier matches on workflow *name*, so a rename silently + unsubscribes the triage job. The README promises failures get reported; this + keeps that promise checkable without running Actions. + """ + + WORKFLOWS = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".github", "workflows" + ) + + def read(self, filename): + with open(os.path.join(self.WORKFLOWS, filename), encoding="utf-8") as fh: + return fh.read() + + def test_notify_failure_watches_the_triage_workflow_by_its_current_name(self): + triage_yml = self.read("zendesk_triage.yml") + match = re.search(r"^name:\s*(.+?)\s*$", triage_yml, re.MULTILINE) + self.assertIsNotNone(match, "zendesk_triage.yml has no top-level name") + name = match.group(1).strip("\"'") + self.assertIn(f'"{name}"', self.read("notify_failure.yml")) + + +if __name__ == "__main__": + unittest.main() diff --git a/zendesk_triage/triage.py b/zendesk_triage/triage.py new file mode 100644 index 0000000..d9d423b --- /dev/null +++ b/zendesk_triage/triage.py @@ -0,0 +1,1250 @@ +#!/usr/bin/env python3 +""" +Daily Zendesk ticket triage with Claude, delivered to Discord. + +Fetches open Zendesk tickets (broad query by default, not just bugs), sends the +whole batch to Claude in a single structured-output request, and posts a Discord +summary that links back to each original ticket and highlights the ones worth +looking into (crashes, data loss, legal requests, security/legislation, etc.). + +Because this repo is public, ticket content is never written to the job summary or +anywhere public: in a normal run the only place ticket detail goes is the Discord +webhook (a private channel) and the links point at Zendesk (which needs auth to +open). Set --dry-run to print the Discord payload locally instead of posting. + +The one exception is --dump-batch, a local debugging flag that writes ticket +content to a file you name. Keep those files out of the repo (see .gitignore) or +write them somewhere like /tmp. + +Categories Claude sorts each ticket into: + bug_report | low_star_review | legal_request | security_or_legislation + | question | feature_request | other + +Config (env vars, or CLI flags for local runs): + ZENDESK_SUBDOMAIN e.g. "mycompany" -> https://mycompany.zendesk.com + ZENDESK_EMAIL agent email for API token auth + ZENDESK_API_TOKEN Zendesk API token + ANTHROPIC_API_KEY Claude API key (read by the SDK automatically) + DISCORD_WEBHOOK_URL Discord incoming webhook + ZENDESK_QUERY (optional) Zendesk search query; see DEFAULT_QUERY + ZENDESK_TRIAGE_MODEL (optional) Claude model id; defaults to claude-opus-4-8 + +Usage: + # real run (CI): reads everything from the environment + python triage.py + + # local dry run: fetch + analyze, print the Discord payload, post nothing + python triage.py --dry-run + + # what the scheduled daily run does: 48h window, skipping unchanged repeats + python triage.py --window-hours 48 --state .triage-state/seen.json + + # or an explicit query, which overrides --window-hours + python triage.py --query "type:ticket status:open tags:bug" --max-tickets 50 + + # local debugging without an ANTHROPIC_API_KEY: classify via the `claude` CLI + python triage.py --backend claude-cli --dry-run --max-tickets 20 + + # or split it in two: dump the batch, classify it by hand, feed it back + python triage.py --dump-batch /tmp/batch.json --max-tickets 20 + python triage.py --backend file --findings /tmp/findings.json --dry-run +""" +import argparse +import json +import math +import os +import re +import subprocess +import sys +import textwrap +import time +from datetime import datetime, timedelta, timezone +from functools import partial + +import anthropic +import requests + +# Open, pending, new, and on-hold tickets, newest first. Broad on purpose: we +# want bug reports AND low-star reviews, legal requests, security/legislation +# questions, and non-English tickets โ€” Claude does the categorising, so we don't +# filter to a single tag here. +DEFAULT_QUERY = "type:ticket status48hours` form, so the exact window lands in the run log. + """ + cutoff = (datetime.now(timezone.utc) - timedelta(hours=hours)).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + return f"type:ticket status{cutoff} order_by:created_at sort:desc" + + +def window_label(hours): + if hours % 24 == 0 and hours >= 24: + days = hours // 24 + return f"created in the past {days} day{'s' if days > 1 else ''}" + return f"created in the past {hours}h" +DEFAULT_MODEL = "claude-opus-4-8" +DEFAULT_MAX_TICKETS = 100 +DESCRIPTION_CHARS = 1500 # per-ticket description sent to Claude (triage only) +# One classification runs ~100 output tokens per ticket, and adaptive thinking draws +# from the same max_tokens budget. 400 keeps a chunk far under the 128K output +# ceiling; batches larger than this are split rather than truncated. +DEFAULT_BATCH_SIZE = 400 +MAX_OUTPUT_TOKENS = 128000 + +# ---- Taxonomy -------------------------------------------------------------- +# +# Single source of truth. The schema enum, the Discord labels, the urgency colours, +# and the system-prompt guidance are all derived from this table, so adding a +# category is one edit and the model can never be given an enum value that the +# prompt never explains. +# +# Percentages come from a 3,662-ticket sample of the 13 months to 2026-08. +# Columns: (name, Discord label, urgency colour or None, guidance for the model) +CATEGORY_SPECS = ( + ("abuse_report", "๐Ÿšจ Abuse report", 0xC0392B, + "One user reporting another account for illegal or abusive content (CSAM, " + "harassment, drugs, impersonation). Usually quotes the offending Session ID. " + "~11% of non-review tickets. Always set worth_looking_into."), + ("security_report", "๐Ÿ”’ Security report", 0xC0392B, + "A vulnerability, exploit, or account-compromise disclosure. Not the same as a " + "policy question. Always set worth_looking_into."), + ("legal_or_data_request", "โš–๏ธ Legal / data request", 0xC0392B, + "GDPR or data-deletion request, subpoena, law-enforcement or court order. " + "Always set worth_looking_into."), + ("bug_report", "๐Ÿž Bug report", None, + "Something in the app is broken or misbehaving."), + ("account_access", "๐Ÿ”‘ Account access", None, + "Lost recovery phrase, locked out, or asking to restore an account. Usually " + "irreversible by design, but track the volume."), + ("policy_question", "๐Ÿ“œ Policy question", None, + "Questions about law, regulation, or policy โ€” 'Chat Control', encryption " + "backdoors, whether Session complies with something."), + ("low_star_review", "โญ Low-star review", None, + "An app-store review of 3 stars or fewer. These often hide a real bug โ€” put " + "the underlying problem in `summary`."), + ("positive_review", "๐Ÿ‘ Positive review", None, + "An app-store review of 4-5 stars with no actionable content."), + ("feature_request", "๐Ÿ’ก Feature request", None, + "Asking for something the app does not do yet."), + ("question", "โ“ Question", None, + "A how-do-I or usage question that is not a bug."), + ("spam_or_solicitation", "๐Ÿ—‘๏ธ Spam / solicitation", None, + "Marketing, token or OTC investment offers, partnership pitches, listing spam."), + ("other", "โ€ข Other", None, + "Genuinely none of the above. Prefer a specific category wherever one fits."), +) +CATEGORIES = [name for name, _, _, _ in CATEGORY_SPECS] +CATEGORY_LABEL = {name: label for name, label, _, _ in CATEGORY_SPECS} +# Categories whose urgency `severity` cannot express. They are not bugs, so the model +# rates them not_applicable โ€” which would otherwise paint the most serious ticket in +# the batch the calmest colour and sort it last. +CATEGORY_COLOR = {name: color for name, _, color, _ in CATEGORY_SPECS if color} +URGENT_CATEGORIES = frozenset(CATEGORY_COLOR) +CATEGORY_GUIDANCE = "\n".join(f"- {name}: {desc}" for name, _, _, desc in CATEGORY_SPECS) + +SEVERITIES = ["crash", "data_loss", "major", "minor", "cosmetic", "not_applicable"] +PLATFORMS = [ + "ios", "android", "desktop_windows", "desktop_macos", "desktop_linux", + "multiple", "unknown", +] + +# Structured-output schema. Kept within the structured-output constraints: +# additionalProperties:false everywhere, every field required, enums for the +# closed sets, no min/max-length constraints. +TICKET_PROPERTIES = { + "id": {"type": "integer", "description": "The Zendesk ticket id, echoed back unchanged."}, + "category": {"type": "string", "enum": CATEGORIES}, + "severity": { + "type": "string", + "enum": SEVERITIES, + "description": "crash/data_loss are the most serious; not_applicable for non-bug tickets.", + }, + "affected_component": { + "type": "string", + "description": "Best guess at the product area, e.g. 'onboarding', 'attachments', 'sync'. Empty string if unknown.", + }, + "summary": { + "type": "string", + "description": "One line stating what is actually broken or what the user actually wants.", + }, + "likely_root_cause": { + "type": "string", + "description": "Short hypothesis for the underlying cause. Empty string if not a bug or unclear.", + }, + "language": { + "type": "string", + "description": "Language the ticket is written in, e.g. 'English', 'German'.", + }, + "priority_rank": { + "type": "integer", + "description": "Relative triage priority, 1 = look at first.", + }, + "worth_looking_into": { + "type": "boolean", + "description": "True if a human should review this soon. Always true for abuse_report, security_report, and legal_or_data_request.", + }, + "cluster": { + "type": "string", + "description": "Short label grouping tickets with the same likely root cause; identical labels mean likely duplicates/clusters. Empty string if standalone.", + }, + "platform": { + "type": "string", + "enum": PLATFORMS, + "description": "Platform the ticket is about. 'multiple' if several, 'unknown' if not stated.", + }, + "app_version": { + "type": "string", + "description": "App version if the ticket states one, e.g. '2.15.2'. Empty string otherwise.", + }, + "reported_session_id": { + "type": "string", + "description": "For abuse reports: the reported account's Session ID (66 hex chars, starts with 05), copied exactly. Empty string if the ticket gives none.", + }, +} +SCHEMA = { + "type": "object", + "additionalProperties": False, + "required": ["tickets"], + "properties": { + "tickets": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": False, + "required": list(TICKET_PROPERTIES.keys()), + "properties": TICKET_PROPERTIES, + }, + } + }, +} + +_SYSTEM_PROMPT_TEMPLATE = textwrap.dedent( + """ + You are a senior support-triage engineer. You are given a batch of Zendesk + support tickets as JSON. For every ticket, return a classification object. + + Tickets arrive in many languages and many are machine-imported app-store + reviews. Classify what the user is actually reporting, not how they said it. + + Categories โ€” use exactly these values: + __CATEGORIES__ + + Guidelines: + - Infer severity from the description: crash and data_loss are the most + serious; use not_applicable for tickets that are not bug reports. + - Group tickets that share a likely root cause under the same short `cluster` + label, even if worded differently or in different languages. Identical + labels flag likely duplicates/clusters. + - `summary` is one plain line: what is actually broken, or what the user + actually wants. Do not restate the ticket subject. + - Mark `worth_looking_into` true for anything a human should see soon: + crashes, data loss, major bugs, and every abuse_report, security_report, + and legal_or_data_request. Be selective otherwise โ€” not everything is major. + - For abuse_report, copy the reported account's Session ID into + `reported_session_id` exactly as written. Do not invent or reformat it. + - Rank by `priority_rank` (1 = first) across the whole batch. + - Echo each ticket `id` back exactly. Return one object per input ticket. + """ +).strip() +# Substituted after dedent so the guidance block keeps its own formatting. +SYSTEM_PROMPT = _SYSTEM_PROMPT_TEMPLATE.replace("__CATEGORIES__", CATEGORY_GUIDANCE) + + +def get_env(name, cli_value=None, required=True): + if cli_value: + return cli_value + value = os.environ.get(name) + if value: + return value + if required: + sys.exit(f"Missing required config: set the {name} environment variable (or pass the matching flag).") + return None + + +def zendesk_session(email, token): + session = requests.Session() + # Zendesk API-token auth: username is "{email}/token", password is the token. + session.auth = (f"{email}/token", token) + session.headers["Accept"] = "application/json" + return session + + +def retry_after_seconds(resp, default): + """Seconds to wait per the Retry-After header, falling back to `default`. + + RFC 9110 allows either a delay in seconds or an HTTP-date; float() on the date + form raises, so anything unparseable falls back rather than crashing the run. + + Negative, NaN, and infinite values fall back too: time.sleep() rejects the first + two outright, so a hostile or buggy proxy sending `Retry-After: -30` would + otherwise take the run down with a ValueError. + """ + raw = resp.headers.get("retry-after") + if raw is None: + return default + try: + seconds = float(raw) + except (TypeError, ValueError): + return default + if not math.isfinite(seconds) or seconds < 0: + return default + return seconds + + +def request_with_retry(session, method, url, attempts=6, **kwargs): + """GET/POST with backoff on 429 and 5xx. + + Lower `attempts` for calls whose result is nice-to-have: the full budget can + burn ~60s of backoff, which is not worth spending on optional data. + """ + if attempts <= 0: + raise ValueError("attempts must be at least 1") + + delay = 1.0 + last_exc = None + resp = None + for attempt in range(attempts): + final = attempt == attempts - 1 + try: + resp = session.request(method, url, timeout=30, **kwargs) + except requests.RequestException as exc: + last_exc = exc + if final: + break + time.sleep(delay) + delay = min(delay * 2, 30) + continue + if resp.status_code == 429 or resp.status_code >= 500: + if final: + break + time.sleep(min(retry_after_seconds(resp, delay), 60)) + delay = min(delay * 2, 30) + continue + return resp + if last_exc: + raise last_exc + return resp + + +def fetch_tickets(session, subdomain, query, max_tickets): + """Fetch tickets via the Zendesk Search API, following pagination. + + Returns (tickets, total_matched). total_matched is the full result count + reported by Zendesk, which can exceed len(tickets) when max_tickets โ€” or + SEARCH_RESULT_LIMIT โ€” caps the batch; the caller surfaces that gap so the + truncation isn't silent. + """ + base = f"https://{subdomain}.zendesk.com/api/v2/search.json" + url = base + params = {"query": query, "per_page": 100} + tickets = [] + total_matched = None + # Whichever bites first: our own runaway guard or Zendesk's hard result limit. + cap = min(max_tickets, SEARCH_RESULT_LIMIT) + while url and len(tickets) < cap: + resp = request_with_retry(session, "GET", url, params=params) + params = None # next_page already carries the query + if resp.status_code == 403: + sys.exit("Zendesk returned 403 โ€” the API token/email may lack search access.") + # 422 past the result limit: `cap` should have stopped us first, so this only + # fires if the account's effective limit is lower than documented. Keep the + # tickets already in hand โ€” a partial digest beats no digest โ€” and let the + # caller report the gap. With nothing in hand there is nothing to salvage. + if resp.status_code == 422 and tickets: + print(f"Note: Zendesk stopped paginating at {len(tickets)} results " + f"(search result limit); analyzing what was fetched.") + break + if resp.status_code >= 400: + sys.exit(f"Zendesk search failed ({resp.status_code}): {resp.text[:300]}") + payload = resp.json() + if total_matched is None: + total_matched = payload.get("count") + for row in payload.get("results", []): + if row.get("result_type") != "ticket": + continue + tickets.append(row) + if len(tickets) >= cap: + break + url = payload.get("next_page") + return tickets, total_matched + + +def fetch_total_unsolved(session, subdomain): + """Count the whole unsolved backlog. Best effort: returns None on failure. + + Context for the digest, not something to hold the run up for โ€” hence the + short retry budget. + """ + url = f"https://{subdomain}.zendesk.com/api/v2/search/count.json" + try: + resp = request_with_retry( + session, "GET", url, attempts=2, params={"query": BACKLOG_QUERY} + ) + if resp.status_code >= 400: + print(f"Note: could not count the unsolved backlog ({resp.status_code}).") + return None + return resp.json().get("count") + except (requests.RequestException, ValueError) as exc: + # request_with_retry re-raises the transport error once its (short) budget is + # spent, and .json() raises on a non-JSON body โ€” neither is a reason to lose + # the digest over one context number, so both land on the documented None. + print(f"Note: could not count the unsolved backlog ({exc}).") + return None + + +# ---- Dedup state ----------------------------------------------------------- +# +# Maps ticket id -> {updated_at, last_reported}. A ticket is re-reported only if +# Zendesk's updated_at has moved since we last showed it, so the daily 48h window +# does not repost yesterday's unchanged tickets. The state lives outside the repo +# (CI restores it from the Actions cache), so every read degrades gracefully: a +# missing or corrupt file just means everything looks new. + + +def empty_state(): + return {"version": STATE_VERSION, "seen": {}} + + +def load_state(path): + if not os.path.exists(path): + print(f"No state file at {path}; treating every ticket in the window as new.") + return empty_state() + try: + with open(path, encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, json.JSONDecodeError) as exc: + print(f"Note: unreadable state file {path} ({exc}); treating every ticket as new.") + return empty_state() + if not isinstance(data, dict) or not isinstance(data.get("seen"), dict): + print(f"Note: unexpected shape in {path}; treating every ticket as new.") + return empty_state() + # A state file written by a different schema version can't be trusted field by + # field, so treat it as a cache miss rather than misreading it. + if data.get("version") != STATE_VERSION: + print(f"Note: {path} is version {data.get('version')!r}, expected {STATE_VERSION}; " + f"treating every ticket as new.") + return empty_state() + print(f"Loaded state for {len(data['seen'])} previously reported tickets.") + return data + + +def partition_by_state(tickets, state): + """Split into (new, changed, unchanged) against saved state. + + `changed` means Zendesk's updated_at differs from what we recorded โ€” note that + any agent action (reply, tag, status change) bumps updated_at, not just an + end-user comment. + """ + seen = state.get("seen", {}) + new, changed, unchanged = [], [], [] + for ticket in tickets: + previous = seen.get(str(ticket.get("id"))) + if previous is None: + new.append(ticket) + elif previous.get("updated_at") != ticket.get("updated_at"): + changed.append(ticket) + else: + unchanged.append(ticket) + return new, changed, unchanged + + +def save_state(path, state, reported, retention_days): + """Record `reported` as seen, prune old entries, write atomically. + + Returns (kept, pruned). + """ + now = datetime.now(timezone.utc) + stamp = now.strftime("%Y-%m-%dT%H:%M:%SZ") + seen = dict(state.get("seen", {})) + for ticket in reported: + seen[str(ticket.get("id"))] = { + "updated_at": ticket.get("updated_at"), + "last_reported": stamp, + } + + # Bound the file: the window is 48h, so anything older than retention is moot. + cutoff = now - timedelta(days=retention_days) + kept = {} + for ticket_id, record in seen.items(): + try: + last = datetime.strptime( + record.get("last_reported", ""), "%Y-%m-%dT%H:%M:%SZ" + ).replace(tzinfo=timezone.utc) + except (TypeError, ValueError): + continue # malformed entry โ€” drop it rather than keep it forever + if last >= cutoff: + kept[ticket_id] = record + + directory = os.path.dirname(path) + if directory: + os.makedirs(directory, exist_ok=True) + temporary = f"{path}.tmp" + with open(temporary, "w", encoding="utf-8") as fh: + json.dump({"version": STATE_VERSION, "updated_at": stamp, "seen": kept}, fh, indent=2) + os.replace(temporary, path) # atomic: a crash mid-write can't corrupt the state + return len(kept), len(seen) - len(kept) + + +# ---- App-store review filtering -------------------------------------------- +# +# AppFollow imports app-store reviews into Zendesk through this channel. In the +# 13-month sample it identified reviews with no false positives (2,656 of 2,656), +# whereas the `app-store` tag was present on only 287 of them โ€” so filter on the +# channel, not on tags. 4-5 star reviews were 59% of *all* tickets and are never +# actionable, so counting them beats paying tokens to classify them. +REVIEW_CHANNEL = "any_channel" +STAR_SUBJECT = re.compile(r"^\s*([โ˜…โ˜†]{1,10})") +DEFAULT_REVIEW_STAR_FLOOR = 3 + + +def squash(value): + """Collapse whitespace so subject/description can be compared meaningfully.""" + return re.sub(r"\s+", " ", value or "").strip() + + +def review_stars(ticket): + """Star count from an AppFollow review subject, or None if not a review subject.""" + match = STAR_SUBJECT.match(ticket.get("subject") or "") + return match.group(1).count("โ˜…") if match else None + + +def is_store_review(ticket): + return (((ticket.get("via") or {}).get("channel") == REVIEW_CHANNEL) + or STAR_SUBJECT.match(ticket.get("subject") or "") is not None) + + +def partition_reviews(tickets, star_floor): + """Split off app-store reviews rated above `star_floor`. + + Reviews whose rating cannot be parsed are kept: spending a few tokens beats + dropping a real complaint. + """ + keep, skipped = [], [] + for ticket in tickets: + stars = review_stars(ticket) + if is_store_review(ticket) and stars is not None and stars > star_floor: + skipped.append(ticket) + else: + keep.append(ticket) + return keep, skipped + + +def is_content_free(ticket): + """True when the description just repeats the subject, carrying no information. + + Twitter DM tickets arrive this way โ€” subject and body are both + "Conversation with " โ€” 15% of non-review tickets in the sample. Sent + as-is they are unclassifiable, so the model invents a category for a handle. + """ + return squash(ticket.get("description")) == squash(ticket.get("subject")) + + +def hydrate_descriptions(session, subdomain, tickets): + """Fill content-free descriptions from the ticket's comments. Returns the count. + + Costs one extra API call per affected ticket, so it only runs for those. + """ + hydrated = 0 + for ticket in tickets: + if not is_content_free(ticket): + continue + url = f"https://{subdomain}.zendesk.com/api/v2/tickets/{ticket['id']}/comments.json" + try: + resp = request_with_retry(session, "GET", url, attempts=2, params={"per_page": 10}) + except requests.RequestException as exc: + # Hydration is an enrichment, never a reason to abort the digest: an + # unreachable comments endpoint just leaves the description as-is. + print(f"Note: could not fetch comments for #{ticket['id']} ({exc}).") + continue + if resp.status_code >= 400: + continue + try: + comments = resp.json().get("comments", []) + except ValueError as exc: + # A 200 carrying an HTML error page (proxy, maintenance) is the same kind + # of non-event as an HTTP error here โ€” enrich what we can, skip the rest. + print(f"Note: unreadable comments payload for #{ticket['id']} ({exc}).") + continue + subject = squash(ticket.get("subject")) + bodies = [squash(c.get("body")) for c in comments] + useful = [b for b in bodies if b and b != subject] + if useful: + ticket["description"] = "\n".join(useful) + hydrated += 1 + if hydrated: + print(f"Recovered {hydrated} content-free description(s) from ticket comments.") + return hydrated + + +def compact_ticket(ticket): + """Reduce a Zendesk ticket to the fields Claude needs for triage.""" + description = (ticket.get("description") or "").strip() + truncated = len(description) > DESCRIPTION_CHARS + if truncated: + description = description[:DESCRIPTION_CHARS] + " โ€ฆ[truncated]" + rating = ticket.get("satisfaction_rating") or {} + return { + "id": ticket.get("id"), + "subject": ticket.get("subject") or "", + "description": description, + "tags": ticket.get("tags") or [], + "priority": ticket.get("priority"), + "status": ticket.get("status"), + "satisfaction_rating": rating.get("score"), + } + + +def build_analysis_prompt(compact_tickets): + return ( + "Classify every ticket in this batch and return one object per ticket.\n\n" + "TICKETS (JSON):\n" + + json.dumps(compact_tickets, ensure_ascii=False) + ) + + +def _cli_field_lines(): + """Describe every schema field for the CLI path, derived from TICKET_PROPERTIES. + + The API path has structured outputs to enforce the shape; the CLI path only has + this text. Hardcoding the field list here is how three fields (platform, + app_version, reported_session_id) silently came back empty on the CLI backend + after being added to the schema. + """ + lines = [] + for name, spec in TICKET_PROPERTIES.items(): + shape = spec.get("type", "string") + if "enum" in spec: + shape += "; one of: " + ", ".join(spec["enum"]) + description = spec.get("description", "") + lines.append(f"- {name} ({shape}){': ' + description if description else ''}") + return "\n".join(lines) + + +# The API path gets the shape enforced by structured outputs. The CLI path has no +# such enforcement, so the shape is spelled out here โ€” from the same schema. +CLI_JSON_INSTRUCTIONS = textwrap.dedent( + """ + Return ONLY a single JSON object โ€” no prose, no explanation, no markdown code + fence. The object has exactly one key, "tickets", whose value is an array with + one object per input ticket, each with exactly these keys: + __FIELDS__ + + Every key is required on every object. Use an empty string for text fields you + cannot fill, and the enum's catch-all value ('unknown', 'not_applicable', 'other') + rather than inventing a new one. + """ +).strip().replace("__FIELDS__", _cli_field_lines()) + + +def extract_json_object(text): + """Pull the outermost JSON object out of model prose (tolerates code fences).""" + start = text.find("{") + end = text.rfind("}") + if start == -1 or end <= start: + sys.exit(f"No JSON object found in the model output:\n{text[:500]}") + try: + return json.loads(text[start : end + 1]) + except json.JSONDecodeError as exc: + sys.exit(f"Model output was not valid JSON ({exc}):\n{text[start : start + 500]}") + + +REQUIRED_FINDING_KEYS = ("id", "category", "severity") + + +def validate_findings(findings, label): + """Exit unless every entry is an object carrying the keys the renderer indexes. + + build_summary_embed does f["category"] / f["severity"] and build_highlight_embed + does f["id"], so a missing key surfaces as a KeyError halfway through building a + Discord payload. Failing here names the offending entry instead. + """ + for position, entry in enumerate(findings): + if not isinstance(entry, dict): + sys.exit(f"{label}: entry {position} is {type(entry).__name__}, expected an object.") + missing = [key for key in REQUIRED_FINDING_KEYS if key not in entry] + if missing: + sys.exit(f"{label}: entry {position} (id={entry.get('id')!r}) is missing " + f"required key(s): {', '.join(missing)}.") + return findings + + +def tickets_from_payload(payload, source): + """Pull the `tickets` list out of a classification payload, or exit clearly. + + Structured outputs guarantee the key and the item shape on the API path, but the + CLI path has neither โ€” a bare KeyError mid-render is a confusing way to learn + that, so both paths are validated here before anything renders them. + """ + found = payload.get("tickets") if isinstance(payload, dict) else None + if not isinstance(found, list): + shape = sorted(payload) if isinstance(payload, dict) else type(payload).__name__ + sys.exit(f"{source} returned no 'tickets' list (got: {shape}).") + return validate_findings(found, source) + + +def analyze_via_claude_cli(model, compact_tickets, timeout=1800): + """Classify the batch with the local `claude` CLI instead of the Anthropic API. + + Local debugging path: it authenticates as Claude Code, so no ANTHROPIC_API_KEY is + needed. There is no structured-output enforcement here, so the response is parsed + defensively and the schema is described in the prompt. + """ + prompt = "\n\n".join( + [SYSTEM_PROMPT, CLI_JSON_INSTRUCTIONS, build_analysis_prompt(compact_tickets)] + ) + cmd = ["claude", "-p", "--output-format", "json"] + if model: + cmd += ["--model", model] + try: + # Prompt goes over stdin: a full batch can exceed the argv size limit. + proc = subprocess.run( + cmd, input=prompt, capture_output=True, text=True, timeout=timeout + ) + except FileNotFoundError: + sys.exit("`claude` not found on PATH. Install Claude Code, or use --backend api.") + except subprocess.TimeoutExpired: + sys.exit(f"`claude` timed out after {timeout}s. Try a smaller --max-tickets.") + if proc.returncode != 0: + sys.exit(f"`claude` failed ({proc.returncode}): {proc.stderr[:500]}") + + envelope = extract_json_object(proc.stdout) + if envelope.get("is_error") or envelope.get("subtype") != "success": + sys.exit(f"`claude` reported an error: {envelope.get('result') or envelope}") + cost = envelope.get("total_cost_usd") + if cost is not None: + print(f"claude CLI reported ${cost:.4f} for this batch.") + result = extract_json_object(envelope.get("result") or "") + return tickets_from_payload(result, "`claude` CLI") + + +def dump_batch(path, compact_tickets, model): + """Write the batch to disk so it can be classified by hand. Contains ticket text.""" + payload = { + "model": model, + "system_prompt": SYSTEM_PROMPT, + "instructions": CLI_JSON_INSTRUCTIONS, + "schema": SCHEMA, + "tickets": compact_tickets, + } + with open(path, "w", encoding="utf-8") as fh: + json.dump(payload, fh, indent=2, ensure_ascii=False) + + +def load_findings(path): + """Read findings written by hand or by another tool: {"tickets": [...]} or [...]. + + Validates the keys the Discord renderer indexes directly, so a hand-edited file + fails here with the offending entry rather than as a KeyError mid-render. + """ + with open(path, encoding="utf-8") as fh: + data = json.load(fh) + findings = data.get("tickets") if isinstance(data, dict) else data + if not isinstance(findings, list): + sys.exit(f"{path}: expected a JSON list, or an object with a 'tickets' list.") + return validate_findings(findings, path) + + +def analyze_in_chunks(analyzer, compact_tickets, batch_size): + """Classify in chunks so a big batch can't blow the output token ceiling. + + Chunking is per-request, so `cluster` labels and `priority_rank` are only + meaningful within a chunk โ€” batches that actually split are large enough that + cross-chunk cluster fidelity is secondary to completing at all. + """ + if len(compact_tickets) <= batch_size: + return analyzer(compact_tickets) + + findings = [] + chunks = (len(compact_tickets) + batch_size - 1) // batch_size + print(f"Batch of {len(compact_tickets)} exceeds {batch_size}; splitting into {chunks} requests.") + for number, start in enumerate(range(0, len(compact_tickets), batch_size), start=1): + chunk = compact_tickets[start : start + batch_size] + print(f" chunk {number}/{chunks}: {len(chunk)} tickets") + findings.extend(analyzer(chunk)) + return findings + + +def analyze(client, model, effort, compact_tickets): + prompt = build_analysis_prompt(compact_tickets) + with client.messages.stream( + model=model, + max_tokens=MAX_OUTPUT_TOKENS, + thinking={"type": "adaptive"}, + output_config={ + "format": {"type": "json_schema", "schema": SCHEMA}, + "effort": effort, + }, + system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": prompt}], + ) as stream: + message = stream.get_final_message() + + if message.stop_reason == "refusal": + sys.exit("Claude refused to process the batch.") + if message.stop_reason == "max_tokens": + # Structured output truncated mid-JSON: json.loads below would fail with a + # baffling parse error, so say what actually went wrong. + sys.exit( + f"Claude hit the {MAX_OUTPUT_TOKENS} output-token limit on a batch of " + f"{len(compact_tickets)} tickets, so the JSON is incomplete. " + f"Lower --batch-size (currently splitting at {DEFAULT_BATCH_SIZE})." + ) + text = next((b.text for b in message.content if b.type == "text"), None) + if not text: + sys.exit("Claude returned no structured output.") + return tickets_from_payload(json.loads(text), "Claude") + + +# ---- Discord rendering ----------------------------------------------------- + +SEVERITY_COLOR = { + "crash": 0xE74C3C, # red + "data_loss": 0xC0392B, # dark red + "major": 0xE67E22, # orange + "minor": 0xF1C40F, # yellow + "cosmetic": 0x95A5A6, # grey + "not_applicable": 0x3498DB, # blue +} +MAX_EMBEDS_PER_MESSAGE = 10 +MAX_EMBED_CHARS_PER_MESSAGE = 6000 # Discord's aggregate limit across one message +MAX_HIGHLIGHTS = 27 # 3 messages of ~9 highlights + a summary embed + + +def ticket_url(subdomain, ticket_id): + return f"https://{subdomain}.zendesk.com/agent/tickets/{ticket_id}" + + +def clip(text, limit): + text = (text or "").strip() + return text if len(text) <= limit else text[: limit - 1] + "โ€ฆ" + + +def build_summary_embed(findings, highlights, subdomain, stats=None): + by_category = {} + by_severity = {} + clusters = {} + for f in findings: + by_category[f["category"]] = by_category.get(f["category"], 0) + 1 + by_severity[f["severity"]] = by_severity.get(f["severity"], 0) + 1 + label = (f.get("cluster") or "").strip() + if label: + clusters.setdefault(label, []).append(f["id"]) + + cat_lines = "\n".join( + f"{CATEGORY_LABEL.get(cat, cat)}: **{count}**" + for cat, count in sorted(by_category.items(), key=lambda kv: -kv[1]) + ) + serious = by_severity.get("crash", 0) + by_severity.get("data_loss", 0) + dup_clusters = {k: v for k, v in clusters.items() if len(v) > 1} + fields = [{"name": "By category", "value": cat_lines or "โ€”", "inline": False}] + if dup_clusters: + cluster_lines = "\n".join( + f"**{clip(label, 40)}** โ€” {len(ids)} tickets (#{', #'.join(str(i) for i in ids[:6])})" + for label, ids in sorted(dup_clusters.items(), key=lambda kv: -len(kv[1]))[:6] + ) + fields.append({"name": "Likely duplicate clusters", "value": clip(cluster_lines, 1024), "inline": False}) + + # Account for the batch honestly: how many of the window we looked at, how many + # we skipped as unchanged, and how big the untriaged backlog is behind it. + stats = stats or {} + matched = stats.get("matched") + skipped = stats.get("skipped_unchanged") or 0 + updated = stats.get("updated_count") or 0 + backlog = stats.get("total_unsolved") + + window = f"Analyzed **{len(findings)}**" + if matched is not None: + window += f" of **{matched}**" + window += f" tickets in the window" + if stats.get("scope"): + window += f" ({stats['scope']})" + window += "." + if skipped: + window += f" Skipped **{skipped}** already reported and unchanged." + reviews = stats.get("skipped_reviews") or 0 + if reviews: + window += f" Skipped **{reviews}** positive app-store review(s)." + lines = [window] + + if backlog is not None: + lines.append(f"Backlog: **{backlog:,}** unsolved tickets in total (not triaged).") + + tail = f"**{len(highlights)}** worth looking into" + tail += f", including **{serious}** crash/data-loss." if serious else "." + if updated: + tail += f" ๐Ÿ”„ **{updated}** changed since last reported." + lines.append(tail) + + return { + "title": "๐Ÿ—‚๏ธ Zendesk triage", + "description": "\n".join(lines), + "color": 0xE67E22 if highlights else 0x2ECC71, + "fields": fields, + } + + +def is_urgent(finding): + return finding.get("category") in URGENT_CATEGORIES + + +def embed_color(finding): + """Colour by category urgency first, then severity. + + An abuse or legal report is not a bug, so the model rates it not_applicable โ€” + which maps to the calmest blue. Category has to win, or the most serious ticket + in the digest looks the most benign. + """ + urgent = CATEGORY_COLOR.get(finding.get("category")) + if urgent: + return urgent + return SEVERITY_COLOR.get(finding.get("severity", "not_applicable"), 0x95A5A6) + + +def build_highlight_embed(finding, subdomain, is_update=False): + tid = finding["id"] + sev = finding.get("severity", "not_applicable") + cat = CATEGORY_LABEL.get(finding.get("category"), finding.get("category", "")) + # ๐Ÿ”„ marks a ticket we already showed that has since changed, so the reader + # knows it is a follow-up rather than a duplicate post. + marker = "๐Ÿ”„ " if is_update else "" + title = f"{marker}#{tid} ยท {clip(finding.get('summary'), 200) or '(no summary)'}" + fields = [ + {"name": "Category", "value": clip(cat, 60) or "โ€”", "inline": True}, + {"name": "Severity", "value": sev, "inline": True}, + {"name": "Language", "value": clip(finding.get("language"), 40) or "โ€”", "inline": True}, + ] + platform = finding.get("platform") + if platform and platform != "unknown": + fields.append({"name": "Platform", "value": clip(platform, 40), "inline": True}) + component = clip(finding.get("affected_component"), 100) + if component: + fields.append({"name": "Component", "value": component, "inline": True}) + version = clip(finding.get("app_version"), 40) + if version: + fields.append({"name": "Version", "value": version, "inline": True}) + # The reported account is the actionable part of an abuse report โ€” surfacing it + # here saves opening the ticket to copy it. + reported = clip(finding.get("reported_session_id"), 100) + if reported: + fields.append({"name": "Reported account", "value": f"`{reported}`", "inline": False}) + root = clip(finding.get("likely_root_cause"), 300) + description = f"Likely cause: {root}" if root else "" + return { + "title": clip(title, 256), + "url": ticket_url(subdomain, tid), + "description": description, + "color": embed_color(finding), + "fields": fields, + } + + +def embed_char_count(embed): + """Characters Discord counts against the per-message embed budget.""" + total = len(embed.get("title") or "") + len(embed.get("description") or "") + for field in embed.get("fields") or []: + total += len(field.get("name") or "") + len(field.get("value") or "") + return total + + +def chunk_entries(entries): + """Group (embed, ticket_ids) pairs into messages within both Discord limits. + + Discord caps a message at 10 embeds *and* 6,000 characters summed across them; + chunking on count alone can produce a payload that is rejected as too large. + """ + chunks, current, current_chars = [], [], 0 + for embed, ids in entries: + size = embed_char_count(embed) + too_many = len(current) >= MAX_EMBEDS_PER_MESSAGE + too_long = current_chars + size > MAX_EMBED_CHARS_PER_MESSAGE + if current and (too_many or too_long): + chunks.append(current) + current, current_chars = [], 0 + current.append((embed, ids)) + current_chars += size + if current: + chunks.append(current) + return chunks + + +def select_highlights(findings): + """Ordered highlights split into (shown, omitted) by the display cap. + + Urgent categories are included even if the model failed to flag them, and sort + ahead of everything else โ€” an abuse or legal report must not be pushed out of the + digest by a queue of ordinary bugs. Shared with state recording so the display + and what gets marked reported can't drift apart. + """ + highlights = [f for f in findings if f.get("worth_looking_into") or is_urgent(f)] + highlights.sort(key=lambda f: (not is_urgent(f), f.get("priority_rank", 9999))) + return highlights[:MAX_HIGHLIGHTS], highlights[MAX_HIGHLIGHTS:] + + +def build_messages(findings, subdomain, stats=None, updated_ids=None): + """Return (messages, coverage). + + coverage[i] is the set of ticket ids message i accounts for, so a partial post + failure can still record exactly the tickets that reached Discord. + """ + updated_ids = updated_ids or set() + shown, omitted = select_highlights(findings) + shown_ids = {f.get("id") for f in shown} + omitted_ids = {f.get("id") for f in omitted} + + # The summary embed accounts for every classified ticket except the highlights + # that didn't fit; those are covered by no message and stay eligible. + summary_ids = {f.get("id") for f in findings} - shown_ids - omitted_ids + entries = [(build_summary_embed(findings, shown + omitted, subdomain, stats), summary_ids)] + entries += [ + (build_highlight_embed(f, subdomain, is_update=f.get("id") in updated_ids), + {f.get("id")}) + for f in shown + ] + + content = None + if omitted: + content = (f"Showing the top {len(shown)} of {len(shown) + len(omitted)} " + f"tickets worth looking into.") + + messages, coverage = [], [] + for index, chunk in enumerate(chunk_entries(entries)): + payload = {"embeds": [embed for embed, _ in chunk]} + if index == 0 and content: + payload["content"] = content + messages.append(payload) + covered = set() + for _, ids in chunk: + covered |= ids + coverage.append(covered) + return messages, coverage + + +def post_to_discord(session, webhook_url, messages): + """POST each message in order; return how many Discord accepted. + + Stops at the first failure and returns the accepted count instead of exiting, so + the caller can record the tickets that did land before signalling the failure โ€” + otherwise a failure on message 3 of 3 reposts messages 1 and 2 on the next run. + """ + for index, payload in enumerate(messages): + resp = request_with_retry(session, "POST", webhook_url, json=payload) + if resp.status_code >= 400: + print(f"Discord webhook failed on message {index + 1}/{len(messages)} " + f"({resp.status_code}): {resp.text[:300]}") + return index + return len(messages) + + +def main(): + parser = argparse.ArgumentParser(description="Triage open Zendesk tickets with Claude and post a Discord summary.") + parser.add_argument("--subdomain", help="Zendesk subdomain (else ZENDESK_SUBDOMAIN).") + parser.add_argument("--email", help="Zendesk agent email (else ZENDESK_EMAIL).") + parser.add_argument("--api-token", help="Zendesk API token (else ZENDESK_API_TOKEN).") + parser.add_argument("--webhook", help="Discord webhook URL (else DISCORD_WEBHOOK_URL).") + parser.add_argument("--query", help="Zendesk search query (else ZENDESK_QUERY, else default). " + "Takes precedence over --window-hours.") + parser.add_argument("--window-hours", type=int, metavar="N", + help="Only analyze unsolved tickets created in the last N hours. " + "The scheduled daily run uses 48.") + parser.add_argument("--model", help="Claude model id (else ZENDESK_TRIAGE_MODEL, else claude-opus-4-8).") + parser.add_argument("--effort", default="medium", choices=["low", "medium", "high", "xhigh", "max"], + help="Claude reasoning effort (default: medium).") + parser.add_argument("--max-tickets", type=int, default=DEFAULT_MAX_TICKETS, + help=f"Max tickets to analyze (default: {DEFAULT_MAX_TICKETS}).") + parser.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE, metavar="N", + help=f"Split batches larger than N tickets across multiple requests, " + f"so a big batch can't exceed the output token ceiling " + f"(default: {DEFAULT_BATCH_SIZE}).") + parser.add_argument("--dry-run", action="store_true", + help="Fetch and analyze, then print the Discord payload instead of posting.") + parser.add_argument("--backend", default="api", choices=["api", "claude-cli", "file"], + help="Where classification happens: the Anthropic API (default), the " + "local `claude` CLI (no API key needed), or a findings file.") + parser.add_argument("--findings", + help="Findings JSON to render instead of classifying (--backend file).") + parser.add_argument("--review-star-floor", type=int, default=DEFAULT_REVIEW_STAR_FLOOR, + metavar="N", + help=f"Classify app-store reviews of N stars or fewer; count the rest " + f"without spending tokens (default: {DEFAULT_REVIEW_STAR_FLOOR}).") + parser.add_argument("--include-positive-reviews", action="store_true", + help="Classify every app-store review, including 4-5 star ones.") + parser.add_argument("--no-hydrate", action="store_true", + help="Skip fetching comments for tickets whose description just " + "repeats the subject (e.g. Twitter DMs).") + parser.add_argument("--state", metavar="PATH", + help="Dedup state file. When set, tickets already reported and " + "unchanged (same Zendesk updated_at) are skipped entirely; " + "changed ones are re-reported and flagged. Written only on a " + "real run, after Discord accepts the post.") + parser.add_argument("--state-retention-days", type=int, default=30, metavar="N", + help="Forget state entries older than N days (default: 30).") + parser.add_argument("--dump-batch", metavar="PATH", + help="Write the batch (tickets, prompt, schema) to PATH and exit, for " + "hand-classification. WARNING: writes ticket content to disk.") + args = parser.parse_args() + + if args.backend == "file" and not args.findings: + sys.exit("--backend file requires --findings PATH.") + + # Subdomain is always needed: it builds the ticket links in the Discord payload. + subdomain = get_env("ZENDESK_SUBDOMAIN", args.subdomain) + # A dump exits before rendering anything, so it never needs the webhook either. + needs_webhook = not (args.dry_run or args.dump_batch) + webhook = get_env("DISCORD_WEBHOOK_URL", args.webhook, required=needs_webhook) + model = args.model or os.environ.get("ZENDESK_TRIAGE_MODEL") or DEFAULT_MODEL + + stats = {} + state = None + classified = [] + updated_ids = set() + + if args.backend == "file": + # Findings already exist, so neither Zendesk nor a model is involved. + findings = load_findings(args.findings) + print(f"Loaded {len(findings)} findings from {args.findings}.") + else: + email = get_env("ZENDESK_EMAIL", args.email) + api_token = get_env("ZENDESK_API_TOKEN", args.api_token) + + # An explicit query wins over --window-hours; warn rather than silently drop it. + explicit_query = args.query or os.environ.get("ZENDESK_QUERY") + if explicit_query: + if args.window_hours: + print("Note: --window-hours ignored because an explicit query was given.") + query = explicit_query + elif args.window_hours: + query = build_window_query(args.window_hours) + stats["scope"] = window_label(args.window_hours) + else: + query = DEFAULT_QUERY + + zd = zendesk_session(email, api_token) + tickets, total_matched = fetch_tickets(zd, subdomain, query, args.max_tickets) + matched = "?" if total_matched is None else total_matched + print(f"Fetched {len(tickets)} of {matched} matching tickets (query: {query!r}).") + if total_matched is not None and total_matched > len(tickets): + # Name whichever cap actually bound, so a truncated digest doesn't send + # someone raising --max-tickets against a limit that isn't ours. + reason = (f"Zendesk's search API returns at most {SEARCH_RESULT_LIMIT} results" + if args.max_tickets >= SEARCH_RESULT_LIMIT + else f"--max-tickets is {args.max_tickets}") + print(f"Note: {total_matched - len(tickets)} matching tickets were not analyzed " + f"({reason}).") + if not tickets: + print("No tickets matched the query; nothing to do.") + return + + stats["matched"] = total_matched + stats["total_unsolved"] = fetch_total_unsolved(zd, subdomain) + + # Drop positive store reviews before anything expensive: they were 59% of all + # tickets in the sample and never actionable. + if not args.include_positive_reviews: + tickets, skipped_reviews = partition_reviews(tickets, args.review_star_floor) + if skipped_reviews: + stats["skipped_reviews"] = len(skipped_reviews) + print(f"Skipped {len(skipped_reviews)} app-store review(s) above " + f"{args.review_star_floor} stars; {len(tickets)} tickets remain.") + if not tickets: + print("Only positive reviews in this window; nothing to report.") + return + + if args.state: + state = load_state(args.state) + new, changed, unchanged = partition_by_state(tickets, state) + print(f"{len(new)} new, {len(changed)} changed since last reported, " + f"{len(unchanged)} unchanged (skipped).") + stats["skipped_unchanged"] = len(unchanged) + stats["updated_count"] = len(changed) + updated_ids = {t.get("id") for t in changed} + # Skipping before the model call means unchanged tickets cost nothing. + tickets = new + changed + if not tickets: + print("Nothing new or changed since the last run; nothing to report.") + return + + # Only after the review and dedup filters, so we never pay comment lookups + # for tickets we are about to discard. + if not args.no_hydrate: + hydrate_descriptions(zd, subdomain, tickets) + + analyzed = tickets + compact = [compact_ticket(t) for t in tickets] + + if args.dump_batch: + dump_batch(args.dump_batch, compact, model) + print(f"Wrote {len(compact)} tickets to {args.dump_batch} โ€” this file contains " + f"ticket content, so keep it out of the repo.") + print("Classify it, then: --backend file --findings --dry-run") + return + + if args.backend == "claude-cli": + analyzer = partial(analyze_via_claude_cli, model) + else: + client = anthropic.Anthropic() # reads ANTHROPIC_API_KEY + analyzer = partial(analyze, client, model, args.effort) + findings = analyze_in_chunks(analyzer, compact, args.batch_size) + + # Keep only findings whose id maps to a fetched ticket, in case of drift. + valid_ids = {t["id"] for t in compact} + findings = [f for f in findings if f.get("id") in valid_ids] + + # A ticket with no classification was never triaged, so it must not be + # recorded as reported โ€” leave it eligible for the next run. + finding_ids = {f.get("id") for f in findings} + classified = [t for t in analyzed if t.get("id") in finding_ids] + if len(classified) < len(analyzed): + print(f"Note: {len(analyzed) - len(classified)} ticket(s) came back without a " + f"classification; they stay eligible for the next run.") + + # Same selection the embeds use, so the console count can't disagree with the + # digest โ€” worth_looking_into alone would miss urgent categories the model + # failed to flag. + shown, omitted = select_highlights(findings) + print(f"{len(findings)} tickets classified; {len(shown) + len(omitted)} worth looking into" + + (f" ({len(omitted)} beyond the display cap)." if omitted else ".")) + + messages, coverage = build_messages(findings, subdomain, stats, updated_ids) + if args.dry_run: + print(json.dumps(messages, indent=2, ensure_ascii=False)) + if args.state: + would = set().union(*coverage) if coverage else set() + print(f"(dry run: would record " + f"{sum(1 for t in classified if t.get('id') in would)} tickets " + f"in {args.state})") + return + + posted = post_to_discord(requests.Session(), webhook, messages) + print(f"Posted {posted} of {len(messages)} Discord message(s).") + + # Record only tickets covered by messages Discord actually accepted, so a partial + # failure neither reposts what landed nor suppresses what didn't. + if args.state and state is not None: + delivered = set().union(*coverage[:posted]) if posted else set() + recorded = [t for t in classified if t.get("id") in delivered] + kept, pruned = save_state(args.state, state, recorded, args.state_retention_days) + print(f"Recorded {len(recorded)} tickets; state now tracks {kept} " + f"({pruned} pruned beyond {args.state_retention_days} days).") + + if posted < len(messages): + sys.exit(f"Aborted after {posted}/{len(messages)} messages; " + f"undelivered tickets stay eligible for the next run.") + + +if __name__ == "__main__": + main()