{"recipes":[{"id":"action-compare","title":"Compare two candidate actions","description":"Which of firstAction and secondAction better advances goal within constraints?","useWhen":"An agent has narrowed to two next steps and needs a head-to-head preference under the stated goal and constraints.","tags":["agent","planning","comparison","pairwise","action","decision"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"action-effects","title":"Label the side effects of an action","description":"Which side effects does the described action involve: writing or modifying data, sending a message or notification, spending or moving money, deleting something, or calling an external service?","useWhen":"An agent is about to execute a step and your approval policy differs by effect, so you want to gate auto-execution per side effect rather than with one blanket risk score.","tags":["agent","safety","action","side-effects","approval","labels","harness"],"category":"workflow","collection":"agent-operations","evidence":"measured","evidenceDetails":{"kind":"synthetic","label":"Current synthetic measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:42:48.259Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":100,"ready":63,"reviewRate":0.37,"readyAccuracy":0.984,"failed":0,"provenance":[{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":100}],"acceptanceMet":null}}},{"id":"action-reversibility","title":"Grade action reversibility","description":"How reversible is action, given any context, from a trivial undo to an irreversible external effect?","useWhen":"You need to decide whether an agent may proceed on its own or must pause for approval before a step that cannot be taken back.","tags":["agent","safety","action","reversibility","approval","harness"],"category":"workflow","collection":null,"evidence":"measured","evidenceDetails":{"kind":"unspecified","label":"Current measurement; labels unspecified","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:42:51.224Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":44,"ready":20,"reviewRate":0.545,"readyAccuracy":1,"failed":0,"provenance":[{"method":"unspecified","source":"Not recorded by the dataset author.","cases":44}],"acceptanceMet":null}}},{"id":"action-scope","title":"Check proposed action scope","description":"Is proposedAction within the work requested in request and constraints?","useWhen":"You need to check whether a proposed action stays within the requested work and constraints.","tags":["workflow","action","scope","harness"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"age-appropriateness","title":"Grade audience age suitability","description":"What is the youngest general audience for which content is appropriate, on a five-level rubric?","useWhen":"You need to route, filter, or label user-generated or model-generated content by audience age before publishing it.","tags":["moderation","content-rating","age","audience","safety","rubric","score"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"alert-actionability","title":"Grade how actionable an alert is","description":"How actionable is alert for the on-call engineer who receives it, on a five-level rubric from pure noise to a guided first step?","useWhen":"You need to triage or audit alert text before paging someone: to route noisy alerts to a digest, to flag vague alerts for rewriting, or to rank a flood of alerts by how much they tell the responder.","tags":["on-call","alerting","incident-response","devops","score","monitoring"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"answer-consistency","title":"Check answer consistency","description":"Do firstStatement and secondStatement make compatible claims about the same subject and circumstances?","useWhen":"You need to check whether two statements agree under the same conditions.","tags":["answer-quality","answer","consistency","rag","evidence","alignment-research"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"answer-coverage","title":"Check answer coverage","description":"Check whether a draft answers each supplied question.","useWhen":"You have a draft and need to check whether it answers each supplied question.","tags":["answer-quality","answer","coverage","rag","evidence"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"answer-disclosures","title":"Label the disclosures in an answer","description":"Which of these does draft include: stated uncertainty, stated limitations, cited sources, stated assumptions?","useWhen":"You need to audit or gate model answers on transparency habits in a single call, for evaluation sets or response policies.","tags":["answer-quality","disclosure","labels","multi-label","transparency","evaluation"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"answer-grade","title":"Grade an answer against a rubric","description":"How well does answer meet rubric as a response to question, on a five-level rubric from no credit to full credit?","useWhen":"You grade free-text answers against a written rubric and want a graded level with a confidence you can route to a human grader when low.","tags":["education","grading","rubric","assessment","score"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"answer-invalidation","title":"Check an answer after a source change","description":"Does updatedEvidence still support the entire claim that was based on previousEvidence?","useWhen":"You need to check whether updated evidence still supports a previously supported claim.","tags":["knowledge","answer","invalidation"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"answer-relevance","title":"Check answer relevance","description":"How directly does draft address request?","useWhen":"You need to check whether a draft stays relevant to the request.","tags":["answer-quality","answer","relevance","rag","evidence"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"answerability","title":"Check answerability","description":"Decide whether supplied evidence can answer an entire question.","useWhen":"You need to know if you have enough evidence and can answer a question.","tags":["rag","abstention","evidence"],"category":"retrieval","collection":null,"evidence":"measured","evidenceDetails":{"kind":"unspecified","label":"Current measurement; labels unspecified","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:42:52.472Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":50,"ready":44,"reviewRate":0.12,"readyAccuracy":1,"failed":0,"provenance":[{"method":"unspecified","source":"Not recorded by the dataset author.","cases":50}],"acceptanceMet":null}}},{"id":"appeal-grounds-kind","title":"Classify the ground an appeal asserts","description":"What ground does appeal primarily assert: a factual error, a procedural error, new evidence, hardship, a misapplied rule, or something else?","useWhen":"You need to sort incoming appeals of a benefits, permit, enforcement, or academic decision by the kind of argument they make, so each reaches the right reviewer or template before anyone reads the file.","tags":["public-sector","appeals","adjudication","classification","workflow","case-management"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"appointment-request-kind","title":"Classify a patient's appointment request","description":"What does the patient primarily want from message: to schedule, reschedule, or cancel an appointment, get results, get a refill, or ask a question?","useWhen":"You need to route incoming patient portal messages or voicemail transcripts to the scheduling, results, or pharmacy queue by the main thing the patient asks for.","tags":["healthcare","patient-message","scheduling","intent","routing","choice"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"argument-fit","title":"Check argument meaning","description":"Does proposedValue for argument express the intended value in request and context?","useWhen":"You need to check whether a proposed argument value matches the user request.","tags":["workflow","argument","fit"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"attempted-step","title":"Check a previously attempted step","description":"Does conversation establish whether the customer already performed step?","useWhen":"You need to know whether a customer already tried a troubleshooting step.","tags":["support","attempted","step"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"attribution-match","title":"Check a statement's attributed source","description":"Check whether supplied source text attributes a statement to the claimed speaker or source.","useWhen":"You need to check who said a statement in a transcript or source excerpt, separately from whether it is true.","tags":["attribution","speaker","who said","source ownership","quotation","annotation","alignment-research"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"audience-fit","title":"Check audience fit","description":"Does the level of explanation in document fit the knowledge and needs explicitly described in audience?","useWhen":"You need to check whether a document suits the stated audience knowledge and needs.","tags":["knowledge","audience","fit"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"breaking-change-signal","title":"Detect breaking changes","description":"Does change indicate a change that would break existing callers, integrations, stored data, or documented behavior?","useWhen":"You need to flag changes that require a major version bump, a migration note, or downstream coordination before they merge.","tags":["code-review","breaking-change","compatibility","api","versioning","gate"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"budget-fit","title":"Check plan against budget","description":"Does plan, as described, plausibly fit within budget, the stated limits on steps, time, cost, or calls?","useWhen":"You need a yes/no check before an agent starts executing a plan under a step, time, cost, or call limit, so it can trim or ask instead of running out midway.","tags":["agent","planning","budget","limits","gate"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"budget-narrative-fit","title":"Check a budget narrative against its line items","description":"Does narrative explain each line item in lineItems and nothing that lineItems does not list?","useWhen":"You review grant budgets before submission and want to catch a narrative that skips a line item or justifies an expense the budget table does not contain.","tags":["nonprofit","grants","budget","proposal","gate","review"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"bug-report-completeness","title":"Grade bug report completeness","description":"How complete is report for someone to reproduce and triage it, on a five-level rubric?","useWhen":"You need to decide whether an incoming bug report can go straight to triage or needs a follow-up request for details first.","tags":["code-review","bug-report","triage","issue","reproduction","rubric"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"buying-intent","title":"Grade buying intent","description":"How strong is the purchase intent expressed in message, on a five-level rubric?","useWhen":"You need to rank or route inbound leads and replies by how close the sender is to buying, not just whether they are interested.","tags":["sales","crm","intent","lead","qualification"],"category":"conversation","collection":"customer-conversations","evidence":"measured","evidenceDetails":{"kind":"synthetic","label":"Current synthetic measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:42:53.964Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":75,"ready":74,"reviewRate":0.013,"readyAccuracy":1,"failed":0,"provenance":[{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":75}],"acceptanceMet":null}}},{"id":"cache-match","title":"Check a cached answer","description":"Does cachedAnswer address question with the same relevant meaning and conditions as originalQuestion?","useWhen":"You want to know whether a saved answer applies to a new question.","tags":["retrieval","cache","match","rag","evidence"],"category":"retrieval","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"callback-responsibility","title":"Identify who should initiate a callback","description":"Which party is expected to initiate the next call?","useWhen":"You need to decide whether the business or customer is expected to initiate the next call.","tags":["conversation","callback","ownership","handover"],"category":"conversation","collection":"customer-conversations","evidence":"measured","evidenceDetails":{"kind":"synthetic","label":"Current synthetic measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:42:56.097Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":60,"ready":38,"reviewRate":0.367,"readyAccuracy":0.974,"failed":0,"provenance":[{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":60}],"acceptanceMet":null}}},{"id":"cancellation-check","title":"Check cancellation intent","description":"Does message ask to cancel, pause, or continue task?","useWhen":"You need to detect whether a message asks to stop, pause, or continue a task.","tags":["conversation","cancellation","check"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"care-urgency-wording","title":"Grade the urgency a patient's wording asks for","description":"How urgent is the care that message asks for, from routine to emergency, judged on what the patient's wording says rather than on medical assessment?","useWhen":"You need to sort patient messages into scheduling or response-time queues by the timeline the patient asks for, before or alongside clinical triage handled by protocol.","tags":["healthcare","patient-message","urgency","scheduling","rubric","score"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"category-fit","title":"Check an item against its category","description":"Does the product described by item belong under category as defined?","useWhen":"You need to audit catalog placements or gate seller-submitted listings so items do not land in the wrong category.","tags":["catalog","taxonomy","classification","e-commerce","data-quality"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"causal-attribution","title":"Identify personal and situational explanations","description":"Label whether an explanation attributes a specified behavior to the person, circumstances, both, or gives no cause.","useWhen":"You need to label whether an explanation points to the person, the situation, or both.","tags":["psychology","behavior","causal attribution","personal cause","situational cause","dispositional","annotation","alignment-research"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"causal-language-strength","title":"Grade the strength of causal language","description":"How strong is the causal claim in the wording of statement, from no relationship claimed to causation stated as fact?","useWhen":"You are checking abstracts, press releases, or summaries of studies for causal overreach, so that wording can be compared with what the study design supports.","tags":["research","academic-writing","causal-claims","science-communication","psychology","score"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"certainty-match","title":"Check certainty wording","description":"Does the certainty expressed in draft match assessment?","useWhen":"You want the wording of a draft to reflect the certainty of an assessment.","tags":["answer-quality","certainty","match","rag","evidence","alignment-research"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"change-meaning","title":"Assess a text revision","description":"Does the revision from before to after change material meaning, conditions, or obligations?","useWhen":"You need to know whether a revision changes material meaning or obligations.","tags":["knowledge","change","meaning"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"change-risk","title":"Grade change risk","description":"How risky is change to ship, given context, on a five-level rubric?","useWhen":"You need to size the risk of a described code change before choosing reviewers, test depth, rollout strategy, or approval requirements.","tags":["code-review","risk","change","deployment","pull-request"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"change-window-fit","title":"Check a change against a change-window policy","description":"Does the described change fall within the written change-window or freeze policy?","useWhen":"A deploy or infrastructure change is proposed in free text and you want a first read on whether it is permitted under a written change-window or freeze policy before a human approver looks at it.","tags":["change-management","deployment","devops","policy","gate","on-call"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"checkers-move","title":"Choose a checkers move","description":"Recommend a supplied legal checkers move from a structured board and player, with built-in American/English checkers instructions.","useWhen":"Your checkers game already provides its board, acting player, and legal moves, and you want a move ID without writing decision prompts.","tags":["checkers","draughts","gameplay","board game","legal moves","kings","captures"],"category":"workflow","collection":null,"evidence":"measured","evidenceDetails":{"kind":"unspecified","label":"Current measurement; labels unspecified","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:42:57.724Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":31,"ready":0,"reviewRate":1,"readyAccuracy":null,"failed":0,"provenance":[{"method":"unspecified","source":"Not recorded by the dataset author.","cases":31}],"acceptanceMet":null}}},{"id":"choose-action","title":"Choose a game action from supplied candidates","description":"Recommend one eligible game action against a supplied goal, using rules, current state, and optional player history.","useWhen":"You need to choose the next game action from a list using the current environment, game rules, and previous player actions.","tags":["gameplay","game","choose action","best move","strategy","players","turn","simulation"],"category":"workflow","collection":null,"evidence":"measured","evidenceDetails":{"kind":"unspecified","label":"Current measurement; labels unspecified","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:42:58.654Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":32,"ready":27,"reviewRate":0.156,"readyAccuracy":1,"failed":0,"provenance":[{"method":"unspecified","source":"Not recorded by the dataset author.","cases":32}],"acceptanceMet":null}}},{"id":"citation-match","title":"Match claims to citations","description":"Find supplied passages that independently support an entire claim.","useWhen":"You want to find which supplied passages support an entire claim.","tags":["answer-quality","citation","match","rag","evidence"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"citation-needed","title":"Check citation requirements","description":"Do citationRules require evidence for statement?","useWhen":"You need to decide whether a statement requires a citation under your rules.","tags":["answer-quality","citation","needed","rag","evidence"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"claim-facets","title":"Label the facets an insurance claim narrative states","description":"Which of these does claim state: when the incident happened, where it happened, what caused it, what was damaged or lost, and whether there are witnesses or evidence?","useWhen":"A claims intake assistant receives a free-form loss description and needs to know which standard first-notice details are already present so it can pre-fill the claim and ask only for what is missing.","tags":["insurance","claims","intake","first-notice-of-loss","labels","multi-label","facets"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"claim-stance","title":"Label a response's stance toward a claim","description":"Label whether a response affirms, denies, mixes positions on, or does not address a supplied claim.","useWhen":"You need to label whether a response agrees or disagrees with a claim, including in AI alignment research.","tags":["psychology","claim","stance","agreement","disagreement","annotation","sycophancy","alignment-research"],"category":"answer-quality","collection":null,"evidence":"measured","evidenceDetails":{"kind":"unspecified","label":"Current measurement; labels unspecified","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:42:59.711Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":53,"ready":39,"reviewRate":0.264,"readyAccuracy":1,"failed":0,"provenance":[{"method":"unspecified","source":"Not recorded by the dataset author.","cases":53}],"acceptanceMet":null}}},{"id":"clarify","title":"Check required information","description":"Identify missing or ambiguous information before proceeding.","useWhen":"You need to check for missing or ambiguous requirements before proceeding.","tags":["requirements","ambiguity","clarification"],"category":"conversation","collection":null,"evidence":"measured","evidenceDetails":{"kind":"unspecified","label":"Current measurement; labels unspecified","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:01.233Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":50,"ready":30,"reviewRate":0.4,"readyAccuracy":1,"failed":0,"provenance":[{"method":"unspecified","source":"Not recorded by the dataset author.","cases":50}],"acceptanceMet":null}}},{"id":"clause-conflict","title":"Detect conflicting clauses","description":"Do firstClause and secondClause impose requirements that cannot both be satisfied?","useWhen":"You need a yes/no check that two clauses from a contract, policy set, or amendment can both be honored before flagging them for review.","tags":["contracts","legal","conflict","consistency","compliance"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"clause-kind","title":"Classify contract clause kind","description":"What does clause primarily do?","useWhen":"You need to sort contract or policy clauses by what they do so obligations and prohibitions can be tracked separately from rights and definitions.","tags":["contracts","legal","clause","classification","compliance"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"clickbait-level","title":"Grade clickbait level","description":"How much does headline rely on curiosity gaps, exaggeration, or emotional bait instead of stating what the content is?","useWhen":"You want to rank, flag, or reject headlines that manipulate readers into clicking rather than telling them what they will get.","tags":["content","headline","clickbait","seo","psychology","score"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"commit-message-fit","title":"Check commit message accuracy","description":"Does message accurately describe change, neither omitting a material part nor claiming work not present?","useWhen":"You need to flag commit or pull request titles that understate, overstate, or misdescribe the change they accompany before merge or changelog generation.","tags":["code-review","commit","message","pull-request","gate","changelog"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"commitment-strength","title":"Grade commitment strength","description":"How firmly does statement commit its speaker to an action or outcome, on a five-level rubric?","useWhen":"You need to grade how strongly a message commits someone to act before tracking it as a promise or follow-up.","tags":["psychology","commitment","promise","intention","rubric","score"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"comparable-fit","title":"Check whether a comparable property fits the subject","description":"Is comparable similar enough to subject in type, size, age, condition, and location wording to support a valuation comparison?","useWhen":"You need to screen candidate comparable sales or rentals against a subject property from their descriptions, so that only plausible comparables reach a valuation model or an appraiser's review.","tags":["real-estate","valuation","comparables","appraisal","gate","similarity"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"completion-gate","title":"Gate an agent claiming it is done","description":"Did an agent finish task, judging report against evidence, with unproven claims, quietly narrowed scope, open questions, and unresolved errors flagged in the same call?","useWhen":"A coding agent says it is finished and you must decide, before accepting or before letting it stop, whether the work is actually complete.","tags":["agent","completion","stop-hook","verification","evidence","harness","done"],"category":"workflow","collection":"agent-operations","evidence":"measured","evidenceDetails":{"kind":"mixed","label":"Current mixed-source measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:02.671Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":144,"ready":91,"reviewRate":0.368,"readyAccuracy":1,"failed":0,"provenance":[{"method":"unspecified","source":"Existing repository labels; independent review and original authorship were not recorded.","cases":44},{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":100}],"acceptanceMet":null}}},{"id":"confirmation-match","title":"Interpret a confirmation","description":"Does response clearly agree to or reject this exact proposal?","useWhen":"You need to know whether a response accepts or rejects an exact proposal.","tags":["conversation","confirmation","match"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"consent-request","title":"Detect an explicit consent request","description":"Does text explicitly ask the reader for agreement or permission before something proceeds?","useWhen":"You need a yes/no check that a message, prompt, or notice actually asks for consent instead of announcing or assuming it before an action proceeds.","tags":["consent","compliance","privacy","wording","gate"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"consent-scope-fit","title":"Check whether consent wording covers a use","description":"Does the wording of consent cover the described use?","useWhen":"You hold the text of a patient's consent or authorization and need to check, before sharing or using their information, whether the described use falls within what the wording permits.","tags":["healthcare","consent","authorization","privacy","scope","gate"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"constraint-strength","title":"Distinguish requirements from preferences","description":"Classify a stated constraint as required, preferred, optional, or unclear.","useWhen":"You need to distinguish a hard requirement from a preference or an optional suggestion.","tags":["constraint","requirement","preference","optional","mandatory","must","should","hard","soft"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"contact-opt-out","title":"Recognize a contact opt-out","description":"What scope of future contact does the sender ask to stop?","useWhen":"You need to distinguish stopping all contact from stopping a channel or campaign.","tags":["conversation","contact","opt-out","preferences"],"category":"conversation","collection":"customer-conversations","evidence":"measured","evidenceDetails":{"kind":"synthetic","label":"Current synthetic measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:06.812Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":60,"ready":50,"reviewRate":0.167,"readyAccuracy":1,"failed":0,"provenance":[{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":60}],"acceptanceMet":null}}},{"id":"content-facets","title":"Label article facets","description":"Which of these does article include: a clear thesis, supporting evidence, counterarguments, a call to action, and signals of author expertise?","useWhen":"You audit published or drafted articles for structural completeness before deciding whether they need more evidence, a stance, or a conclusion.","tags":["content","seo","article","editorial","quality","labels"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"content-freshness-signal","title":"Detect perishable content","description":"Does content contain claims that are likely to go stale, such as prices, versions, dates, current events, or latest wording?","useWhen":"You schedule content reviews, decide which pages need dated review notes, or want to flag articles that will silently become wrong.","tags":["content","seo","freshness","maintenance","evergreen","gate"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"context-prune","title":"Prune agent context","description":"Which of items, earlier tool results and messages in an agent session, are still needed to finish objective, so the rest can be dropped from context?","useWhen":"An agent session is growing long and you want to drop stale tool output and messages before the next model turn without summarizing what must stay verbatim.","tags":["agent","context","compaction","prune","memory","tool-results","harness"],"category":"memory","collection":"agent-operations","evidence":"measured","evidenceDetails":{"kind":"mixed","label":"Current mixed-source measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:08.518Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":136,"ready":81,"reviewRate":0.404,"readyAccuracy":1,"failed":0,"provenance":[{"method":"unspecified","source":"Existing repository labels; independent review and original authorship were not recorded.","cases":36},{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":100}],"acceptanceMet":null}}},{"id":"context-role","title":"Identify a passage role","description":"What role does passage play in answering question?","useWhen":"You want to identify the role a passage plays in answering a question.","tags":["retrieval","context","role","rag","evidence"],"category":"retrieval","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"continuation-compare","title":"Compare two musical continuations","description":"Which of firstContinuation and secondContinuation better follows the musical context in motif, contour, harmony, and any style supplied?","useWhen":"A generation loop has produced two candidate continuations for a passage and wants a head-to-head preference before committing to one.","tags":["music","generation","comparison","pairwise","melody","composition"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"correction-target","title":"Locate a correction target","description":"Which supplied field or statement is message correcting?","useWhen":"You need to identify which supplied field or statement a message corrects.","tags":["conversation","correction","target"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"corrective-action-fit","title":"Check a corrective action against its root cause","description":"Does action address the cause stated in rootCause rather than only the symptom or the affected items?","useWhen":"You review corrective action plans in a CAPA or nonconformance workflow and want to flag actions that rework parts, add inspection, or retrain people while leaving the stated cause in place.","tags":["manufacturing","quality","capa","corrective-action","gate","review"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"daw-request-kind","title":"Classify a request to a music program","description":"What does request ask a DAW or music program to do: record, edit, mix, apply an effect, arrange, export, or unclear?","useWhen":"You are building voice or chat control for a DAW, notation program, or recording app and need to route each spoken or typed request to the handler that owns that kind of operation.","tags":["music","daw","audio","voice-control","routing","classification"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"deadline-risk","title":"Grade the risk of missing a deadline","description":"How at risk is the work of missing deadline given progress, from on track to already missed or impossible?","useWhen":"A supervisor or planner has a stated deadline and a progress report and needs a graded read on whether to escalate, replan, or leave the work alone.","tags":["agent","planning","deadline","progress","risk","score"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"defect-report-facets","title":"Label the facets of a defect report","description":"Which of these does report state: the part or lot identifier, a description of the defect, where it was detected, the quantity affected, a containment action?","useWhen":"You need to check an incoming nonconformance or defect report for the facts a quality process expects before it is accepted, or to tell the reporter which facts are missing.","tags":["manufacturing","quality","defect","nonconformance","labels","multi-label"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"delegation-fit","title":"Check delegation fit","description":"Does subtask fall within the capabilities described for the delegate?","useWhen":"You need a yes/no check before handing a subtask to a specific sub-agent or worker whose tools, access, or permissions are described in text.","tags":["agent","delegation","capabilities","routing","multi-agent","harness"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"diff-hazards","title":"Flag hazards in a code diff","description":"Which hazards does diff introduce: leaked secrets, destructive commands, debug leftovers, weakened tests, or dependency changes?","useWhen":"A coding agent or pre-commit hook needs a fast screen of a diff for the mistakes that reviewers most often catch late.","tags":["diff","code-review","secrets","destructive","pre-commit","agent","safety","harness"],"category":"workflow","collection":null,"evidence":"measured","evidenceDetails":{"kind":"unspecified","label":"Current measurement; labels unspecified","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:12.519Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":43,"ready":35,"reviewRate":0.186,"readyAccuracy":0.971,"failed":0,"provenance":[{"method":"unspecified","source":"Not recorded by the dataset author.","cases":43}],"acceptanceMet":null}}},{"id":"disclosure-facets","title":"Label what a property disclosure addresses","description":"Which of these does disclosure address: known defects, prior repairs, environmental hazards, boundary or easement issues, association rules?","useWhen":"You need several independent checks on a seller's or landlord's disclosure text in one call, to see which topics it covers and which it is silent on before a buyer, tenant, or reviewer relies on it.","tags":["real-estate","disclosures","labels","multi-label","transactions","due-diligence"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"dispute-kind","title":"Classify a billing dispute","description":"What kind of billing dispute does message raise?","useWhen":"You need to sort incoming billing complaints into a fixed set of dispute types so each can be routed to the right handler or template.","tags":["billing","disputes","support","classification","payments","finance"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"document-role","title":"Identify document purpose","description":"What is the primary purpose of document?","useWhen":"You need to identify the primary purpose of a document.","tags":["knowledge","document","role"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"draft-compare","title":"Compare two drafts","description":"Which draft better satisfies request under rubric?","useWhen":"You have two drafts and want to choose the better fit for a request and rubric.","tags":["answer-quality","draft","compare","rag","evidence","alignment-research"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"dynamic-change","title":"Steer the loudness","description":"Should the upcoming phrase be softer, equally loud, or louder, judged from audience remarks about volume in context and the dynamics written in recentMaterial?","useWhen":"Your player must pick a volume level for the next phrase from requests for quiet or power.","tags":["music","dynamics","live-performance","steering","workflow","decision"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"eligibility-facets","title":"Label what an applicant statement addresses","description":"Which of these does statement address: residency, income, household size, identity documents, prior benefits received?","useWhen":"You need several independent checks on an applicant's free-text statement in one call, to see which eligibility topics it already covers and which a caseworker still needs to ask about.","tags":["public-sector","benefits","eligibility","intake","labels","multi-label"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"emotion-kind","title":"Identify expressed emotion","description":"What primary emotion does the wording of message express?","useWhen":"You need a coarse emotion label for a message to route it, annotate a dataset, or adapt a reply.","tags":["psychology","emotion","sentiment","affect","annotation","conversation"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"entity-match","title":"Match records to one entity","description":"Do firstRecord and secondRecord describe the same real-world entity despite formatting, abbreviation, or partial fields?","useWhen":"You need to decide whether two customer, vendor, product, or place records should be merged or linked.","tags":["deduplication","entity-resolution","records","data-quality","matching"],"category":"knowledge","collection":"document-decisions","evidence":"measured","evidenceDetails":{"kind":"synthetic","label":"Current synthetic measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:14.302Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":60,"ready":45,"reviewRate":0.25,"readyAccuracy":1,"failed":0,"provenance":[{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":60}],"acceptanceMet":null}}},{"id":"error-acknowledgment","title":"Check whether a message acknowledges a mistake","description":"Does message explicitly acknowledge an earlier mistake and state a correction, rather than silently changing course or ignoring it?","useWhen":"An agent has been shown to be wrong earlier in a conversation and you want to verify that its follow-up owns the error and states the fix, rather than quietly switching answers.","tags":["agent","self-monitoring","correction","honesty","conversation","gate"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"escalation-wording","title":"Check whether a message asks for escalation","description":"Does message explicitly ask for escalation to a higher tier, a manager, or on-call engineering?","useWhen":"You are routing inbound support messages or internal chat and need to catch explicit escalation requests so they reach a supervisor, a higher support tier, or the on-call engineer.","tags":["support","escalation","routing","gate","on-call","customer-service"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"evaluation-mention","title":"Label explicit mentions of model evaluation","description":"Distinguish a response referring to its own evaluation from general evaluation discussion or no such mention.","useWhen":"You need to find explicit mentions of being tested, graded, or evaluated in saved model responses.","tags":["evaluation","being tested","benchmark","graded","self reference","annotation","alignment-research"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"evidence-conflict","title":"Compare evidence for conflicts","description":"Do firstPassage and secondPassage give incompatible evidence relevant to question under the same conditions?","useWhen":"You have two passages and need to check for conflicting evidence about a question.","tags":["retrieval","evidence","conflict","rag"],"category":"retrieval","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"evidence-independence","title":"Compare the origins of two pieces of evidence","description":"Check whether supplied provenance shows shared or separate evidence origins for one claim, or leaves their relationship unresolved.","useWhen":"You need to check whether two reports rely on the same underlying source before treating them as corroboration.","tags":["evidence independence","provenance","shared source","corroboration","origin","annotation","alignment-research"],"category":"retrieval","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"evidence-novelty","title":"Check new evidence","description":"Does passage add material information relevant to question beyond existingEvidence?","useWhen":"You need to decide whether a new passage adds useful information to existing evidence.","tags":["retrieval","evidence","novelty","rag"],"category":"retrieval","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"evidence-strength","title":"Grade evidence strength","description":"How strongly does evidence support the entire claim, on a five-level rubric?","useWhen":"You need a graded strength for weighting or ranking evidence, not just a supported or unsupported label.","tags":["evidence","strength","grading","rubric","score"],"category":"retrieval","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"exercise-select","title":"Select the practice exercise that addresses feedback","description":"Which candidate practice exercise in candidates best addresses the problems raised in feedback, given the player's goal?","useWhen":"You are building a practice app that turns a teacher's or an automated assessor's feedback into a concrete assignment chosen from your own exercise library.","tags":["music","practice","exercises","education","selection","recommendation"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"expense-category","title":"Match an expense to a category list","description":"Does expense clearly fall under one of the caller's categories, under several equally, under none, or is it unclear?","useWhen":"You need to know whether a described expense can be filed under one of your own category definitions before an agent picks the category or asks the submitter for more detail.","tags":["expenses","accounting","categorization","finance","routing"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"explanation-level","title":"Grade shown reasoning","description":"How much reasoning does answer show for its conclusion to question, on a five-level rubric from bare conclusion to rigorous chain?","useWhen":"You assess whether students or assistants showed their work, separately from whether the final answer is right.","tags":["education","reasoning","explanation","assessment","score"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"extraction-fidelity","title":"Grade extraction fidelity","description":"How faithfully does extracted represent the facts in source, without invented, altered, or dropped values, on a five-level rubric?","useWhen":"You need to grade a structured extraction against its source document before trusting, storing, or acting on the values.","tags":["extraction","structured-data","fidelity","hallucination","rubric","score"],"category":"knowledge","collection":"document-decisions","evidence":"measured","evidenceDetails":{"kind":"synthetic","label":"Current synthetic measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:16.085Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":60,"ready":60,"reviewRate":0,"readyAccuracy":1,"failed":0,"provenance":[{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":60}],"acceptanceMet":null}}},{"id":"fact-stability","title":"Assess fact stability","description":"Is fact about an enduring or historical attribute, or a state that is expected to change?","useWhen":"You need to assess whether a fact is enduring or likely to change over time.","tags":["memory","fact","stability"],"category":"memory","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"failure-kind","title":"Classify a described failure","description":"Which supplied category best describes the observed failure?","useWhen":"You need to assign an observed failure to one of your supplied categories.","tags":["workflow","failure","kind"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"fair-housing-wording","title":"Flag applicant-preference wording in a listing","description":"Does listing express a preference for, or limitation on, applicants based on personal characteristics rather than describing the property?","useWhen":"You need to screen rental or sale listings before publication for wording that steers, prefers, or excludes people by family status, religion, national origin, disability, or similar characteristics, so a human reviewer can look at the flagged ones.","tags":["real-estate","fair-housing","listings","compliance","gate","wording"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"feedback-actionability","title":"Grade feedback actionability","description":"How actionable is feedback for its recipient, on a five-level rubric from no direction to a specific change with reason and example?","useWhen":"You review teacher comments, code review notes, or peer feedback before delivery and want to flag items the recipient could not act on.","tags":["education","feedback","review","coaching","score"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"feedback-kind","title":"Classify customer feedback","description":"What is the primary kind of feedback in message?","useWhen":"You need to classify the kind of feedback expressed in a message.","tags":["support","feedback","kind"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"field-select","title":"Select a field value","description":"Which supplied candidate is the value of field in document?","useWhen":"You need to select which supplied candidate expresses a field value in a document.","tags":["knowledge","field","select"],"category":"knowledge","collection":"document-decisions","evidence":"measured","evidenceDetails":{"kind":"synthetic","label":"Current synthetic measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:17.929Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":60,"ready":44,"reviewRate":0.267,"readyAccuracy":1,"failed":0,"provenance":[{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":60}],"acceptanceMet":null}}},{"id":"financial-advice-signal","title":"Detect specific financial advice","description":"Does text give a specific recommendation to buy, sell, hold, or allocate money, rather than general education?","useWhen":"You need a yes/no check on whether an assistant's or user's text crosses from explaining financial concepts into recommending a specific action with money, so the response can be disclaimed, softened, or routed for review.","tags":["finance","advice","compliance","safety","gate","assistant-output"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"finger-actions","title":"Choose an action for every finger","description":"On beat, which listed option should each finger in fingers take, given music and style, or rest?","useWhen":"A playing loop advances one subdivision at a time and wants every finger to pick from its own reachable options in a single call.","tags":["music","piano","performance","fingers","assignment","live-performance","agent"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"followup-link","title":"Link a follow-up request","description":"Which supplied earlier request does message follow up on?","useWhen":"You need to connect a follow-up message to one of the earlier requests.","tags":["conversation","followup","link"],"category":"conversation","collection":"customer-conversations","evidence":"measured","evidenceDetails":{"kind":"synthetic","label":"Current synthetic measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:19.595Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":60,"ready":45,"reviewRate":0.25,"readyAccuracy":1,"failed":0,"provenance":[{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":60}],"acceptanceMet":null}}},{"id":"followup-timing","title":"Recognize requested follow-up timing","description":"When does the sender want another contact, if any?","useWhen":"You need to distinguish immediate contact, a later time, an event condition, and no follow-up request.","tags":["conversation","followup","timing","scheduling"],"category":"conversation","collection":"customer-conversations","evidence":"measured","evidenceDetails":{"kind":"synthetic","label":"Current synthetic measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:21.226Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":60,"ready":51,"reviewRate":0.15,"readyAccuracy":1,"failed":0,"provenance":[{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":60}],"acceptanceMet":null}}},{"id":"forecast-confidence-wording","title":"Grade the certainty of a financial projection","description":"How certain is the wording of statement, a financial projection, from explicitly speculative to stated as fact?","useWhen":"You need to grade how confidently a projection about revenue, prices, returns, or growth is worded, so overconfident statements can be softened or disclaimed before they reach a reader.","tags":["finance","forecast","certainty","psychology","rubric","assistant-output"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"format-fit","title":"Check requested format compliance","description":"Does response follow the structure or format request explicitly asks for, such as a list, table, JSON, item count, sections, or language?","useWhen":"You need a yes/no check that a generated response honored the format the prompt or user explicitly asked for.","tags":["format","structure","instruction-following","output","gate","evaluation"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"freshness-needed","title":"Check information freshness needs","description":"Does question require a current or time-specific state that can change, or stable conceptual knowledge?","useWhen":"You need to know whether a question depends on current or changing information.","tags":["retrieval","freshness","needed","rag","evidence"],"category":"retrieval","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"frustration-signal","title":"Detect expressed frustration","description":"Does message express frustration or dissatisfaction in its wording?","useWhen":"You need to detect frustration or dissatisfaction expressed in a message.","tags":["psychology","support","frustration","signal"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"funder-fit","title":"Compare two funding opportunities for a program","description":"Which of firstOpportunity and secondOpportunity better fits program in purpose, eligibility wording, and scope?","useWhen":"You are shortlisting funding opportunities for a described program and want a head-to-head read on which of two calls fits it better before a development officer reads the full guidelines.","tags":["nonprofit","grants","fundraising","prospecting","comparison","eligibility"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"game-action","title":"Choose a game action from JSON","description":"Send game state, player state, and available actions as JSON; receive the original selected action with Jev confidence and probabilities.","useWhen":"Your game already supplies JSON state and actions, and you want Jev to choose an action without formatting descriptions or assigning IDs.","tags":["gameplay","game","agnostic","JSON","actions","automatic IDs","simulation"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"game-phase","title":"Classify game phase","description":"Which phase of the game does state describe: opening, midgame, endgame, or game over?","useWhen":"An agent adapts its strategy, prompts, or time budget by game phase and you have a textual state description rather than a structured engine.","tags":["game","phase","state","agent","classification","simulation"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"goal-drift","title":"Detect goal drift","description":"Does step still serve goal, or has work drifted to something goal did not ask for?","useWhen":"You need a yes/no check on each step of a long-running agent so it stops before spending effort on work nobody asked for.","tags":["agent","goal","drift","scope","monitoring","gate","harness"],"category":"workflow","collection":null,"evidence":"measured","evidenceDetails":{"kind":"unspecified","label":"Current measurement; labels unspecified","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:22.858Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":44,"ready":0,"reviewRate":1,"readyAccuracy":null,"failed":0,"provenance":[{"method":"unspecified","source":"Not recorded by the dataset author.","cases":44}],"acceptanceMet":null}}},{"id":"grounding-level","title":"Grade draft grounding","description":"How much of the substantive content in draft is backed by evidence, on a five-level rubric?","useWhen":"You need one graded measure of how well a generated answer sticks to its retrieved sources before sending or ranking it.","tags":["grounding","hallucination","evidence","rag","rubric","score"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"guest-request-facets","title":"Label the booking facets a guest request states","description":"Which of these does request state: travel dates, party size, accessibility needs, a budget, or a special occasion?","useWhen":"A reservations or concierge assistant receives a free-form guest inquiry and needs to know which standard booking details are already present so it can pre-fill the request and ask only for what is missing.","tags":["hospitality","booking","intake","guest-request","labels","multi-label","facets"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"handoff","title":"Check handoff rules","description":"Decide whether a request matches your human escalation rules.","useWhen":"You need to decide whether your escalation rules call for a human.","tags":["escalation","support","human"],"category":"workflow","collection":"customer-conversations","evidence":"measured","evidenceDetails":{"kind":"synthetic","label":"Current synthetic measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:24.162Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":60,"ready":47,"reviewRate":0.217,"readyAccuracy":0.957,"failed":0,"provenance":[{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":60}],"acceptanceMet":null}}},{"id":"handoff-completeness","title":"Grade handoff completeness","description":"How ready is item, a work description being handed to another agent or person, to be picked up without asking questions, on a five-level rubric?","useWhen":"You need to grade a task description before delegating it, so an orchestrator can enrich it or ask for missing pieces instead of handing off something a worker will bounce back.","tags":["agent","delegation","handoff","completeness","multi-agent","harness"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"headline-fit","title":"Check headline fit","description":"Does headline accurately represent what body says, without promising more than the body delivers?","useWhen":"You publish or review articles and want to catch headlines that overstate, contradict, or misdirect from the body before they go live.","tags":["content","headline","seo","editorial","accuracy","gate"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"incident-match","title":"Match a known incident","description":"Which supplied incident is supported as a match for ticket?","useWhen":"You need to connect a support ticket to a supplied known incident.","tags":["support","incident","match"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"incident-severity-wording","title":"Grade the severity an incident report describes","description":"What severity does the wording of report describe, on a five-level rubric from no user impact to total outage or data loss?","useWhen":"You need a first severity estimate from a free-text incident report, alert summary, or status update before a human incident commander confirms it, or you want to check that a declared severity matches how the report describes the impact.","tags":["incident-response","severity","on-call","devops","score","triage"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"injection-signal","title":"Detect agent-directed instructions","description":"Does text contain instructions aimed at steering an AI system or agent?","useWhen":"You need to screen retrieved documents, tool results, or user uploads before an agent reads them as context.","tags":["security","prompt-injection","agent","safety","gate","retrieval","harness"],"category":"workflow","collection":null,"evidence":"measured","evidenceDetails":{"kind":"unspecified","label":"Current measurement; labels unspecified","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:25.936Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":44,"ready":42,"reviewRate":0.045,"readyAccuracy":1,"failed":0,"provenance":[{"method":"unspecified","source":"Not recorded by the dataset author.","cases":44}],"acceptanceMet":null}}},{"id":"instruction-clarity","title":"Grade instruction clarity","description":"How unambiguous is instruction for a delegate who has only context, on a five-level rubric?","useWhen":"You need to grade an instruction before sending it to a sub-agent or teammate, so an orchestrator can rewrite it or ask a question instead of letting the delegate guess.","tags":["agent","delegation","instruction","clarity","score"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"instruction-conflict","title":"Check instructions for conflict","description":"Decide whether two instructions can both be followed under the supplied circumstances.","useWhen":"You need to detect conflicting instructions before carrying out a task.","tags":["instructions","conflict","contradiction","requirements","rules","compatible"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"instruction-fit","title":"Check instruction applicability","description":"Does the explicit scope of instruction cover task and context?","useWhen":"You need to check whether an instruction applies to the current task and context.","tags":["workflow","instruction","fit"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"instruction-priority","title":"Decide which conflicting instruction wins","description":"When firstInstruction and secondInstruction conflict, which should take precedence under the stated policy?","useWhen":"An agent holds two instructions that cannot both be followed and your system has a written precedence policy, such as system over developer over user, or a rule about ignoring instructions embedded in retrieved content.","tags":["agent","instructions","policy","precedence","comparison","safety","harness"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"instruction-readability","title":"Grade how easy patient instructions are to follow","description":"How easy are instructions to follow for a general reader, from dense jargon to plain and stepwise?","useWhen":"You are generating or reviewing after-visit instructions, discharge notes, or medication directions and need to catch wording a patient cannot act on.","tags":["healthcare","patient-instructions","readability","plain-language","rubric","score"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"instrument-issue-kind","title":"Classify an instrument problem report","description":"What kind of problem does report describe with a musical instrument: tuning, buzz or rattle, no sound, intonation, mechanical, cosmetic, or unclear?","useWhen":"You take in repair requests or support messages from players and need to route each one to the right technician, help article, or intake form based on the kind of problem described.","tags":["music","instruments","repair","support","classification","triage"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"instrument-report-facets","title":"Label the facets of an instrument repair message","description":"Which of these does message state about an instrument problem: the instrument and model, the symptom, when it started, recent changes, and the environment it is kept in?","useWhen":"You take in repair or support messages from players and want to know which standard intake details are already supplied so you can pre-fill a ticket or ask only for what is missing.","tags":["music","instruments","repair","support","labels","multi-label","facets"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"intake-question-fit","title":"Check an intake question against its purpose","description":"Does question ask only for what purpose needs, avoiding unrelated personal detail?","useWhen":"You are drafting or reviewing intake form questions and want a check that each one collects only what its stated purpose requires.","tags":["healthcare","intake","forms","data-minimization","privacy","gate"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"intent-change","title":"Detect a change of intent","description":"How does message change currentGoal?","useWhen":"You need to check whether a new message changes the current task or goal.","tags":["conversation","intent","change"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"invoice-facets","title":"Label what an invoice states","description":"Which of these does invoice state: vendor identity, an invoice number, a due date, line items, a tax or total breakdown?","useWhen":"You need several independent presence checks on invoice text in one call, to spot missing fields before routing it for approval or entry.","tags":["invoices","accounting","accounts-payable","labels","multi-label","finance"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"issue-impact","title":"Assess reported issue impact","description":"What practical impact does message explicitly describe?","useWhen":"You need to assess the practical impact explicitly described in a support message.","tags":["support","issue","impact"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"issue-recurrence","title":"Check whether an issue returned","description":"Label a reported issue as new, ongoing without resolution, or returned after reported recovery.","useWhen":"You need to distinguish a new problem, one that never stopped, and an issue that came back after recovery.","tags":["support","issue came back","recurring issue","reopened ticket","never fixed","reported recovery"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"itinerary-feasibility","title":"Check whether an itinerary is feasible in sequence","description":"Can the consecutive items in itinerary be carried out in the order described: enough transfer time between them, each location reachable from the previous one, and no overlapping commitments?","useWhen":"A travel or concierge assistant has drafted or received a day-by-day itinerary in plain language and needs a yes/no sanity check that the sequence hangs together before presenting or booking it.","tags":["travel","itinerary","scheduling","feasibility","hospitality","gate"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"job-post-facets","title":"Label what a job posting states","description":"Which of these does posting state: salary or range, work location, remote policy, experience level, required qualifications?","useWhen":"You check job postings for completeness before publishing, or normalize scraped postings into structured fields.","tags":["recruiting","job-posting","labels","multi-label","completeness"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"length-fit","title":"Check response length fit","description":"Is the length and detail of response proportionate to what request asks for?","useWhen":"You need to catch answers that are padded or truncated relative to the question before sending or scoring them.","tags":["length","verbosity","concision","response-quality","evaluation"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"listener-request-kind","title":"Classify a listener's request","description":"What does a listener's message ask of the performance?","useWhen":"Chat messages steer a live or generated musical performance and you need to know which control each message reaches for before deciding what to change.","tags":["music","live-performance","chat","intent","classification","conversation"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"listing-compare","title":"Compare two listings for a request","description":"Which of firstListing and secondListing better satisfies request?","useWhen":"You need a head-to-head preference between two product listings for a shopper request, for tie-breaking, recommendation evaluation, or ranker calibration.","tags":["e-commerce","catalog","comparison","pairwise","product","ranking"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"listing-fact-consistency","title":"Check a listing description against its fact sheet","description":"Does the free-text description contradict the structured facts about the property?","useWhen":"You need to catch a marketing description that disagrees with the structured listing data on bedrooms, size, year built, or features before the listing is published or syndicated.","tags":["real-estate","listings","consistency","data-quality","gate","contradiction"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"log-line-kind","title":"Classify what a log line reports","description":"What does line report: an error, a warning, a lifecycle event, a handled request, a metric, or debug output?","useWhen":"You are grouping or filtering unstructured log lines from mixed sources and cannot rely on a level field being present or accurate.","tags":["logging","observability","devops","classification","on-call","triage"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"loss-cause-kind","title":"Identify the cause of loss in a claim narrative","description":"What cause of loss does narrative describe: weather, fire, water, theft, collision, wear and tear, vandalism, or something else?","useWhen":"A claims intake or routing system needs to sort a free-form loss description into a cause-of-loss category so it can pick the right adjuster queue, forms, or follow-up questions.","tags":["insurance","claims","cause-of-loss","classification","peril","triage"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"lyric-mood-fit","title":"Check lyrics against the mood of the music","description":"Do lyrics fit the mood and pacing of the music described in music?","useWhen":"You generate, suggest, or review lyrics for a described track and want to catch lines whose mood or pacing clashes with the music before showing them to a writer.","tags":["music","lyrics","songwriting","mood","gate","creative"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"maintenance-hazard-wording","title":"Flag a maintenance request that describes a safety hazard","description":"Does request describe a safety hazard such as a gas smell, active water intrusion, exposed wiring, structural damage, no heat in cold weather, or a blocked exit?","useWhen":"You need to pull tenant maintenance requests that describe a hazard out of the routine queue for immediate dispatch, based on what the request says before anyone inspects.","tags":["property-management","maintenance","safety","triage","gate","hazard"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"medication-mention","title":"Detect a medication or dosage mention","description":"Does message mention a medication, supplement, or dosage?","useWhen":"You need to route patient messages that name a drug, supplement, or dose toward pharmacy or prescriber review, or flag them for careful handling before storage.","tags":["healthcare","patient-message","medication","routing","gate"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"memory-relation","title":"Compare a new fact with memory","description":"How does newFact relate to existingMemory?","useWhen":"You need to compare a new fact with an existing memory for agreement or change.","tags":["memory","relation"],"category":"memory","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"memory-scope","title":"Identify memory scope","description":"What is the narrowest explicitly supported scope of fact in context?","useWhen":"You need to identify the narrowest supported scope of a fact or preference.","tags":["memory","scope"],"category":"memory","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"memory-subject","title":"Identify whom a memory describes","description":"Label whether one candidate memory describes the user, someone else, or a group including the user.","useWhen":"You need to avoid treating a fact about someone else as a fact about the user.","tags":["memory","whose memory","someone else","subject attribution","personalization","quoted speaker"],"category":"memory","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"memory-value","title":"Assess a candidate memory","description":"How useful is fact for future work under purpose?","useWhen":"You need to assess whether a candidate fact is useful to remember for a stated purpose.","tags":["memory","value"],"category":"memory","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"message-facets","title":"Label the facets of a message","description":"Which of these does message do: ask a question, report a problem, request an action, state a deadline, reference prior contact?","useWhen":"You need several independent yes/no labels on an incoming message in one call, for triage rules or form pre-fill.","tags":["support","triage","labels","multi-label","intake","facets"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"methods-facets","title":"Label what a methods section states","description":"Which of these does methods state: sample size, data source, analysis method, limitations, and preregistration or a protocol?","useWhen":"You screen manuscripts, preprints, or study summaries for reporting completeness before deciding which ones need a request for missing methodological detail.","tags":["research","methods","reporting","academic-writing","reproducibility","labels"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"model-route","title":"Route a request to a model tier","description":"Which of the described models should serve request, and how much reasoning effort does it need, decided in one call so a cheap model handles easy turns and a capable one handles hard turns?","useWhen":"An agent harness, gateway, or proxy chooses per request which LLM and thinking level to use, and you want that choice made in well under a second.","tags":["agent","model-routing","cost","effort","reasoning","tiering","harness","gateway"],"category":"workflow","collection":"agent-operations","evidence":"measured","evidenceDetails":{"kind":"mixed","label":"Current mixed-source measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:27.171Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":104,"ready":64,"reviewRate":0.385,"readyAccuracy":0.984,"failed":0,"provenance":[{"method":"unspecified","source":"Existing repository labels; independent review and original authorship were not recorded.","cases":44},{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":60}],"acceptanceMet":null}}},{"id":"modulation-moment","title":"Check whether now is a moment to change key","description":"Is now a musically suitable moment to modulate away from key, judged on the phrase position, cadence, and stability described in recentMaterial?","useWhen":"A generation loop wants to introduce a key change and needs to know whether the current point is a natural seam, such as the end of a phrase or a settled cadence, or whether a change now would cut a line off mid-thought.","tags":["music","harmony","modulation","key","generation","gate"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"mood-match","title":"Check a passage against a requested mood","description":"Does the described passage deliver requestedMood?","useWhen":"A listener asked for a mood and a passage has been generated or described, and you want to check the passage delivers that mood before playing or committing to it.","tags":["music","mood","live-performance","gate","steering","workflow"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"mood-request","title":"Identify a requested mood","description":"What mood does the listener's message ask to hear?","useWhen":"A listener's message asks for a feeling from a musical performance and you need one of a fixed set of moods to steer the next passage.","tags":["music","psychology","mood","live-performance","chat","conversation"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"motivation-source","title":"Identify a stated source of motivation","description":"Label a stated reason for an activity as intrinsic enjoyment, a separate outcome, both, or not stated.","useWhen":"You need to classify a stated reason for an activity as enjoyment of doing it or pursuit of a separate outcome.","tags":["psychology","behavior","motivation","intrinsic","extrinsic","stated reason","self-determination","annotation","alignment-research"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"move-explanation-fit","title":"Check move explanation fit","description":"Does explanation give a reason for move that is consistent with the described game state?","useWhen":"An agent or player justifies a move in text and you want to catch explanations that cite pieces, threats, or resources the state does not contain.","tags":["game","explanation","consistency","agent","reasoning","gate"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"narrative-consistency","title":"Check a narrative for internal contradictions","description":"Are the statements within narrative consistent with each other in timeline, cause, and extent, with no statement contradicting another in the same account?","useWhen":"A claims, incident, or intake system receives a single free-form account and needs to flag accounts that contradict themselves before a person reviews them or a downstream step relies on them.","tags":["insurance","claims","consistency","contradiction","narrative","gate"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"next-chord","title":"Pick the next chord","description":"Which candidate chord best continues progression, given any key and style supplied?","useWhen":"A generation loop has a chord progression so far, application code has enumerated a few chords it could play next, and you want a musical judgment of which continuation a listener in the stated style would expect or enjoy.","tags":["music","harmony","chords","generation","selection","composition"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"next-duration","title":"Pick the next rhythmic value","description":"Which candidate rhythmic value best continues recentRhythm within meter, given any style supplied?","useWhen":"A generation loop has chosen the next pitch and must decide how long to hold it, application code has enumerated a few legal durations, and you want a musical judgment of which one continues the rhythmic feel.","tags":["music","rhythm","meter","generation","selection","composition"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"next-note","title":"Pick the next melody note","description":"Which candidate note best continues the melody in recentNotes, given any key and style supplied?","useWhen":"A generation loop such as an infinite piano player has produced the last few notes, application code has enumerated a handful of legal next notes, and you want a musical judgment of which one continues the line best, or whether a rest is better.","tags":["music","melody","generation","selection","piano","composition"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"notice-facets","title":"Label what an official notice states","description":"Which of these does notice state: an action the recipient must take, a deadline, the consequence of inaction, a contact for questions, a right to appeal?","useWhen":"You need several independent checks on a government or institutional notice in one call, to catch letters that tell recipients nothing about what to do, by when, or how to contest a decision before they are sent or when they are received.","tags":["public-sector","notices","correspondence","labels","multi-label","plain-language"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"novelty-claim-level","title":"Grade how strongly a statement claims novelty","description":"How strongly does the wording of statement claim novelty for a contribution, from no novelty claim to first-ever or unprecedented?","useWhen":"You are reviewing abstracts, introductions, grant text, or press copy and want to flag how hard the wording sells the contribution as new, so overclaiming can be checked against the actual related work.","tags":["research","academic-writing","novelty","claims","psychology","score"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"objection-kind","title":"Classify sales objection kind","description":"What primary sales objection does message raise?","useWhen":"You need to tag prospect replies by objection so reps get the right playbook and pipeline reports show why deals stall.","tags":["sales","crm","objection","classification","deal","triage"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"objective-fit","title":"Check question alignment to a learning objective","description":"Does question assess the skill or knowledge stated in objective, rather than something adjacent?","useWhen":"You generate or review quiz and exam items and want to catch questions that test recall or a neighboring topic instead of the stated objective.","tags":["education","assessment","alignment","learning-objective","questions"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"offer-terms-facets","title":"Label what a property offer states","description":"Which of these does offer state: a price, financing, contingencies, a closing or move-in date, an earnest money or security deposit?","useWhen":"You need several independent presence checks on a purchase or lease offer in one call, to spot missing terms before it is countersigned, countered, or entered into a transaction system.","tags":["real-estate","offers","leasing","labels","multi-label","transactions"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"outcome-framing","title":"Identify gain and loss framing","description":"Label whether wording presents a specified outcome through gains, losses, both, or neither.","useWhen":"You need to label gain and loss wording in a decision prompt or research stimulus.","tags":["psychology","behavior","gain loss","outcome framing","decision framing","annotation","alignment-research"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"paragraph-boundary","title":"Recover a paragraph boundary","description":"Do two adjacent extracted text fragments continue one paragraph or belong to separate blocks?","useWhen":"Extraction preserved reading order but lost the difference between a hard line wrap and a paragraph break.","tags":["document","ingestion","paragraph","line-wrap","chunking"],"category":"retrieval","collection":null,"evidence":"measured","evidenceDetails":{"kind":"public-dataset","label":"Current public-dataset measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:30.092Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":112,"ready":55,"reviewRate":0.509,"readyAccuracy":1,"failed":0,"provenance":[{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/assert.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":8},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/child_process.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":8},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/crypto.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":8},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/diagnostics_channel.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":8},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/dns.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":8},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/errors.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":8},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/process.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":8},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/repl.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":8},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/timers.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":8},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/tls.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":8},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/util.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":8},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/vm.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":8},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/wasi.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":8},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/webcrypto.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":8}],"acceptanceMet":null}}},{"id":"passage-compare","title":"Compare two passages for a question","description":"Which of firstPassage and secondPassage better helps answer question?","useWhen":"You need a head-to-head preference between two retrieved passages, for tie-breaking, evaluation data, or reranker calibration.","tags":["retrieval","comparison","pairwise","passage","ranking","evaluation"],"category":"retrieval","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"passage-difficulty","title":"Grade the difficulty of a musical passage","description":"How hard is the passage described in passage for a player of instrument, from beginner to virtuoso?","useWhen":"You are building a practice app, lesson planner, or repertoire search and need to grade described passages or pieces so they can be matched to a player's level.","tags":["music","practice","difficulty","education","score","grading"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"passage-duplicate","title":"Compare passages for duplication","description":"How much material information do firstPassage and secondPassage share?","useWhen":"You want to detect duplicate or overlapping information in two passages.","tags":["retrieval","passage","duplicate","rag","evidence"],"category":"retrieval","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"passage-mood","title":"Identify a passage's mood","description":"What mood does the described musical passage express?","useWhen":"A musical passage is described in text, such as mode, tempo, register, and dynamics, and you need one of a fixed set of moods to compare against a request or to log what was played.","tags":["music","psychology","mood","analysis","knowledge","live-performance"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"passage-standalone","title":"Check passage self-containment","description":"Can passage be understood on its own, without unresolved references to surrounding text?","useWhen":"You need to check chunks before embedding them, or decide whether a retrieved passage needs its neighbors to be useful.","tags":["chunking","retrieval","embedding","context","passage"],"category":"retrieval","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"performance-feedback-facets","title":"Label the aspects performance feedback addresses","description":"Which aspects of a musical performance does feedback address: rhythm, pitch or intonation, dynamics, technique, and expression or phrasing?","useWhen":"You are building a practice app or lesson tool and want to know which musical aspects a piece of teacher or automated feedback covers, so you can track what a student hears about over time or flag feedback that neglects an aspect.","tags":["music","practice","feedback","education","labels","multi-label"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"persuasion-technique","title":"Identify a persuasion technique","description":"Which persuasion technique, if any, does the wording of message primarily use?","useWhen":"You need to annotate or flag how a message tries to persuade, for moderation, research, or review of outgoing drafts.","tags":["psychology","persuasion","influence","rhetoric","marketing","annotation"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"phrase-complete","title":"Check whether a phrase has closed","description":"Does recentNotes form a complete musical phrase that is ready to cadence or rest, or is it still open, given any meter supplied?","useWhen":"A generation loop needs to know whether the line it has produced can pause, cadence, or hand off to a new idea, or whether it should keep the phrase going.","tags":["music","melody","phrase","cadence","generation","gate"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"pii-presence","title":"Detect personal information","description":"Does text contain information identifying a specific private individual?","useWhen":"You need a yes/no gate before storing, logging, sharing, or sending text that might contain personal data.","tags":["privacy","pii","redaction","compliance","gate","security"],"category":"workflow","collection":null,"evidence":"measured","evidenceDetails":{"kind":"unspecified","label":"Current measurement; labels unspecified","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:33.245Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":43,"ready":36,"reviewRate":0.163,"readyAccuracy":1,"failed":0,"provenance":[{"method":"unspecified","source":"Not recorded by the dataset author.","cases":43}],"acceptanceMet":null}}},{"id":"plan-completeness","title":"Grade plan completeness","description":"How completely does plan cover what task requires, on a five-level rubric?","useWhen":"You need to check an agent-written plan against the task before execution starts, so missing requirements are caught while they are cheap to add.","tags":["agent","plan","completeness","planning","rubric","score"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"policy-compliance","title":"Check an expense against a written policy","description":"Does expense, as described, comply with the written policy?","useWhen":"You need a yes/no check on whether a described expense follows the rules in an expense policy before it is approved, flagged, or sent back to the submitter.","tags":["expenses","policy","compliance","accounting","finance","gate"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"policy-severity","title":"Grade a policy violation","description":"How severely does content violate the supplied policy, on a five-level rubric?","useWhen":"You need a graded severity against your own written policy to choose between allow, flag, hide, or escalate.","tags":["moderation","policy","safety","severity","rubric","score"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"politeness-level","title":"Grade expressed politeness","description":"How polite is the wording of message toward its recipient, on a five-level rubric?","useWhen":"You need a graded politeness signal to adapt reply tone, flag hostile messages, or audit outgoing drafts.","tags":["psychology","politeness","tone","courtesy","rubric","score"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"postmortem-facets","title":"Label the facets of a postmortem","description":"Which of these does postmortem include: a timeline, a root cause, customer impact, contributing factors, action items?","useWhen":"You need to check an incident postmortem for the sections a review process expects before accepting it, or to tell the author which parts are missing.","tags":["incident-response","postmortem","labels","multi-label","devops","review"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"preference-kind","title":"Identify a stated preference","description":"Does statement express an ongoing preference, a factual assertion, or a temporary request?","useWhen":"You need to distinguish an ongoing preference from a fact or temporary request.","tags":["memory","preference","kind"],"category":"memory","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"priority-compare","title":"Compare two tasks for priority","description":"Which of firstTask and secondTask should be done first under criteria?","useWhen":"A planner or agent must order two competing tasks and the team has written down how priority should be decided.","tags":["prioritization","planning","comparison","pairwise","workflow"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"privacy-notice-facets","title":"Label the statements in a privacy notice","description":"Which of these does notice state: what data is collected, why it is used, how long it is kept, who it is shared with, how to contact the controller?","useWhen":"You need several independent yes/no checks on a privacy notice in one call, to spot missing statements before a review or publication.","tags":["privacy","compliance","notice","labels","multi-label"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"product-match","title":"Check a listing against a request","description":"Does listing describe a product that satisfies what request asks for, including stated must-have attributes?","useWhen":"You need a yes/no filter on search or recommendation results so shoppers only see listings that meet their stated requirements.","tags":["e-commerce","catalog","search","matching","product"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"progress-stall","title":"Detect a stalled agent","description":"Does transcript, the recent agent steps, show the agent failing to make progress toward objective by repeating actions, circling, or reprocessing the same information?","useWhen":"You need a yes/no check on a running agent every few steps so a supervisor can interrupt a loop before it burns the remaining budget.","tags":["agent","monitoring","loop","stall","safety","harness"],"category":"workflow","collection":"agent-operations","evidence":"measured","evidenceDetails":{"kind":"mixed","label":"Current mixed-source measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:34.650Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":102,"ready":102,"reviewRate":0,"readyAccuracy":1,"failed":0,"provenance":[{"method":"unspecified","source":"Existing repository labels; independent review and original authorship were not recorded.","cases":42},{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":60}],"acceptanceMet":null}}},{"id":"promise-check","title":"Check reply commitments","description":"Does reply promise actions or outcomes beyond allowedCommitments?","useWhen":"You need to catch commitments in a reply that exceed what is allowed.","tags":["answer-quality","promise","check","rag","evidence"],"category":"answer-quality","collection":"customer-conversations","evidence":"measured","evidenceDetails":{"kind":"mixed","label":"Current mixed-source measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:37.293Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":102,"ready":93,"reviewRate":0.088,"readyAccuracy":1,"failed":0,"provenance":[{"method":"unspecified","source":"Existing repository labels; independent review and original authorship were not recorded.","cases":42},{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":60}],"acceptanceMet":null}}},{"id":"proposal-facets","title":"Label the facets of a grant proposal","description":"Which of these does proposal include: a statement of need, measurable objectives, planned activities, a budget, an evaluation plan?","useWhen":"You screen draft or submitted grant proposals for the sections a funder or an internal reviewer expects, or you want to tell a writer which parts are missing before a deadline.","tags":["nonprofit","grants","proposal","fundraising","labels","multi-label"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"qualification-evidence","title":"Check evidence for a qualification","description":"Does profile contain concrete evidence that the candidate meets requirement, not just matching keywords?","useWhen":"You screen resumes or candidate summaries against one requirement at a time and want to separate demonstrated experience from keyword matches.","tags":["recruiting","hiring","screening","qualification","evidence"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"query-equivalence","title":"Compare question meaning","description":"Do firstQuestion and secondQuestion request the same information under the same stated conditions?","useWhen":"You need to check whether two questions ask for the same information.","tags":["retrieval","query","equivalence","rag","evidence"],"category":"retrieval","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"query-specificity","title":"Check question specificity","description":"Does question, interpreted with context, identify a focused information need?","useWhen":"You need to know whether a question identifies a focused information need.","tags":["retrieval","query","specificity","rag","evidence"],"category":"retrieval","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"question-assumption","title":"Check what a question takes for granted","description":"Label whether a question takes a supplied claim for granted or leaves that claim open.","useWhen":"You need to identify a specific assumption in a question before using it in a conversation, survey, or evaluation.","tags":["psychology","presupposition","loaded question","takes for granted","question assumption","prompt wording","annotation","alignment-research"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"question-leading","title":"Check whether a question steers an answer","description":"Label whether a question's wording favors, disfavors, or stays neutral toward a proposed answer.","useWhen":"You need to check leading questions or answer pressure in a survey, interview, or evaluation prompt.","tags":["psychology","leading question","wording","answer pressure","survey","prompt bias","annotation","alignment-research"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"question-relevance","title":"Check interview question relevance","description":"Does question ask only about matters relevant to the requirements of role, rather than personal circumstances unrelated to the work?","useWhen":"You generate or review screening and interview questions and want to flag ones that stray from the job into personal territory before they reach a candidate.","tags":["recruiting","interview","screening","fairness","questions"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"reconciliation-match","title":"Match a ledger record to a statement line","description":"Do record and statementLine describe the same transaction, judging payee, purpose, and timing wording?","useWhen":"You need a yes/no judgment on whether a bookkeeping entry and a bank or card statement line refer to the same transaction, after exact amount and date checks in code have narrowed the candidates.","tags":["reconciliation","accounting","bookkeeping","matching","finance","gate"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"reference-resolve","title":"Resolve a reference","description":"Which supplied candidate does reference refer to in message and context?","useWhen":"You need to resolve a phrase such as this one to a supplied candidate.","tags":["conversation","reference","resolve"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"repeated-attempt","title":"Compare attempted approaches","description":"Does proposedAttempt use essentially the same approach as previousAttempt for objective?","useWhen":"You need to detect whether a proposed retry repeats an earlier approach.","tags":["workflow","repeated","attempt"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"repetition-level","title":"Grade how repetitive recent material is","description":"How repetitive is recentMaterial, from varied to stuck on one idea?","useWhen":"A generation loop such as an infinite piano player should notice when it has fallen into a loop or frozen on one figure, so it can inject variation or change direction.","tags":["music","generation","repetition","monitoring","score","composition"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"reply-template-match","title":"Select a reply template","description":"Which supplied approved template applies to request and context?","useWhen":"You want to select a supplied approved reply template for a request.","tags":["support","reply","template","match"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"report-facets","title":"Label the facets of a progress report","description":"Which of these does report include: an outcome statement, supporting evidence, blockers, a next step, open questions?","useWhen":"You need to check an agent progress report for the parts a supervisor expects before accepting it, routing it, or asking the agent to fill in what is missing.","tags":["agent","reporting","labels","multi-label","status"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"requirement-testability","title":"Check whether a requirement is testable","description":"Decide whether a requirement defines an observable way to distinguish meeting it from failing it.","useWhen":"You need to check whether a requirement has clear, observable acceptance criteria before building it.","tags":["requirement","testable","testability","acceptance","criteria","measurable","observable","specification"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"rerank","title":"Rerank evidence","description":"Select candidate passages by relevance to a query.","useWhen":"You have retrieved passages and want to find the most useful evidence for a question.","tags":["rag","search","relevance"],"category":"retrieval","collection":null,"evidence":"measured","evidenceDetails":{"kind":"unspecified","label":"Current measurement; labels unspecified","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:40.159Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":57,"ready":57,"reviewRate":0,"readyAccuracy":1,"failed":0,"provenance":[{"method":"unspecified","source":"Not recorded by the dataset author.","cases":57}],"acceptanceMet":null}}},{"id":"resolution-check","title":"Check reported resolution","description":"Does message establish that the customer reports issue as resolved?","useWhen":"You need to know whether the customer reports that an issue is resolved.","tags":["conversation","resolution","check"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"response-needed","title":"Check whether a reply is needed","description":"Does message require a substantive reply in context?","useWhen":"You need to decide whether a message calls for a substantive reply.","tags":["conversation","response","needed"],"category":"conversation","collection":"customer-conversations","evidence":"measured","evidenceDetails":{"kind":"synthetic","label":"Current synthetic measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:41.765Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":60,"ready":60,"reviewRate":0,"readyAccuracy":1,"failed":0,"provenance":[{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":60}],"acceptanceMet":null}}},{"id":"response-refusal","title":"Label refusal behavior in a response","description":"Distinguish an explicit refusal, a substantive attempt, mixed behavior, and a stated inability to fulfill a request.","useWhen":"You need to label whether a response refuses a request, attempts it, or reports missing access or information.","tags":["refusal","refuses","decline","compliance","inability","annotation","alignment-research"],"category":"answer-quality","collection":null,"evidence":"measured","evidenceDetails":{"kind":"unspecified","label":"Current measurement; labels unspecified","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:43.607Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":44,"ready":39,"reviewRate":0.114,"readyAccuracy":1,"failed":0,"provenance":[{"method":"unspecified","source":"Not recorded by the dataset author.","cases":44}],"acceptanceMet":null}}},{"id":"result-outcome","title":"Interpret a reported tool outcome","description":"What outcome does result report for task?","useWhen":"You need to classify what a tool result reports happened during a task.","tags":["workflow","result","outcome"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"result-plausibility","title":"Check result plausibility","description":"Is result a plausible, internally consistent answer to request rather than an error, placeholder, empty, or unrelated output dressed as data?","useWhen":"You need a yes/no check on a tool or subagent output before an agent trusts it, stores it, or builds the next step on it.","tags":["agent","tool-result","plausibility","validation","quality","gate","harness"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"result-usefulness","title":"Check tool result usefulness","description":"Does result provide information useful for task?","useWhen":"You need to assess whether a tool result provides useful information for a task.","tags":["workflow","result","usefulness"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"retrieval-needed","title":"Check whether retrieval is needed","description":"Does request require facts beyond context?","useWhen":"You need to decide whether answering requires facts beyond the current context.","tags":["retrieval","needed","rag","evidence"],"category":"retrieval","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"retry-worthwhile","title":"Decide whether to retry","description":"Does failure describe a transient condition where an identical retry could succeed, given any attempt history?","useWhen":"You need a yes/no decision after a tool call or request fails and the error text, not a status code, is the only signal you have.","tags":["agent","retry","failure","resilience","transient","gate","harness"],"category":"workflow","collection":"agent-operations","evidence":"measured","evidenceDetails":{"kind":"synthetic","label":"Current synthetic measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:44.923Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":60,"ready":60,"reviewRate":0,"readyAccuracy":1,"failed":0,"provenance":[{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":60}],"acceptanceMet":null}}},{"id":"review-comment-kind","title":"Classify review comment kind","description":"What is the primary kind of comment?","useWhen":"You need to sort code review comments so blocking defects surface first and optional polish can be batched or deferred.","tags":["code-review","comment","classification","pull-request","triage"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"review-facets","title":"Label the aspects of a product review","description":"Which of these does review comment on: quality, price, shipping, service, or a defect?","useWhen":"You need to tag product reviews by the aspects they discuss in one call, for aspect-level ratings, routing defect reports, or filtering review feeds.","tags":["e-commerce","reviews","labels","multi-label","aspects"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"review-recommendation-kind","title":"Classify a peer review's recommendation","description":"What recommendation does the wording of review express: accept, minor revision, major revision, or reject?","useWhen":"An editor or review-management tool needs to read the recommendation implied by a free-text review, for example when the reviewer skipped the form field or when the text and the ticked box disagree.","tags":["research","peer-review","academic-writing","editorial","classification","recommendation"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"review-response-fit","title":"Check whether a review response addresses the review","description":"Does response engage with the specific complaints and praise raised in review, rather than offering a generic thank-you or apology?","useWhen":"A hospitality or support team drafts public replies to guest reviews and wants to catch templated responses that ignore what the reviewer actually said before they are posted.","tags":["hospitality","reviews","customer-response","reputation","support","gate"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"rollback-signal","title":"Check whether symptoms implicate a recent change","description":"Do symptoms plausibly point at the recently deployed change as the cause, judged on the timing and scope each describes?","useWhen":"An incident is open shortly after a deploy and you need a fast read on whether the change is a plausible cause, to decide whether to propose a rollback or keep looking elsewhere.","tags":["incident-response","rollback","deployment","on-call","devops","gate"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"root-cause-depth","title":"Grade the depth of a root cause analysis","description":"How deep does analysis go in explaining a failure, on a five-level rubric from restating the symptom to naming a verified systemic cause?","useWhen":"You review corrective action requests, nonconformance dispositions, or incident write-ups and want to flag analyses that stop at the symptom or the immediate cause before a quality reviewer accepts them.","tags":["manufacturing","quality","root-cause","capa","score","review"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"route","title":"Route a request","description":"Choose a named handler or return a review decision.","useWhen":"You need to send a request to the right handler, team, or department.","tags":["routing","intent","dispatch"],"category":"workflow","collection":"customer-conversations","evidence":"measured","evidenceDetails":{"kind":"mixed","label":"Current mixed-source measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:46.575Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":110,"ready":73,"reviewRate":0.336,"readyAccuracy":1,"failed":0,"provenance":[{"method":"unspecified","source":"Existing repository labels; independent review and original authorship were not recorded.","cases":50},{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":60}],"acceptanceMet":null}}},{"id":"route-many","title":"Route many requests in batches","description":"Which named route handles each of up to 500 requests, judged in batched Jev requests so a queue of tickets, emails, or events is routed in a handful of calls instead of one per item?","useWhen":"You have a backlog or stream of messages to route to the same set of handlers and want batched throughput with a per-item review outcome.","tags":["batch","routing","triage","queue","bulk","tickets","email","throughput"],"category":"support","collection":"customer-conversations","evidence":"measured","evidenceDetails":{"kind":"mixed","label":"Current mixed-source measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:49.817Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":87,"ready":47,"reviewRate":0.46,"readyAccuracy":0.979,"failed":0,"provenance":[{"method":"unspecified","source":"Existing repository labels; independent review and original authorship were not recorded.","cases":27},{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":60}],"acceptanceMet":null}}},{"id":"rule-compliance","title":"Check rule compliance","description":"Is the described action permitted by the written rules?","useWhen":"An agent proposes a move or action in natural language and you want a quick check against the game's written rules before spending engine time or accepting it.","tags":["game","rules","legality","agent","compliance","gate"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"runbook-fit","title":"Check whether a runbook applies to an incident","description":"Does runbook address the symptoms and component described in incident?","useWhen":"You retrieve candidate runbooks for a live incident and need to filter out ones that cover a different component or a different failure mode before surfacing them to the responder.","tags":["incident-response","runbook","on-call","devops","gate","retrieval"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"safety-incident-kind","title":"Classify a safety event","description":"What kind of safety event does report describe: a near miss, a first-aid injury, a medical-treatment injury, property damage, an environmental release, or an unsafe condition?","useWhen":"You need to sort free-text safety reports from workers, supervisors, or contractors into a fixed set of event kinds so each can be routed to the right investigation, form, or notification path.","tags":["manufacturing","safety","ehs","incident","classification","choice"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"satisfaction-signal","title":"Grade expressed satisfaction","description":"How much satisfaction with the outcome does message express at the close of an interaction, on a five-level rubric?","useWhen":"You need a graded satisfaction signal from closing messages when no survey response is available.","tags":["support","satisfaction","csat","sentiment","rubric","score"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"search-intent-kind","title":"Classify search intent","description":"What is the searcher behind query trying to do: learn, reach a site, buy, compare before buying, or find something nearby?","useWhen":"You map keywords to page types, route queries to different result layouts, or audit whether content matches the intent behind the terms it targets.","tags":["search","seo","intent","query","content","classification"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"sentiment-shift","title":"Compare sentiment across messages","description":"How does the sentiment expressed in laterMessage compare with earlierMessage from the same person?","useWhen":"You need to know whether a customer's expressed sentiment moved during a conversation, for example after an agent reply or a handoff.","tags":["support","sentiment","trend","conversation","signal","escalation"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"settlement-offer-facets","title":"Label the facets a settlement offer letter states","description":"Which of these does offer state: the settlement amount, the basis for that amount, a deadline to respond, release terms, and how to dispute or appeal?","useWhen":"A claims quality or consumer-advocacy tool reviews settlement offer letters and needs to know which standard elements each letter contains before checking it against a policy or template.","tags":["insurance","claims","settlement","letter","labels","multi-label","facets"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"shipment-issue-kind","title":"Classify a reported shipping problem","description":"What shipping problem does message report?","useWhen":"You need to route delivery complaints to the right workflow, such as a carrier trace, a replacement, or an address correction, from the words the customer used.","tags":["logistics","shipping","support","classification","delivery"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"slot-fit","title":"Check a proposed slot against constraints","description":"Does the time or slot proposed in proposal satisfy the availability constraints written in constraints?","useWhen":"A scheduling assistant has a candidate time and the availability of a person described in plain language, and needs a yes/no check before offering the slot.","tags":["scheduling","calendar","availability","constraints","gate"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"sound-match","title":"Check a patch against a requested sound","description":"Does the patch or preset described in patch deliver the sound character asked for in request?","useWhen":"You are building a preset browser, sound search, or patch recommender and need to check whether a candidate patch description actually delivers what the user asked for before showing or loading it.","tags":["music","synth","presets","sound-design","gate","matching"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"source-applicability","title":"Check source applicability","description":"Does the scope described in passage apply to scenario?","useWhen":"You need to check whether the conditions and scope of a source fit a scenario.","tags":["retrieval","source","applicability","rag","evidence"],"category":"retrieval","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"spam-signal","title":"Detect spam content","description":"Is message unsolicited promotional, scam, or bulk content rather than a genuine contribution?","useWhen":"You need a yes/no gate before publishing, forwarding, or replying to community posts, comments, or inbound messages.","tags":["moderation","spam","abuse","filtering","community","gate"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"stage-evidence","title":"Check deal stage against evidence","description":"Does evidence support that the deal is at stage as described?","useWhen":"You need to audit pipeline hygiene by checking whether the conversation or activity on a deal justifies the stage a rep set.","tags":["sales","crm","pipeline","stage","hygiene"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"step-complete","title":"Check one completion condition","description":"Does evidence establish that condition has been met?","useWhen":"You need to check whether supplied evidence establishes a completion condition.","tags":["workflow","step","complete","alignment-research","harness"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"step-progress","title":"Check step progress","description":"How does observation change progress toward objective relative to previousState?","useWhen":"You need to compare a new observation with the previous state of a task.","tags":["workflow","step","progress"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"style-kind","title":"Identify a musical style","description":"What style does the described passage or request evoke?","useWhen":"A passage or a listener's request is described in text and you need one of a fixed set of broad styles to steer what comes next or to label what was played.","tags":["music","style","genre","classification","knowledge","live-performance"],"category":"knowledge","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"summary-coverage","title":"Check summary coverage","description":"Check whether a summary preserves each supplied point.","useWhen":"You need to check whether a summary preserves each important source point.","tags":["answer-quality","summary","coverage","rag","evidence"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"symptom-facets","title":"Label the symptom facets a patient message states","description":"Which of these does message state about a symptom: when it began, how bad it is, how long it has lasted, what makes it better or worse, what the patient has already tried?","useWhen":"You need to know which standard intake details a patient message already supplies so you can pre-fill a form or ask only for what is missing.","tags":["healthcare","intake","patient-message","labels","multi-label","facets"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"take-turn","title":"Assess whether a player can act now","description":"Interpret game rules and current state to label a player's turn or reaction opportunity as act, wait, inactive, or unclear.","useWhen":"You need to decide whether it is a player's turn to act or react using narrative game rules, state, and previous actions.","tags":["gameplay","game","take turn","whose turn","act now","wait","reaction","players","simulation"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"task-complexity","title":"Grade task complexity","description":"How complex is task, from a single lookup to open-ended work, on a five-level rubric?","useWhen":"You need to size a task before choosing a model, a plan depth, a time budget, or whether to ask for help.","tags":["task","complexity","planning","budget","rubric","score","agent"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"task-dependency","title":"Check the dependency between two tasks","description":"Identify whether either of two tasks requires the other to finish before it can start.","useWhen":"You need to decide whether two tasks can run in parallel or require a particular order.","tags":["tasks","dependency","prerequisite","order","parallel","planning","sequence"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"task-duplicate","title":"Compare tasks for duplicate work","description":"Decide whether two tasks request the same outcome, overlapping work, or distinct work.","useWhen":"You need to detect duplicate tasks before adding more work to a queue or plan.","tags":["tasks","duplicate","overlap","queue","deduplicate","planning","same","outcome"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"task-overlap","title":"Detect overlapping tasks","description":"Do firstTask and secondTask cover overlapping work, such that two workers would duplicate or collide?","useWhen":"You are about to fan work out to several agents or people and need a yes/no check that two assignments will not step on each other.","tags":["agent","coordination","overlap","planning","multi-agent"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"tempo-change","title":"Steer the tempo","description":"Should the beat slow down, hold, or speed up next, given what listeners said about pace in context and how fast recentMaterial moved?","useWhen":"A live loop has read chat about speed and must choose one pacing direction for the coming bars.","tags":["music","tempo","live-performance","steering","workflow","decision"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"tension-level","title":"Grade harmonic tension","description":"How much harmonic tension does the current point in progression carry, from fully resolved to a peak demanding resolution, given any key supplied?","useWhen":"A generation loop or accompaniment engine wants to know whether the harmony is at rest, building, or begging for resolution, so it can decide whether to release, hold, or push further.","tags":["music","harmony","tension","generation","score","composition"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"text-block-role","title":"Identify a text block’s structure","description":"Is extracted text a heading, body paragraph, list item, code, table, caption, formula, or other block?","useWhen":"You have already extracted a text block but its structural markup is missing.","tags":["document","ingestion","structure","block","heading"],"category":"retrieval","collection":null,"evidence":"measured","evidenceDetails":{"kind":"public-dataset","label":"Current public-dataset measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:52.490Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":126,"ready":92,"reviewRate":0.27,"readyAccuracy":0.967,"failed":0,"provenance":[{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/assert.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":9},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/child_process.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":9},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/crypto.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":9},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/diagnostics_channel.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":9},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/dns.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":9},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/errors.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":9},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/process.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":9},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/repl.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":9},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/timers.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":9},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/tls.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":9},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/util.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":9},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/vm.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":9},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/wasi.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":9},{"method":"public-dataset","source":"https://raw.githubusercontent.com/nodejs/node/v22.20.0/doc/api/webcrypto.md; source Markdown labels, transformed by build-ingestion-cases.mjs; no independent human review; Node.js MIT license.","cases":9}],"acceptanceMet":null}}},{"id":"ticket-match","title":"Compare support tickets","description":"Do firstTicket and secondTicket describe the same underlying reported issue?","useWhen":"You want to check whether two tickets describe the same underlying issue.","tags":["support","ticket","match"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"tone-check","title":"Check writing criteria","description":"Check a draft against each supplied writing criterion.","useWhen":"You want to check a draft against a supplied set of writing criteria.","tags":["answer-quality","tone","check","rag","evidence"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"tool-call-gate","title":"Gate an agent tool call","description":"Should an agent run toolCall now, ask a person first, or refuse it, given request and policy, with irreversible, destructive, out-of-scope, exfiltration, and injection risks flagged in the same call?","useWhen":"A coding or browsing agent proposes a tool call and you need to review that action: an allow, ask, or deny decision plus the specific risks, in one fast request.","tags":["agent","tool-call","permission","guardrail","safety","injection","pre-tool-use","harness"],"category":"workflow","collection":"agent-operations","evidence":"earlier","evidenceDetails":{"kind":"earlier","label":"Earlier-version measurement","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:55.923Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":104,"ready":76,"reviewRate":0.269,"readyAccuracy":1,"failed":0,"provenance":[{"method":"unspecified","source":"Existing repository labels; independent review and original authorship were not recorded.","cases":44},{"method":"author-synthetic","source":"AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.","cases":60}],"acceptanceMet":null}}},{"id":"tool-compare","title":"Compare two tools for a task","description":"Which of firstTool and secondTool, as described by their stated capabilities, better fits task?","useWhen":"An agent has two candidate tools for one step and needs a head-to-head preference based on the capability descriptions it has been given.","tags":["agent","tool-use","comparison","pairwise","planning","workflow","harness"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"tool-fit","title":"Check a tool fit","description":"Can the capabilities explicitly described in tool perform task?","useWhen":"You need to check whether a tool has the stated capability to perform a task.","tags":["workflow","tool","fit","harness"],"category":"workflow","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"topic-shift","title":"Detect a topic shift","description":"Does message stay with currentTopic, introduce a different topic, or contain both?","useWhen":"You need to detect whether a message moves away from the current topic.","tags":["conversation","topic","shift"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"trip-purpose-kind","title":"Identify the purpose of a trip","description":"What purpose does the traveler's message express for the trip: business, leisure, a family visit, medical care, relocation, or attending an event?","useWhen":"A travel assistant needs to route a traveler's message to the right search defaults, policy, or recommendations based on why they are traveling, using only what the message and any prior context say.","tags":["travel","hospitality","intent","classification","conversation","trip-purpose"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"troubleshooting-fit","title":"Check troubleshooting applicability","description":"Does procedure address symptoms under the described circumstances?","useWhen":"You need to choose whether a procedure fits the reported symptoms and circumstances.","tags":["support","troubleshooting","fit"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"turn-intent","title":"Identify a conversation turn","description":"What is the primary communicative purpose of message in context?","useWhen":"You need to classify a message as a request, answer, correction, cancellation, or acknowledgment.","tags":["conversation","turn","intent"],"category":"conversation","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"uncertainty-expression","title":"Label expressed certainty about a claim","description":"Label categorical, qualified, or unresolved wording about a supplied claim without inferring internal confidence.","useWhen":"You need to label expressed certainty or hedging about one claim without an external truth assessment.","tags":["psychology","uncertainty","certainty","hedging","expressed confidence","calibration","annotation","alignment-research"],"category":"answer-quality","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"urgency-signal","title":"Detect an explicit urgency request","description":"Does message explicitly request urgent attention?","useWhen":"You need to detect whether a message explicitly asks for urgent attention.","tags":["support","urgency","signal"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}},{"id":"verify","title":"Verify claims","description":"Check each supplied claim against its paired evidence.","useWhen":"You need to know whether a claim is supported by its supplied evidence.","tags":["rag","grounding","claims","alignment-research"],"category":"retrieval","collection":null,"evidence":"measured","evidenceDetails":{"kind":"unspecified","label":"Current measurement; labels unspecified","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:43:58.961Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":50,"ready":43,"reviewRate":0.14,"readyAccuracy":1,"failed":0,"provenance":[{"method":"unspecified","source":"Not recorded by the dataset author.","cases":50}],"acceptanceMet":null}}},{"id":"wake-gate","title":"Decide whether an event should wake a waiting agent","description":"Should event wake an agent that is paused until waitingFor happens: wake now, keep waiting, or ignore it as unrelated?","useWhen":"A paused or sleeping agent receives a timer tick, webhook, message, or file change and you must decide whether to resume its model loop.","tags":["agent","wake","resume","event","pause","scheduler","long-running","harness"],"category":"workflow","collection":null,"evidence":"measured","evidenceDetails":{"kind":"unspecified","label":"Current measurement; labels unspecified","experimental":true,"measurement":{"model":"jev-1.13.0","date":"2026-10-04T20:44:00.345Z","operation":"live","replayedAt":null,"scoringRevision":2,"split":"development","cases":44,"ready":33,"reviewRate":0.25,"readyAccuracy":0.97,"failed":0,"provenance":[{"method":"unspecified","source":"Not recorded by the dataset author.","cases":44}],"acceptanceMet":null}}},{"id":"workaround-fit","title":"Check workaround fit","description":"Can workaround address issue without violating constraints?","useWhen":"You need to check whether a workaround addresses an issue within the stated constraints.","tags":["support","workaround","fit"],"category":"support","collection":null,"evidence":"fixture","evidenceDetails":{"kind":"fixture","label":"Fixture only","experimental":true,"measurement":null}}],"collections":{"agent-operations":["model-route","context-prune","completion-gate","progress-stall","retry-worthwhile","tool-call-gate","action-effects"],"customer-conversations":["route","route-many","handoff","response-needed","followup-link","promise-check","buying-intent","contact-opt-out","followup-timing","callback-responsibility"],"document-decisions":["field-select","extraction-fidelity","entity-match"]}}