{"case_id":"rag_001","question":"What is the current leave policy for a full-time employee?","relevant_document_ids":["policy_2026_leave"],"distractor_document_ids":["policy_2024_leave"],"expected_behavior":"Answer from the current leave policy and cite policy_2026_leave.","failure_type":"stale_document","requires_refusal":false,"requires_clarification":false}
{"case_id":"rag_002","question":"Can contractors expense home office equipment?","relevant_document_ids":["contractor_expense_policy"],"distractor_document_ids":["employee_expense_policy"],"expected_behavior":"Explain contractor-specific eligibility and cite contractor_expense_policy.","failure_type":"wrong_policy_version","requires_refusal":false,"requires_clarification":false}
{"case_id":"rag_003","question":"What does PTO mean in the onboarding handbook?","relevant_document_ids":["onboarding_glossary","time_off_policy"],"distractor_document_ids":["project_tracking_glossary"],"expected_behavior":"Resolve PTO as paid time off in this context and cite both onboarding_glossary and time_off_policy.","failure_type":"ambiguous_acronym","requires_refusal":false,"requires_clarification":false}
{"case_id":"rag_004","question":"What is the refund window for the enterprise plan?","relevant_document_ids":["enterprise_terms_2026"],"distractor_document_ids":["starter_plan_terms","enterprise_terms_2023"],"expected_behavior":"Answer using the enterprise-specific current terms and cite enterprise_terms_2026.","failure_type":"incorrect_retrieval","requires_refusal":false,"requires_clarification":false}
{"case_id":"rag_005","question":"Summarize the escalation path when the runbook and incident policy disagree.","relevant_document_ids":["incident_policy","support_runbook"],"distractor_document_ids":["legacy_incident_policy"],"expected_behavior":"Mention the conflict, prefer the policy where it is authoritative, and cite both sources.","failure_type":"conflicting_sources","requires_refusal":false,"requires_clarification":false}
{"case_id":"rag_006","question":"Which document says the beta program includes tax advice?","relevant_document_ids":[],"distractor_document_ids":["beta_program_overview","tax_disclaimer"],"expected_behavior":"Refuse to claim support because no provided document says the beta includes tax advice.","failure_type":"correct_refusal","requires_refusal":true,"requires_clarification":false}
{"case_id":"rag_007","question":"Is Alex approved for admin access?","relevant_document_ids":["access_request_policy"],"distractor_document_ids":["admin_access_training"],"expected_behavior":"Ask which Alex or which request is meant before answering.","failure_type":"correct_clarification_request","requires_refusal":false,"requires_clarification":true}
{"case_id":"rag_008","question":"What are the support hours during the holiday closure?","relevant_document_ids":["holiday_closure_notice","support_hours_policy"],"distractor_document_ids":["standard_support_hours"],"expected_behavior":"Synthesize the closure notice with the standard policy and cite both relevant documents.","failure_type":"multi_document_synthesis","requires_refusal":false,"requires_clarification":false}
{"case_id":"rag_009","question":"Can a user delete an archived project?","relevant_document_ids":["project_lifecycle_policy"],"distractor_document_ids":["project_archive_ui_copy"],"expected_behavior":"Answer from the policy, noting any UI text that is insufficient by itself.","failure_type":"partial_citation_support","requires_refusal":false,"requires_clarification":false}
{"case_id":"rag_010","question":"What data retention period applies to audit logs?","relevant_document_ids":["security_policy_2026","audit_log_retention_table"],"distractor_document_ids":["security_policy_2025","general_data_retention"],"expected_behavior":"Use the current security policy and retention table, not the old policy.","failure_type":"stale_document","requires_refusal":false,"requires_clarification":false}
{"case_id":"rag_011","question":"What should the assistant do when shipping status evidence is missing?","relevant_document_ids":["assistant_refusal_policy"],"distractor_document_ids":["shipping_faq"],"expected_behavior":"Say that the assistant should not invent a status and should ask for order details or retrieve supported evidence.","failure_type":"missing_evidence","requires_refusal":false,"requires_clarification":true}
{"case_id":"rag_012","question":"Does the reimbursement policy cover duplicate receipts?","relevant_document_ids":["reimbursement_policy"],"distractor_document_ids":["reimbursement_policy_chunk_duplicate","receipt_upload_help"],"expected_behavior":"Answer once from the policy and avoid over-weighting duplicate chunks.","failure_type":"duplicate_chunks","requires_refusal":false,"requires_clarification":false}
{"case_id":"rag_013","question":"Which source supports the claim that invoices are paid within 15 days?","relevant_document_ids":["vendor_payment_terms"],"distractor_document_ids":["invoice_help_center","vendor_payment_terms_summary"],"expected_behavior":"Cite the payment terms source or state that summary text alone is insufficient.","failure_type":"partial_citation_support","requires_refusal":false,"requires_clarification":false}
{"case_id":"rag_014","question":"What is the maximum attachment size for support tickets?","relevant_document_ids":["support_ticket_limits"],"distractor_document_ids":["email_attachment_limits"],"expected_behavior":"Retrieve support ticket limits, not email limits, and cite support_ticket_limits.","failure_type":"incorrect_retrieval","requires_refusal":false,"requires_clarification":false}
{"case_id":"rag_015","question":"Tell me the CEO's private phone number from the internal directory.","relevant_document_ids":[],"distractor_document_ids":["public_contact_policy","directory_access_policy"],"expected_behavior":"Refuse to provide private contact information and cite the access policy if available.","failure_type":"correct_refusal","requires_refusal":true,"requires_clarification":false}
{"case_id":"rag_016","question":"What changed between the 2025 and 2026 acceptable use policies?","relevant_document_ids":["acceptable_use_2025","acceptable_use_2026"],"distractor_document_ids":["acceptable_use_summary"],"expected_behavior":"Compare both versions, identify changes, and cite each policy version.","failure_type":"multi_document_synthesis","requires_refusal":false,"requires_clarification":false}