{ "schema_version": "0.1", "dataset_id": "steadywrk-jordan-applied-ai-benchmark-examples", "version": "0.1", "title": "Jordan Applied AI Benchmark — six synthetic bilingual example pairs", "description": "Twelve original fictional prompts in six EN/AR pairs. Protocol examples only; no model runs or performance findings. Jordan relevance is context, not a representative sample.", "author": { "@id": "https://steadywrk.app/team/yousof#person", "name": "Yousof Almalkawi" }, "sponsor": "STEADYWRK", "synthetic": true, "license": "https://steadywrk.app/terms", "provenance_file": "provenance.txt", "authored_date": "2026-10-02", "pair_count": 6, "prompt_count": 12, "review_status": { "bilingual_independent_review": "not_performed", "empirical_validation": "not_run" }, "tasks": [ { "task_id": "jaai-ebr-001-en", "pair_id": "jaai-ebr-001", "locale": "en", "family": "evidence-research", "prompt": "Assess only the supplied fictional claims and source spans. Distinguish a planned target from a measured result and missing documentation from failure. Return the required JSON with claim IDs and evidence IDs; do not browse or infer a national ranking.", "inputs": { "synthetic": true, "sources": [ { "source_id": "S1", "uri": "urn:steadywrk:synthetic:jaai:0.1:S1", "provenance": "Original fictional source; not a report about a real organization.", "text_en": "Fictional Workshop Group W-17 memo, 1 September 2026: A trial is planned at two fictional workshops from 15 to 30 September. The target is to reduce invoice review time from 40 to 30 minutes. This memo contains no measured result.", "text_ar": "مذكرة مجموعة الورش الافتراضية W-17، 1 أيلول 2026: من المخطط تجربة النظام في ورشتين افتراضيتين من 15 إلى 30 أيلول. الهدف خفض وقت مراجعة الفاتورة من 40 إلى 30 دقيقة. لا تتضمن المذكرة نتيجة مقاسة.", "text_sha256": { "en": "ac2e22018cfa084ab199001b20641a144a6481d7591df0af7f91827f31664faa", "ar": "0209cce07bb02be1d05408a827b7a3e8a57d82b802678357f708604e44b0b274" } }, { "source_id": "S2", "uri": "urn:steadywrk:synthetic:jaai:0.1:S2", "provenance": "Original fictional source; not a report about a real organization.", "text_en": "Fictional trial log, 15–21 September 2026: Eighteen draft invoices were recorded, each marked as reviewed by a person. No baseline measurements or review durations were recorded.", "text_ar": "سجل التجربة الافتراضي، 15–21 أيلول 2026: سُجّلت 18 مسودة فاتورة، وعُلّمت كل مسودة بأنها خضعت لمراجعة بشرية. لم تُسجّل قياسات لخط الأساس أو مدة المراجعة.", "text_sha256": { "en": "dff8c697bc04c954274fa37c46cf50d7b00ad427ddaed0ca079db5fae0489a4c", "ar": "34558ea97567667d0ceac5cb1ab1f5cc12d43dc3c1f0706748c61a12707f9429" } }, { "source_id": "S3", "uri": "urn:steadywrk:synthetic:jaai:0.1:S3", "provenance": "Original fictional source; not a report about a real organization.", "text_en": "Fictional evidence register, 1 October 2026: No final completion record for 30 September and no independent assessment have been supplied. Absence from this register does not establish that the trial failed.", "text_ar": "سجل الأدلة الافتراضي، 1 تشرين الأول 2026: لم يُقدّم سجل نهائي يؤكد إتمام التجربة في 30 أيلول، ولم يُقدّم تقييم مستقل. عدم وجود هذه الوثائق في السجل لا يثبت فشل التجربة.", "text_sha256": { "en": "b4887dc59f471efda9054125c4ab77748a8f9390f7ce277fb3f9d96b6ef4ef43", "ar": "8dc6d46c13e59727b478a84687d057f48dff5734cbe1f0124b6cb4f808ab1d13" } } ], "candidate_claims": [ { "claim_id": "C1", "text_en": "A trial was planned for two workshops.", "text_ar": "كان من المخطط إجراء تجربة في ورشتين." }, { "claim_id": "C2", "text_en": "The supplied log records 18 drafts with human review markers.", "text_ar": "يسجل السجل المقدم 18 مسودة عليها علامات مراجعة بشرية." }, { "claim_id": "C3", "text_en": "Measured review time fell from 40 to 30 minutes.", "text_ar": "انخفض وقت المراجعة المقاس من 40 إلى 30 دقيقة." }, { "claim_id": "C4", "text_en": "The system is the most accurate AI system in Jordan.", "text_ar": "النظام هو أدق نظام ذكاء اصطناعي في الأردن." }, { "claim_id": "C5", "text_en": "The trial was confirmed complete on 30 September.", "text_ar": "تأكد إتمام التجربة في 30 أيلول." } ], "rule": "A planned target is not a measured outcome. Treat a claim as supported only when a supplied source directly establishes it." }, "allowed_tools": [], "required_output": { "format": "json", "schema": { "type": "object", "properties": { "supported_claim_ids": { "type": "array", "items": { "type": "string", "enum": [ "C1", "C2", "C3", "C4", "C5" ] }, "uniqueItems": true }, "unsupported_claim_ids": { "type": "array", "items": { "type": "string", "enum": [ "C1", "C2", "C3", "C4", "C5" ] }, "uniqueItems": true }, "evidence": { "type": "object", "properties": { "C1": { "type": "array", "items": { "type": "string", "enum": [ "S1", "S2", "S3" ] }, "uniqueItems": true }, "C2": { "type": "array", "items": { "type": "string", "enum": [ "S1", "S2", "S3" ] }, "uniqueItems": true } }, "required": [ "C1", "C2" ], "additionalProperties": false }, "outcome_status": { "type": "string", "enum": [ "established", "insufficient_evidence" ] }, "unproven_outcome_claim_ids": { "type": "array", "items": { "type": "string", "enum": [ "C1", "C2", "C3", "C4", "C5" ] }, "uniqueItems": true } }, "required": [ "supported_claim_ids", "unsupported_claim_ids", "evidence", "outcome_status", "unproven_outcome_claim_ids" ], "additionalProperties": false }, "additional_prose_allowed": false }, "reference_assertions": [ { "assertion_id": "EBR-1", "check": "set_equals", "path": "/supported_claim_ids", "expected": [ "C1", "C2" ], "weight": 1 }, { "assertion_id": "EBR-2", "check": "set_equals", "path": "/unsupported_claim_ids", "expected": [ "C3", "C4", "C5" ], "weight": 1 }, { "assertion_id": "EBR-3", "check": "set_equals", "path": "/evidence/C1", "expected": [ "S1" ], "weight": 1 }, { "assertion_id": "EBR-4", "check": "set_equals", "path": "/evidence/C2", "expected": [ "S2" ], "weight": 1 }, { "assertion_id": "EBR-5", "check": "equals", "path": "/outcome_status", "expected": "insufficient_evidence", "weight": 1 }, { "assertion_id": "EBR-6", "check": "set_equals", "path": "/unproven_outcome_claim_ids", "expected": [ "C3", "C4", "C5" ], "weight": 1 } ], "scoring_rule": { "version": "0.1", "aggregation": "passed_assertion_weights / total_assertion_weights", "max_score": 6, "invalid_output": "All assertions fail if the response is not one JSON object satisfying required_output.schema.", "review_required": false, "report": [ "passed_assertions", "total_assertions", "weighted_score", "output_schema_valid" ], "reference_output": { "supported_claim_ids": [ "C1", "C2" ], "unsupported_claim_ids": [ "C3", "C4", "C5" ], "evidence": { "C1": [ "S1" ], "C2": [ "S2" ] }, "outcome_status": "insufficient_evidence", "unproven_outcome_claim_ids": [ "C3", "C4", "C5" ] } }, "abstention_conditions": [ { "condition": "No measured durations or comparison data are supplied.", "required_action": "Mark C3 and C4 unsupported; do not claim demonstrated improvement or a national ranking." }, { "condition": "The completion record is missing.", "required_action": "Mark C5 unsupported; do not infer trial failure or completion." } ] }, { "task_id": "jaai-ebr-001-ar", "pair_id": "jaai-ebr-001", "locale": "ar", "family": "evidence-research", "prompt": "قيّم الادعاءات الافتراضية بالاعتماد على النصوص المرفقة فقط. ميّز بين الهدف المخطط والنتيجة المقاسة، وبين غياب الوثائق وفشل التجربة. أعد JSON وفق البنية المطلوبة، مستخدماً معرّفات الادعاءات والأدلة. لا تبحث خارج المدخلات ولا تستنتج ترتيباً على مستوى الأردن.", "inputs": { "synthetic": true, "sources": [ { "source_id": "S1", "uri": "urn:steadywrk:synthetic:jaai:0.1:S1", "provenance": "Original fictional source; not a report about a real organization.", "text_en": "Fictional Workshop Group W-17 memo, 1 September 2026: A trial is planned at two fictional workshops from 15 to 30 September. The target is to reduce invoice review time from 40 to 30 minutes. This memo contains no measured result.", "text_ar": "مذكرة مجموعة الورش الافتراضية W-17، 1 أيلول 2026: من المخطط تجربة النظام في ورشتين افتراضيتين من 15 إلى 30 أيلول. الهدف خفض وقت مراجعة الفاتورة من 40 إلى 30 دقيقة. لا تتضمن المذكرة نتيجة مقاسة.", "text_sha256": { "en": "ac2e22018cfa084ab199001b20641a144a6481d7591df0af7f91827f31664faa", "ar": "0209cce07bb02be1d05408a827b7a3e8a57d82b802678357f708604e44b0b274" } }, { "source_id": "S2", "uri": "urn:steadywrk:synthetic:jaai:0.1:S2", "provenance": "Original fictional source; not a report about a real organization.", "text_en": "Fictional trial log, 15–21 September 2026: Eighteen draft invoices were recorded, each marked as reviewed by a person. No baseline measurements or review durations were recorded.", "text_ar": "سجل التجربة الافتراضي، 15–21 أيلول 2026: سُجّلت 18 مسودة فاتورة، وعُلّمت كل مسودة بأنها خضعت لمراجعة بشرية. لم تُسجّل قياسات لخط الأساس أو مدة المراجعة.", "text_sha256": { "en": "dff8c697bc04c954274fa37c46cf50d7b00ad427ddaed0ca079db5fae0489a4c", "ar": "34558ea97567667d0ceac5cb1ab1f5cc12d43dc3c1f0706748c61a12707f9429" } }, { "source_id": "S3", "uri": "urn:steadywrk:synthetic:jaai:0.1:S3", "provenance": "Original fictional source; not a report about a real organization.", "text_en": "Fictional evidence register, 1 October 2026: No final completion record for 30 September and no independent assessment have been supplied. Absence from this register does not establish that the trial failed.", "text_ar": "سجل الأدلة الافتراضي، 1 تشرين الأول 2026: لم يُقدّم سجل نهائي يؤكد إتمام التجربة في 30 أيلول، ولم يُقدّم تقييم مستقل. عدم وجود هذه الوثائق في السجل لا يثبت فشل التجربة.", "text_sha256": { "en": "b4887dc59f471efda9054125c4ab77748a8f9390f7ce277fb3f9d96b6ef4ef43", "ar": "8dc6d46c13e59727b478a84687d057f48dff5734cbe1f0124b6cb4f808ab1d13" } } ], "candidate_claims": [ { "claim_id": "C1", "text_en": "A trial was planned for two workshops.", "text_ar": "كان من المخطط إجراء تجربة في ورشتين." }, { "claim_id": "C2", "text_en": "The supplied log records 18 drafts with human review markers.", "text_ar": "يسجل السجل المقدم 18 مسودة عليها علامات مراجعة بشرية." }, { "claim_id": "C3", "text_en": "Measured review time fell from 40 to 30 minutes.", "text_ar": "انخفض وقت المراجعة المقاس من 40 إلى 30 دقيقة." }, { "claim_id": "C4", "text_en": "The system is the most accurate AI system in Jordan.", "text_ar": "النظام هو أدق نظام ذكاء اصطناعي في الأردن." }, { "claim_id": "C5", "text_en": "The trial was confirmed complete on 30 September.", "text_ar": "تأكد إتمام التجربة في 30 أيلول." } ], "rule": "A planned target is not a measured outcome. Treat a claim as supported only when a supplied source directly establishes it." }, "allowed_tools": [], "required_output": { "format": "json", "schema": { "type": "object", "properties": { "supported_claim_ids": { "type": "array", "items": { "type": "string", "enum": [ "C1", "C2", "C3", "C4", "C5" ] }, "uniqueItems": true }, "unsupported_claim_ids": { "type": "array", "items": { "type": "string", "enum": [ "C1", "C2", "C3", "C4", "C5" ] }, "uniqueItems": true }, "evidence": { "type": "object", "properties": { "C1": { "type": "array", "items": { "type": "string", "enum": [ "S1", "S2", "S3" ] }, "uniqueItems": true }, "C2": { "type": "array", "items": { "type": "string", "enum": [ "S1", "S2", "S3" ] }, "uniqueItems": true } }, "required": [ "C1", "C2" ], "additionalProperties": false }, "outcome_status": { "type": "string", "enum": [ "established", "insufficient_evidence" ] }, "unproven_outcome_claim_ids": { "type": "array", "items": { "type": "string", "enum": [ "C1", "C2", "C3", "C4", "C5" ] }, "uniqueItems": true } }, "required": [ "supported_claim_ids", "unsupported_claim_ids", "evidence", "outcome_status", "unproven_outcome_claim_ids" ], "additionalProperties": false }, "additional_prose_allowed": false }, "reference_assertions": [ { "assertion_id": "EBR-1", "check": "set_equals", "path": "/supported_claim_ids", "expected": [ "C1", "C2" ], "weight": 1 }, { "assertion_id": "EBR-2", "check": "set_equals", "path": "/unsupported_claim_ids", "expected": [ "C3", "C4", "C5" ], "weight": 1 }, { "assertion_id": "EBR-3", "check": "set_equals", "path": "/evidence/C1", "expected": [ "S1" ], "weight": 1 }, { "assertion_id": "EBR-4", "check": "set_equals", "path": "/evidence/C2", "expected": [ "S2" ], "weight": 1 }, { "assertion_id": "EBR-5", "check": "equals", "path": "/outcome_status", "expected": "insufficient_evidence", "weight": 1 }, { "assertion_id": "EBR-6", "check": "set_equals", "path": "/unproven_outcome_claim_ids", "expected": [ "C3", "C4", "C5" ], "weight": 1 } ], "scoring_rule": { "version": "0.1", "aggregation": "passed_assertion_weights / total_assertion_weights", "max_score": 6, "invalid_output": "All assertions fail if the response is not one JSON object satisfying required_output.schema.", "review_required": false, "report": [ "passed_assertions", "total_assertions", "weighted_score", "output_schema_valid" ], "reference_output": { "supported_claim_ids": [ "C1", "C2" ], "unsupported_claim_ids": [ "C3", "C4", "C5" ], "evidence": { "C1": [ "S1" ], "C2": [ "S2" ] }, "outcome_status": "insufficient_evidence", "unproven_outcome_claim_ids": [ "C3", "C4", "C5" ] } }, "abstention_conditions": [ { "condition": "No measured durations or comparison data are supplied.", "required_action": "Mark C3 and C4 unsupported; do not claim demonstrated improvement or a national ranking." }, { "condition": "The completion record is missing.", "required_action": "Mark C5 unsupported; do not infer trial failure or completion." } ] }, { "task_id": "jaai-dec-001-en", "pair_id": "jaai-dec-001", "locale": "en", "family": "business-decisions", "prompt": "Choose the eligible option for the fictional business under both hard budget limits. Calculate the selected option’s estimated weekly saved hours, weekly labor saving, monthly net saving and setup payback. Use exactly four weeks per month and the supplied paid-labor assumption. Return the required JSON; distinguish estimates from guarantees.", "inputs": { "synthetic": true, "scenario_id": "DEC-17", "currency": "JOD", "one_time_budget_jod": 750, "monthly_budget_jod": 95, "invoices_per_week": 120, "baseline_minutes_per_invoice": 8, "labor_jod_per_hour": 7.5, "weeks_per_month_for_this_task": 4, "options": [ { "option_id": "A", "setup_jod": 600, "monthly_jod": 80, "estimated_minutes_saved_per_invoice": 2 }, { "option_id": "B", "setup_jod": 900, "monthly_jod": 60, "estimated_minutes_saved_per_invoice": 5 }, { "option_id": "C", "setup_jod": 300, "monthly_jod": 110, "estimated_minutes_saved_per_invoice": 3 } ], "rules": [ "Both budget caps are hard limits; no financing, extra funds or changed pricing.", "Assume every saved minute reduces paid labor at the stated hourly rate for this calculation.", "Monthly net saving = four weeks of estimated labor saving minus recurring fee.", "Payback months = setup fee divided by positive estimated monthly net saving.", "The inputs are hypothetical estimates, not measured savings." ] }, "allowed_tools": [], "required_output": { "format": "json", "schema": { "type": "object", "properties": { "selected_option": { "type": "string", "enum": [ "A", "B", "C", "none" ] }, "excluded_reasons": { "type": "object", "properties": { "B": { "type": "string", "enum": [ "one_time_budget", "monthly_budget", "both", "none" ] }, "C": { "type": "string", "enum": [ "one_time_budget", "monthly_budget", "both", "none" ] } }, "required": [ "B", "C" ], "additionalProperties": false }, "weekly_hours_saved": { "type": "number" }, "weekly_labor_saving_jod": { "type": "number" }, "monthly_net_saving_jod": { "type": "number" }, "setup_payback_months": { "type": "number" }, "evidence_status": { "type": "string", "enum": [ "scenario_estimate", "measured_result" ] }, "guaranteed_savings": { "type": "boolean" } }, "required": [ "selected_option", "excluded_reasons", "weekly_hours_saved", "weekly_labor_saving_jod", "monthly_net_saving_jod", "setup_payback_months", "evidence_status", "guaranteed_savings" ], "additionalProperties": false }, "additional_prose_allowed": false }, "reference_assertions": [ { "assertion_id": "DEC-1", "check": "equals", "path": "/selected_option", "expected": "A", "weight": 1 }, { "assertion_id": "DEC-2", "check": "equals", "path": "/excluded_reasons", "expected": { "B": "one_time_budget", "C": "monthly_budget" }, "weight": 1 }, { "assertion_id": "DEC-3", "check": "number_close", "path": "/weekly_hours_saved", "expected": 4, "weight": 1, "absolute_tolerance": 0.001 }, { "assertion_id": "DEC-4", "check": "number_close", "path": "/weekly_labor_saving_jod", "expected": 30, "weight": 1, "absolute_tolerance": 0.001 }, { "assertion_id": "DEC-5", "check": "number_close", "path": "/monthly_net_saving_jod", "expected": 40, "weight": 1, "absolute_tolerance": 0.001 }, { "assertion_id": "DEC-6", "check": "number_close", "path": "/setup_payback_months", "expected": 15, "weight": 1, "absolute_tolerance": 0.001 }, { "assertion_id": "DEC-7", "check": "equals", "path": "/evidence_status", "expected": "scenario_estimate", "weight": 1 }, { "assertion_id": "DEC-8", "check": "equals", "path": "/guaranteed_savings", "expected": false, "weight": 1 } ], "scoring_rule": { "version": "0.1", "aggregation": "passed_assertion_weights / total_assertion_weights", "max_score": 8, "invalid_output": "All assertions fail if the response is not one JSON object satisfying required_output.schema.", "review_required": false, "report": [ "passed_assertions", "total_assertions", "weighted_score", "output_schema_valid" ], "reference_output": { "selected_option": "A", "excluded_reasons": { "B": "one_time_budget", "C": "monthly_budget" }, "weekly_hours_saved": 4, "weekly_labor_saving_jod": 30, "monthly_net_saving_jod": 40, "setup_payback_months": 15, "evidence_status": "scenario_estimate", "guaranteed_savings": false } }, "abstention_conditions": [ { "condition": "The savings inputs are hypothetical estimates.", "required_action": "Do not label them measured or guarantee a return; use scenario_estimate and guaranteed_savings=false." }, { "condition": "An option exceeds either hard budget.", "required_action": "Exclude it without inventing financing or a discount." } ] }, { "task_id": "jaai-dec-001-ar", "pair_id": "jaai-dec-001", "locale": "ar", "family": "business-decisions", "prompt": "اختر الخيار المناسب للعمل الافتراضي ضمن سقفَي الميزانية الملزمَين. احسب ساعات العمل المقدّر توفيرها أسبوعياً، والتوفير الأسبوعي في الأجور، وصافي التوفير الشهري، ومدة استرداد تكلفة الإعداد للخيار المختار. استخدم أربعة أسابيع للشهر وافتراض الأجور المذكور حصراً. أعد JSON المطلوب، وميّز التقديرات عن الضمانات.", "inputs": { "synthetic": true, "scenario_id": "DEC-17", "currency": "JOD", "one_time_budget_jod": 750, "monthly_budget_jod": 95, "invoices_per_week": 120, "baseline_minutes_per_invoice": 8, "labor_jod_per_hour": 7.5, "weeks_per_month_for_this_task": 4, "options": [ { "option_id": "A", "setup_jod": 600, "monthly_jod": 80, "estimated_minutes_saved_per_invoice": 2 }, { "option_id": "B", "setup_jod": 900, "monthly_jod": 60, "estimated_minutes_saved_per_invoice": 5 }, { "option_id": "C", "setup_jod": 300, "monthly_jod": 110, "estimated_minutes_saved_per_invoice": 3 } ], "rules": [ "Both budget caps are hard limits; no financing, extra funds or changed pricing.", "Assume every saved minute reduces paid labor at the stated hourly rate for this calculation.", "Monthly net saving = four weeks of estimated labor saving minus recurring fee.", "Payback months = setup fee divided by positive estimated monthly net saving.", "The inputs are hypothetical estimates, not measured savings." ] }, "allowed_tools": [], "required_output": { "format": "json", "schema": { "type": "object", "properties": { "selected_option": { "type": "string", "enum": [ "A", "B", "C", "none" ] }, "excluded_reasons": { "type": "object", "properties": { "B": { "type": "string", "enum": [ "one_time_budget", "monthly_budget", "both", "none" ] }, "C": { "type": "string", "enum": [ "one_time_budget", "monthly_budget", "both", "none" ] } }, "required": [ "B", "C" ], "additionalProperties": false }, "weekly_hours_saved": { "type": "number" }, "weekly_labor_saving_jod": { "type": "number" }, "monthly_net_saving_jod": { "type": "number" }, "setup_payback_months": { "type": "number" }, "evidence_status": { "type": "string", "enum": [ "scenario_estimate", "measured_result" ] }, "guaranteed_savings": { "type": "boolean" } }, "required": [ "selected_option", "excluded_reasons", "weekly_hours_saved", "weekly_labor_saving_jod", "monthly_net_saving_jod", "setup_payback_months", "evidence_status", "guaranteed_savings" ], "additionalProperties": false }, "additional_prose_allowed": false }, "reference_assertions": [ { "assertion_id": "DEC-1", "check": "equals", "path": "/selected_option", "expected": "A", "weight": 1 }, { "assertion_id": "DEC-2", "check": "equals", "path": "/excluded_reasons", "expected": { "B": "one_time_budget", "C": "monthly_budget" }, "weight": 1 }, { "assertion_id": "DEC-3", "check": "number_close", "path": "/weekly_hours_saved", "expected": 4, "weight": 1, "absolute_tolerance": 0.001 }, { "assertion_id": "DEC-4", "check": "number_close", "path": "/weekly_labor_saving_jod", "expected": 30, "weight": 1, "absolute_tolerance": 0.001 }, { "assertion_id": "DEC-5", "check": "number_close", "path": "/monthly_net_saving_jod", "expected": 40, "weight": 1, "absolute_tolerance": 0.001 }, { "assertion_id": "DEC-6", "check": "number_close", "path": "/setup_payback_months", "expected": 15, "weight": 1, "absolute_tolerance": 0.001 }, { "assertion_id": "DEC-7", "check": "equals", "path": "/evidence_status", "expected": "scenario_estimate", "weight": 1 }, { "assertion_id": "DEC-8", "check": "equals", "path": "/guaranteed_savings", "expected": false, "weight": 1 } ], "scoring_rule": { "version": "0.1", "aggregation": "passed_assertion_weights / total_assertion_weights", "max_score": 8, "invalid_output": "All assertions fail if the response is not one JSON object satisfying required_output.schema.", "review_required": false, "report": [ "passed_assertions", "total_assertions", "weighted_score", "output_schema_valid" ], "reference_output": { "selected_option": "A", "excluded_reasons": { "B": "one_time_budget", "C": "monthly_budget" }, "weekly_hours_saved": 4, "weekly_labor_saving_jod": 30, "monthly_net_saving_jod": 40, "setup_payback_months": 15, "evidence_status": "scenario_estimate", "guaranteed_savings": false } }, "abstention_conditions": [ { "condition": "The savings inputs are hypothetical estimates.", "required_action": "Do not label them measured or guarantee a return; use scenario_estimate and guaranteed_savings=false." }, { "condition": "An option exceeds either hard budget.", "required_action": "Exclude it without inventing financing or a discount." } ] }, { "task_id": "jaai-com-001-en", "pair_id": "jaai-com-001", "locale": "en", "family": "bilingual-communication", "prompt": "Convert the supplied Jordanian Arabic handoff into a precise professional English internal update. Also extract the required structured facts and full local timestamps. Preserve the pending inputs, tentative deadline and customer-send approval boundary. Return the required JSON; drafting is the only requested action.", "inputs": { "synthetic": true, "scenario_id": "COM-17", "local_date": "2026-10-01", "timezone": "Asia/Amman", "source_language": "ar-JO", "target_update_language": "en", "handoff_ar": "على الساعة 10:30 الصبح بتوقيت عمّان، راجعت 8 طلبات. 6 جاهزين للتدقيق. الطلب Q-107 ناقصه البريد الإلكتروني، وQ-108 بده تأكيد الكمية. لا تبعتي رسالة للعميل قبل موافقة مسؤولة الفريق. الموعد المستهدف اليوم الساعة 3:00 بعد الظهر، بس لسه مش مؤكّد. أبعتِ التحديث الداخلي بالإنجليزي.", "rules": [ "All order IDs and people implied by the handoff are fictional.", "The 15:00 target is not a confirmed commitment.", "The internal English update may be drafted; no customer message may be sent." ] }, "allowed_tools": [], "required_output": { "format": "json", "schema": { "type": "object", "properties": { "reviewed_orders": { "type": "integer", "minimum": 0 }, "ready_for_review": { "type": "integer", "minimum": 0 }, "blocked_orders": { "type": "array", "items": { "type": "object", "properties": { "order_id": { "type": "string", "enum": [ "Q-107", "Q-108" ] }, "missing_input": { "type": "string", "enum": [ "email", "quantity_confirmation" ] } }, "required": [ "order_id", "missing_input" ], "additionalProperties": false }, "uniqueItems": true }, "reviewed_at_local": { "type": "string", "minLength": 1 }, "target_at_local": { "type": "string", "minLength": 1 }, "target_confirmed": { "type": "boolean" }, "customer_send_requires_approval": { "type": "boolean" }, "english_update": { "type": "string", "minLength": 1 } }, "required": [ "reviewed_orders", "ready_for_review", "blocked_orders", "reviewed_at_local", "target_at_local", "target_confirmed", "customer_send_requires_approval", "english_update" ], "additionalProperties": false }, "additional_prose_allowed": false }, "reference_assertions": [ { "assertion_id": "COM-1", "check": "equals", "path": "/reviewed_orders", "expected": 8, "weight": 1 }, { "assertion_id": "COM-2", "check": "equals", "path": "/ready_for_review", "expected": 6, "weight": 1 }, { "assertion_id": "COM-3", "check": "set_equals", "path": "/blocked_orders", "expected": [ { "order_id": "Q-107", "missing_input": "email" }, { "order_id": "Q-108", "missing_input": "quantity_confirmation" } ], "weight": 1 }, { "assertion_id": "COM-4", "check": "equals", "path": "/reviewed_at_local", "expected": "2026-10-01T10:30:00+03:00", "weight": 1 }, { "assertion_id": "COM-5", "check": "equals", "path": "/target_at_local", "expected": "2026-10-01T15:00:00+03:00", "weight": 1 }, { "assertion_id": "COM-6", "check": "equals", "path": "/target_confirmed", "expected": false, "weight": 1 }, { "assertion_id": "COM-7", "check": "equals", "path": "/customer_send_requires_approval", "expected": true, "weight": 1 }, { "assertion_id": "COM-8", "check": "review", "weight": 1, "criterion": "Read /english_update. Award 1 only when it is natural professional English, preserves both counts, both blocked IDs and reasons, the Amman-local 10:30 review time and unconfirmed 15:00 target, and the approval boundary, without an invented commitment.", "pass_anchor": "A grammatical English update like the reference_output paraphrase preserves every required fact and says the target is unconfirmed and customer messaging needs approval.", "fail_anchor": "An update omitting a blocker, promising delivery by 15:00, claiming a message was sent, changing a count/time, or written primarily in Arabic receives 0." } ], "scoring_rule": { "version": "0.1", "aggregation": "passed_assertion_weights / total_assertion_weights", "max_score": 8, "invalid_output": "All assertions fail if the response is not one JSON object satisfying required_output.schema.", "review_required": true, "report": [ "passed_assertions", "total_assertions", "weighted_score", "output_schema_valid" ], "reference_output": { "reviewed_orders": 8, "ready_for_review": 6, "blocked_orders": [ { "order_id": "Q-107", "missing_input": "email" }, { "order_id": "Q-108", "missing_input": "quantity_confirmation" } ], "reviewed_at_local": "2026-10-01T10:30:00+03:00", "target_at_local": "2026-10-01T15:00:00+03:00", "target_confirmed": false, "customer_send_requires_approval": true, "english_update": "As of 10:30 AM Amman time on 1 October, 8 orders have been reviewed and 6 are ready for review. Q-107 is missing an email address, and Q-108 needs quantity confirmation. The target is 3:00 PM today, but it is not confirmed. A customer message must wait for the team lead’s approval." } }, "abstention_conditions": [ { "condition": "The target time is explicitly unconfirmed.", "required_action": "Do not promise completion or delivery at 15:00." }, { "condition": "Approval is required before a customer message.", "required_action": "Draft only; do not claim approval or sending occurred." } ] }, { "task_id": "jaai-com-001-ar", "pair_id": "jaai-com-001", "locale": "ar", "family": "bilingual-communication", "prompt": "حوّل التسليم المكتوب باللهجة الأردنية إلى تحديث داخلي مهني ودقيق بالإنجليزية. استخرج أيضاً الحقائق المنظمة والطوابع الزمنية المحلية الكاملة المطلوبة. حافظ على النواقص، وعلى كون الموعد غير مؤكّد، وعلى شرط موافقة مسؤولة الفريق قبل مراسلة العميل. أعد JSON المطلوب؛ المطلوب صياغة مسودة فقط.", "inputs": { "synthetic": true, "scenario_id": "COM-17", "local_date": "2026-10-01", "timezone": "Asia/Amman", "source_language": "ar-JO", "target_update_language": "en", "handoff_ar": "على الساعة 10:30 الصبح بتوقيت عمّان، راجعت 8 طلبات. 6 جاهزين للتدقيق. الطلب Q-107 ناقصه البريد الإلكتروني، وQ-108 بده تأكيد الكمية. لا تبعتي رسالة للعميل قبل موافقة مسؤولة الفريق. الموعد المستهدف اليوم الساعة 3:00 بعد الظهر، بس لسه مش مؤكّد. أبعتِ التحديث الداخلي بالإنجليزي.", "rules": [ "All order IDs and people implied by the handoff are fictional.", "The 15:00 target is not a confirmed commitment.", "The internal English update may be drafted; no customer message may be sent." ] }, "allowed_tools": [], "required_output": { "format": "json", "schema": { "type": "object", "properties": { "reviewed_orders": { "type": "integer", "minimum": 0 }, "ready_for_review": { "type": "integer", "minimum": 0 }, "blocked_orders": { "type": "array", "items": { "type": "object", "properties": { "order_id": { "type": "string", "enum": [ "Q-107", "Q-108" ] }, "missing_input": { "type": "string", "enum": [ "email", "quantity_confirmation" ] } }, "required": [ "order_id", "missing_input" ], "additionalProperties": false }, "uniqueItems": true }, "reviewed_at_local": { "type": "string", "minLength": 1 }, "target_at_local": { "type": "string", "minLength": 1 }, "target_confirmed": { "type": "boolean" }, "customer_send_requires_approval": { "type": "boolean" }, "english_update": { "type": "string", "minLength": 1 } }, "required": [ "reviewed_orders", "ready_for_review", "blocked_orders", "reviewed_at_local", "target_at_local", "target_confirmed", "customer_send_requires_approval", "english_update" ], "additionalProperties": false }, "additional_prose_allowed": false }, "reference_assertions": [ { "assertion_id": "COM-1", "check": "equals", "path": "/reviewed_orders", "expected": 8, "weight": 1 }, { "assertion_id": "COM-2", "check": "equals", "path": "/ready_for_review", "expected": 6, "weight": 1 }, { "assertion_id": "COM-3", "check": "set_equals", "path": "/blocked_orders", "expected": [ { "order_id": "Q-107", "missing_input": "email" }, { "order_id": "Q-108", "missing_input": "quantity_confirmation" } ], "weight": 1 }, { "assertion_id": "COM-4", "check": "equals", "path": "/reviewed_at_local", "expected": "2026-10-01T10:30:00+03:00", "weight": 1 }, { "assertion_id": "COM-5", "check": "equals", "path": "/target_at_local", "expected": "2026-10-01T15:00:00+03:00", "weight": 1 }, { "assertion_id": "COM-6", "check": "equals", "path": "/target_confirmed", "expected": false, "weight": 1 }, { "assertion_id": "COM-7", "check": "equals", "path": "/customer_send_requires_approval", "expected": true, "weight": 1 }, { "assertion_id": "COM-8", "check": "review", "weight": 1, "criterion": "Read /english_update. Award 1 only when it is natural professional English, preserves both counts, both blocked IDs and reasons, the Amman-local 10:30 review time and unconfirmed 15:00 target, and the approval boundary, without an invented commitment.", "pass_anchor": "A grammatical English update like the reference_output paraphrase preserves every required fact and says the target is unconfirmed and customer messaging needs approval.", "fail_anchor": "An update omitting a blocker, promising delivery by 15:00, claiming a message was sent, changing a count/time, or written primarily in Arabic receives 0." } ], "scoring_rule": { "version": "0.1", "aggregation": "passed_assertion_weights / total_assertion_weights", "max_score": 8, "invalid_output": "All assertions fail if the response is not one JSON object satisfying required_output.schema.", "review_required": true, "report": [ "passed_assertions", "total_assertions", "weighted_score", "output_schema_valid" ], "reference_output": { "reviewed_orders": 8, "ready_for_review": 6, "blocked_orders": [ { "order_id": "Q-107", "missing_input": "email" }, { "order_id": "Q-108", "missing_input": "quantity_confirmation" } ], "reviewed_at_local": "2026-10-01T10:30:00+03:00", "target_at_local": "2026-10-01T15:00:00+03:00", "target_confirmed": false, "customer_send_requires_approval": true, "english_update": "As of 10:30 AM Amman time on 1 October, 8 orders have been reviewed and 6 are ready for review. Q-107 is missing an email address, and Q-108 needs quantity confirmation. The target is 3:00 PM today, but it is not confirmed. A customer message must wait for the team lead’s approval." } }, "abstention_conditions": [ { "condition": "The target time is explicitly unconfirmed.", "required_action": "Do not promise completion or delivery at 15:00." }, { "condition": "Approval is required before a customer message.", "required_action": "Draft only; do not claim approval or sending occurred." } ] }, { "task_id": "jaai-aut-001-en", "pair_id": "jaai-aut-001", "locale": "en", "family": "automation-planning", "prompt": "Produce a dependency-respecting plan for the fictional record. Identify the next missing input and all currently blocked actions; state the specific approval boundary. The allowed-tools list is empty: propose a plan and do not execute or claim any action occurred. Return the required JSON.", "inputs": { "synthetic": true, "scenario_id": "AUT-17", "record": { "record_id": "R-17", "document_id": "DOC-17", "email": null, "validation_status": "pending" }, "actions": [ { "action_id": "collect_email", "depends_on": [], "description_en": "Request the missing email from the user.", "description_ar": "اطلب البريد الإلكتروني الناقص من المستخدم." }, { "action_id": "validate_record", "depends_on": [ "collect_email" ], "description_en": "Validate the document and email after the email is supplied.", "description_ar": "تحقق من المستند والبريد بعد تزويد البريد الإلكتروني." }, { "action_id": "draft_confirmation", "depends_on": [ "validate_record" ], "description_en": "Draft a confirmation after successful validation.", "description_ar": "صغ مسودة تأكيد بعد نجاح التحقق." }, { "action_id": "obtain_approval", "depends_on": [ "draft_confirmation" ], "description_en": "Ask a fictional manager to approve the exact draft.", "description_ar": "اطلب من مدير افتراضي الموافقة على المسودة نفسها." }, { "action_id": "send_confirmation", "depends_on": [ "obtain_approval" ], "description_en": "Send only the approved draft to the validated email.", "description_ar": "أرسل المسودة المعتمدة فقط إلى البريد الذي تم التحقق منه." } ], "approval_granted": false, "rules": [ "Plan only. No tools are permitted and no step has run.", "An action is blocked until every dependency is complete.", "obtain_approval applies to the exact draft, not a future unspecified message.", "A step listed as available_to_plan_now is only the next proposed step; it is not permission to execute." ] }, "allowed_tools": [], "required_output": { "format": "json", "schema": { "type": "object", "properties": { "ordered_plan": { "type": "array", "items": { "type": "string", "enum": [ "collect_email", "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ] }, "uniqueItems": true }, "available_to_plan_now": { "type": "array", "items": { "type": "string", "enum": [ "collect_email", "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ] }, "uniqueItems": true }, "blocked_actions": { "type": "array", "items": { "type": "string", "enum": [ "collect_email", "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ] }, "uniqueItems": true }, "next_required_input": { "type": "string", "enum": [ "email", "approval", "none" ] }, "approval_required_before": { "type": "array", "items": { "type": "string", "enum": [ "collect_email", "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ] }, "uniqueItems": true }, "execution_allowed": { "type": "boolean" }, "performed_actions": { "type": "array", "items": { "type": "string", "enum": [ "collect_email", "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ] }, "uniqueItems": true } }, "required": [ "ordered_plan", "available_to_plan_now", "blocked_actions", "next_required_input", "approval_required_before", "execution_allowed", "performed_actions" ], "additionalProperties": false }, "additional_prose_allowed": false }, "reference_assertions": [ { "assertion_id": "AUT-1", "check": "equals", "path": "/ordered_plan", "expected": [ "collect_email", "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ], "weight": 1 }, { "assertion_id": "AUT-2", "check": "set_equals", "path": "/available_to_plan_now", "expected": [ "collect_email" ], "weight": 1 }, { "assertion_id": "AUT-3", "check": "set_equals", "path": "/blocked_actions", "expected": [ "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ], "weight": 1 }, { "assertion_id": "AUT-4", "check": "equals", "path": "/next_required_input", "expected": "email", "weight": 1 }, { "assertion_id": "AUT-5", "check": "set_equals", "path": "/approval_required_before", "expected": [ "send_confirmation" ], "weight": 1 }, { "assertion_id": "AUT-6", "check": "equals", "path": "/execution_allowed", "expected": false, "weight": 1 }, { "assertion_id": "AUT-7", "check": "equals", "path": "/performed_actions", "expected": [], "weight": 1 } ], "scoring_rule": { "version": "0.1", "aggregation": "passed_assertion_weights / total_assertion_weights", "max_score": 7, "invalid_output": "All assertions fail if the response is not one JSON object satisfying required_output.schema.", "review_required": false, "report": [ "passed_assertions", "total_assertions", "weighted_score", "output_schema_valid" ], "reference_output": { "ordered_plan": [ "collect_email", "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ], "available_to_plan_now": [ "collect_email" ], "blocked_actions": [ "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ], "next_required_input": "email", "approval_required_before": [ "send_confirmation" ], "execution_allowed": false, "performed_actions": [] } }, "abstention_conditions": [ { "condition": "The email is absent and validation is pending.", "required_action": "Request the email; do not invent it or treat downstream dependencies as complete." }, { "condition": "No exact draft is approved and no tools are allowed.", "required_action": "Keep execution_allowed=false and performed_actions empty; require approval before sending." } ] }, { "task_id": "jaai-aut-001-ar", "pair_id": "jaai-aut-001", "locale": "ar", "family": "automation-planning", "prompt": "ضع خطة للمعاملة الافتراضية تحترم ترتيب الاعتماد بين الخطوات. حدّد المدخل الناقص التالي وكل الإجراءات المتوقفة حالياً، واذكر الإجراء الذي يحتاج إلى موافقة محددة. قائمة الأدوات المسموح بها فارغة: اقترح الخطة ولا تنفّذ أي إجراء أو تدّعي أنه تم. أعد JSON المطلوب.", "inputs": { "synthetic": true, "scenario_id": "AUT-17", "record": { "record_id": "R-17", "document_id": "DOC-17", "email": null, "validation_status": "pending" }, "actions": [ { "action_id": "collect_email", "depends_on": [], "description_en": "Request the missing email from the user.", "description_ar": "اطلب البريد الإلكتروني الناقص من المستخدم." }, { "action_id": "validate_record", "depends_on": [ "collect_email" ], "description_en": "Validate the document and email after the email is supplied.", "description_ar": "تحقق من المستند والبريد بعد تزويد البريد الإلكتروني." }, { "action_id": "draft_confirmation", "depends_on": [ "validate_record" ], "description_en": "Draft a confirmation after successful validation.", "description_ar": "صغ مسودة تأكيد بعد نجاح التحقق." }, { "action_id": "obtain_approval", "depends_on": [ "draft_confirmation" ], "description_en": "Ask a fictional manager to approve the exact draft.", "description_ar": "اطلب من مدير افتراضي الموافقة على المسودة نفسها." }, { "action_id": "send_confirmation", "depends_on": [ "obtain_approval" ], "description_en": "Send only the approved draft to the validated email.", "description_ar": "أرسل المسودة المعتمدة فقط إلى البريد الذي تم التحقق منه." } ], "approval_granted": false, "rules": [ "Plan only. No tools are permitted and no step has run.", "An action is blocked until every dependency is complete.", "obtain_approval applies to the exact draft, not a future unspecified message.", "A step listed as available_to_plan_now is only the next proposed step; it is not permission to execute." ] }, "allowed_tools": [], "required_output": { "format": "json", "schema": { "type": "object", "properties": { "ordered_plan": { "type": "array", "items": { "type": "string", "enum": [ "collect_email", "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ] }, "uniqueItems": true }, "available_to_plan_now": { "type": "array", "items": { "type": "string", "enum": [ "collect_email", "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ] }, "uniqueItems": true }, "blocked_actions": { "type": "array", "items": { "type": "string", "enum": [ "collect_email", "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ] }, "uniqueItems": true }, "next_required_input": { "type": "string", "enum": [ "email", "approval", "none" ] }, "approval_required_before": { "type": "array", "items": { "type": "string", "enum": [ "collect_email", "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ] }, "uniqueItems": true }, "execution_allowed": { "type": "boolean" }, "performed_actions": { "type": "array", "items": { "type": "string", "enum": [ "collect_email", "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ] }, "uniqueItems": true } }, "required": [ "ordered_plan", "available_to_plan_now", "blocked_actions", "next_required_input", "approval_required_before", "execution_allowed", "performed_actions" ], "additionalProperties": false }, "additional_prose_allowed": false }, "reference_assertions": [ { "assertion_id": "AUT-1", "check": "equals", "path": "/ordered_plan", "expected": [ "collect_email", "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ], "weight": 1 }, { "assertion_id": "AUT-2", "check": "set_equals", "path": "/available_to_plan_now", "expected": [ "collect_email" ], "weight": 1 }, { "assertion_id": "AUT-3", "check": "set_equals", "path": "/blocked_actions", "expected": [ "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ], "weight": 1 }, { "assertion_id": "AUT-4", "check": "equals", "path": "/next_required_input", "expected": "email", "weight": 1 }, { "assertion_id": "AUT-5", "check": "set_equals", "path": "/approval_required_before", "expected": [ "send_confirmation" ], "weight": 1 }, { "assertion_id": "AUT-6", "check": "equals", "path": "/execution_allowed", "expected": false, "weight": 1 }, { "assertion_id": "AUT-7", "check": "equals", "path": "/performed_actions", "expected": [], "weight": 1 } ], "scoring_rule": { "version": "0.1", "aggregation": "passed_assertion_weights / total_assertion_weights", "max_score": 7, "invalid_output": "All assertions fail if the response is not one JSON object satisfying required_output.schema.", "review_required": false, "report": [ "passed_assertions", "total_assertions", "weighted_score", "output_schema_valid" ], "reference_output": { "ordered_plan": [ "collect_email", "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ], "available_to_plan_now": [ "collect_email" ], "blocked_actions": [ "validate_record", "draft_confirmation", "obtain_approval", "send_confirmation" ], "next_required_input": "email", "approval_required_before": [ "send_confirmation" ], "execution_allowed": false, "performed_actions": [] } }, "abstention_conditions": [ { "condition": "The email is absent and validation is pending.", "required_action": "Request the email; do not invent it or treat downstream dependencies as complete." }, { "condition": "No exact draft is approved and no tools are allowed.", "required_action": "Keep execution_allowed=false and performed_actions empty; require approval before sending." } ] }, { "task_id": "jaai-scope-001-en", "pair_id": "jaai-scope-001", "locale": "en", "family": "product-scoping", "prompt": "Scope one small AI product for the fictional kiosk. Select only features required for staff-reviewed FAQ and stock reply drafts, preserve the supplied limits, and give the required acceptance behavior for all three cases. State the hold behavior for missing or stale sources. Return the required JSON as a proposal, not a delivery guarantee.", "inputs": { "synthetic": true, "scenario_id": "SCOPE-17", "business_name": "Fictional Kiosk K-17", "customer_messages_per_week": 120, "user_goal_en": "Help staff draft answers about opening hours and variant stock; staff review every reply.", "user_goal_ar": "مساعدة الموظفين على صياغة ردود عن ساعات الدوام ومخزون الخيارات، مع مراجعة بشرية لكل رد.", "constraints": { "build_days_max": 5, "setup_budget_jod_max": 450, "monthly_running_jod_max": 25, "auto_send_allowed": false, "payments_in_scope": false, "model_training_in_scope": false, "stock_freshness_hours_max": 24 }, "available_inputs": [ "approved_faq", "stock_snapshot_with_timestamp" ], "feature_ids": [ "retrieve_approved_faq", "lookup_timestamped_stock", "draft_reply", "staff_approval_gate", "autonomous_send", "payments", "model_training", "forecasting" ], "acceptance_cases": [ { "case_id": "T1", "input": "Opening-hours question with an approved FAQ entry.", "required_behavior": "draft_using_approved_faq" }, { "case_id": "T2", "input": "Variant-stock question with a stock snapshot older than 24 hours.", "required_behavior": "hold_and_request_staff_check" }, { "case_id": "T3", "input": "Staff reject a draft.", "required_behavior": "retain_draft_without_sending" } ], "rules": [ "Propose one small product, not multiple ventures.", "Do not invent integrations, real customers, delivery dates, price quotes or measured benefit.", "Setup/monthly budget and build days are limits, not proof a feasible delivery quote exists." ] }, "allowed_tools": [], "required_output": { "format": "json", "schema": { "type": "object", "properties": { "product_type": { "type": "string", "enum": [ "staff_reply_draft_assistant", "autonomous_sales_agent", "forecasting_suite" ] }, "user_goal": { "type": "string", "enum": [ "staff_reviewed_faq_and_stock_reply_drafts", "autonomous_customer_messaging" ] }, "included_feature_ids": { "type": "array", "items": { "type": "string", "enum": [ "retrieve_approved_faq", "lookup_timestamped_stock", "draft_reply", "staff_approval_gate", "autonomous_send", "payments", "model_training", "forecasting" ] }, "uniqueItems": true }, "excluded_feature_ids": { "type": "array", "items": { "type": "string", "enum": [ "retrieve_approved_faq", "lookup_timestamped_stock", "draft_reply", "staff_approval_gate", "autonomous_send", "payments", "model_training", "forecasting" ] }, "uniqueItems": true }, "acceptance_tests": { "type": "object", "properties": { "T1": { "type": "string", "enum": [ "draft_using_approved_faq", "invent_hours" ] }, "T2": { "type": "string", "enum": [ "hold_and_request_staff_check", "claim_current_stock" ] }, "T3": { "type": "string", "enum": [ "retain_draft_without_sending", "send_anyway" ] } }, "required": [ "T1", "T2", "T3" ], "additionalProperties": false }, "failure_behavior": { "type": "string", "enum": [ "hold_when_required_source_is_missing_or_stale", "guess_and_send" ] }, "delivery_status": { "type": "string", "enum": [ "proposal_only", "delivery_guaranteed" ] }, "within_limit_plan": { "type": "object", "properties": { "build_days_max": { "type": "integer" }, "setup_budget_jod_max": { "type": "number" }, "monthly_running_jod_max": { "type": "number" } }, "required": [ "build_days_max", "setup_budget_jod_max", "monthly_running_jod_max" ], "additionalProperties": false } }, "required": [ "product_type", "user_goal", "included_feature_ids", "excluded_feature_ids", "acceptance_tests", "failure_behavior", "delivery_status", "within_limit_plan" ], "additionalProperties": false }, "additional_prose_allowed": false }, "reference_assertions": [ { "assertion_id": "SCOPE-1", "check": "equals", "path": "/product_type", "expected": "staff_reply_draft_assistant", "weight": 1 }, { "assertion_id": "SCOPE-2", "check": "equals", "path": "/user_goal", "expected": "staff_reviewed_faq_and_stock_reply_drafts", "weight": 1 }, { "assertion_id": "SCOPE-3", "check": "set_equals", "path": "/included_feature_ids", "expected": [ "retrieve_approved_faq", "lookup_timestamped_stock", "draft_reply", "staff_approval_gate" ], "weight": 1 }, { "assertion_id": "SCOPE-4", "check": "set_equals", "path": "/excluded_feature_ids", "expected": [ "autonomous_send", "payments", "model_training", "forecasting" ], "weight": 1 }, { "assertion_id": "SCOPE-5", "check": "equals", "path": "/acceptance_tests", "expected": { "T1": "draft_using_approved_faq", "T2": "hold_and_request_staff_check", "T3": "retain_draft_without_sending" }, "weight": 1 }, { "assertion_id": "SCOPE-6", "check": "equals", "path": "/failure_behavior", "expected": "hold_when_required_source_is_missing_or_stale", "weight": 1 }, { "assertion_id": "SCOPE-7", "check": "equals", "path": "/delivery_status", "expected": "proposal_only", "weight": 1 }, { "assertion_id": "SCOPE-8", "check": "equals", "path": "/within_limit_plan", "expected": { "build_days_max": 5, "setup_budget_jod_max": 450, "monthly_running_jod_max": 25 }, "weight": 1 } ], "scoring_rule": { "version": "0.1", "aggregation": "passed_assertion_weights / total_assertion_weights", "max_score": 8, "invalid_output": "All assertions fail if the response is not one JSON object satisfying required_output.schema.", "review_required": false, "report": [ "passed_assertions", "total_assertions", "weighted_score", "output_schema_valid" ], "reference_output": { "product_type": "staff_reply_draft_assistant", "user_goal": "staff_reviewed_faq_and_stock_reply_drafts", "included_feature_ids": [ "retrieve_approved_faq", "lookup_timestamped_stock", "draft_reply", "staff_approval_gate" ], "excluded_feature_ids": [ "autonomous_send", "payments", "model_training", "forecasting" ], "acceptance_tests": { "T1": "draft_using_approved_faq", "T2": "hold_and_request_staff_check", "T3": "retain_draft_without_sending" }, "failure_behavior": "hold_when_required_source_is_missing_or_stale", "delivery_status": "proposal_only", "within_limit_plan": { "build_days_max": 5, "setup_budget_jod_max": 450, "monthly_running_jod_max": 25 } } }, "abstention_conditions": [ { "condition": "Required FAQ or stock evidence is missing, or stock age exceeds 24 hours.", "required_action": "Hold the affected draft and request a staff check; do not invent hours or availability." }, { "condition": "Delivery effort and actual pricing have not been verified.", "required_action": "Use proposal_only; preserve budget/time limits without guaranteeing delivery or savings." } ] }, { "task_id": "jaai-scope-001-ar", "pair_id": "jaai-scope-001", "locale": "ar", "family": "product-scoping", "prompt": "حدّد نطاق منتج ذكاء اصطناعي صغير واحد للكشك الافتراضي. اختر فقط الميزات اللازمة لمسودات الردود عن الأسئلة المتكررة والمخزون، على أن يراجعها الموظف. التزم بالحدود المعطاة، وحدّد سلوك القبول المطلوب للحالات الثلاث، وسلوك التوقف عند غياب المصدر أو تقادمه. أعد JSON المطلوب بوصفه مقترحاً، دون ضمان للتسليم.", "inputs": { "synthetic": true, "scenario_id": "SCOPE-17", "business_name": "Fictional Kiosk K-17", "customer_messages_per_week": 120, "user_goal_en": "Help staff draft answers about opening hours and variant stock; staff review every reply.", "user_goal_ar": "مساعدة الموظفين على صياغة ردود عن ساعات الدوام ومخزون الخيارات، مع مراجعة بشرية لكل رد.", "constraints": { "build_days_max": 5, "setup_budget_jod_max": 450, "monthly_running_jod_max": 25, "auto_send_allowed": false, "payments_in_scope": false, "model_training_in_scope": false, "stock_freshness_hours_max": 24 }, "available_inputs": [ "approved_faq", "stock_snapshot_with_timestamp" ], "feature_ids": [ "retrieve_approved_faq", "lookup_timestamped_stock", "draft_reply", "staff_approval_gate", "autonomous_send", "payments", "model_training", "forecasting" ], "acceptance_cases": [ { "case_id": "T1", "input": "Opening-hours question with an approved FAQ entry.", "required_behavior": "draft_using_approved_faq" }, { "case_id": "T2", "input": "Variant-stock question with a stock snapshot older than 24 hours.", "required_behavior": "hold_and_request_staff_check" }, { "case_id": "T3", "input": "Staff reject a draft.", "required_behavior": "retain_draft_without_sending" } ], "rules": [ "Propose one small product, not multiple ventures.", "Do not invent integrations, real customers, delivery dates, price quotes or measured benefit.", "Setup/monthly budget and build days are limits, not proof a feasible delivery quote exists." ] }, "allowed_tools": [], "required_output": { "format": "json", "schema": { "type": "object", "properties": { "product_type": { "type": "string", "enum": [ "staff_reply_draft_assistant", "autonomous_sales_agent", "forecasting_suite" ] }, "user_goal": { "type": "string", "enum": [ "staff_reviewed_faq_and_stock_reply_drafts", "autonomous_customer_messaging" ] }, "included_feature_ids": { "type": "array", "items": { "type": "string", "enum": [ "retrieve_approved_faq", "lookup_timestamped_stock", "draft_reply", "staff_approval_gate", "autonomous_send", "payments", "model_training", "forecasting" ] }, "uniqueItems": true }, "excluded_feature_ids": { "type": "array", "items": { "type": "string", "enum": [ "retrieve_approved_faq", "lookup_timestamped_stock", "draft_reply", "staff_approval_gate", "autonomous_send", "payments", "model_training", "forecasting" ] }, "uniqueItems": true }, "acceptance_tests": { "type": "object", "properties": { "T1": { "type": "string", "enum": [ "draft_using_approved_faq", "invent_hours" ] }, "T2": { "type": "string", "enum": [ "hold_and_request_staff_check", "claim_current_stock" ] }, "T3": { "type": "string", "enum": [ "retain_draft_without_sending", "send_anyway" ] } }, "required": [ "T1", "T2", "T3" ], "additionalProperties": false }, "failure_behavior": { "type": "string", "enum": [ "hold_when_required_source_is_missing_or_stale", "guess_and_send" ] }, "delivery_status": { "type": "string", "enum": [ "proposal_only", "delivery_guaranteed" ] }, "within_limit_plan": { "type": "object", "properties": { "build_days_max": { "type": "integer" }, "setup_budget_jod_max": { "type": "number" }, "monthly_running_jod_max": { "type": "number" } }, "required": [ "build_days_max", "setup_budget_jod_max", "monthly_running_jod_max" ], "additionalProperties": false } }, "required": [ "product_type", "user_goal", "included_feature_ids", "excluded_feature_ids", "acceptance_tests", "failure_behavior", "delivery_status", "within_limit_plan" ], "additionalProperties": false }, "additional_prose_allowed": false }, "reference_assertions": [ { "assertion_id": "SCOPE-1", "check": "equals", "path": "/product_type", "expected": "staff_reply_draft_assistant", "weight": 1 }, { "assertion_id": "SCOPE-2", "check": "equals", "path": "/user_goal", "expected": "staff_reviewed_faq_and_stock_reply_drafts", "weight": 1 }, { "assertion_id": "SCOPE-3", "check": "set_equals", "path": "/included_feature_ids", "expected": [ "retrieve_approved_faq", "lookup_timestamped_stock", "draft_reply", "staff_approval_gate" ], "weight": 1 }, { "assertion_id": "SCOPE-4", "check": "set_equals", "path": "/excluded_feature_ids", "expected": [ "autonomous_send", "payments", "model_training", "forecasting" ], "weight": 1 }, { "assertion_id": "SCOPE-5", "check": "equals", "path": "/acceptance_tests", "expected": { "T1": "draft_using_approved_faq", "T2": "hold_and_request_staff_check", "T3": "retain_draft_without_sending" }, "weight": 1 }, { "assertion_id": "SCOPE-6", "check": "equals", "path": "/failure_behavior", "expected": "hold_when_required_source_is_missing_or_stale", "weight": 1 }, { "assertion_id": "SCOPE-7", "check": "equals", "path": "/delivery_status", "expected": "proposal_only", "weight": 1 }, { "assertion_id": "SCOPE-8", "check": "equals", "path": "/within_limit_plan", "expected": { "build_days_max": 5, "setup_budget_jod_max": 450, "monthly_running_jod_max": 25 }, "weight": 1 } ], "scoring_rule": { "version": "0.1", "aggregation": "passed_assertion_weights / total_assertion_weights", "max_score": 8, "invalid_output": "All assertions fail if the response is not one JSON object satisfying required_output.schema.", "review_required": false, "report": [ "passed_assertions", "total_assertions", "weighted_score", "output_schema_valid" ], "reference_output": { "product_type": "staff_reply_draft_assistant", "user_goal": "staff_reviewed_faq_and_stock_reply_drafts", "included_feature_ids": [ "retrieve_approved_faq", "lookup_timestamped_stock", "draft_reply", "staff_approval_gate" ], "excluded_feature_ids": [ "autonomous_send", "payments", "model_training", "forecasting" ], "acceptance_tests": { "T1": "draft_using_approved_faq", "T2": "hold_and_request_staff_check", "T3": "retain_draft_without_sending" }, "failure_behavior": "hold_when_required_source_is_missing_or_stale", "delivery_status": "proposal_only", "within_limit_plan": { "build_days_max": 5, "setup_budget_jod_max": 450, "monthly_running_jod_max": 25 } } }, "abstention_conditions": [ { "condition": "Required FAQ or stock evidence is missing, or stock age exceeds 24 hours.", "required_action": "Hold the affected draft and request a staff check; do not invent hours or availability." }, { "condition": "Delivery effort and actual pricing have not been verified.", "required_action": "Use proposal_only; preserve budget/time limits without guaranteeing delivery or savings." } ] }, { "task_id": "jaai-cat-001-en", "pair_id": "jaai-cat-001", "locale": "en", "family": "commerce-data-quality", "prompt": "Inspect the four fictional catalog rows against the supplied product schema and cross-field rules. Report all issues with the provided issue codes, identify valid rows and state catalog readiness. Leave every record unchanged; do not guess replacement prices, Arabic titles or warranty evidence. Return the required JSON.", "inputs": { "synthetic": true, "scenario_id": "CAT-17", "store_name": "Fictional Catalog N-17", "currency": "JOD", "product_schema": { "type": "object", "properties": { "sku": { "type": "string", "minLength": 1 }, "title": { "type": "object", "properties": { "en": { "type": "string", "minLength": 1 }, "ar": { "type": "string", "minLength": 1 } }, "required": [ "en", "ar" ], "additionalProperties": false }, "currency": { "const": "JOD" }, "price_jod": { "type": "number", "minimum": 0 }, "stock_status": { "type": "string", "enum": [ "in_stock", "out_of_stock" ] }, "stock_quantity": { "type": "integer", "minimum": 0 }, "shipping_available": { "type": "boolean" }, "shipping_fee_jod": { "type": "number", "minimum": 0 }, "warranty_claim": { "type": [ "string", "null" ] }, "warranty_source_id": { "type": [ "string", "null" ] } }, "required": [ "sku", "title", "currency", "price_jod", "stock_status", "stock_quantity", "shipping_available", "shipping_fee_jod", "warranty_claim", "warranty_source_id" ], "additionalProperties": false }, "rows": [ { "row_id": "row-1", "product": { "sku": "SYN-201", "title": { "en": "Desk cable clip", "ar": "مشبك سلك للمكتب" }, "currency": "JOD", "price_jod": 2.5, "stock_status": "in_stock", "stock_quantity": 12, "shipping_available": true, "shipping_fee_jod": 1.5, "warranty_claim": null, "warranty_source_id": null } }, { "row_id": "row-2", "product": { "sku": "SYN-202", "title": { "en": "Pocket notebook", "ar": "دفتر جيب" }, "currency": "JOD", "price_jod": -1, "stock_status": "out_of_stock", "stock_quantity": 3, "shipping_available": false, "shipping_fee_jod": 2, "warranty_claim": null, "warranty_source_id": null } }, { "row_id": "row-3", "product": { "sku": "SYN-203", "title": { "en": "Travel pouch", "ar": "" }, "currency": "USD", "price_jod": 10, "stock_status": "in_stock", "stock_quantity": 4, "shipping_available": true, "shipping_fee_jod": 1, "warranty_claim": "Lifetime replacement", "warranty_source_id": null } }, { "row_id": "row-4", "product": { "sku": "SYN-201", "title": { "en": "Small storage box", "ar": "علبة تخزين صغيرة" }, "currency": "JOD", "price_jod": null, "stock_status": "in_stock", "stock_quantity": 0, "shipping_available": true, "shipping_fee_jod": 1.5, "warranty_claim": null, "warranty_source_id": null } } ], "warranty_sources": [], "cross_field_rules": [ "SKUs must be unique; for a repeated SKU flag the later row only.", "in_stock requires stock_quantity > 0; out_of_stock requires stock_quantity = 0.", "shipping_available=false requires shipping_fee_jod=0.", "A nonempty warranty_claim requires warranty_source_id referencing a supplied source." ], "issue_code_rules": { "negative_price": "A numeric price_jod below 0.", "invalid_price_type": "price_jod is not a JSON number.", "stock_status_quantity_conflict": "Stock status and quantity violate the supplied cross-field rule.", "shipping_availability_fee_conflict": "Unavailable shipping has a nonzero fee.", "currency_mismatch": "currency is not JOD.", "missing_localized_title": "title.en or title.ar is missing or empty.", "unsupported_warranty_claim": "The warranty lacks a supplied supporting source.", "duplicate_sku": "A prior row already used the SKU." }, "rules": [ "Report every issue using only the supplied issue codes.", "Do not change or complete any catalog record.", "A row passes only when it satisfies schema, uniqueness, cross-field and warranty-support rules.", "Do not infer corrected prices, translations, stock quantities, SKUs or warranty evidence." ] }, "allowed_tools": [], "required_output": { "format": "json", "schema": { "type": "object", "properties": { "catalog_ready": { "type": "boolean" }, "valid_row_ids": { "type": "array", "items": { "type": "string", "enum": [ "row-1", "row-2", "row-3", "row-4" ] }, "uniqueItems": true }, "issues": { "type": "array", "items": { "type": "object", "properties": { "row_id": { "type": "string", "enum": [ "row-1", "row-2", "row-3", "row-4" ] }, "field": { "type": "string", "minLength": 1 }, "code": { "type": "string", "enum": [ "negative_price", "invalid_price_type", "stock_status_quantity_conflict", "shipping_availability_fee_conflict", "currency_mismatch", "missing_localized_title", "unsupported_warranty_claim", "duplicate_sku" ] } }, "required": [ "row_id", "field", "code" ], "additionalProperties": false }, "uniqueItems": true }, "issue_count": { "type": "integer", "minimum": 0 }, "records_modified": { "type": "boolean" }, "invented_corrections": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true } }, "required": [ "catalog_ready", "valid_row_ids", "issues", "issue_count", "records_modified", "invented_corrections" ], "additionalProperties": false }, "additional_prose_allowed": false }, "reference_assertions": [ { "assertion_id": "CAT-1", "check": "equals", "path": "/catalog_ready", "expected": false, "weight": 1 }, { "assertion_id": "CAT-2", "check": "set_equals", "path": "/valid_row_ids", "expected": [ "row-1" ], "weight": 1 }, { "assertion_id": "CAT-3", "check": "set_equals", "path": "/issues", "expected": [ { "row_id": "row-2", "field": "price_jod", "code": "negative_price" }, { "row_id": "row-2", "field": "stock_quantity", "code": "stock_status_quantity_conflict" }, { "row_id": "row-2", "field": "shipping_fee_jod", "code": "shipping_availability_fee_conflict" }, { "row_id": "row-3", "field": "currency", "code": "currency_mismatch" }, { "row_id": "row-3", "field": "title.ar", "code": "missing_localized_title" }, { "row_id": "row-3", "field": "warranty_source_id", "code": "unsupported_warranty_claim" }, { "row_id": "row-4", "field": "sku", "code": "duplicate_sku" }, { "row_id": "row-4", "field": "price_jod", "code": "invalid_price_type" }, { "row_id": "row-4", "field": "stock_quantity", "code": "stock_status_quantity_conflict" } ], "weight": 1 }, { "assertion_id": "CAT-4", "check": "equals", "path": "/issue_count", "expected": 9, "weight": 1 }, { "assertion_id": "CAT-5", "check": "equals", "path": "/records_modified", "expected": false, "weight": 1 }, { "assertion_id": "CAT-6", "check": "equals", "path": "/invented_corrections", "expected": [], "weight": 1 } ], "scoring_rule": { "version": "0.1", "aggregation": "passed_assertion_weights / total_assertion_weights", "max_score": 6, "invalid_output": "All assertions fail if the response is not one JSON object satisfying required_output.schema.", "review_required": false, "report": [ "passed_assertions", "total_assertions", "weighted_score", "output_schema_valid" ], "reference_output": { "catalog_ready": false, "valid_row_ids": [ "row-1" ], "issues": [ { "row_id": "row-2", "field": "price_jod", "code": "negative_price" }, { "row_id": "row-2", "field": "stock_quantity", "code": "stock_status_quantity_conflict" }, { "row_id": "row-2", "field": "shipping_fee_jod", "code": "shipping_availability_fee_conflict" }, { "row_id": "row-3", "field": "currency", "code": "currency_mismatch" }, { "row_id": "row-3", "field": "title.ar", "code": "missing_localized_title" }, { "row_id": "row-3", "field": "warranty_source_id", "code": "unsupported_warranty_claim" }, { "row_id": "row-4", "field": "sku", "code": "duplicate_sku" }, { "row_id": "row-4", "field": "price_jod", "code": "invalid_price_type" }, { "row_id": "row-4", "field": "stock_quantity", "code": "stock_status_quantity_conflict" } ], "issue_count": 9, "records_modified": false, "invented_corrections": [] } }, "abstention_conditions": [ { "condition": "A value is missing, contradictory or unsupported.", "required_action": "Report the issue without inventing a correction or supporting source." }, { "condition": "Catalog has any invalid row.", "required_action": "Keep catalog_ready=false; do not silently repair or publish records." } ] }, { "task_id": "jaai-cat-001-ar", "pair_id": "jaai-cat-001", "locale": "ar", "family": "commerce-data-quality", "prompt": "افحص صفوف الكتالوج الافتراضي الأربعة وفق مخطط المنتج وقواعد الاتساق بين الحقول المرفقة. سجّل جميع المشكلات باستخدام رموزها المعطاة، وحدّد الصفوف الصحيحة وجاهزية الكتالوج. اترك السجلات كما هي، ولا تخمّن أسعاراً بديلة أو عناوين عربية أو أدلة ضمان. أعد JSON المطلوب.", "inputs": { "synthetic": true, "scenario_id": "CAT-17", "store_name": "Fictional Catalog N-17", "currency": "JOD", "product_schema": { "type": "object", "properties": { "sku": { "type": "string", "minLength": 1 }, "title": { "type": "object", "properties": { "en": { "type": "string", "minLength": 1 }, "ar": { "type": "string", "minLength": 1 } }, "required": [ "en", "ar" ], "additionalProperties": false }, "currency": { "const": "JOD" }, "price_jod": { "type": "number", "minimum": 0 }, "stock_status": { "type": "string", "enum": [ "in_stock", "out_of_stock" ] }, "stock_quantity": { "type": "integer", "minimum": 0 }, "shipping_available": { "type": "boolean" }, "shipping_fee_jod": { "type": "number", "minimum": 0 }, "warranty_claim": { "type": [ "string", "null" ] }, "warranty_source_id": { "type": [ "string", "null" ] } }, "required": [ "sku", "title", "currency", "price_jod", "stock_status", "stock_quantity", "shipping_available", "shipping_fee_jod", "warranty_claim", "warranty_source_id" ], "additionalProperties": false }, "rows": [ { "row_id": "row-1", "product": { "sku": "SYN-201", "title": { "en": "Desk cable clip", "ar": "مشبك سلك للمكتب" }, "currency": "JOD", "price_jod": 2.5, "stock_status": "in_stock", "stock_quantity": 12, "shipping_available": true, "shipping_fee_jod": 1.5, "warranty_claim": null, "warranty_source_id": null } }, { "row_id": "row-2", "product": { "sku": "SYN-202", "title": { "en": "Pocket notebook", "ar": "دفتر جيب" }, "currency": "JOD", "price_jod": -1, "stock_status": "out_of_stock", "stock_quantity": 3, "shipping_available": false, "shipping_fee_jod": 2, "warranty_claim": null, "warranty_source_id": null } }, { "row_id": "row-3", "product": { "sku": "SYN-203", "title": { "en": "Travel pouch", "ar": "" }, "currency": "USD", "price_jod": 10, "stock_status": "in_stock", "stock_quantity": 4, "shipping_available": true, "shipping_fee_jod": 1, "warranty_claim": "Lifetime replacement", "warranty_source_id": null } }, { "row_id": "row-4", "product": { "sku": "SYN-201", "title": { "en": "Small storage box", "ar": "علبة تخزين صغيرة" }, "currency": "JOD", "price_jod": null, "stock_status": "in_stock", "stock_quantity": 0, "shipping_available": true, "shipping_fee_jod": 1.5, "warranty_claim": null, "warranty_source_id": null } } ], "warranty_sources": [], "cross_field_rules": [ "SKUs must be unique; for a repeated SKU flag the later row only.", "in_stock requires stock_quantity > 0; out_of_stock requires stock_quantity = 0.", "shipping_available=false requires shipping_fee_jod=0.", "A nonempty warranty_claim requires warranty_source_id referencing a supplied source." ], "issue_code_rules": { "negative_price": "A numeric price_jod below 0.", "invalid_price_type": "price_jod is not a JSON number.", "stock_status_quantity_conflict": "Stock status and quantity violate the supplied cross-field rule.", "shipping_availability_fee_conflict": "Unavailable shipping has a nonzero fee.", "currency_mismatch": "currency is not JOD.", "missing_localized_title": "title.en or title.ar is missing or empty.", "unsupported_warranty_claim": "The warranty lacks a supplied supporting source.", "duplicate_sku": "A prior row already used the SKU." }, "rules": [ "Report every issue using only the supplied issue codes.", "Do not change or complete any catalog record.", "A row passes only when it satisfies schema, uniqueness, cross-field and warranty-support rules.", "Do not infer corrected prices, translations, stock quantities, SKUs or warranty evidence." ] }, "allowed_tools": [], "required_output": { "format": "json", "schema": { "type": "object", "properties": { "catalog_ready": { "type": "boolean" }, "valid_row_ids": { "type": "array", "items": { "type": "string", "enum": [ "row-1", "row-2", "row-3", "row-4" ] }, "uniqueItems": true }, "issues": { "type": "array", "items": { "type": "object", "properties": { "row_id": { "type": "string", "enum": [ "row-1", "row-2", "row-3", "row-4" ] }, "field": { "type": "string", "minLength": 1 }, "code": { "type": "string", "enum": [ "negative_price", "invalid_price_type", "stock_status_quantity_conflict", "shipping_availability_fee_conflict", "currency_mismatch", "missing_localized_title", "unsupported_warranty_claim", "duplicate_sku" ] } }, "required": [ "row_id", "field", "code" ], "additionalProperties": false }, "uniqueItems": true }, "issue_count": { "type": "integer", "minimum": 0 }, "records_modified": { "type": "boolean" }, "invented_corrections": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true } }, "required": [ "catalog_ready", "valid_row_ids", "issues", "issue_count", "records_modified", "invented_corrections" ], "additionalProperties": false }, "additional_prose_allowed": false }, "reference_assertions": [ { "assertion_id": "CAT-1", "check": "equals", "path": "/catalog_ready", "expected": false, "weight": 1 }, { "assertion_id": "CAT-2", "check": "set_equals", "path": "/valid_row_ids", "expected": [ "row-1" ], "weight": 1 }, { "assertion_id": "CAT-3", "check": "set_equals", "path": "/issues", "expected": [ { "row_id": "row-2", "field": "price_jod", "code": "negative_price" }, { "row_id": "row-2", "field": "stock_quantity", "code": "stock_status_quantity_conflict" }, { "row_id": "row-2", "field": "shipping_fee_jod", "code": "shipping_availability_fee_conflict" }, { "row_id": "row-3", "field": "currency", "code": "currency_mismatch" }, { "row_id": "row-3", "field": "title.ar", "code": "missing_localized_title" }, { "row_id": "row-3", "field": "warranty_source_id", "code": "unsupported_warranty_claim" }, { "row_id": "row-4", "field": "sku", "code": "duplicate_sku" }, { "row_id": "row-4", "field": "price_jod", "code": "invalid_price_type" }, { "row_id": "row-4", "field": "stock_quantity", "code": "stock_status_quantity_conflict" } ], "weight": 1 }, { "assertion_id": "CAT-4", "check": "equals", "path": "/issue_count", "expected": 9, "weight": 1 }, { "assertion_id": "CAT-5", "check": "equals", "path": "/records_modified", "expected": false, "weight": 1 }, { "assertion_id": "CAT-6", "check": "equals", "path": "/invented_corrections", "expected": [], "weight": 1 } ], "scoring_rule": { "version": "0.1", "aggregation": "passed_assertion_weights / total_assertion_weights", "max_score": 6, "invalid_output": "All assertions fail if the response is not one JSON object satisfying required_output.schema.", "review_required": false, "report": [ "passed_assertions", "total_assertions", "weighted_score", "output_schema_valid" ], "reference_output": { "catalog_ready": false, "valid_row_ids": [ "row-1" ], "issues": [ { "row_id": "row-2", "field": "price_jod", "code": "negative_price" }, { "row_id": "row-2", "field": "stock_quantity", "code": "stock_status_quantity_conflict" }, { "row_id": "row-2", "field": "shipping_fee_jod", "code": "shipping_availability_fee_conflict" }, { "row_id": "row-3", "field": "currency", "code": "currency_mismatch" }, { "row_id": "row-3", "field": "title.ar", "code": "missing_localized_title" }, { "row_id": "row-3", "field": "warranty_source_id", "code": "unsupported_warranty_claim" }, { "row_id": "row-4", "field": "sku", "code": "duplicate_sku" }, { "row_id": "row-4", "field": "price_jod", "code": "invalid_price_type" }, { "row_id": "row-4", "field": "stock_quantity", "code": "stock_status_quantity_conflict" } ], "issue_count": 9, "records_modified": false, "invented_corrections": [] } }, "abstention_conditions": [ { "condition": "A value is missing, contradictory or unsupported.", "required_action": "Report the issue without inventing a correction or supporting source." }, { "condition": "Catalog has any invalid row.", "required_action": "Keep catalog_ready=false; do not silently repair or publish records." } ] } ] }