{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://steadywrk.app/research/jordan-applied-ai-benchmark/v0.1/task-schema.json",
  "title": "Jordan Applied AI Benchmark v0.1 synthetic example-pack schema",
  "type": "object",
  "required": [
    "schema_version",
    "dataset_id",
    "version",
    "title",
    "description",
    "author",
    "sponsor",
    "synthetic",
    "license",
    "provenance_file",
    "authored_date",
    "pair_count",
    "prompt_count",
    "review_status",
    "tasks"
  ],
  "additionalProperties": false,
  "properties": {
    "schema_version": {
      "const": "0.1"
    },
    "dataset_id": {
      "const": "steadywrk-jordan-applied-ai-benchmark-examples"
    },
    "version": {
      "const": "0.1"
    },
    "title": {
      "type": "string",
      "minLength": 1
    },
    "description": {
      "type": "string",
      "minLength": 1
    },
    "author": {
      "type": "object",
      "properties": {
        "@id": {
          "const": "https://steadywrk.app/team/yousof#person"
        },
        "name": {
          "const": "Yousof Almalkawi"
        }
      },
      "required": [
        "@id",
        "name"
      ],
      "additionalProperties": false
    },
    "sponsor": {
      "const": "STEADYWRK"
    },
    "synthetic": {
      "const": true
    },
    "license": {
      "const": "https://steadywrk.app/terms"
    },
    "provenance_file": {
      "const": "provenance.txt"
    },
    "authored_date": {
      "type": "string",
      "pattern": "^\\d{4}-\\d{2}-\\d{2}$"
    },
    "pair_count": {
      "const": 6
    },
    "prompt_count": {
      "const": 12
    },
    "review_status": {
      "type": "object",
      "properties": {
        "bilingual_independent_review": {
          "const": "not_performed"
        },
        "empirical_validation": {
          "const": "not_run"
        }
      },
      "required": [
        "bilingual_independent_review",
        "empirical_validation"
      ],
      "additionalProperties": false
    },
    "tasks": {
      "type": "array",
      "minItems": 12,
      "maxItems": 12,
      "items": {
        "$ref": "#/$defs/task"
      },
      "allOf": [
        {
          "contains": {
            "properties": {
              "family": {
                "const": "evidence-research"
              }
            },
            "required": [
              "family"
            ]
          },
          "minContains": 2,
          "maxContains": 2
        },
        {
          "contains": {
            "properties": {
              "family": {
                "const": "business-decisions"
              }
            },
            "required": [
              "family"
            ]
          },
          "minContains": 2,
          "maxContains": 2
        },
        {
          "contains": {
            "properties": {
              "family": {
                "const": "bilingual-communication"
              }
            },
            "required": [
              "family"
            ]
          },
          "minContains": 2,
          "maxContains": 2
        },
        {
          "contains": {
            "properties": {
              "family": {
                "const": "automation-planning"
              }
            },
            "required": [
              "family"
            ]
          },
          "minContains": 2,
          "maxContains": 2
        },
        {
          "contains": {
            "properties": {
              "family": {
                "const": "product-scoping"
              }
            },
            "required": [
              "family"
            ]
          },
          "minContains": 2,
          "maxContains": 2
        },
        {
          "contains": {
            "properties": {
              "family": {
                "const": "commerce-data-quality"
              }
            },
            "required": [
              "family"
            ]
          },
          "minContains": 2,
          "maxContains": 2
        },
        {
          "contains": {
            "properties": {
              "locale": {
                "const": "en"
              }
            },
            "required": [
              "locale"
            ]
          },
          "minContains": 6,
          "maxContains": 6
        },
        {
          "contains": {
            "properties": {
              "locale": {
                "const": "ar"
              }
            },
            "required": [
              "locale"
            ]
          },
          "minContains": 6,
          "maxContains": 6
        }
      ]
    }
  },
  "$defs": {
    "task": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "task_id",
        "pair_id",
        "locale",
        "family",
        "prompt",
        "inputs",
        "allowed_tools",
        "required_output",
        "reference_assertions",
        "scoring_rule",
        "abstention_conditions"
      ],
      "properties": {
        "task_id": {
          "type": "string",
          "pattern": "^jaai-(ebr|dec|com|aut|scope|cat)-001-(en|ar)$"
        },
        "pair_id": {
          "type": "string",
          "pattern": "^jaai-(ebr|dec|com|aut|scope|cat)-001$"
        },
        "locale": {
          "type": "string",
          "enum": [
            "en",
            "ar"
          ]
        },
        "family": {
          "type": "string",
          "enum": [
            "evidence-research",
            "business-decisions",
            "bilingual-communication",
            "automation-planning",
            "product-scoping",
            "commerce-data-quality"
          ]
        },
        "prompt": {
          "type": "string",
          "minLength": 1
        },
        "inputs": {
          "type": "object",
          "required": [
            "synthetic"
          ],
          "properties": {
            "synthetic": {
              "const": true
            }
          }
        },
        "allowed_tools": {
          "type": "array",
          "maxItems": 0
        },
        "required_output": {
          "type": "object",
          "properties": {
            "format": {
              "const": "json"
            },
            "schema": {
              "type": "object"
            },
            "additional_prose_allowed": {
              "const": false
            }
          },
          "required": [
            "format",
            "schema",
            "additional_prose_allowed"
          ],
          "additionalProperties": false
        },
        "reference_assertions": {
          "type": "array",
          "minItems": 1,
          "items": {
            "$ref": "#/$defs/assertion"
          }
        },
        "scoring_rule": {
          "type": "object",
          "properties": {
            "version": {
              "const": "0.1"
            },
            "aggregation": {
              "const": "passed_assertion_weights / total_assertion_weights"
            },
            "max_score": {
              "type": "integer",
              "minimum": 1
            },
            "invalid_output": {
              "type": "string",
              "minLength": 1
            },
            "review_required": {
              "type": "boolean"
            },
            "report": {
              "type": "array",
              "items": {
                "type": "string",
                "minLength": 1
              },
              "uniqueItems": true
            },
            "reference_output": {
              "type": "object"
            }
          },
          "required": [
            "version",
            "aggregation",
            "max_score",
            "invalid_output",
            "review_required",
            "report",
            "reference_output"
          ],
          "additionalProperties": false
        },
        "abstention_conditions": {
          "type": "array",
          "minItems": 1,
          "items": {
            "type": "object",
            "properties": {
              "condition": {
                "type": "string",
                "minLength": 1
              },
              "required_action": {
                "type": "string",
                "minLength": 1
              }
            },
            "required": [
              "condition",
              "required_action"
            ],
            "additionalProperties": false
          }
        }
      }
    },
    "assertion": {
      "type": "object",
      "required": [
        "assertion_id",
        "check",
        "weight"
      ],
      "additionalProperties": false,
      "properties": {
        "assertion_id": {
          "type": "string",
          "minLength": 1
        },
        "check": {
          "type": "string",
          "enum": [
            "equals",
            "set_equals",
            "number_close",
            "review"
          ]
        },
        "weight": {
          "const": 1
        },
        "path": {
          "type": "string",
          "pattern": "^/"
        },
        "expected": {},
        "absolute_tolerance": {
          "type": "number",
          "minimum": 0
        },
        "criterion": {
          "type": "string",
          "minLength": 1
        },
        "pass_anchor": {
          "type": "string",
          "minLength": 1
        },
        "fail_anchor": {
          "type": "string",
          "minLength": 1
        }
      },
      "allOf": [
        {
          "if": {
            "properties": {
              "check": {
                "enum": [
                  "equals",
                  "set_equals",
                  "number_close"
                ]
              }
            }
          },
          "then": {
            "required": [
              "path",
              "expected"
            ]
          }
        },
        {
          "if": {
            "properties": {
              "check": {
                "const": "number_close"
              }
            }
          },
          "then": {
            "required": [
              "absolute_tolerance"
            ]
          }
        },
        {
          "if": {
            "properties": {
              "check": {
                "const": "review"
              }
            }
          },
          "then": {
            "required": [
              "criterion",
              "pass_anchor",
              "fail_anchor"
            ]
          }
        }
      ]
    }
  }
}