{
  "name": "RiseHush Foundation Models evaluation protocol",
  "version": "2026-07-26",
  "canonicalArticle": "https://risehush.com/journal/foundation-models-quality-pipeline/",
  "scope": "A repeatable field contract for validating availability, latency, rejection, fallback, and local persistence in the RiseHush on-device quote-generation path.",
  "resultsStatus": "Protocol only. This file contains no physical-device measurements and must not be cited as a benchmark result.",
  "recordBeforeEachRun": [
    "deviceModel",
    "osVersionAndBuild",
    "appBuild",
    "localeIdentifier",
    "appleIntelligenceState",
    "systemModelAvailability",
    "modelAssetsReady",
    "networkState",
    "entryPoint",
    "fixtureId"
  ],
  "recordAfterEachRun": [
    "startedAt",
    "elapsedMilliseconds",
    "terminalBranch",
    "partialSnapshotsObserved",
    "deterministicGateResult",
    "noveltyGateResult",
    "judgeOverall",
    "distinctivenessOverall",
    "rejectionReason",
    "fallbackShown",
    "persisted",
    "persistedSource",
    "persistedLanguageCode",
    "notes"
  ],
  "entryPoints": [
    {
      "id": "foreground-stream",
      "budgetSeconds": 45,
      "expected": "Partial snapshots may appear; rejected provisional text must not persist."
    },
    {
      "id": "siri",
      "budgetSeconds": 20,
      "expected": "A bounded response or reason-specific curated fallback."
    },
    {
      "id": "background-pool",
      "budgetSeconds": 120,
      "expected": "Accepted local rows only; once model assets are ready, no RiseHush inference endpoint is required."
    },
    {
      "id": "widget-read",
      "budgetSeconds": 0,
      "expected": "Reads a persisted row and never starts Foundation Models generation in the extension."
    }
  ],
  "requiredScenarios": [
    {
      "id": "ready-supported-locale",
      "setup": "Compatible device, Apple Intelligence enabled, system model available, supported locale.",
      "assert": [
        "The generation path can start.",
        "The interface remains responsive during the model turn.",
        "Only a candidate accepted by every required gate can persist."
      ]
    },
    {
      "id": "ready-airplane-mode",
      "setup": "Model and required language assets already ready, then enable Airplane Mode.",
      "assert": [
        "No RiseHush inference endpoint is required.",
        "A successful result follows the same local quality and persistence path."
      ]
    },
    {
      "id": "device-ineligible",
      "setup": "Hardware or OS without eligible Apple Intelligence support.",
      "assert": [
        "Generation reports unavailable.",
        "The curated library remains usable.",
        "No placeholder generated row is inserted."
      ]
    },
    {
      "id": "model-assets-not-ready",
      "setup": "Compatible device while the system model reports assets unavailable or preparing.",
      "assert": [
        "The app distinguishes not-ready from permanent device ineligibility.",
        "The curated library remains usable.",
        "No generated row is inserted."
      ]
    },
    {
      "id": "unsupported-locale",
      "setup": "Active locale unsupported by direct Foundation Models generation.",
      "assert": [
        "The direct path does not silently claim support.",
        "Where local Translation assets support the pair, record the translated fallback as a distinct branch.",
        "Do not report translated output as having repeated the full target-language quality stack."
      ]
    },
    {
      "id": "deterministic-rejections",
      "fixtures": [
        "empty",
        "wrapped-in-quotes",
        "multiple-sentences",
        "too-short",
        "too-long",
        "wrong-language",
        "banned-residue",
        "known-cliche"
      ],
      "assert": [
        "Every fixture is rejected before persistence.",
        "The rejection reason is recorded without storing the draft text in diagnostics."
      ]
    },
    {
      "id": "semantic-near-duplicate",
      "setup": "Candidate at or above the configured local similarity threshold against a catalog fixture.",
      "assert": [
        "The candidate is rejected before judging and persistence."
      ]
    },
    {
      "id": "judge-or-distinctiveness-rejection",
      "setup": "Candidate below the current acceptance threshold.",
      "assert": [
        "A streamed candidate aborts instead of animating a replacement.",
        "A finite pool may try another candidate.",
        "Rejected text is not persisted."
      ]
    },
    {
      "id": "mapped-runtime-errors",
      "fixtures": [
        "guardrail",
        "refusal",
        "rate-limit",
        "context-limit",
        "structured-parsing",
        "concurrent-request",
        "timeout"
      ],
      "assert": [
        "The branch terminates within its time budget.",
        "No unbounded inline retry occurs.",
        "No generated row is inserted."
      ]
    },
    {
      "id": "persistence-failure",
      "setup": "Use a test store that fails save after insertion.",
      "assert": [
        "Transient inserted models are removed.",
        "The caller receives a persistence failure.",
        "No half-saved generated quote remains."
      ]
    }
  ],
  "reportingContract": {
    "requiredAggregatesWhenResultsExist": [
      "sampleCount",
      "acceptedCount",
      "rejectionCountsByReason",
      "fallbackCountsByReason",
      "timeoutRate",
      "latencyP50Milliseconds",
      "latencyP95Milliseconds",
      "toolCallCorrectnessRate",
      "persistenceFailureCount"
    ],
    "prohibitions": [
      "Do not infer live-device results from source wiring or a test script.",
      "Do not publish prompt text, generated text, saved quotes, or private personalization signals in diagnostics.",
      "Do not compare model versions without recording the exact OS build and readiness state.",
      "Do not call translated fallback equivalent to direct target-language generation."
    ]
  },
  "primaryDocumentation": [
    "https://developer.apple.com/documentation/FoundationModels",
    "https://developer.apple.com/documentation/foundationmodels/systemlanguagemodel",
    "https://developer.apple.com/documentation/foundationmodels/supporting-languages-and-locales-with-foundation-models",
    "https://developer.apple.com/documentation/foundationmodels/generating-swift-data-structures-with-guided-generation",
    "https://developer.apple.com/documentation/foundationmodels/analyzing-the-runtime-performance-of-your-foundation-models-app"
  ]
}