{
  "question": "Does Apple's Speech framework (SpeechAnalyzer / SpeechTranscriber) return different transcripts on macOS 27 than on macOS 26, and does it differ by Mac hardware tier?",
  "answer": "No on both counts. Across macOS 26.5.1 and 27.0.1, and across an M2 Pro and an M4 Pro, the framework returns the same transcripts. Word-level output is identical between the two machines on all 300 clips.",
  "date": "2026-10-12",
  "corpus": {
    "name": "LibriSpeech test-clean",
    "source": "https://www.openslr.org/resources/12/test-clean.tar.gz",
    "clips": 300,
    "selection": "first 300 utterances by id from the 2620-utterance manifest",
    "audio": "16 kHz mono WAV, converted with ffmpeg",
    "audioSeconds": 2641.09,
    "referenceWords": 7083
  },
  "scoring": {
    "metric": "word error rate, corpus-level (total edit distance / total reference words)",
    "normalizer": "lowercase, strip punctuation except apostrophes (compare.py in this directory)",
    "note": "This is a simpler normalizer than the one behind the 2.12% figure published for the full 2620-clip test-clean split in 2026-07, and the subset is 300 clips, so these percentages are internally comparable to each other and NOT comparable to that number.",
    "primaryEvidence": "byte-identical transcript counts, not WER: an unchanged model reproduces the same string, a changed one does not"
  },
  "runs": [
    {
      "id": "published-speechtranscriber-m2pro-macos26",
      "module": "SpeechTranscriber",
      "chip": "Apple M2 Pro",
      "memoryGB": 32,
      "os": "macOS 26.5.1 (25F80)",
      "werPercent": 1.67,
      "file": "https://lyonesse.app/data/speech-benchmark/raw-transcripts-apple.json.gz",
      "note": "The Round 1 transcripts, published 2026-07-13. Scored here on the same 300-clip subset with the normalizer above."
    },
    {
      "id": "speechtranscriber-m2pro-macos27",
      "module": "SpeechTranscriber",
      "chip": "Apple M2 Pro",
      "memoryGB": 32,
      "os": "macOS 27.0.1 (26A434)",
      "werPercent": 1.64,
      "file": "speechtranscriber-m2pro-macos27.json"
    },
    {
      "id": "speechtranscriber-m4pro-macos27",
      "module": "SpeechTranscriber",
      "chip": "Apple M4 Pro",
      "memoryGB": 24,
      "os": "macOS 27.0.1 (26A434)",
      "werPercent": 1.64,
      "file": "speechtranscriber-m4pro-macos27.json"
    },
    {
      "id": "speechtranscriber-m4pro-macos27-dictation-enabled",
      "module": "SpeechTranscriber",
      "chip": "Apple M4 Pro",
      "memoryGB": 24,
      "os": "macOS 27.0.1 (26A434)",
      "werPercent": 1.64,
      "file": "speechtranscriber-m4pro-macos27-dictation-enabled.json",
      "note": "Repeat of the run above with system dictation switched on. Byte-identical to it on all 300 clips."
    },
    {
      "id": "dictationtranscriber-m2pro-macos27",
      "module": "DictationTranscriber(.longDictation)",
      "chip": "Apple M2 Pro",
      "memoryGB": 32,
      "os": "macOS 27.0.1 (26A434)",
      "werPercent": 7.13,
      "file": "dictationtranscriber-m2pro-macos27.json"
    },
    {
      "id": "dictationtranscriber-m4pro-macos27",
      "module": "DictationTranscriber(.longDictation)",
      "chip": "Apple M4 Pro",
      "memoryGB": 24,
      "os": "macOS 27.0.1 (26A434)",
      "werPercent": 7.12,
      "file": "dictationtranscriber-m4pro-macos27.json"
    }
  ],
  "pairwise": [
    { "a": "speechtranscriber-m2pro-macos27", "b": "speechtranscriber-m4pro-macos27", "byteIdentical": "293/300", "identicalIgnoringPunctuation": "300/300" },
    { "a": "speechtranscriber-m2pro-macos27", "b": "published-speechtranscriber-m2pro-macos26", "byteIdentical": "280/300", "identicalIgnoringPunctuation": "296/300" },
    { "a": "speechtranscriber-m4pro-macos27", "b": "published-speechtranscriber-m2pro-macos26", "byteIdentical": "277/300", "identicalIgnoringPunctuation": "296/300" },
    { "a": "speechtranscriber-m4pro-macos27", "b": "speechtranscriber-m4pro-macos27-dictation-enabled", "byteIdentical": "300/300", "identicalIgnoringPunctuation": "300/300" },
    { "a": "dictationtranscriber-m2pro-macos27", "b": "dictationtranscriber-m4pro-macos27", "byteIdentical": "298/300", "identicalIgnoringPunctuation": "299/300" },
    { "a": "speechtranscriber-m2pro-macos27", "b": "dictationtranscriber-m2pro-macos27", "byteIdentical": "1/300", "identicalIgnoringPunctuation": "108/300" }
  ],
  "wordLevelDifferencesVsMacos26": [
    { "id": "1089-134691-0024", "reference": "stephanos dedalos", "macos26": "Stephano stedlos.", "macos27": "Stephanos dedlos." },
    { "id": "121-127105-0009", "reference": "she has been dead these twenty years", "macos26": "She has been dead these twenty years.", "macos27": "She has been dead these 20 years." },
    { "id": "121-127105-0016", "reference": "probably not till the second post", "macos26": "Probably not till the 2nd post.", "macos27": "Probably not till the second post." },
    { "id": "1284-1180-0015", "reference": "we are traveling replied ojo and we stopped at your house just to rest and refresh ourselves", "macos26": "We are traveling, replied Ojo, and we stop at your house just to rest and refresh ourselves.", "macos27": "We are traveling, replied Ojo, and we stopped at your house just to rest and refresh ourselves." }
  ],
  "environment": {
    "note": "Full machine state as measured, for anyone reproducing this.",
    "m2pro": {
      "chip": "Apple M2 Pro",
      "memory": "32GB",
      "os": "macOS 27.0.1 (26A434)",
      "toolchain": "Xcode 27.0 (27A266a), macOS SDK 27.0",
      "qualifiesForAfmCoreAdvanced": false,
      "qualificationNote": "Apple states its most advanced on-device model (AFM Core Advanced) requires a Mac with M3 or later and at least 12GB of unified memory.",
      "appleIntelligence": "enabled",
      "siriAiBeta": "not enabled",
      "systemDictationToggle": "on"
    },
    "m4pro": {
      "chip": "Apple M4 Pro",
      "memory": "24GB",
      "os": "macOS 27.0.1 (26A434)",
      "toolchain": "Command Line Tools 27.0.0.0.1788430756, macOS SDK 27.0, Swift 6.4",
      "qualifiesForAfmCoreAdvanced": true,
      "appleIntelligence": "enabled",
      "siriAiBeta": "not enabled",
      "systemDictationToggle": "off for speechtranscriber-m4pro-macos27 and the DictationTranscriber run; on for speechtranscriber-m4pro-macos27-dictation-enabled",
      "assetStatus": "SpeechTranscriber reported 'supported' before the first run and 'installed' afterwards; no asset download was observed in either run"
    }
  },
  "transcriberConfiguration": {
    "locale": "en_US",
    "speechTranscriber": "transcriptionOptions: [], reportingOptions: [], attributeOptions: []",
    "dictationTranscriber": "preset: .longDictation",
    "analyzer": "SpeechAnalyzer(inputAudioFile:modules:finishAfterFile: true)",
    "note": "Changing any of these changes the output and breaks comparability with the published Round 1 transcripts."
  },
  "reproduce": [
    "Download LibriSpeech test-clean and convert to 16 kHz mono WAV, id-sorted (see compare.py and run.swift in this directory).",
    "swiftc -O run.swift -o run",
    "./run manifest-test-clean.json 300 out.json            # SpeechTranscriber",
    "./run manifest-test-clean.json 300 out-dict.json dictation",
    "python3 compare.py out.json out-dict.json"
  ]
}