nonvacuity.sh 95 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110
  1. #!/bin/sh
  2. # nonvacuity.sh -- prove the runtime checks can actually fail.
  3. #
  4. # A test that has never been seen red is not a test. This script breaks the
  5. # runtime on purpose, once per check, and asserts that the check goes red and
  6. # says something useful about the breakage. Then it restores the source and
  7. # asserts everything is green again.
  8. #
  9. # Each mutation below is a real bug that was in this file at some point, not an
  10. # invented one. That is the point: these are the mistakes we actually make
  11. # with 16-bit ModRM, so these are the ones the checks have to catch.
  12. #
  13. # audit_helpers.py name-versus-decode: catches a wrong ModRM that still
  14. # decodes cleanly, and both halves of a name that admits
  15. # an operand at all - `34 02' where the name promises
  16. # `34 01', and `35 01' where the row's opcode gate admits
  17. # only 34h. Both rows are new, and a row nobody has seen
  18. # reject anything accepts whatever it is shown.
  19. # audit_helpers.py coverage: catches a helper that has silently
  20. # dropped OUT of the audit, which is a green report about
  21. # a subject nobody looked at
  22. # run_com_tests.sh the .COM layout: catches a header that cannot be
  23. # located, a runtime size that disagrees with the image,
  24. # and an entry jump that starts in the wrong place
  25. # check_runtime.py golden: catches the same thing in the built
  26. # image
  27. # check_runtime.py decode sweep: catches a wrong instruction LENGTH
  28. # check_runtime.py branch targets: catches a wrong fixup
  29. # check_runtime.py entry goldens: catches a broken prologue
  30. # probe/modrm11.py the mod=11 table: catches the ModRM column itself
  31. # going wrong, which no amount of decoding will show
  32. # check_framedisp.py the BP disp rule: catches a displacement that reads
  33. # a different address than the symbol table named
  34. # tests/comimage.py the image layout: catches a sweep region built from a
  35. # literal instead of from two independent readings of the
  36. # file - the old RT_SZ went on reporting PASS while its
  37. # region began inside the runtime - and the layout itself
  38. # being written down a second time, now that one file
  39. # defines it and all three readers import it
  40. # rt_exec.py the BP contract: catches an entry that borrows BP to
  41. # reach its argument and does not hand it back. The bytes
  42. # are well formed, the golden is satisfied, the audit says
  43. # every helper emits what its name says, and the machine
  44. # triple-faults on the second call - so nothing short of
  45. # running it, or of stating the register contract
  46. # explicitly, can see it.
  47. # rt_exec.py the string entries: catches rdstr storing its first
  48. # character where the length byte belongs (a missing IncSi
  49. # puts `hello' back as 05 65 6c 6c 6f), and strstore
  50. # running LOOP with CX = 0 when the source is empty - 65536
  51. # iterations that wrap the segment and walk through the
  52. # driver's own case record. Both are same-size,
  53. # matrix-green slips that only a byte dump can see.
  54. # run_com_exec.py the frame rule: catches a local STRING packed with the
  55. # INTEGER rule, so its characters grow up over the saved BP
  56. # and the return address; readln into such a local then
  57. # runs the string's own bytes as an address. The fix moves
  58. # no well-formed instruction, so only execution - or the
  59. # size it pins in the compile matrix - can tell.
  60. # run_com_exec.py the string arms: catches the destination limit loaded
  61. # into AX instead of CX, the assignment pushes swapped so
  62. # the destination is read as its own source, and a string
  63. # read dispatched to the integer reader. Every one is the
  64. # right shape reading the wrong thing, and the MovCxi
  65. # helper is COMP in the audit, so no byte check can see it.
  66. # check_8086.py the 8086 opcodes: catches a conditional branch or a
  67. # SETcc emitted as `0F 8x'/`0F 9x', which are 386-only.
  68. # Nothing else here can: qemu-system-i386's lowest CPU
  69. # model is 486, where both are ordinary instructions, and
  70. # FCML's -m16 mode is a 386 too. Clause H additionally
  71. # catches the branch polarity being inverted, which is
  72. # still a legal opcode and therefore invisible to every
  73. # shape-based check.
  74. # run_com_exec.py behaviour: catches a*b that emits an ADD, `>'
  75. # and `>=' swapped, REPEAT..UNTIL that stops after one
  76. # pass, two procedures whose parameters collide, a left
  77. # operand overwritten by the right one's code (SaveLeft),
  78. # and a boolean `not' lowered as the integer one. Every
  79. # one of them sat in a fixture that COMPILED and was
  80. # never RUN, with every byte-level check green.
  81. # run_compile_tests.sh refusal: catches a boolean READ slipping
  82. # through to the runtime. TP3 refuses it with error 66
  83. # (TPSRC8 prderrtp; its runtime has no xrdbool), and
  84. # this compiler used to accept it and let rdbool store
  85. # a WORD into the one-byte variable. The mutation is
  86. # the one arm of IoCall's type dispatch that refuses it,
  87. # and only the COMPILE MATRIX can see a refusal: an ERR
  88. # fixture is never executed, so nothing that runs
  89. # programs on it can fail the check.
  90. # run_exec86.py behaviour (Exec86): catches the interpreter's OWN
  91. # reading of the machine: JE inverted in Exec86's Cond,
  92. # which swaps the two arms of every `=' in every program
  93. # while the emitted bytes, the sizes and every
  94. # byte-level check stay green. The image is fine; only
  95. # the thing reading it is wrong, so nothing that looks at
  96. # the image can fail this one.
  97. # runtest.py the R key: catches CmdRun poking nothing into
  98. # the interpreter at all, which faults on the first step
  99. # and prints "interpreter fault" instead of the guest's
  100. # answer. It is the only case here that needs a rebuilt
  101. # SHELL rather than a rebuilt compiler or runtime.
  102. #
  103. # The mod=11 cases do not need a rebuild -- they read the probe sources
  104. # directly -- so they are cheap, and they are the ones that matter most: the
  105. # table they guard is the one thing in this project that was wrong in the
  106. # documentation while the code was right, and a table that is wrong in the
  107. # code produces bytes that decode perfectly.
  108. #
  109. # Usage: tests/nonvacuity.sh (from shell/; leaves Runtime.mod restored)
  110. set -u
  111. cd "$(dirname "$0")/.." || exit 1
  112. GM2=/home/eric/bin/Modula2/Gm2/bin/gm2
  113. SAVED=../tmp/nonvacuity.Runtime.mod
  114. PROBE=../tmp/nonvacuity.rtprobe
  115. DUMP=../tmp/nonvacuity.dump
  116. # Exec86.mod is UNTRACKED, so unlike Runtime.mod git cannot put a botched
  117. # mutation back: the copy taken here is the only correct one in existence.
  118. # Shell.mod is tracked but is mutated by the R-key case below, and a trap that
  119. # restored only the sources would leave tpshell BUILT FROM the mutation - so
  120. # the trap rebuilds too. All three copies live in the project's own tmp/ and
  121. # are taken here, before the first mutation, rather than beside each case.
  122. #
  123. # comimage.py and comtest.py are the image layout and one of its three readers;
  124. # both are mutated by cases further down, and neither is recoverable from git
  125. # once the mutation is staged over - so they join the trap here for the same
  126. # reason, and an interrupted run cannot leave the layout half-written.
  127. #
  128. # Compiler.mod is the file mutated by the most cases, and it used to be saved
  129. # BESIDE the first case that needed it - the BP displacement section halfway
  130. # down this file, which is well AFTER this trap is set. An interrupted run
  131. # therefore left Compiler.mod half-mutated, with `git status` showing a source
  132. # that no case had finished with. It joins the copies above, taken before the
  133. # first mutation and covered by the same trap.
  134. mkdir -p ../tmp
  135. SAVED_E=../tmp/nonvacuity.Exec86.mod
  136. SAVED_SH=../tmp/nonvacuity.Shell.mod
  137. SAVED_CI=../tmp/nonvacuity.comimage.py
  138. SAVED_T=../tmp/nonvacuity.comtest.py
  139. SAVED_C=../tmp/nonvacuity.Compiler.mod
  140. cp Runtime.mod "$SAVED" || exit 1
  141. cp Exec86.mod "$SAVED_E" || exit 1
  142. cp Shell.mod "$SAVED_SH" || exit 1
  143. cp tests/comimage.py "$SAVED_CI" || exit 1
  144. cp tests/comtest.py "$SAVED_T" || exit 1
  145. cp Compiler.mod "$SAVED_C" || exit 1
  146. restore_all () {
  147. cp "$SAVED" Runtime.mod
  148. cp "$SAVED_E" Exec86.mod
  149. cp "$SAVED_SH" Shell.mod
  150. cp "$SAVED_CI" tests/comimage.py
  151. cp "$SAVED_T" tests/comtest.py
  152. cp "$SAVED_C" Compiler.mod
  153. "$GM2" -fiso -c Runtime.mod >/dev/null 2>&1
  154. "$GM2" -fiso -c Exec86.mod >/dev/null 2>&1
  155. # The shell is rebuilt as well: a test that runs against a binary built
  156. # from a half-restored tree is reporting on the mutation, not on the code.
  157. # make rebuilds Compiler.o out of the Compiler.mod restored just above.
  158. make >/dev/null 2>&1
  159. }
  160. trap restore_all EXIT
  161. #
  162. # An unhandled signal does NOT run an EXIT trap: bash dies on the spot. That is
  163. # what the comment above promises ("an interrupted run cannot leave the layout
  164. # half-written") and what this file did not actually do. A handler that only
  165. # `exit's is enough, because `exit' from a trap DOES fire the EXIT trap -- and
  166. # it has to exit, or the run would go on using a tree it has just put back.
  167. #
  168. # Measured on bash 5.2: TERM and HUP fire these and restore everything, so
  169. # `kill' and a session teardown are covered. INT is trapped too, but in the
  170. # usual case it never reaches this script: a foreground child that dies of
  171. # SIGINT makes bash treat the signal as the CHILD's and continue the script
  172. # instead. So Ctrl-C does not abort this run at all -- it kills the current
  173. # case's child and the next case starts -- and SIGTERM is what actually stops
  174. # it. 129/130/143 = 128+SIGHUP/INT/TERM.
  175. trap 'exit 129' HUP
  176. trap 'exit 130' INT
  177. trap 'exit 143' TERM
  178. pass=0
  179. fail=0
  180. # mutate <file> <sed-expr> -- apply a deliberate breakage and INSIST it landed.
  181. #
  182. # Four cases in this file were already dead when first run, all the same way:
  183. # the helper they name had been renamed or reformatted since the case was
  184. # written, the sed matched nothing, the source was unchanged, and the check
  185. # correctly passed - so the harness reported "NOT NON-VACUOUS" and, worse, a
  186. # reader skimming the output could take "the check still passed" for a passing
  187. # test. A case that cannot fire is worse than no case: it is a claim of
  188. # coverage that was never tested.
  189. #
  190. # So the mutation is verified, not assumed. If the file is byte-identical
  191. # afterwards, that is reported as a FAILURE of the harness, naming the sed, and
  192. # the case is not run - because running it would only produce a meaningless
  193. # green. The message says what to do (fix the sed) rather than what it found.
  194. mutate () {
  195. mf=$1
  196. msed=$2
  197. cp "$mf" ../tmp/nonvacuity.mut.bak
  198. sed -i "$msed" "$mf"
  199. if cmp -s "$mf" ../tmp/nonvacuity.mut.bak; then
  200. echo " BROKEN CASE: the mutation did not change $mf"
  201. echo " sed: $msed"
  202. echo " the named code has probably been renamed or reformatted -"
  203. echo " fix this case, it is asserting nothing"
  204. fail=$((fail + 1))
  205. return 1
  206. fi
  207. return 0
  208. }
  209. # rebuild <label> -- re-emit the runtime and dump it
  210. rebuild () {
  211. "$GM2" -fiso -c Runtime.mod >/dev/null 2>&1 || return 1
  212. "$GM2" -fiso -o "$PROBE" tests/RtProbe.mod Runtime.o Posix.o \
  213. >/dev/null 2>&1 || return 1
  214. "$PROBE" > "$DUMP" || return 1
  215. return 0
  216. }
  217. # expect_red <label> <pattern> <checker-cmd...>
  218. # <pattern> is a grep the failure output must match, so a check cannot
  219. # "pass" by failing for some unrelated reason.
  220. expect_red () {
  221. label=$1
  222. want=$2
  223. shift 2
  224. if out=$("$@" 2>&1); then
  225. echo "NOT NON-VACUOUS: $label -- the check still passed"
  226. fail=$((fail + 1))
  227. elif ! printf '%s\n' "$out" | grep -qi "$want"; then
  228. echo "WRONG FAILURE: $label -- went red, but not for the stated reason"
  229. printf '%s\n' "$out" | sed 's/^/ /'
  230. fail=$((fail + 1))
  231. else
  232. echo " ok: $label"
  233. printf '%s\n' "$out" | grep -im1 "$want" | sed 's/^/ /'
  234. pass=$((pass + 1))
  235. fi
  236. }
  237. echo "== each mutation must turn the named check red"
  238. echo
  239. # --- 1. name-versus-decode -------------------------------------------
  240. # MovSiBx was `89 DC`, which is MOV SP,BX. Two bytes either way, decodes
  241. # cleanly, and no structural check can see it.
  242. cp "$SAVED" Runtime.mod
  243. mutate Runtime.mod 's|B (0DEH) END MovSiBx|B (0DCH) END MovSiBx|'
  244. expect_red "audit_helpers catches MovSiBx emitting MOV SP,BX" \
  245. "MovSiBx" python3 tests/audit_helpers.py
  246. # CmpSiBx had the identical mistake, which is how you know a single fix is
  247. # not enough -- the same misreading was written twice.
  248. cp "$SAVED" Runtime.mod
  249. mutate Runtime.mod 's|B (39H) ; B (0DEH) END CmpSiBx|B (39H) ; B (0DCH) END CmpSiBx|'
  250. expect_red "audit_helpers catches CmpSiBx emitting CMP SP,BX" \
  251. "CmpSiBx" python3 tests/audit_helpers.py
  252. # --- 2. golden, and entry goldens ------------------------------------
  253. # MovDlAl was `88 C0` = MOV AL,AL instead of MOV DL,AL. This is the case that
  254. # motivated runtime.golden: the sweep stayed in sync, every branch target
  255. # stayed on a boundary, no entry's first bytes moved, and the size did not
  256. # change. The target helper was MovAlDh when this case was written, which is
  257. # the fourth way a case here can rot - see the note on `mutate` below.
  258. cp "$SAVED" Runtime.mod
  259. mutate Runtime.mod 's|PROCEDURE MovDlAl ; BEGIN B (88H) ; B (0C2H)|PROCEDURE MovDlAl ; BEGIN B (88H) ; B (0C0H)|'
  260. rebuild
  261. expect_red "runtime.golden catches MOV AL,AL" \
  262. "mov al,al" python3 tests/check_runtime.py "$DUMP"
  263. # initmem opened with the mis-emitted MovSiAx, so its entry golden was the
  264. # thing that noticed the prologue was a no-op.
  265. cp "$SAVED" Runtime.mod
  266. mutate Runtime.mod 's|B (0F0H) END MovSiAx|B (0C0H) END MovSiAx|'
  267. rebuild
  268. expect_red "check_runtime catches a broken initmem prologue" \
  269. "mov ax,ax" python3 tests/check_runtime.py "$DUMP"
  270. # --- 3. decode sweep / length ----------------------------------------
  271. # StDiDl was `88 97` = [BX+disp16],DL: mod=10, so the instruction needs a
  272. # disp16 it was not given, and the sweep loses sync two bytes later.
  273. cp "$SAVED" Runtime.mod
  274. mutate Runtime.mod 's|PROCEDURE StDiDl ; BEGIN B (88H) ; B (15H)|PROCEDURE StDiDl ; BEGIN B (88H) ; B (97H)|'
  275. rebuild
  276. expect_red "decode sweep catches a mod=10 byte move with no displacement" \
  277. "mov byte ptr \[bx+5b5fh\],dl" python3 tests/check_runtime.py "$DUMP"
  278. # --- 4. branch targets ------------------------------------------------
  279. # FixUp measures a rel8 from the end of the instruction, one byte past the
  280. # displacement field. Drop the +1 and every short branch lands one byte into
  281. # its target, which for a 3-byte instruction means the middle of it. The
  282. # bytes themselves are all perfectly well formed -- only the fixups are
  283. # wrong -- so this is the one failure mode the golden cannot be expected to
  284. # catch on its own.
  285. cp "$SAVED" Runtime.mod
  286. mutate Runtime.mod 's|rel := (t + 100H - (fix \[i\].place + 1)) MOD 100H|rel := (t + 100H - fix [i].place) MOD 100H|'
  287. rebuild
  288. expect_red "branch check catches rel8 fixups measured from the wrong byte" \
  289. "not an instruction boundary" \
  290. python3 tests/check_runtime.py "$DUMP"
  291. echo
  292. echo "== everything restored and green again"
  293. cp "$SAVED" Runtime.mod
  294. if rebuild; then
  295. if python3 tests/audit_helpers.py >/dev/null 2>&1 &&
  296. python3 tests/check_runtime.py "$DUMP" >/dev/null 2>&1; then
  297. echo " ok: both checks pass on the restored source"
  298. pass=$((pass + 1))
  299. else
  300. echo "NOT RESTORED: a check is red after restoring Runtime.mod"
  301. fail=$((fail + 1))
  302. fi
  303. else
  304. echo "NOT RESTORED: the runtime would not rebuild"
  305. fail=$((fail + 1))
  306. fi
  307. echo
  308. echo "== the mod=11 table (probe/modrm11.py)"
  309. # These mutate the probe's own sources, not the runtime, so there is no
  310. # rebuild in the loop. SAVED_PY / SAVED_S are restored after each case.
  311. SAVED_PY=../tmp/nonvacuity.modrm11.py
  312. SAVED_S=../tmp/nonvacuity.modrm11.s
  313. cp tests/probe/modrm11.py "$SAVED_PY" || exit 1
  314. cp tests/probe/modrm11.s "$SAVED_S" || exit 1
  315. M11="python3 tests/probe/modrm11.py"
  316. restore_probe () {
  317. cp "$SAVED_PY" tests/probe/modrm11.py
  318. cp "$SAVED_S" tests/probe/modrm11.s
  319. }
  320. # 1. one cell of the table moved
  321. mutate tests/probe/modrm11.py 's|"Si", "Di"\]$|"Bp", "Di"]|'
  322. expect_red "anchor pins a moved table cell" \
  323. "anchor ADD SI, 2" $M11
  324. restore_probe
  325. # 2. the table this project actually shipped: AX dropped off the front and a
  326. # duplicate BX invented at the end, which shifts every code down by one
  327. mutate tests/probe/modrm11.py 's|^REG = .*$|REG = ["Cx", "Dx", "Bx", "Sp", "Bp", "Si", "Di", "Bx"]|'
  328. expect_red "the table shifted by one (AX dropped, BX duplicated)" \
  329. "anchor MOV SP, BP" $M11
  330. restore_probe
  331. # 3. the .s edited to contradict the table. This is the case that shows why
  332. # the hard-coded EXPECT bytes exist: the assembler encodes the new claim
  333. # correctly, so comparing the .s against `as` alone can never fail here.
  334. mutate tests/probe/modrm11.s 's|movw %sp, %di # reg 100|movw %bp, %di # reg 100|'
  335. expect_red "probe source edited away from the recorded bytes" \
  336. "expected 89 E7" $M11
  337. restore_probe
  338. # 4. the 8-bit list edited, which is a different table from the word one
  339. mutate tests/probe/modrm11.s 's|movb %al, %dl # 88 C2 -> DL := AL|movb %al, %bl # was DL|'
  340. expect_red "the 8-bit register list edited" \
  341. "expected 88 C2" $M11
  342. restore_probe
  343. # 5. an anchor's recorded byte corrupted, so the anchor can no longer
  344. # corroborate itself
  345. mutate tests/probe/modrm11.py 's|"8B EC", "8B E5"|"8B ED", "8B E5"|'
  346. expect_red "anchor byte no longer matches the emitted code" \
  347. "expected 8B ED" $M11
  348. restore_probe
  349. if $M11 >/dev/null 2>&1; then
  350. echo " ok: modrm11.py passes on the restored probe sources"
  351. pass=$((pass + 1))
  352. else
  353. echo "NOT RESTORED: modrm11.py is red after restoring its sources"
  354. $M11 2>&1 | sed 's/^/ /'
  355. fail=$((fail + 1))
  356. fi
  357. echo
  358. echo "== the BP displacement rule (check_framedisp.py)"
  359. # This one is about Compiler.mod rather than the runtime, and it needs the
  360. # whole toolchain rebuilt (comtest, not rtprobe), so it gets its own rebuild.
  361. # SAVED_C itself is taken at the top of this file, beside the EXIT trap, so an
  362. # interrupted run restores Compiler.mod as well.
  363. rebuild_compiler () {
  364. $GM2 -fiso -c Compiler.mod >/dev/null 2>&1 || return 1
  365. $GM2 -fiso -fgen-module-list=tests/ct.lst -o /dev/null \
  366. tests/ComTest.mod TextBuf.o Posix.o Compiler.o Runtime.o Linker.o \
  367. >/dev/null 2>&1
  368. $GM2 -fiso -fuse-list=tests/ct.lst -o comtest \
  369. tests/ComTest.mod TextBuf.o Posix.o Compiler.o Runtime.o Linker.o \
  370. >/dev/null 2>&1 || return 1
  371. return 0
  372. }
  373. # 1. the original bug: `off MOD 100H`, always disp8. Restores the code that
  374. # was there before EmBpDisp existed. t28's [BP+128] read becomes [BP-128],
  375. # which is the failure this whole check is named after.
  376. #
  377. # regf is KEPT. It is the other half of EmBpDisp's job - the ModR/M reg
  378. # field, which the DI forms (regf 7) need - and it was added after this
  379. # case was written. Hard-coding 46H here again would break the DI forms at
  380. # the same time and turn one mutation into two faults; the check would
  381. # still go red, but for a reason the label does not name.
  382. python3 - "$SAVED_C" <<'PYEOF'
  383. import sys
  384. p = 'Compiler.mod'
  385. s = open(p).read()
  386. old = """ IF off <= 127 THEN
  387. Ebyte (VAL (BYTE, 40H + regf * 8 + 6H)) ; Ebyte (VAL (BYTE, off))
  388. ELSE
  389. Ebyte (VAL (BYTE, 80H + regf * 8 + 6H)) ; Eword (off)
  390. END
  391. END EmBpDisp ;"""
  392. new = """ Ebyte (VAL (BYTE, 40H + regf * 8 + 6H)) ; Ebyte (VAL (BYTE, off MOD 100H))
  393. END EmBpDisp ;"""
  394. assert old in s, "EmBpDisp body not found -- update this mutation"
  395. open(p, 'w').write(s.replace(old, new))
  396. PYEOF
  397. if rebuild_compiler; then
  398. expect_red "displacement truncation reads a different address" \
  399. "no 8B access at \[BP+128\]" python3 tests/check_framedisp.py
  400. else
  401. echo " FAIL: the compiler would not rebuild with the truncation"
  402. fail=$((fail + 1))
  403. fi
  404. cp "$SAVED_C" Compiler.mod
  405. # 2. the other half of the rule: always use the 4-byte form, ignoring the
  406. # <= 127 case. This is over-cautious rather than wrong, so the checker must
  407. # still be happy -- which is worth asserting, because a check that only
  408. # ever fails on a smaller encoding is a check that pins one answer instead
  409. # of the rule.
  410. python3 - <<'PYEOF'
  411. p = 'Compiler.mod'
  412. s = open(p).read()
  413. old = """ IF off <= 127 THEN
  414. Ebyte (VAL (BYTE, 40H + regf * 8 + 6H)) ; Ebyte (VAL (BYTE, off))
  415. ELSE
  416. Ebyte (VAL (BYTE, 80H + regf * 8 + 6H)) ; Eword (off)
  417. END"""
  418. new = """ Ebyte (VAL (BYTE, 80H + regf * 8 + 6H)) ; Eword (off)"""
  419. assert old in s, "EmBpDisp branch not found -- update this mutation"
  420. open(p, 'w').write(s.replace(old, new))
  421. PYEOF
  422. if rebuild_compiler; then
  423. if python3 tests/check_framedisp.py >/dev/null 2>&1; then
  424. echo " ok: always-disp16 is accepted, so the check pins the rule"
  425. echo " and not one particular encoding"
  426. pass=$((pass + 1))
  427. else
  428. echo " FAIL: check_framedisp rejects a safe, over-long encoding"
  429. python3 tests/check_framedisp.py 2>&1 | sed 's/^/ /'
  430. fail=$((fail + 1))
  431. fi
  432. else
  433. echo " FAIL: the compiler would not rebuild with always-disp16"
  434. fail=$((fail + 1))
  435. fi
  436. cp "$SAVED_C" Compiler.mod
  437. if rebuild_compiler; then
  438. if python3 tests/check_framedisp.py >/dev/null 2>&1; then
  439. echo " ok: check_framedisp passes on the restored source"
  440. pass=$((pass + 1))
  441. else
  442. echo "NOT RESTORED: check_framedisp is red after restoring Compiler.mod"
  443. python3 tests/check_framedisp.py 2>&1 | sed 's/^/ /'
  444. fail=$((fail + 1))
  445. fi
  446. else
  447. echo "NOT RESTORED: the compiler would not rebuild"
  448. fail=$((fail + 1))
  449. fi
  450. # --- the region check_framedisp sweeps, and the one file it comes from -----
  451. echo
  452. echo "== the image layout, measured once (tests/comimage.py)"
  453. # check_framedisp.py decodes instructions over a REGION of the image, and a
  454. # wrong region does not stop it: the byte patterns it searches for are in the
  455. # image wherever they happen to be, and a decode that starts mid-instruction
  456. # reached the same offsets anyway. That is how RT_SZ = 391 - a literal that had
  457. # drifted from the runtime it described - went on reporting PASS while its sweep
  458. # began inside the runtime, part way through an instruction. Nothing in the
  459. # check could see where its own region began.
  460. #
  461. # So the region is two independent readings of the file - where the entry jump
  462. # says execution starts, and where the program header says the runtime ends -
  463. # and the two must AGREE or the region is undetermined. Both readings, and the
  464. # layout itself, live in tests/comimage.py, which is also where comtest.py and
  465. # the independent checker in run_com_tests.sh now get the copy each of them
  466. # used to keep: a duplicated constant that has silently drifted is not an
  467. # independent check, it is a second source of truth that lies, and it lies in
  468. # the direction of looking like the code under test is broken.
  469. #
  470. # None of this needs a rebuild - comimage.py is Python the checkers import -
  471. # so these cases are cheap, and they turn the READERS red one at a time.
  472. # 1. entry_target answers with the literal the region used to be built from.
  473. # The two readings now disagree, and there is no measured region to sweep.
  474. python3 - <<'PYEOF'
  475. p = 'tests/comimage.py'
  476. s = open(p).read()
  477. old = ' return ENT_SZ + int.from_bytes(d[1:ENT_SZ], "little", signed=True)'
  478. new = ' return ENT_SZ + 391 # the literal RT_SZ was'
  479. assert s.count(old) == 1, "entry_target's return not found -- update this mutation"
  480. open(p, 'w').write(s.replace(old, new))
  481. PYEOF
  482. expect_red "a sweep region that is not where execution starts is rejected" \
  483. "neither region is swept" python3 tests/check_framedisp.py
  484. cp "$SAVED_CI" tests/comimage.py
  485. # 2 and 3. The header's own equation, one byte out, so the header cannot be
  486. # located at all - the case where a checker that fell back on a remembered
  487. # offset would report a confident number instead. Two readers, two
  488. # different sentences: each must say in its own words that it does not know
  489. # where the program's code begins.
  490. python3 - <<'PYEOF'
  491. p = 'tests/comimage.py'
  492. s = open(p).read()
  493. old = ' if w(HDR_DS_WORD) != off + 0x1000 + LOAD_BIAS: # hdrDS'
  494. new = ' if w(HDR_DS_WORD) != off + 0x1000 + LOAD_BIAS + 1: # hdrDS'
  495. assert s.count(old) == 1, "the hdrDS equation not found -- update this mutation"
  496. open(p, 'w').write(s.replace(old, new))
  497. PYEOF
  498. expect_red "check_framedisp reports a header it cannot locate" \
  499. "no program header found" python3 tests/check_framedisp.py
  500. expect_red "check_8086 reports the same unlocatable header" \
  501. "could not locate the code region" python3 tests/check_8086.py
  502. cp "$SAVED_CI" tests/comimage.py
  503. # 4-7. The single copy itself, now that there is one. A second find_header,
  504. # a second HDR_SZ, a caller that reaches the helper through another
  505. # consumer instead of through comimage, and - the case that exists
  506. # because of rt_exec.py's "matched nothing" lesson - the definition the
  507. # audit counts. A rule that matches nothing looks exactly like a rule
  508. # that passes, so a deleted definition must be a red report rather than a
  509. # silent green one.
  510. python3 - <<'PYEOF'
  511. p = 'tests/comtest.py'
  512. s = open(p).read()
  513. old = 'def check_com(path, expected_src_len):'
  514. new = ('def find_header(d):\n'
  515. ' return None # a second copy of the helper\n'
  516. '\n\n'
  517. 'def check_com(path, expected_src_len):')
  518. assert s.count(old) == 1, "check_com not found -- update this mutation"
  519. open(p, 'w').write(s.replace(old, new))
  520. PYEOF
  521. expect_red "the audit reports a find_header copied out of comimage" \
  522. "comtest.py defines its own find_header" python3 tests/check_comimage.py
  523. cp "$SAVED_T" tests/comtest.py
  524. python3 - <<'PYEOF'
  525. p = 'tests/comtest.py'
  526. s = open(p).read()
  527. old = 'from comimage import (ENT_SZ, HDR_SZ, LOAD_BIAS, HEAD, HDR_DS_WORD,'
  528. new = ('HDR_SZ = 16 # a second copy of the layout\n'
  529. 'from comimage import (ENT_SZ, HDR_SZ, LOAD_BIAS, HEAD, HDR_DS_WORD,')
  530. assert s.count(old) == 1, "the comimage import not found -- update this mutation"
  531. open(p, 'w').write(s.replace(old, new))
  532. PYEOF
  533. expect_red "the audit reports a layout constant defined elsewhere" \
  534. "comtest.py defines its own HDR_SZ" python3 tests/check_comimage.py
  535. cp "$SAVED_T" tests/comtest.py
  536. python3 - <<'PYEOF'
  537. p = 'tests/comimage.py'
  538. s = open(p).read()
  539. old = 'def find_header(d):'
  540. new = 'def _find_header(d): # the definition is gone'
  541. assert s.count(old) == 1, "def find_header not found -- update this mutation"
  542. open(p, 'w').write(s.replace(old, new))
  543. PYEOF
  544. expect_red "the audit counts its own subject rather than matching nothing" \
  545. "defines find_header 0 times" python3 tests/check_comimage.py
  546. cp "$SAVED_CI" tests/comimage.py
  547. # 7. The import dropped while the CALLS stay - the shape a refactor makes when
  548. # one reader starts taking the helper from another reader instead. It would
  549. # NameError the moment comtest ran, but the audit says so first, and in the
  550. # vocabulary of the layout rather than in the vocabulary of Python: the
  551. # failure this is guarding is "two sources of truth", not "undefined name".
  552. python3 - <<'PYEOF'
  553. p = 'tests/comtest.py'
  554. s = open(p).read()
  555. old = ('from comimage import (ENT_SZ, HDR_SZ, LOAD_BIAS, HEAD, HDR_DS_WORD,\n'
  556. ' HDR_HEAP_WORD, find_header)')
  557. new = '# MUTATION: the shared import is gone, but the calls remain\n'
  558. assert s.count(old) == 1, "the comimage import not found -- update this mutation"
  559. open(p, 'w').write(s.replace(old, new))
  560. PYEOF
  561. expect_red "the audit reports a caller that does not take it from comimage" \
  562. "comtest.py calls find_header without importing it from comimage" \
  563. python3 tests/check_comimage.py
  564. cp "$SAVED_T" tests/comtest.py
  565. # Green again, on all three readers at once: the case above mutated the one
  566. # file all of them import, so restoring it has to satisfy each of them, not
  567. # just the one that was last run.
  568. for c in tests/check_framedisp.py tests/check_comimage.py tests/check_8086.py; do
  569. if python3 "$c" >/dev/null 2>&1; then
  570. echo " ok: $c passes on the restored sources"
  571. pass=$((pass + 1))
  572. else
  573. echo "NOT RESTORED: $c is red after restoring comimage.py and comtest.py"
  574. python3 "$c" 2>&1 | tail -3 | sed 's/^/ /'
  575. fail=$((fail + 1))
  576. fi
  577. done
  578. # --- 3. the operator bugs the nine dead fixtures were hiding ------------
  579. echo
  580. echo "== the bugs the never-executed fixtures were hiding"
  581. echo
  582. # A different KIND of case from everything above. The others break the code
  583. # and assert a byte-level check notices; these break the code and assert a
  584. # BEHAVIOURAL check notices, which is the only kind that could have found them.
  585. # Each of the original four shipped with a green compile matrix, a passing .COM
  586. # layout check, a passing golden and a passing emitter audit:
  587. #
  588. # OpMul = 1 `a * b` emitted ADD AX,CX. `*' and `+' both numbered
  589. # their operator 1, and BinOpEmit cannot see which
  590. # precedence level called it, so every multiplication
  591. # dispatched to the addition. The constant-folding arm was
  592. # correct, which is why `n * n' with n a CONST was right and
  593. # `a * a' with a a variable was not -- and t08_const is the
  594. # only fixture that ever multiplied.
  595. # 9Dh / 9FH `>' got SETGE and `>=' got SETG: swapped, one letter
  596. # apart in the mnemonic. Only a==b could see it.
  597. # JNZ -> body REPEAT..UNTIL looped back while the condition was TRUE,
  598. # which is WHILE, so the body ran once and stopped.
  599. #
  600. # They are mutated back to the original defect and run_com_exec.py must go red
  601. # on the exact fixture that pins the behaviour. The fourth (HideLocals) is a
  602. # scoping bug rather than an operator bug; it was hiding in the same place.
  603. #
  604. # Two joined them when the operand-lifetime and `not' fixes landed, each run
  605. # red by hand before it was written down here, and each with the same property
  606. # as the four above -- green everywhere except execution:
  607. #
  608. # SaveLeft's park `(p > q) or (q > p)' evaluated `(q > p) or (q > p)'. A
  609. # kind-2 value exists only in AX, and the right operand's
  610. # code is emitted before the two are ever brought together,
  611. # so without the push the left one is simply gone.
  612. # neglevel's split `not' on a boolean emitted the INTEGER `not', which left
  613. # 0FFFEh where a boolean belongs, and wrbool reads anything
  614. # non-zero as TRUE -- so `not (a = a)' answered TRUE and so
  615. # did every other `not'.
  616. mutate_compiler () { # reuse mutate's verified-change discipline on Compiler.mod
  617. mf=Compiler.mod
  618. msed=$1
  619. cp "$SAVED_C" ../tmp/nonvacuity.mut2.bak
  620. sed -i "$msed" "$mf"
  621. if cmp -s "$mf" ../tmp/nonvacuity.mut2.bak; then
  622. echo " BROKEN CASE: the mutation did not change Compiler.mod"
  623. echo " sed: $msed"
  624. echo " the named code has probably been renamed or reformatted -"
  625. echo " fix this case, it is asserting nothing"
  626. fail=$((fail + 1))
  627. return 1
  628. fi
  629. return 0
  630. }
  631. # The SETcc swap and the HideLocals removal are done in python rather than
  632. # with sed: both need to match source text containing `*` and `(` in a way that
  633. # is tedious and fragile as a regex, and a case whose only failure mode is a
  634. # malformed sed is a case that silently asserts nothing.
  635. #
  636. # And a python helper fails in a way sed does not: a syntax error in the helper
  637. # is a non-zero exit, `if mutate_foo; then` is simply false, and the case is
  638. # SKIPPED -- with no failure counted and nothing on stdout but whatever python
  639. # printed. That is how the SETcc case spent its first run: an apostrophe in an
  640. # assert message ("the `>' arm") closed the string early, python died, the
  641. # compiler was never broken, and the suite still reported 0 failed. A skipped
  642. # case and a passing case look the same in the total. So each helper below
  643. # fails LOUDLY: a non-zero exit from python is reported as a BROKEN CASE and
  644. # counted, never swallowed.
  645. #
  646. # Each one also counts its targets before replacing. `assert s != before' only
  647. # says the file changed; with two edits it would pass if just one of them
  648. # landed, and with two identical HideLocals call sites it would happily delete
  649. # the wrong one -- still a changed file, still a working compiler, still green
  650. # for the wrong reason.
  651. mutate_cc_swap () {
  652. if python3 - <<'PYX'
  653. p = 'Compiler.mod'
  654. s = open(p).read()
  655. GT = 'EmSetcc (9FH) ; (* > SETG *)' # the greater-than arm
  656. GE = 'EmSetcc (9DH) ; (* >= SETGE *)' # the greater-equal arm
  657. assert s.count(GT) == 1, 'expected 1 greater-than arm, found %d' % s.count(GT)
  658. assert s.count(GE) == 1, 'expected 1 greater-equal arm, found %d' % s.count(GE)
  659. s = s.replace(GT, GT.replace('9FH', '9DH'))
  660. s = s.replace(GE, GE.replace('9DH', '9FH'))
  661. open(p, 'w').write(s)
  662. PYX
  663. then
  664. return 0
  665. fi
  666. echo " BROKEN CASE: the SETcc swap did not apply"
  667. echo " the two EmSetcc arms are probably renamed or reformatted -"
  668. echo " fix this case, it is asserting nothing"
  669. fail=$((fail + 1))
  670. return 1
  671. }
  672. mutate_no_hidelocals () {
  673. if python3 - <<'PYX'
  674. p = 'Compiler.mod'
  675. s = open(p).read()
  676. # Two HideLocals calls exist. Only the body-exit one may go: deleting the
  677. # FORWARD one instead would still change the file, still rebuild, and still
  678. # leave t13_proc compiling, so the case would go green for the wrong reason.
  679. HL = ' HideLocals (nestMark) ; (* parameters and locals stop here *)\n'
  680. assert s.count(HL) == 1, 'expected 1 body-exit HideLocals, found %d' % s.count(HL)
  681. open(p, 'w').write(s.replace(HL, ''))
  682. PYX
  683. then
  684. return 0
  685. fi
  686. echo " BROKEN CASE: HideLocals was not removed"
  687. echo " the call or its comment has probably been reformatted -"
  688. echo " fix this case, it is asserting nothing"
  689. fail=$((fail + 1))
  690. return 1
  691. }
  692. cp "$SAVED_C" Compiler.mod
  693. if mutate_compiler 's|^ op := OpMul ; DropCh| op := OpAdd ; DropCh|'; then
  694. if rebuild_compiler; then
  695. expect_red "execution catches '*' emitting an ADD (t08_const, n*n)" \
  696. "t08_const" python3 tests/run_com_exec.py t08_const
  697. # t08 only ever multiplied two CONSTANTS, which is the one path that was
  698. # never wrong, because BinOpEmit folds it. So the case above is close
  699. # to vacuous: it proves the mutation changed the binary, not that the
  700. # emitted multiply is covered. The check that matters needs a
  701. # VARIABLE operand, and no shipped fixture has one -- which is why the
  702. # bug survived at all. So this writes a throwaway fixture that
  703. # multiplies a variable, runs it, and asserts the multiply is right.
  704. # The fixture is deleted afterwards; it is here to close the coverage
  705. # hole, not to become a permanent test (that is what a real fixture
  706. # with a `*' in it would be for).
  707. cat > tests/fixtures/zzmul.pas <<'ZZEOF'
  708. program zzmul;
  709. var a : integer ;
  710. begin
  711. a := 7 ;
  712. writeln (a * 6)
  713. end.
  714. ZZEOF
  715. printf '42\r\n' > tests/fixtures/zzmul.out
  716. expect_red "execution catches '*' on a VARIABLE (the unfolded path)" \
  717. "zzmul" python3 tests/run_com_exec.py zzmul
  718. rm -f tests/fixtures/zzmul.pas tests/fixtures/zzmul.out
  719. else
  720. echo " FAIL: the compiler would not rebuild with OpAdd for '*'"
  721. fail=$((fail + 1))
  722. fi
  723. fi
  724. cp "$SAVED_C" Compiler.mod
  725. cp "$SAVED_C" Compiler.mod
  726. if mutate_cc_swap; then
  727. if rebuild_compiler; then
  728. expect_red "execution catches '>' and '>=' swapped (t09_if)" \
  729. "t09_if" python3 tests/run_com_exec.py t09_if
  730. else
  731. echo " FAIL: the compiler would not rebuild with the SETcc swap"
  732. fail=$((fail + 1))
  733. fi
  734. fi
  735. cp "$SAVED_C" Compiler.mod
  736. cp "$SAVED_C" Compiler.mod
  737. if mutate_compiler 's| DropC (EmJcc (84H, L1)) ; (\* JZ -> body again \*)| zj := EmJcc (85H, L1) ;|'; then
  738. if rebuild_compiler; then
  739. expect_red "execution catches REPEAT..UNTIL exiting after one pass (t12)" \
  740. "t12_repeat" python3 tests/run_com_exec.py t12_repeat
  741. else
  742. echo " FAIL: the compiler would not rebuild with the JNZ repeat"
  743. fail=$((fail + 1))
  744. fi
  745. fi
  746. cp "$SAVED_C" Compiler.mod
  747. # The fourth: sibling procedures shared one parameter namespace, because a
  748. # finished procedure's symbols were left at a level Search still accepts.
  749. # Removing HideLocals puts two procedures' `a : integer' back in collision.
  750. cp "$SAVED_C" Compiler.mod
  751. if mutate_no_hidelocals; then
  752. if rebuild_compiler; then
  753. expect_red "a duplicate parameter in two procedures is a compile error again" \
  754. "ERROR 41" python3 tests/run_com_exec.py t13_proc
  755. else
  756. echo " FAIL: the compiler would not rebuild without HideLocals"
  757. fail=$((fail + 1))
  758. fi
  759. else
  760. echo " FAIL: could not remove HideLocals to test the scoping fix"
  761. fail=$((fail + 1))
  762. fi
  763. # M6: the LEFT operand, parked for the right one. kind 2 means "this value
  764. # exists in AX and nowhere else", and the right-hand operand's code is emitted
  765. # between recognizing the operator and using both operands - so without
  766. # SaveLeft's push the left value is overwritten before it is ever read, and
  767. # LoadPair's kind-4 shape can never trigger. `(p > q) or (q > p)' then
  768. # evaluates `(q > p) or (q > p)'. Every byte, every size and the whole compile
  769. # matrix stay green: nothing is malformed, the value is simply the wrong one.
  770. cp "$SAVED_C" Compiler.mod
  771. if python3 - <<'PYX'
  772. p = 'Compiler.mod'
  773. s = open(p).read()
  774. # Both guards are matched, and each must be unique. `IF left.kind = 2 THEN'
  775. # reads like it can only be SaveLeft's, but LoadAtom and friends also branch
  776. # on kind, so an unasserted replace could disable one of THEM instead and the
  777. # case would still go red - for a reason the label does not name.
  778. #
  779. # The kind-5 ("a string element whose address is in DI") guard did not exist
  780. # when this case was written; it arrived with `s[i]`. Making only the kind-2
  781. # park unreachable would leave that one alive, so the mutation would no longer
  782. # be "SaveLeft parks nothing" - it would be a half-disabled SaveLeft, and the
  783. # case would be testing something narrower than it says.
  784. g2 = ' IF left.kind = 2 THEN\n'
  785. g5 = ' ELSIF left.kind = 5 THEN\n'
  786. assert s.count(g2) == 1, 'the kind-2 guard found %d times -- update this mutation' % s.count(g2)
  787. assert s.count(g5) == 1, 'the kind-5 guard found %d times -- update this mutation' % s.count(g5)
  788. s = s.replace(g2, ' IF left.kind = 99 THEN\n')
  789. s = s.replace(g5, ' ELSIF left.kind = 99 THEN\n')
  790. open(p, 'w').write(s)
  791. PYX
  792. then
  793. if rebuild_compiler; then
  794. expect_red "execution catches a left operand clobbered by the right one" \
  795. "t34_arith" python3 tests/run_com_exec.py t34_arith
  796. else
  797. echo " FAIL: the compiler would not rebuild without SaveLeft's push"
  798. fail=$((fail + 1))
  799. fi
  800. else
  801. echo " BROKEN CASE: the SaveLeft mutation did not apply"
  802. fail=$((fail + 1))
  803. fi
  804. cp "$SAVED_C" Compiler.mod
  805. # M7: the type split in TPSRC9's neglevel, wrong half. A boolean NOT lowered
  806. # as the INTEGER one leaves 0FFFEh/0FFFFh, and wrbool tests [BP+4] <> 0 - so
  807. # `not (a = a)' answers TRUE, and so does every other `not'. One instruction,
  808. # same length, same sizes, and the compile matrix cannot see it because both
  809. # halves emit well-formed code for a type the parser already accepted.
  810. cp "$SAVED_C" Compiler.mod
  811. if python3 - <<'PYX'
  812. p = 'Compiler.mod'
  813. s = open(p).read()
  814. old = """ IF r.cls = TBool THEN
  815. LoadAtom (r) ;
  816. EmXorAl01 () ;"""
  817. new = """ IF r.cls = TBool THEN
  818. LoadAtom (r) ;
  819. EmNotAx () ; (* MUTATION: integer NOT on a boolean *)"""
  820. assert s.count(old) == 1, 'the TBool arm of ParseNeg found %d times -- update this mutation' % s.count(old)
  821. open(p, 'w').write(s.replace(old, new))
  822. PYX
  823. then
  824. if rebuild_compiler; then
  825. expect_red "execution catches a boolean NOT lowered as an integer one" \
  826. "t35_not" python3 tests/run_com_exec.py t35_not
  827. else
  828. echo " FAIL: the compiler would not rebuild with EmNotAx for a boolean"
  829. fail=$((fail + 1))
  830. fi
  831. else
  832. echo " BROKEN CASE: the neglevel type-split mutation did not apply"
  833. fail=$((fail + 1))
  834. fi
  835. cp "$SAVED_C" Compiler.mod
  836. # Bug 36, argument order: the fix has FOUR independent halves - three parsers
  837. # that push each argument the moment it is parsed instead of after the whole
  838. # list, and the callee that reads its parameters backwards from BP+4 - and
  839. # t36_argclobber is the one fixture that tells them apart. Before these four
  840. # cases existed, its only proof was the pre-fix run quoted in the commit
  841. # message, which is a fact about the past rather than a check: it cannot be
  842. # re-derived, and nothing stops the four halves from drifting apart.
  843. #
  844. # Each revert below was run to a red BEFORE it was written here, and the four
  845. # fail on four DIFFERENT rows, which is what makes them four cases and not one
  846. # wearing four hats: the callee revert turns `5 3 7' into `7 3 5' (every
  847. # argument lands on its neighbour's parameter), the deferred ParseCallArgs
  848. # zeroes the rows built from computed arguments, ParseCall changes the
  849. # function call's result (19 -> 10) and IoCall changes the last write
  850. # (`FALSE 1' -> `TRUE 544', the first argument printed as the second
  851. # argument's value). A revert that left t36 green is reported by expect_red
  852. # as NOT NON-VACUOUS; a revert that did not land is a BROKEN CASE.
  853. # revert_procedure <name> -- put <name> back the way it was before bug 36 was
  854. # fixed, spliced out of the tree the fix landed on top of. The old code's
  855. # source of truth is the old code: quoting a copy of it in this file is a copy
  856. # that can drift from the tag it claims to reproduce, and a revert that
  857. # reproduces nothing is a case asserting nothing.
  858. revert_procedure () {
  859. python3 - "$1" <<'PYX'
  860. import subprocess, sys
  861. name = sys.argv[1]
  862. cur = open('Compiler.mod').read()
  863. r = subprocess.run(['git', 'show', 'v-TP3-IMAGELAYOUT:shell/Compiler.mod'],
  864. capture_output=True)
  865. assert r.returncode == 0, 'cannot read the pre-fix tree: %s' % r.stderr.decode()[:200]
  866. pre = r.stdout.decode()
  867. def span(text, where):
  868. # The DEFINITION, not the FORWARD declaration a few screens earlier: the
  869. # anchor runs to the end of the line, and a FORWARD line ends `FORWARD ;'
  870. # instead of nothing.
  871. a, b = 'PROCEDURE %s (idx : CARDINAL) ;\n' % name, 'END %s ;' % name
  872. na, nb = text.count(a), text.count(b)
  873. assert na == 1 and nb == 1, \
  874. '%s: %d definitions, %d ends in %s -- update this case' % (name, na, nb, where)
  875. i = text.index(a)
  876. j = text.index(b, i) + len(b)
  877. return text[i:j], i, j
  878. new, i, j = span(cur, 'Compiler.mod')
  879. old, _, _ = span(pre, 'the pre-fix tree')
  880. assert old != new, 'the pre-fix %s is identical to the current one' % name
  881. open('Compiler.mod', 'w').write(cur[:i] + old + cur[j:])
  882. PYX
  883. }
  884. # The callee half: no remap, so parameters are read in declaration order from
  885. # BP+4 while the arguments are pushed in parse order - the FIRST argument ends
  886. # up in the LAST parameter's slot, and one-argument calls (all of
  887. # Runtime.mod's own cases) cannot tell the two layouts apart.
  888. cp "$SAVED_C" Compiler.mod
  889. if python3 - <<'PYX'
  890. p = 'Compiler.mod'
  891. s = open(p).read()
  892. old = """ i := nestMark ;
  893. WHILE i < symTop DO
  894. symtab [i].off := parmOff + 2 - symtab [i].off ;
  895. INC (i)
  896. END ;
  897. """
  898. assert s.count(old) == 1, 'the ProcFunc parameter remap found %d times -- update this mutation' % s.count(old)
  899. open(p, 'w').write(s.replace(old, ''))
  900. PYX
  901. then
  902. if rebuild_compiler; then
  903. expect_red "t36 catches the callee laid out in declaration order (no remap)" \
  904. "t36_argclobber" python3 tests/run_com_exec.py t36_argclobber
  905. else
  906. echo " FAIL: the compiler would not rebuild without the parameter remap"
  907. fail=$((fail + 1))
  908. fi
  909. else
  910. echo " BROKEN CASE: the ProcFunc remap mutation did not apply"
  911. fail=$((fail + 1))
  912. fi
  913. cp "$SAVED_C" Compiler.mod
  914. # The caller half, site 1 of 3: a call used as a statement. The old code
  915. # read the whole list into args[] and pushed it backwards afterwards, so a
  916. # computed argument (which exists only in AX) was gone before the push loop
  917. # reached it.
  918. cp "$SAVED_C" Compiler.mod
  919. if revert_procedure ParseCallArgs; then
  920. if rebuild_compiler; then
  921. expect_red "t36 catches ParseCallArgs pushing after the whole list" \
  922. "t36_argclobber" python3 tests/run_com_exec.py t36_argclobber
  923. else
  924. echo " FAIL: the compiler would not rebuild with ParseCallArgs deferred"
  925. fail=$((fail + 1))
  926. fi
  927. else
  928. echo " BROKEN CASE: the ParseCallArgs revert did not apply"
  929. fail=$((fail + 1))
  930. fi
  931. cp "$SAVED_C" Compiler.mod
  932. # Site 2 of 3: the same call reached from ParseAtom, when a FUNCTION is
  933. # called inside an expression. A separate site, because it is a separate
  934. # procedure - and t36's `19' row is the only thing that notices this one.
  935. cp "$SAVED_C" Compiler.mod
  936. if revert_procedure ParseCall; then
  937. if rebuild_compiler; then
  938. expect_red "t36 catches ParseCall pushing after the whole list" \
  939. "t36_argclobber" python3 tests/run_com_exec.py t36_argclobber
  940. else
  941. echo " FAIL: the compiler would not rebuild with ParseCall deferred"
  942. fail=$((fail + 1))
  943. fi
  944. else
  945. echo " BROKEN CASE: the ParseCall revert did not apply"
  946. fail=$((fail + 1))
  947. fi
  948. cp "$SAVED_C" Compiler.mod
  949. # Site 3 of 3: write/writeln/read/readln, where the argument list is also
  950. # where an inline string literal gets its call. Reverting this procedure
  951. # also reverts the per-argument dispatch inside it (the BYTE read landed
  952. # here), which is harmless for t36 and is why the splice is the whole
  953. # procedure rather than the push alone: half a revert is a state the tree
  954. # was never in.
  955. cp "$SAVED_C" Compiler.mod
  956. if revert_procedure IoCall; then
  957. if rebuild_compiler; then
  958. expect_red "t36 catches IoCall printing the last argument's value" \
  959. "t36_argclobber" python3 tests/run_com_exec.py t36_argclobber
  960. else
  961. echo " FAIL: the compiler would not rebuild with IoCall's loops split"
  962. fail=$((fail + 1))
  963. fi
  964. else
  965. echo " BROKEN CASE: the IoCall revert did not apply"
  966. fail=$((fail + 1))
  967. fi
  968. cp "$SAVED_C" Compiler.mod
  969. # The boolean read (t38): TP3 refuses `readln (b)' with error 66 - prderrtp
  970. # rejects the whole class and the runtime has no xrdbool - and this compiler
  971. # used to accept it and dispatch to rdbool, which stores a WORD into the
  972. # one-byte variable, its second byte landing on the variable AFTER b (t37's
  973. # overflow, unobserved only because no fixture read a boolean). The fix is
  974. # one arm of IoCall's type dispatch, so the mutation restores exactly that
  975. # arm. The check is the COMPILE MATRIX rather than any run: an ERR fixture
  976. # is never executed, so nothing that runs programs on it can see a refusal.
  977. cp "$SAVED_C" Compiler.mod
  978. if python3 - <<'PYX'
  979. p = 'Compiler.mod'
  980. s = open(p).read()
  981. old = """ ELSIF acls = TBool THEN
  982. (* TP3 refuses boolean I/O: TPSRC8 prderrtp is error 66,
  983. "I/O not allowed here", and the runtime has no xrdbool
  984. at all. rdbool stores a WORD, so one byte of an
  985. accepted read would land on the variable AFTER b - the
  986. exact overflow t37_readbyte pins for a byte. TP3's
  987. answer to the whole class is the error; so is ours. *)
  988. Err (EIoHere) ;
  989. RETURN"""
  990. assert s.count(old) == 1, 'the boolean-read error arm found %d times -- update this mutation' % s.count(old)
  991. rep = """ ELSIF acls = TBool THEN
  992. ent := TU_RdBool"""
  993. open(p, 'w').write(s.replace(old, rep))
  994. PYX
  995. then
  996. if rebuild_compiler; then
  997. expect_red "t38 red when a boolean read dispatches to TU_RdBool" \
  998. "t38_readbool" tests/run_compile_tests.sh
  999. else
  1000. echo " FAIL: the compiler would not rebuild with the boolean read accepted"
  1001. fail=$((fail + 1))
  1002. fi
  1003. else
  1004. echo " BROKEN CASE: the boolean-read mutation did not apply"
  1005. fail=$((fail + 1))
  1006. fi
  1007. cp "$SAVED_C" Compiler.mod
  1008. # The string variable (t39). A STRING is the first declared type bigger than
  1009. # a word that this compiler accepts, and it is accepted in two places: DefVar
  1010. # exempts TString from its `size > 2' catch-all so `var s : string [10]' can
  1011. # exist at all, and the two arms below USE it once it does. The three
  1012. # mutations are the three places the feature lives, each restored to the
  1013. # exact pre-fix state (all three used to be ENoLib, or never existed):
  1014. #
  1015. # 1. the DEPARSE of assignment: before the fix, `s := 'hello'' fell out of
  1016. # Assignment's size check. Removing the string arm puts it back there.
  1017. # 2. the WRITE: before the fix, `writeln (s)' was refused by IoCall's
  1018. # type dispatch. Restoring the refusal arm is removing the feature.
  1019. # 3. the RUNTIME: wrstr's print loop is only as good as its length byte,
  1020. # and only rt_exec.py can see a wrstr that reads no length at all.
  1021. # 1. Assignment's string arm is gone, so `s := 'hello'' trips the old
  1022. # `size > 2' refusal again. The arm is spliced between its guard and the
  1023. # `size > 2' check that follows it, so a comment inside cannot break the
  1024. # match - but the GUARD is matched by text, and it has to be the whole
  1025. # guard. It grew an `AND (r.cls = TString)' when `s[i] := c' made a
  1026. # TString destination reach a second arm, and ParseSub grew a look-alike
  1027. # `IF symtab [r.idx].cls = TString THEN' for `s[i]'. The shorter anchor
  1028. # therefore now finds ParseSub's, some 800 lines earlier, and the splice
  1029. # below deletes everything between the two - which does not test the
  1030. # refusal, it tests whether the compiler still compiles. Hence the assert
  1031. # and the spelled-out guard.
  1032. cp "$SAVED_C" Compiler.mod
  1033. if python3 - <<'PYX'
  1034. p = 'Compiler.mod'
  1035. s = open(p).read()
  1036. g = ' IF (symtab [r.idx].cls = TString) AND (r.cls = TString) THEN\n'
  1037. assert s.count(g) == 1, 'the string-assignment guard found %d times' % s.count(g)
  1038. a = s.index(g)
  1039. m = ' RETURN\n END ;\n IF symtab [r.idx].size > 2 THEN\n'
  1040. i = s.index(m, a)
  1041. new = g + ' Err (ENoLib) ; (* MUTATION: the arm is refused *)\n RETURN\n'
  1042. open(p, 'w').write(s[:a] + new + s[i + len(' RETURN\n'):])
  1043. PYX
  1044. then
  1045. if rebuild_compiler; then
  1046. expect_red "t39 red when the string-assignment arm is refused" \
  1047. "t39_strvar" tests/run_compile_tests.sh
  1048. else
  1049. echo " FAIL: the compiler would not rebuild with the string assignment refused"
  1050. fail=$((fail + 1))
  1051. fi
  1052. else
  1053. echo " BROKEN CASE: the string-assignment mutation did not apply"
  1054. fail=$((fail + 1))
  1055. fi
  1056. cp "$SAVED_C" Compiler.mod
  1057. # 2. The write arm of IoCall, restored to its pre-fix refusal.
  1058. cp "$SAVED_C" Compiler.mod
  1059. if python3 - <<'PYX'
  1060. p = 'Compiler.mod'
  1061. s = open(p).read()
  1062. old = """ EmPushVarAddr (symtab [arg.idx].local, symtab [arg.idx].off) ;
  1063. ent := TU_WrStr"""
  1064. assert s.count(old) == 1, 'the string-write arm found %d times -- update this mutation' % s.count(old)
  1065. rep = """ Err (ENoLib) ;
  1066. RETURN"""
  1067. open(p, 'w').write(s.replace(old, rep))
  1068. PYX
  1069. then
  1070. if rebuild_compiler; then
  1071. expect_red "t39 red when a string write is refused again (ENoLib)" \
  1072. "t39_strvar" tests/run_compile_tests.sh
  1073. else
  1074. echo " FAIL: the compiler would not rebuild with the string write refused"
  1075. fail=$((fail + 1))
  1076. fi
  1077. else
  1078. echo " BROKEN CASE: the string-write mutation did not apply"
  1079. fail=$((fail + 1))
  1080. fi
  1081. cp "$SAVED_C" Compiler.mod
  1082. # 3. The runtime half: wrstr never reads the length byte, so every string -
  1083. # "hello" and the empty one alike - prints nothing. The compiler, the
  1084. # golden and the audit all stay green; only executing the entry finds it.
  1085. cp "$SAVED" Runtime.mod
  1086. if python3 - <<'PYX'
  1087. p = 'Runtime.mod'
  1088. s = open(p).read()
  1089. old = """ MovBxBp4 ; (* BX := the caller's string address *)
  1090. XorCxCx ;
  1091. MovClBx ; (* CX := the length byte at [BX] *)"""
  1092. assert s.count(old) == 1, 'the wrstr length load found %d times -- update this mutation' % s.count(old)
  1093. rep = """ MovBxBp4 ; (* BX := the caller's string address *)
  1094. XorCxCx ; (* CX is never loaded: prints nothing *)"""
  1095. open(p, 'w').write(s.replace(old, rep))
  1096. PYX
  1097. then
  1098. if rebuild; then
  1099. expect_red "the runtime cases catch a wrstr that reads no length" \
  1100. "wrstr" python3 tests/rt_exec.py
  1101. else
  1102. echo " FAIL: the runtime would not rebuild with the length load dropped"
  1103. fail=$((fail + 1))
  1104. fi
  1105. else
  1106. echo " BROKEN CASE: the wrstr length-load mutation did not apply"
  1107. fail=$((fail + 1))
  1108. fi
  1109. cp "$SAVED" Runtime.mod
  1110. # Re-emit the object: the `rebuild` above compiled Runtime.o from the MUTATED
  1111. # source, and the restored-compiler check next links that object - without
  1112. # this recompile it would report on the mutation, not on the restored code.
  1113. "$GM2" -fiso -c Runtime.mod >/dev/null 2>&1 || {
  1114. echo " FAIL: Runtime.mod would not recompile after the restore"
  1115. fail=$((fail + 1))
  1116. }
  1117. if rebuild_compiler; then
  1118. if python3 tests/run_com_exec.py >/dev/null 2>&1; then
  1119. # No count: the matrix grows every time a fixture is added, and a
  1120. # number here would be right only until the next one.
  1121. echo " ok: every executed fixture passes on the restored compiler"
  1122. pass=$((pass + 1))
  1123. else
  1124. echo "NOT RESTORED: run_com_exec.py is red after restoring Compiler.mod"
  1125. python3 tests/run_com_exec.py 2>&1 | grep -i fail | head -3 | sed 's/^/ /'
  1126. fail=$((fail + 1))
  1127. fi
  1128. else
  1129. echo "NOT RESTORED: the compiler would not rebuild"
  1130. fail=$((fail + 1))
  1131. fi
  1132. echo
  1133. echo "== the string READ and ASSIGNMENT entries (t40, t41)"
  1134. # t39 above gave a STRING variable a frame and printed it. This milestone puts
  1135. # one on the RECEIVING end of the two operations that move data into a string
  1136. # and between two strings - readln(s)/read(s), and s := t - and each one is
  1137. # three things at once: a compiler arm that chooses a runtime entry, the
  1138. # convention that carries the limit, and the entry itself. The cases below
  1139. # break each of the six in turn. Every mutation is a fault, or a plausible
  1140. # neighbour of one, that was run to a red BY HAND before it was written here.
  1141. #
  1142. # The frame rule is the odd one out: it is not a new entry at all, but a
  1143. # pre-existing compiler bug that only this milestone could reach. A local
  1144. # STRING was packed with the INTEGER rule - the symbol took the pre-decrement
  1145. # locFree, so a string[5] local sat at BP-2 and its characters grew UP into
  1146. # BP+3, over the saved BP at [BP] and the return address at [BP+2]. `readln(s)'
  1147. # into such a local stored 'hello' over its own return address, and t40 then ran
  1148. # the string's own bytes as an address: Exec86 said `fault at IP=0x6F6C', which
  1149. # is 'l','o'. The fix packs locals top-anchored, so the first INTEGER (size 2)
  1150. # is still at -2 - byte for byte what the old rule produced - while a string[5]
  1151. # (size 6) goes to -6..-1, inside the frame. t39's size (217) and t40's
  1152. # behaviour are the two independent things that pin it.
  1153. # --- the two runtime entries ---------------------------------------------
  1154. # rdstr stored the first character where the length byte belongs, because the
  1155. # IncSi that steps SI past dest[0] was missing. The length byte then
  1156. # overwrote it: `hello' read back as 05 65 6c 6c 6f. Same length, same sizes,
  1157. # and the whole compile matrix stays green - only rt_exec.py's byte dump can
  1158. # see it.
  1159. cp "$SAVED" Runtime.mod
  1160. if python3 - <<'PYX'
  1161. p = 'Runtime.mod'
  1162. lines = open(p).read().split('\n')
  1163. out, i, n = [], 0, 0
  1164. while i < len(lines):
  1165. if lines[i].startswith(' IncSi ;') and 'dest + 1' in lines[i]:
  1166. out.append(' (* MUTATION: the IncSi that puts chars at dest+1 is gone *)')
  1167. i += 1
  1168. while i < len(lines) and not lines[i].rstrip().endswith('*)'):
  1169. i += 1
  1170. i += 1
  1171. n += 1
  1172. continue
  1173. out.append(lines[i])
  1174. i += 1
  1175. assert n == 1, 'the rdstr IncSi found %d times -- update this mutation' % n
  1176. open(p, 'w').write('\n'.join(out))
  1177. PYX
  1178. then
  1179. if rebuild; then
  1180. expect_red "rt_exec catches rdstr storing its first character at dest[0]" \
  1181. "rdstr reads a line into a string" python3 tests/rt_exec.py
  1182. else
  1183. echo " FAIL: the runtime would not rebuild with rdstr's IncSi removed"
  1184. fail=$((fail + 1))
  1185. fi
  1186. else
  1187. echo " BROKEN CASE: the rdstr IncSi mutation did not apply"
  1188. fail=$((fail + 1))
  1189. fi
  1190. cp "$SAVED" Runtime.mod
  1191. # strstore's empty-source guard. A source of length 0 sets CX := 0, and LOOP
  1192. # with CX = 0 runs 65536 times rather than zero - so `s := ''' copied 64K,
  1193. # wrapped the segment, and walked through the driver's own case record. That is
  1194. # why the pattern below matches the CASE, not a byte count: rt_exec.py reports
  1195. # this one as a malformed record, which is the honest shape of "the entry wrote
  1196. # where it had no business writing".
  1197. if python3 - <<'PYX'
  1198. p = 'Runtime.mod'
  1199. lines = open(p).read().split('\n')
  1200. out, i, n = [], 0, 0
  1201. while i < len(lines):
  1202. if lines[i].startswith(' Jcxz8 ("ss_done")'):
  1203. out.append(' (* MUTATION: the empty-source guard is gone *)')
  1204. i += 1
  1205. while i < len(lines) and not lines[i].rstrip().endswith('*)'):
  1206. i += 1
  1207. i += 1
  1208. n += 1
  1209. continue
  1210. out.append(lines[i])
  1211. i += 1
  1212. assert n == 1, 'the strstore Jcxz8 found %d times -- update this mutation' % n
  1213. open(p, 'w').write('\n'.join(out))
  1214. PYX
  1215. then
  1216. if rebuild; then
  1217. expect_red "rt_exec catches a strstore that runs LOOP with CX=0" \
  1218. "strstore copies an empty string" python3 tests/rt_exec.py
  1219. else
  1220. echo " FAIL: the runtime would not rebuild with the empty-source guard removed"
  1221. fail=$((fail + 1))
  1222. fi
  1223. else
  1224. echo " BROKEN CASE: the strstore empty-source guard mutation did not apply"
  1225. fail=$((fail + 1))
  1226. fi
  1227. cp "$SAVED" Runtime.mod
  1228. # Re-emit the object: `rebuild` above compiled Runtime.o from the MUTATED
  1229. # source, and the compiler rebuild below links that object - without this
  1230. # recompile the restored-compiler check at the end would report on the mutation.
  1231. "$GM2" -fiso -c Runtime.mod >/dev/null 2>&1 || {
  1232. echo " FAIL: Runtime.mod would not recompile after the restore"
  1233. fail=$((fail + 1))
  1234. }
  1235. # --- the frame rule -------------------------------------------------------
  1236. # The pre-fix layout, exactly: the symbol took locFree itself instead of its
  1237. # top, and the frame began at 0FFFEh. For INTEGER locals that is byte-for-byte
  1238. # the same -2, -4, ... the new rule produces, which is why the old rule survived
  1239. # so long unnoticed; for a STRING it puts characters where the return address
  1240. # is. Two independent readers go red together, so both are asserted: the
  1241. # compile matrix pins the LAYOUT (t39's code size drops 217 -> 215), and t40
  1242. # pins that the program RUNS (with the overlap its readln never sees the line
  1243. # terminator and the machine hangs).
  1244. cp "$SAVED_C" Compiler.mod
  1245. if python3 - <<'PYX'
  1246. p = 'Compiler.mod'
  1247. s = open(p).read()
  1248. a = '(locFree - size + 1) MOD 10000H'
  1249. assert s.count(a) == 2, 'the top-anchored base found %d times -- update this mutation' % s.count(a)
  1250. s = s.replace(a, 'locFree') # the old pre-decrement base
  1251. b = 'locFree := 0FFFFH ;'
  1252. assert s.count(b) == 2, 'the frame top found %d times -- update this mutation' % s.count(b)
  1253. s = s.replace(b, 'locFree := 0FFFEH ;') # the old starting point
  1254. open(p, 'w').write(s)
  1255. PYX
  1256. then
  1257. if rebuild_compiler; then
  1258. expect_red "the compile matrix catches a local string packed over the frame" \
  1259. "expected: OK 217" tests/run_compile_tests.sh
  1260. expect_red "t40 red when readln into a local string overwrites its return address" \
  1261. "never halted" python3 tests/run_com_exec.py t40_readln
  1262. else
  1263. echo " FAIL: the compiler would not rebuild with the old frame layout"
  1264. fail=$((fail + 1))
  1265. fi
  1266. else
  1267. echo " BROKEN CASE: the frame-layout revert did not apply"
  1268. fail=$((fail + 1))
  1269. fi
  1270. cp "$SAVED_C" Compiler.mod
  1271. # --- the compiler's arms --------------------------------------------------
  1272. # The limit both entries read is MOV CX,imm16. MOV AX,imm16 is one opcode away,
  1273. # the same length, and the audit's MovCxi row is COMP - "no single sequence to
  1274. # disassemble" - so nothing in the byte-level checks can tell them apart. Only
  1275. # execution can: t41's `s := t' truncates to 4 under CX and does not under AX.
  1276. # (t40 passes either way, because its CX happens to survive large enough for the
  1277. # five characters it reads - exactly the kind of luck a behavioural case is for.)
  1278. cp "$SAVED_C" Compiler.mod
  1279. if mutate_compiler 's| Ebyte (0B9H) ; Eword (imm)| Ebyte (0B8H) ; Eword (imm)|'; then
  1280. if rebuild_compiler; then
  1281. expect_red "t41 red when the destination limit is loaded into AX, not CX" \
  1282. "t41_strassign" python3 tests/run_com_exec.py t41_strassign
  1283. else
  1284. echo " FAIL: the compiler would not rebuild with MovCxi emitting MOV AX"
  1285. fail=$((fail + 1))
  1286. fi
  1287. fi
  1288. cp "$SAVED_C" Compiler.mod
  1289. # Assignment pushes the source first, then the destination, so the source sits
  1290. # at [BP+6] and the destination at [BP+4]. Swapped, strstore takes the
  1291. # destination as its own source - t41's `worl' rows become copies of what was
  1292. # already in the destination, which looks like it works and prints the wrong
  1293. # bytes.
  1294. cp "$SAVED_C" Compiler.mod
  1295. if python3 - <<'PYX'
  1296. p = 'Compiler.mod'
  1297. s = open(p).read()
  1298. src = (' EmPushVarAddr (symtab [src.idx].local,\n'
  1299. ' (symtab [src.idx].off + src.boff) MOD 10000H) ;\n'
  1300. ' EmPushVarAddr (symtab [r.idx].local,\n'
  1301. ' (symtab [r.idx].off + r.boff) MOD 10000H) ;')
  1302. dst = (' EmPushVarAddr (symtab [r.idx].local,\n'
  1303. ' (symtab [r.idx].off + r.boff) MOD 10000H) ;\n'
  1304. ' EmPushVarAddr (symtab [src.idx].local,\n'
  1305. ' (symtab [src.idx].off + src.boff) MOD 10000H) ;')
  1306. assert s.count(src) == 1, 'the assignment pushes found %d times -- update this mutation' % s.count(src)
  1307. open(p, 'w').write(s.replace(src, dst))
  1308. PYX
  1309. then
  1310. if rebuild_compiler; then
  1311. expect_red "t41 red when the assignment pushes are swapped" \
  1312. "t41_strassign" python3 tests/run_com_exec.py t41_strassign
  1313. else
  1314. echo " FAIL: the compiler would not rebuild with the pushes swapped"
  1315. fail=$((fail + 1))
  1316. fi
  1317. else
  1318. echo " BROKEN CASE: the assignment push-order mutation did not apply"
  1319. fail=$((fail + 1))
  1320. fi
  1321. cp "$SAVED_C" Compiler.mod
  1322. # The read dispatch's string arm. TU_RdStr and TU_RdInt sit next to each other
  1323. # in the same table and both take one address; TU_RdInt parses the line as a
  1324. # number, which stores nothing on `world', so readln(s) leaves s empty and the
  1325. # program prints a blank line. A refusal would at least say so; this arm looks
  1326. # like it worked and answers with the wrong bytes.
  1327. cp "$SAVED_C" Compiler.mod
  1328. if python3 - <<'PYX'
  1329. p = 'Compiler.mod'
  1330. s = open(p).read()
  1331. old = (' EmMovCxi ((symtab [arg.idx].size - 1) MOD 10000H) ;\n'
  1332. ' ent := TU_RdStr')
  1333. assert s.count(old) == 1, 'the string-read arm found %d times -- update this mutation' % s.count(old)
  1334. open(p, 'w').write(s.replace(old, old.replace('TU_RdStr', 'TU_RdInt')))
  1335. PYX
  1336. then
  1337. if rebuild_compiler; then
  1338. expect_red "t40 red when a string read dispatches to TU_RdInt" \
  1339. "t40_readln" python3 tests/run_com_exec.py t40_readln
  1340. else
  1341. echo " FAIL: the compiler would not rebuild with the read dispatch on TU_RdInt"
  1342. fail=$((fail + 1))
  1343. fi
  1344. else
  1345. echo " BROKEN CASE: the string-read dispatch mutation did not apply"
  1346. fail=$((fail + 1))
  1347. fi
  1348. cp "$SAVED_C" Compiler.mod
  1349. if rebuild_compiler; then
  1350. if python3 tests/run_com_exec.py >/dev/null 2>&1; then
  1351. echo " ok: every executed fixture passes on the restored sources"
  1352. pass=$((pass + 1))
  1353. else
  1354. echo "NOT RESTORED: run_com_exec.py is red after restoring the sources"
  1355. python3 tests/run_com_exec.py 2>&1 | grep -i fail | head -3 | sed 's/^/ /'
  1356. fail=$((fail + 1))
  1357. fi
  1358. else
  1359. echo "NOT RESTORED: the compiler would not rebuild"
  1360. fail=$((fail + 1))
  1361. fi
  1362. echo
  1363. echo "== the second execution oracle (run_exec86.py)"
  1364. # Everything above runs the image under qemu-system-i386. This one runs the
  1365. # SAME image inside shell/Exec86.mod, this project's own in-process 8086
  1366. # interpreter, and requires its output to agree with BOTH the hand-derived .out
  1367. # and qemu, byte for byte. Two execution checks that could only ever agree
  1368. # with each other would be one check: the point of this one is that it was
  1369. # written against the 8086's own reference rather than against qemu, whose
  1370. # lowest CPU model is a 486 and whose `0F 84' is an ordinary JZ.
  1371. #
  1372. # So the only mutation worth writing here is one qemu cannot make at all - the
  1373. # interpreter's own reading of the machine. Cond is that reading: nibble 4 is
  1374. # JE, and inverting it swaps the two arms of every `=' in every program. The
  1375. # emitted bytes, the sizes and the compile matrix are untouched, because
  1376. # nothing is emitted here - the fault is in who interprets it.
  1377. #
  1378. # Exec86.mod is recompiled explicitly rather than left to the harness:
  1379. # ensure_exec86run() notices the source changed and RELINKS, but does not
  1380. # recompile it, so a mutated source with a stale object would link the good
  1381. # interpreter straight back in and stay green - which is the trap its own
  1382. # docstring records, and the reason this case compiles first.
  1383. #
  1384. # SAVED_E itself was taken at the top of this file, next to the EXIT trap that
  1385. # puts it back.
  1386. if python3 - <<'PYX'
  1387. p = 'Exec86.mod'
  1388. s = open(p).read()
  1389. old = "| 4H : r := ZF"
  1390. new = "| 4H : r := NOT ZF (* MUTATION: JE inverted *)"
  1391. assert s.count(old) == 1, 'the JE arm of Cond found %d times -- update this mutation' % s.count(old)
  1392. open(p, 'w').write(s.replace(old, new))
  1393. PYX
  1394. then
  1395. if $GM2 -fiso -c Exec86.mod >/dev/null 2>&1; then
  1396. expect_red "the interpreter's own oracle catches JE inverted" \
  1397. "t33_cmpops" python3 tests/run_exec86.py t33_cmpops
  1398. else
  1399. echo " FAIL: Exec86.mod would not compile with JE inverted"
  1400. fail=$((fail + 1))
  1401. fi
  1402. else
  1403. echo " BROKEN CASE: the Cond JE mutation did not apply"
  1404. fail=$((fail + 1))
  1405. fi
  1406. cp "$SAVED_E" Exec86.mod
  1407. # And the check on the RESTORED interpreter, because a green above could also
  1408. # come from a mutation that never took and an object left mutated by a run
  1409. # that died in between.
  1410. if $GM2 -fiso -c Exec86.mod >/dev/null 2>&1; then
  1411. if python3 tests/run_exec86.py >/dev/null 2>&1; then
  1412. echo " ok: run_exec86 green on the restored interpreter"
  1413. pass=$((pass + 1))
  1414. else
  1415. echo "NOT RESTORED: run_exec86.py is red after restoring Exec86.mod"
  1416. python3 tests/run_exec86.py 2>&1 | grep -i "fail" | head -3 | sed 's/^/ /'
  1417. fail=$((fail + 1))
  1418. fi
  1419. else
  1420. echo "NOT RESTORED: Exec86.mod would not recompile"
  1421. fail=$((fail + 1))
  1422. fi
  1423. echo
  1424. echo "== the R key: compile, poke, run, report (runtest.py)"
  1425. # The only check in the project that exercises CmdRun. Everything above looks
  1426. # at bytes, or at what qemu says the image does; this one drives the shell
  1427. # through a pty the way a person would, presses R, and compares the GUEST's
  1428. # output against the fixture's hand-derived .out - so it fails on things no
  1429. # byte check can see and no file records either, because R writes no file at
  1430. # all (which is itself asserted).
  1431. #
  1432. # The mutation is the poke loop's count. With n = 0 nothing is copied into
  1433. # the interpreter, loadHi stays where Clear86 left it - 0100h - and Run86
  1434. # faults on its very first step, "execution left the loaded image", before the
  1435. # guest has executed a single instruction. That is fast and deterministic, which
  1436. # matters: the obvious alternative (poking the image somewhere else) leaves the
  1437. # machine executing zeros and buys a step-limit timeout instead of a finding.
  1438. cp "$SAVED_SH" Shell.mod
  1439. if python3 - <<'PYX'
  1440. p = 'Shell.mod'
  1441. s = open(p).read()
  1442. old = " n := LinkSize () ;"
  1443. new = " n := 0 ; (* MUTATION: nothing is poked *)"
  1444. assert s.count(old) == 1, 'the poke count in CmdRun found %d times -- update this mutation' % s.count(old)
  1445. open(p, 'w').write(s.replace(old, new))
  1446. PYX
  1447. then
  1448. if make > ../tmp/nonvacuity.make.log 2>&1; then
  1449. expect_red "the R check catches an image that was never poked" \
  1450. "AH=4Ch exit" python3 tests/runtest.py
  1451. else
  1452. echo " FAIL: the shell would not rebuild with the poke loop neutered"
  1453. tail -5 ../tmp/nonvacuity.make.log | sed 's/^/ /'
  1454. fail=$((fail + 1))
  1455. fi
  1456. else
  1457. echo " BROKEN CASE: the CmdRun poke mutation did not apply"
  1458. fail=$((fail + 1))
  1459. fi
  1460. cp "$SAVED_SH" Shell.mod
  1461. if make > ../tmp/nonvacuity.make.log 2>&1; then
  1462. if python3 tests/runtest.py >/dev/null 2>&1; then
  1463. echo " ok: runtest green on the restored shell"
  1464. pass=$((pass + 1))
  1465. else
  1466. echo "NOT RESTORED: runtest.py is red after restoring Shell.mod"
  1467. python3 tests/runtest.py 2>&1 | grep -i "fail" | head -3 | sed 's/^/ /'
  1468. fail=$((fail + 1))
  1469. fi
  1470. else
  1471. echo "NOT RESTORED: the shell would not rebuild"
  1472. tail -5 ../tmp/nonvacuity.make.log | sed 's/^/ /'
  1473. fail=$((fail + 1))
  1474. fi
  1475. echo
  1476. echo "== 8086 legality of the conditional lowering (check_8086.py)"
  1477. # This whole section exists because of a fault that every other check in the
  1478. # project was blind to, and being blunt about WHY is the point of listing it.
  1479. #
  1480. # EmJcc and EmSetcc emitted `0F 8x rel16' and `0F 9x rel8'. `0F' is a
  1481. # 386-and-later opcode prefix; the 8086 has none. So EVERY conditional branch
  1482. # and EVERY comparison in EVERY compiled program was an illegal instruction on
  1483. # the machine this compiler targets. And all of the following were green while
  1484. # it was: the compile matrix, the .COM layout checker, the runtime golden, the
  1485. # emitter audit, and the whole execution suite answering correctly under qemu.
  1486. #
  1487. # Two reasons, and the second is the one worth keeping:
  1488. # 1. qemu-system-i386 has no 8086 model. Its lowest is 486, where `0F 84' is
  1489. # a perfectly ordinary JZ. So the execution oracle CANNOT see this class
  1490. # of fault, ever.
  1491. # 2. FCML is this project's INDEPENDENT disassembler, and FCML's -m16 mode is
  1492. # a 386. The one tool whose job is to say "this is not a real instruction"
  1493. # was architecturally guaranteed to agree with the bug.
  1494. #
  1495. # So a check was needed that asks the question nothing else was asking. The
  1496. # five mutations below are the mutations that check has to catch, and each was
  1497. # run by hand and confirmed red BEFORE this section existed.
  1498. #
  1499. # M4 and M5 are the interesting pair. M4 is a fault this project actually
  1500. # introduced while fixing the above: EmJcc jumps TO its target, but the 8086
  1501. # shape steps OVER a 3-byte EJMP, so the naive port inverts every conditional
  1502. # in every program. It was caught by execution, not by any byte check. M5 is
  1503. # the partial version -- the inversion dropped for the IF and CASE sites only,
  1504. # kept for FOR -- and MEASURING that is what produced clause H, because the
  1505. # check was blind to M5 as first written: IF declares nibble 4, CASE declares
  1506. # 5, they negate into each other, and both are declared, so nothing complained
  1507. # while every conditional in every program took the wrong path. FOR declares
  1508. # C and F, whose negations D and E are declared for nothing at all, so the
  1509. # whole-suite version was caught by luck.
  1510. #
  1511. # Note these need `rebuild_compiler' only to refresh comtest; check_8086.py
  1512. # relinks the fixtures itself. Getting that wrong is how this section's first
  1513. # draft reported a mutation as VACUOUS: the build was skipped, a stale comtest
  1514. # ran the GOOD compiler, and the check passed. A helper that exits non-zero
  1515. # makes `if mutate_x; then' false and the case is silently skipped, which looks
  1516. # exactly like a pass -- so rebuild_compiler is checked, not assumed.
  1517. # M1: the original defect, restored verbatim.
  1518. cp "$SAVED_C" Compiler.mod
  1519. if python3 - <<'PYX'
  1520. p = 'Compiler.mod'
  1521. s = open(p).read()
  1522. old = """ Ebyte (JccShortInv (cc)) ; (* Jcc_s, taken when cc does NOT hold *)
  1523. Ebyte (03H) ; (* rel8: step over the 3-byte EJMP *)
  1524. RETURN EmJmpNear (target) (* target = 0 => forward, see above *)"""
  1525. new = """ Ebyte (0FH) ; Ebyte (cc) ; Eword (0) ;
  1526. IF target = 0 THEN
  1527. RETURN nPatch
  1528. END ;
  1529. RETURN 0"""
  1530. assert s.count(old) == 1, 'EmJcc body found %d times -- update this mutation' % s.count(old)
  1531. open(p, 'w').write(s.replace(old, new))
  1532. PYX
  1533. then
  1534. if rebuild_compiler; then
  1535. expect_red "check_8086 catches a 0F 8x branch (the original bug)" \
  1536. "0F 84" python3 tests/check_8086.py
  1537. else
  1538. echo " FAIL: the compiler would not rebuild with the 0F branch"
  1539. fail=$((fail + 1))
  1540. fi
  1541. else
  1542. echo " BROKEN CASE: the 0F branch mutation did not apply"
  1543. fail=$((fail + 1))
  1544. fi
  1545. cp "$SAVED_C" Compiler.mod
  1546. # M2: the other original defect, SETcc plus the MOV AH,0 it needed.
  1547. cp "$SAVED_C" Compiler.mod
  1548. if python3 - <<'PYX'
  1549. p = 'Compiler.mod'
  1550. s = open(p).read()
  1551. old = """ Ebyte (0B8H) ; Eword (1) ; (* MOV AX,#0001 *)
  1552. Ebyte (JccShort (cc)) ; (* taken when the comparison HOLDS *)
  1553. Ebyte (01H) ; (* rel8: step over the DEC AX *)
  1554. Ebyte (48H) (* DEC AX *)"""
  1555. new = """ Ebyte (0FH) ; Ebyte (cc) ; Ebyte (0C0H) ;
  1556. EmMovAh0 ()"""
  1557. assert s.count(old) == 1, 'EmSetcc body found %d times -- update this mutation' % s.count(old)
  1558. open(p, 'w').write(s.replace(old, new))
  1559. PYX
  1560. then
  1561. if rebuild_compiler; then
  1562. expect_red "check_8086 catches a 0F 9x SETcc (the original bug)" \
  1563. "0F 9F C0" python3 tests/check_8086.py
  1564. else
  1565. echo " FAIL: the compiler would not rebuild with the 0F SETcc"
  1566. fail=$((fail + 1))
  1567. fi
  1568. else
  1569. echo " BROKEN CASE: the 0F SETcc mutation did not apply"
  1570. fail=$((fail + 1))
  1571. fi
  1572. cp "$SAVED_C" Compiler.mod
  1573. # M4: the polarity inversion, everywhere. Note the expected text is clause H's,
  1574. # not a shape complaint: every byte here is a legal 8086 shape, which is the
  1575. # entire reason a separate clause had to be written.
  1576. cp "$SAVED_C" Compiler.mod
  1577. if python3 - <<'PYX'
  1578. p = 'Compiler.mod'
  1579. s = open(p).read()
  1580. old = ' Ebyte (JccShortInv (cc)) ; (* Jcc_s, taken when cc does NOT hold *)'
  1581. new = ' Ebyte (JccShort (cc)) ; (* MUTATION: inversion dropped *)'
  1582. assert s.count(old) == 1, 'EmJcc Ebyte found %d times -- update this mutation' % s.count(old)
  1583. open(p, 'w').write(s.replace(old, new))
  1584. PYX
  1585. then
  1586. if rebuild_compiler; then
  1587. expect_red "clause H catches the branch polarity inverted everywhere" \
  1588. "but reading the source says" python3 tests/check_8086.py
  1589. else
  1590. echo " FAIL: the compiler would not rebuild with the inversion dropped"
  1591. fail=$((fail + 1))
  1592. fi
  1593. else
  1594. echo " BROKEN CASE: the polarity mutation did not apply"
  1595. fail=$((fail + 1))
  1596. fi
  1597. cp "$SAVED_C" Compiler.mod
  1598. # M5: the same inversion dropped for the IF and CASE sites ONLY. This one is
  1599. # the reason clause H exists: it was run by hand and the check stayed GREEN,
  1600. # and FOR's C/F nibbles were the only reason the whole-suite version (M4)
  1601. # happened to be caught.
  1602. cp "$SAVED_C" Compiler.mod
  1603. if python3 - <<'PYX'
  1604. p = 'Compiler.mod'
  1605. s = open(p).read()
  1606. old = ' Ebyte (JccShortInv (cc)) ; (* Jcc_s, taken when cc does NOT hold *)'
  1607. new = """ IF (cc = 84H) OR (cc = 85H) THEN
  1608. Ebyte (JccShort (cc)) (* MUTATION: no inversion here *)
  1609. ELSE
  1610. Ebyte (JccShortInv (cc))
  1611. END ;"""
  1612. assert s.count(old) == 1, 'EmJcc Ebyte found %d times -- update this mutation' % s.count(old)
  1613. open(p, 'w').write(s.replace(old, new))
  1614. PYX
  1615. then
  1616. if rebuild_compiler; then
  1617. expect_red "clause H catches the inversion dropped for IF and CASE only" \
  1618. "t22_case" python3 tests/check_8086.py
  1619. else
  1620. echo " FAIL: the compiler would not rebuild with the partial inversion"
  1621. fail=$((fail + 1))
  1622. fi
  1623. else
  1624. echo " BROKEN CASE: the partial-inversion mutation did not apply"
  1625. fail=$((fail + 1))
  1626. fi
  1627. cp "$SAVED_C" Compiler.mod
  1628. # And the check on the RESTORED compiler, so a green above cannot come from a
  1629. # mutation that failed to take and left the real defect in place.
  1630. if rebuild_compiler; then
  1631. if python3 tests/check_8086.py >/dev/null 2>&1; then
  1632. echo " ok: check_8086 green on the restored compiler"
  1633. pass=$((pass + 1))
  1634. else
  1635. echo "NOT RESTORED: check_8086.py is red after restoring Compiler.mod"
  1636. python3 tests/check_8086.py 2>&1 | grep -m3 -- ' - ' | sed 's/^/ /'
  1637. fail=$((fail + 1))
  1638. fi
  1639. else
  1640. echo "NOT RESTORED: the compiler would not rebuild"
  1641. fail=$((fail + 1))
  1642. fi
  1643. echo
  1644. echo "== the emitter-name audit of Compiler.mod (audit_helpers.py)"
  1645. # These need no rebuild: the audit reads the SOURCE, not the built object, so
  1646. # they are the cheapest cases here and they cover the module the audit used
  1647. # not to look at at all. That is the point of the section: the audit reported
  1648. # "every helper agrees with its name" for a module it had never examined, and
  1649. # EmXchgAxCx was `93` (XCHG BX,AX) under a name that says XCHG AX,CX for the
  1650. # whole life of the project. Two of these five are for faults that were real.
  1651. SAVED_C2=../tmp/nonvacuity.Compiler.mod.2
  1652. SAVED_R2=../tmp/nonvacuity.Runtime.mod.2
  1653. cp Compiler.mod "$SAVED_C2" || exit 1
  1654. cp Runtime.mod "$SAVED_R2" || exit 1
  1655. restore_audit_sources () {
  1656. cp "$SAVED_C2" Compiler.mod
  1657. cp "$SAVED_R2" Runtime.mod
  1658. }
  1659. AUD="python3 tests/audit_helpers.py"
  1660. # 1. THE fault. 91h is XCHG AX,CX; 93h is XCHG BX,AX. Both are one byte, so
  1661. # the compile matrix never moved and the byte counts never moved.
  1662. cp "$SAVED_C2" Compiler.mod
  1663. mutate Compiler.mod 's|^ Ebyte (91H)$| Ebyte (93H)|'
  1664. expect_red "audit catches XchgAxCx emitting XCHG BX,AX" \
  1665. "exchanges Ax and Bx" $AUD
  1666. restore_audit_sources
  1667. # 2. the coverage check itself. A parameter list that find_helpers does not
  1668. # accept is exactly how the real emitter was missed, and the inventory is
  1669. # scanned separately on purpose so this can be caught. Without the
  1670. # independent scan this case is silent, because both lists would come from
  1671. # the same parser and agree that the helper does not exist.
  1672. cp "$SAVED_C2" Compiler.mod
  1673. mutate Compiler.mod 's|^PROCEDURE EmXchgAxCx () ;$|PROCEDURE EmXchgAxCx (why : CARDINAL) ;|'
  1674. expect_red "audit reports an emitter it cannot reach, rather than skipping it" \
  1675. "never examined it" $AUD
  1676. restore_audit_sources
  1677. # 3. EmXchgAxDx was named EmMoveAxDx, which said MOV where the bytes say XCHG.
  1678. # 93h here is XCHG AX,BX - one letter away, the exact class of mistake the
  1679. # name is supposed to make impossible.
  1680. cp "$SAVED_C2" Compiler.mod
  1681. mutate Compiler.mod 's|^ Ebyte (92H)$| Ebyte (93H)|'
  1682. expect_red "audit catches XchgAxDx emitting XCHG BX,AX" \
  1683. "XchgAxDx" $AUD
  1684. restore_audit_sources
  1685. # 4. CmpArgW0's [BP+2] written as the 386 SIB form, which decodes on a 8086 as
  1686. # [SI+24h]. A real bug: the runtime was clearing the wrong memory.
  1687. cp "$SAVED_R2" Runtime.mod
  1688. mutate Runtime.mod 's| B (83H) ; B (7EH) ; B (2) ; B (0) ;| B (83H) ; B (7CH) ; B (24) ; B (0) ; B (0) ;|'
  1689. expect_red "audit catches the [SI+24h] encoding of [BP+2]" \
  1690. "memory base is 'si" $AUD
  1691. restore_audit_sources
  1692. # 5. MovAxSp is POP then PUSH, because MOV AX,[SP] does not exist on an 8086.
  1693. # Dropping the POP leaves the stack one word short - a fault in the shape,
  1694. # not in a byte value.
  1695. cp "$SAVED_C2" Compiler.mod
  1696. python3 - <<'PYEOF'
  1697. p='Compiler.mod'; s=open(p).read()
  1698. a=" Ebyte (58H) ; (* POP AX *)\n"
  1699. assert s.count(a)==1, "EmMovAxSp POP line not found -- update this mutation"
  1700. open(p,'w').write(s.replace(a, ""))
  1701. PYEOF
  1702. expect_red "audit catches MovAxSp with its POP missing" \
  1703. "MovAxSp" $AUD
  1704. restore_audit_sources
  1705. # 6. The two-instruction shape: IDIV is CWD then IDIV, and dropping the CWD
  1706. # leaves an un-sign-extended dividend in DX:AX. Both are still present as
  1707. # a two-step spec, so a missing step has to be visible.
  1708. cp "$SAVED_C2" Compiler.mod
  1709. mutate Compiler.mod 's| Ebyte (99H) ; Ebyte (0F7H) ; Ebyte (0F9H)| Ebyte (0F7H) ; Ebyte (0F9H)|'
  1710. expect_red "audit catches IDiv without the CWD that extends the dividend" \
  1711. "IDivAxCx" $AUD
  1712. restore_audit_sources
  1713. # 7. The byte under EmXorAl01's name, and the row in PATTERNS that was added
  1714. # to admit it. `34 01' is XOR AL,#01 - the boolean NOT, and the one emitter
  1715. # in Compiler.mod with no ModRM byte at all. A grammar row nobody has ever
  1716. # seen reject anything cannot be trusted to accept only the truth, so here
  1717. # is the rejection: `34 02' is the same length, decodes just as cleanly, and
  1718. # `not x' would flip bit 1 instead of bit 0.
  1719. cp "$SAVED_C2" Compiler.mod
  1720. mutate Compiler.mod 's|^ Ebyte (34H) ; Ebyte (01H)| Ebyte (34H) ; Ebyte (02H)|'
  1721. expect_red "audit catches EmXorAl01 emitting xor al,#2 under a name saying #1" \
  1722. "XorAl01" $AUD
  1723. restore_audit_sources
  1724. # 7b. The other half of that row: the opcode gate. 34h is XOR AL,#imm and
  1725. # 35h is XOR AX,#imm - a word, not a byte, under a name that says AL. The
  1726. # row is pinned to {034h} on purpose, and a pin that has never been asked
  1727. # to hold is a pin. The report is the one thing the row above cannot
  1728. # produce: with no reading at all, the helper falls out of the grammar.
  1729. cp "$SAVED_C2" Compiler.mod
  1730. mutate Compiler.mod 's|^ Ebyte (34H) ; Ebyte (01H)| Ebyte (35H) ; Ebyte (01H)|'
  1731. expect_red "the opcode gate rejects XOR AX under an XOR AL name" \
  1732. "no name pattern accepts it" $AUD
  1733. restore_audit_sources
  1734. if $AUD >/dev/null 2>&1; then
  1735. echo " ok: the audit passes on both restored sources"
  1736. pass=$((pass + 1))
  1737. else
  1738. echo "NOT RESTORED: the audit is red after restoring the sources"
  1739. $AUD 2>&1 | sed 's/^/ /'
  1740. fail=$((fail + 1))
  1741. fi
  1742. echo
  1743. echo "== the BP contract rt_exec.py checks before it starts a machine"
  1744. # wrchar and wrbool both borrowed BP to reach their argument -- [SP] is not
  1745. # encodable in 16-bit mode -- and neither saved it. The driver's cursor into
  1746. # the case record lives in BP precisely because BP is the one register an entry
  1747. # may keep, so "wrchar borrowed it and did not give it back" sent the second
  1748. # call to a garbage address, the machine triple-faulted, and the run printed the
  1749. # record header twice and hung. Both the golden and the audit call that shape
  1750. # CORRECT: the bytes are well formed, every branch is on a boundary, the size
  1751. # is unchanged, and the decode says exactly what it says. So the rule is now
  1752. # checked directly, and these two cases are what make that check more than a
  1753. # claim.
  1754. #
  1755. # rt_exec.py needs the whole runtime rebuilt and then boots 36 machines, so this
  1756. # section is the slow one. The baseline comes first and is asserted: a case
  1757. # that mutates a red tree proves nothing.
  1758. SAVED_R3=../tmp/nonvacuity.Runtime.mod.3
  1759. cp Runtime.mod "$SAVED_R3" || exit 1
  1760. if rebuild; then
  1761. if python3 tests/rt_exec.py >/dev/null 2>&1; then
  1762. echo " ok: baseline - rt_exec.py passes on the unmutated runtime"
  1763. pass=$((pass + 1))
  1764. else
  1765. echo " FAIL: the baseline is already red, so the cases below prove"
  1766. echo " nothing - fix the baseline before reading them"
  1767. python3 tests/rt_exec.py 2>&1 | tail -3 | sed 's/^/ /'
  1768. fail=$((fail + 1))
  1769. fi
  1770. else
  1771. echo " FAIL: could not rebuild the runtime for the baseline"
  1772. fail=$((fail + 1))
  1773. fi
  1774. # 1. THE fault: drop the PUSH, exactly as EmitWrChar was written. This is the
  1775. # shape the behavioural case found, so the byte-level check must find it too
  1776. # -- a check that only the expensive test can trip is a check that has not
  1777. # been made to earn its place.
  1778. cp "$SAVED_R3" Runtime.mod
  1779. python3 - <<'PYEOF'
  1780. p = 'Runtime.mod'
  1781. s = open(p).read()
  1782. a = ' M ("wrchar") ;\n PushBp ; MovBpSp ;\n'
  1783. assert s.count(a) == 1, "EmitWrChar prologue not found exactly once"
  1784. open(p, 'w').write(s.replace(a, ' M ("wrchar") ;\n MovBpSp ;\n'))
  1785. PYEOF
  1786. if rebuild; then
  1787. expect_red "the BP contract catches wrchar borrowing BP unsaved" \
  1788. "borrows BP but does not open with" python3 tests/rt_exec.py
  1789. else
  1790. echo " FAIL: the runtime would not rebuild with the PUSH removed"
  1791. fail=$((fail + 1))
  1792. fi
  1793. cp "$SAVED_R3" Runtime.mod
  1794. # 2. The mirror image: push it and never pop it. A different one-instruction
  1795. # omission with the same consequence for a caller, and the reason the rule
  1796. # asks for the 5D and not just the 55.
  1797. cp "$SAVED_R3" Runtime.mod
  1798. mutate Runtime.mod 's|^ MovSpBp ; PopBp ;$| MovSpBp ;|'
  1799. if rebuild; then
  1800. expect_red "the BP contract catches a BP that is pushed and never popped" \
  1801. "pushes BP but never pops it" python3 tests/rt_exec.py
  1802. else
  1803. echo " FAIL: the runtime would not rebuild with the POP removed"
  1804. fail=$((fail + 1))
  1805. fi
  1806. cp "$SAVED_R3" Runtime.mod
  1807. # 3. The scan's other silent failure: a rule that matches nothing looks exactly
  1808. # like a rule that passes. The pattern the check looks for is changed to one
  1809. # the blob does not contain -- MOV BP,DI, which the runtime has no reason to
  1810. # emit -- so the check has examined thirteen entries and matched nothing and
  1811. # MUST say so rather than report a clean sweep. This is the general shape of
  1812. # the fault this project keeps making: a check whose SUBJECT has drifted
  1813. # reports a confident answer about the wrong thing.
  1814. cp "$SAVED_R3" Runtime.mod
  1815. SAVED_X=../tmp/nonvacuity.rt_exec.py
  1816. cp tests/rt_exec.py "$SAVED_X" || exit 1
  1817. mutate tests/rt_exec.py 's|^MOV_BP_SP = b"\\x8b\\xec" .*$|MOV_BP_SP = b"\\x8b\\xed" # MOV BP,DI: never emitted|' \
  1818. expect_red "a check that matched nothing is a failure, not a pass" \
  1819. "matched nothing" python3 tests/rt_exec.py
  1820. cp "$SAVED_X" tests/rt_exec.py
  1821. if rebuild; then
  1822. if python3 tests/rt_exec.py >/dev/null 2>&1; then
  1823. echo " ok: rt_exec.py passes on the restored source"
  1824. pass=$((pass + 1))
  1825. else
  1826. echo "NOT RESTORED: rt_exec.py is red after restoring Runtime.mod"
  1827. python3 tests/rt_exec.py 2>&1 | tail -3 | sed 's/^/ /'
  1828. fail=$((fail + 1))
  1829. fi
  1830. else
  1831. echo "NOT RESTORED: the runtime would not rebuild"
  1832. fail=$((fail + 1))
  1833. fi
  1834. # 4. The GOLDEN and the entry goldens, which are the other half of the same
  1835. # rule. rt_exec.py states the contract; check_runtime.py pins the bytes.
  1836. # They are separate mechanisms and the case below is what shows the golden
  1837. # one works on its own -- a re-baseline of runtime.golden would otherwise
  1838. # have absorbed the new prologues silently, and the next person to bless it
  1839. # would have no way to know the PUSH and POP were ever missing.
  1840. cp "$SAVED_R3" Runtime.mod
  1841. python3 - <<'PYEOF'
  1842. p = 'Runtime.mod'
  1843. s = open(p).read()
  1844. a = ' M ("wrchar") ;\n PushBp ; MovBpSp ;\n MovAlArg4 ;\n MovSpBp ; PopBp ;\n'
  1845. assert s.count(a) == 1, "EmitWrChar frame not found exactly once"
  1846. open(p, 'w').write(s.replace(a, ' M ("wrchar") ;\n MovBpSp ;\n MovAlArg2 ;\n MovSpBp ;\n'))
  1847. PYEOF
  1848. if rebuild; then
  1849. expect_red "the entry golden catches wrchar without its PUSH BP" \
  1850. "entry wrchar starts 8B EC" python3 tests/check_runtime.py "$DUMP"
  1851. else
  1852. echo " FAIL: the runtime would not rebuild with wrchar's frame removed"
  1853. fail=$((fail + 1))
  1854. fi
  1855. cp "$SAVED_R3" Runtime.mod
  1856. echo
  1857. echo "== the .COM layout check, and the runtime size it now measures"
  1858. # The checker used to RESTATE the runtime's size as a literal. It had drifted
  1859. # from the runtime it described, so the header was read out of the code stream
  1860. # and EVERY linked fixture "failed" on it - a whole file of "failures" that were
  1861. # not findings at all. A duplicated constant that has drifted does not fail
  1862. # loudly; it re-reports the same falsehood, in which the real failures hide.
  1863. # The size is now MEASURED from the image.
  1864. #
  1865. # These cases corrupt a real emitted .COM and require the checker to notice.
  1866. # They need the images, so they are built once and copied; the checker has a
  1867. # --check-only mode for exactly this, because its scratch directory is normally
  1868. # deleted on exit and a check that has only ever seen the truth is not a check.
  1869. KEEPDIR=../tmp/nonvacuity.com
  1870. rm -rf "$KEEPDIR"
  1871. TP_COM_KEEP=1 tests/run_com_tests.sh >../tmp/nonvacuity.com.log 2>&1
  1872. KEEP=$(sed -n 's/^TP_COM_KEEP=1: images left in //p' \
  1873. ../tmp/nonvacuity.com.log | tail -1)
  1874. if [ -z "$KEEP" ] || [ ! -d "$KEEP" ]; then
  1875. echo " FAIL: could not obtain emitted .COM images for the layout cases"
  1876. fail=$((fail + 1))
  1877. else
  1878. COMCHK="tests/run_com_tests.sh --check-only"
  1879. # 0. The baseline. Every case below is a claim that a specific assertion
  1880. # turns red, and none of them means anything if the copies of untouched
  1881. # images already fail. (The stale RT_SZ produced exactly that: a whole
  1882. # file of "failures" that were not findings.) So this is asserted first,
  1883. # and a failure here is reported as a broken baseline rather than a red
  1884. # test.
  1885. #
  1886. # The baseline is also WHERE THE HEADER OFFSET COMES FROM. Cases 1 and 2
  1887. # used to carry it as the literal 435 and 439, which were ENT_SZ + the
  1888. # runtime size at the time -- 432. Two 4-byte changes to the runtime
  1889. # later, both cases were editing the wrong bytes: 435 had become four
  1890. # bytes inside the runtime itself, so case 1 had stopped testing "the
  1891. # header cannot be found" and had started testing "the checker also
  1892. # notices you scribbled on the code", and case 2 had become a no-op
  1893. # that wrote the header where it already was. Both still went red, for
  1894. # reasons the messages did not name, which is the whole problem: a
  1895. # hard-coded offset is a claim about the world that expires silently.
  1896. #
  1897. # So the offset is read back out of the checker's own report on the
  1898. # untouched images, and the expected numbers below are ARITHMETIC ON IT.
  1899. # If the runtime grows again, these cases still test what they say.
  1900. rm -rf "$KEEPDIR"; mkdir -p "$KEEPDIR"
  1901. cp "$KEEP"/*.COM "$KEEP"/raw.txt "$KEEPDIR"/
  1902. BASE_OUT=$($COMCHK "$KEEPDIR" 2>&1)
  1903. if [ $? -eq 0 ]; then
  1904. echo " ok: baseline - untouched copies of the real images all pass"
  1905. pass=$((pass + 1))
  1906. else
  1907. echo " FAIL: the baseline is already red, so the cases below prove"
  1908. echo " nothing - fix the baseline before reading them"
  1909. printf '%s\n' "$BASE_OUT" | grep FAIL | head -3 | sed 's/^/ /'
  1910. fail=$((fail + 1))
  1911. fi
  1912. BASE_RT=$(printf '%s\n' "$BASE_OUT" \
  1913. | sed -n 's/.*measured runtime size: \([0-9][0-9]*\) bytes.*/\1/p' \
  1914. | head -1)
  1915. if [ -z "$BASE_RT" ]; then
  1916. echo " FAIL: the checker did not report the runtime size it measured,"
  1917. echo " so the cases below cannot find the header to edit"
  1918. fail=$((fail + 1))
  1919. BASE_RT=0
  1920. fi
  1921. # The layout constants are ENT_SZ 3 and HDR_SZ 16 (tests/comimage.py), so the
  1922. # header sits at hdrOff = 3 + rtSz and the entry jump's displacement must be
  1923. # (hdrOff + 16) - 3. The wanted number is therefore computed from the header
  1924. # OFFSET, not from rtSz: the two differ by 3, and getting that backwards
  1925. # produces a plausible-looking expectation three bytes out, which is how
  1926. # this case came to expect "want 452" and then be reported as not proving
  1927. # what it said.
  1928. BASE_OFF=$((3 + BASE_RT))
  1929. # Case 2 claims a runtime four bytes LONGER, so the header - and with it the
  1930. # demanded jump target - moves four bytes further on.
  1931. SHIFTED_WANT=$((BASE_OFF + 4 + 16 - 3))
  1932. echo " (measured runtime size $BASE_RT, header at image offset $BASE_OFF)"
  1933. # 1. Break hdrDS so it no longer ties the header to its own offset. The
  1934. # header must become UNFINDABLE and be reported as such - a checker that
  1935. # fell back to a remembered offset would report a confident number here,
  1936. # which is the failure mode the measurement was introduced to remove.
  1937. rm -rf "$KEEPDIR"; mkdir -p "$KEEPDIR"
  1938. cp "$KEEP"/*.COM "$KEEP"/raw.txt "$KEEPDIR"/
  1939. python3 - "$KEEPDIR/t01_minimal.COM" "$BASE_OFF" <<'PYEOF'
  1940. import sys
  1941. p, off = sys.argv[1], int(sys.argv[2])
  1942. d = bytearray(open(p, 'rb').read())
  1943. d[off + 4:off + 6] = (0x1234).to_bytes(2, 'little') # hdrDS, no longer self-consistent
  1944. open(p, 'wb').write(bytes(d))
  1945. PYEOF
  1946. expect_red "a header that cannot be located is reported, not assumed" \
  1947. "no program header found" $COMCHK "$KEEPDIR"
  1948. # 2. A complete, self-consistent header four bytes later, so the measured
  1949. # runtime size becomes $((BASE_RT + 4)) instead of $BASE_RT. This is the
  1950. # positive half of the same check: the derivation must FOLLOW the file,
  1951. # and the entry jump assertion - expressed in terms of the measurement -
  1952. # must follow it too, demanding $SHIFTED_WANT rather than $((BASE_OFF + 16 - 3)).
  1953. #
  1954. # hdrCS is copied from the header already in the file rather than written
  1955. # as a literal. It used to be the literal 464+100h, which was the image
  1956. # length when the image was 720 bytes; four bytes of runtime later it was
  1957. # 464+100h against a 724-byte image, so this case was ALSO failing on
  1958. # hdrCS, for a reason three lines below the one it was written to test.
  1959. rm -rf "$KEEPDIR"; mkdir -p "$KEEPDIR"
  1960. cp "$KEEP"/*.COM "$KEEP"/raw.txt "$KEEPDIR"/
  1961. python3 - "$KEEPDIR/t01_minimal.COM" "$BASE_OFF" "$SHIFTED_WANT" <<'PYEOF'
  1962. import sys
  1963. p, off, want = sys.argv[1], int(sys.argv[2]), int(sys.argv[3])
  1964. d = bytearray(open(p, 'rb').read())
  1965. hdrCS = int.from_bytes(d[off + 2:off + 4], 'little') # unchanged: still the image end
  1966. off += 4
  1967. ds = off + 0x1000 + 0x100
  1968. w = [1, hdrCS, ds, ds + 4, 0, 0, 0, 0]
  1969. for i, x in enumerate(w):
  1970. d[off + 2 * i:off + 2 * i + 2] = x.to_bytes(2, 'little')
  1971. open(p, 'wb').write(bytes(d))
  1972. PYEOF
  1973. expect_red "the measured runtime size follows the image ($BASE_RT -> $((BASE_RT + 4)))" \
  1974. "want $SHIFTED_WANT" $COMCHK "$KEEPDIR"
  1975. # 3. The entry jump's opcode. One byte, and the only assertion in the
  1976. # project that can see where execution STARTS.
  1977. rm -rf "$KEEPDIR"; mkdir -p "$KEEPDIR"
  1978. cp "$KEEP"/*.COM "$KEEP"/raw.txt "$KEEPDIR"/
  1979. python3 - "$KEEPDIR/t01_minimal.COM" <<'PYEOF'
  1980. import sys
  1981. p = sys.argv[1]
  1982. d = bytearray(open(p, 'rb').read())
  1983. d[0] = 0xEA
  1984. open(p, 'wb').write(bytes(d))
  1985. PYEOF
  1986. expect_red "the entry jump must be E9, not a near JMP" \
  1987. "not the E9 of the entry jump" $COMCHK "$KEEPDIR"
  1988. # 4. The entry jump's target, moved one instruction earlier. A .COM that
  1989. # lands in the middle of the prologue runs, prints something and exits
  1990. # cleanly, so no size or structure check can see this.
  1991. rm -rf "$KEEPDIR"; mkdir -p "$KEEPDIR"
  1992. cp "$KEEP"/*.COM "$KEEP"/raw.txt "$KEEPDIR"/
  1993. python3 - "$KEEPDIR/t01_minimal.COM" <<'PYEOF'
  1994. import sys
  1995. p = sys.argv[1]
  1996. d = bytearray(open(p, 'rb').read())
  1997. d[1:3] = (100).to_bytes(2, 'little')
  1998. open(p, 'wb').write(bytes(d))
  1999. PYEOF
  2000. expect_red "the entry jump must land on the first instruction" \
  2001. "entry jump rel16=100" $COMCHK "$KEEPDIR"
  2002. rm -rf "$KEEPDIR"
  2003. fi
  2004. echo
  2005. echo "non-vacuity: $pass ok, $fail failed"
  2006. [ "$fail" -eq 0 ]