audit_helpers.py 50 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990
  1. #!/usr/bin/env python3
  2. """audit_helpers.py -- check that Runtime.mod's one-liner emitter procedures
  3. emit the instruction their name claims.
  4. Why this exists
  5. ---------------
  6. Runtime.mod hand-assembles 8086 by writing raw bytes. Most of it is wrapped
  7. up as one-line helpers:
  8. PROCEDURE MovSiBx ; BEGIN B (89H) ; B (0DCH) END MovSiBx ;
  9. and those names are the only documentation of what the bytes mean. That is
  10. exactly the setup for a silent, expensive mistake: write the right opcode and
  11. the wrong ModRM, and nothing complains. It happened twice here, in the same
  12. direction both times:
  13. MovSiBx emitted 89 DC -- which is MOV SP,BX, not MOV SI,BX
  14. CmpSiBx emitted 39 DC -- ditto, CMP SP,BX
  15. The comment next to CmpSiBx even spelled the ModRM out as "11 011 100" and
  16. still got it wrong, because 11 011 100 in mod=11 means rm=100=SP; SI is
  17. rm=100 only in mod=00, where it means [SI]. Both bugs shipped together into
  18. EmitWrInt, which then stored decimal digits through an SI register it had
  19. never initialised. The structural checks in check_runtime.py could not see
  20. any of this: the bytes decoded cleanly, the sweep stayed in sync, the branch
  21. targets were all on boundaries and none of the entry prologues moved. Only a
  22. *name* versus a *decode* comparison finds it, because only that knows what
  23. the author was trying to say.
  24. So: for every helper whose body is a literal byte list, disassemble those
  25. bytes with FCML and require the decode to match the name. The name grammar
  26. is deliberately narrow and mechanical:
  27. <Op><Dest><Src> e.g. MovSiBx, StDiAx, CmpSpBx, MovDlSi
  28. <Op><Reg> e.g. XorAxAx, NegAx, NotDx, IncSi, DecSi
  29. with Op in {Mov, Lea, St, Ld, Cmp, Add, Sub, Xor, And, Or, Not, Inc, Dec,
  30. Push, Pop, Int, Jmp, Jne, Jge, Jnl, Jle, Jl, Je, Jae, Jbe, Jb, Ja, JaE...}
  31. and the register/operand words below. A helper whose name does not parse is
  32. reported as UNPARSED rather than silently skipped -- a name the grammar does
  33. not understand is a name we are not checking, and that must be visible.
  34. This is a source-level check, so it needs no re-baselining: it is a function
  35. of the current source, and it gets stricter as more names are added to the
  36. grammar. It also runs without building anything.
  37. Usage: audit_helpers.py [MODULE.mod] # default ../Runtime.mod
  38. audit_helpers.py -v # print every helper, passing or not
  39. """
  40. import os
  41. import re
  42. import sys
  43. HERE = os.path.dirname(os.path.abspath(__file__))
  44. SHELL = os.path.dirname(HERE)
  45. sys.path.insert(0, HERE)
  46. import disasm16 # noqa: E402 (path set above)
  47. # An emitter may be declared with or without a parameter list, so the
  48. # parentheses are optional. This is not cosmetic: the version that required
  49. # `PROCEDURE Name ;` matched ZERO of Compiler.mod's 22 emitters, because they
  50. # are all declared `PROCEDURE EmName () ;`. So the audit reported "everything
  51. # agrees" for a module where it had examined nothing - a check that cannot fail
  52. # is not a check.
  53. RE_PROC_HEAD = re.compile(
  54. r"^PROCEDURE\s+(\w+)\s*(?:\(\s*\))?\s*;", re.MULTILINE)
  55. RE_ANY_PROC = re.compile(r"^PROCEDURE\s+(\w+)", re.MULTILINE)
  56. # --- what the one-line grammar deliberately does NOT reach ----------------
  57. #
  58. # The name audit checks procedures whose body is a list of CONSTANT bytes, so
  59. # that the bytes can be disassembled and compared against the name. Three
  60. # kinds of emitter cannot be checked that way, and all three are listed here
  61. # with their reason. A name in this table is a documented exclusion; a name
  62. # that is merely absent is a finding, and the coverage check below is what
  63. # tells the two apart.
  64. #
  65. # 1. PARAMETERISED operands -- a byte that is a parameter, not a literal.
  66. # `SubAl (v)` emits 2C v, which disassembles to "sub al, 0" whatever v is,
  67. # so the name can be checked only by also trusting the source order. The
  68. # grammar could learn this (the pattern Int([0-9A-Fa-f]+) already handles
  69. # the analogous case for INT); it does not yet, and until it does these are
  70. # unchecked BY THE AUDIT, not verified.
  71. # 2. COMPUTED or DISPATCHED bytes -- a ModRM chosen by a comparison, a
  72. # rel16 with a patch slot, a form that depends on which kind of variable it
  73. # is. There is no single byte sequence to disassemble.
  74. # 3. DATA, not code -- the B(...) calls here emit the D_ block, so running
  75. # them through a disassembler yields "add [bx+si],al" and a decode error.
  76. # tests/check_runtime.py pins these bytes against runtime.golden, so they
  77. # are pinned; only the name-to-opcode correspondence does not apply.
  78. #
  79. # Named per module because the same name can be a one-liner in one and not the
  80. # other. The Vx family in Runtime.mod is the reason a name is worth listing
  81. # separately: those four ARE audited, by audit_vx_helpers, because their shape
  82. # is fixed even though their immediate is a data offset.
  83. PARAM = "parameterised operand - the byte is an argument, not a literal"
  84. COMP = "computed or dispatched bytes - no single sequence to disassemble"
  85. DATA = "emits DATA (the D_ block), not instructions"
  86. NON_ONE_LINE = {
  87. "Runtime.mod": {
  88. # Real (MBF) entry helpers whose operands are data/field offsets or
  89. # immediates; each is a fixed sequence, pinned by rt_exec's MBF cases.
  90. "MovAxImm": COMP, "Jmp8": COMP, "AndDhImm": COMP,
  91. "StDiOffBx": COMP, "StDiOffDx": COMP,
  92. # The MBF core's parameterised moves and data-byte helpers. Each is
  93. # a fixed instruction/sequence; real_add/real_sub's 16 rt_exec cases
  94. # pin the behaviour (alignment, add/sub, normalisation, sign).
  95. "AdcBxImm": COMP, "AdcDxImm": COMP, "AddAhImm": COMP,
  96. "AndDataImm": COMP, "Call": COMP, "CmpClImm": COMP,
  97. "LdAlData": COMP, "LdClData": COMP, "LdCxDiOff": COMP,
  98. "LdCxSiOff": COMP, "LdDiDiOff": COMP, "LdDxSiOff": COMP,
  99. "LdSiDiOff": COMP, "MovClImm": COMP, "OrDhImm": COMP,
  100. "OrDiImm": COMP, "StDataCl": COMP, "StDataDi": COMP, "StDataImm": COMP,
  101. "SubClImm": COMP, "TestDhImm": COMP, "TestDataImm": COMP,
  102. "XorDhData": COMP, "XorDiData": COMP, "XorDataDh": COMP,
  103. "XorDataImm": COMP, "PushBp4": COMP,
  104. "MovBxImm": PARAM, "MovCxImm": PARAM, "MovDxImm": PARAM,
  105. "MovDl": PARAM, "MovAlD": PARAM, "MovAh": PARAM,
  106. "SubAl": PARAM, "CmpAl": PARAM, "AddDl": PARAM, "J8": COMP,
  107. "Jcc": COMP, "CmpCxV": PARAM,
  108. "B": DATA, "C8": DATA,
  109. },
  110. "Compiler.mod": {
  111. "AddSp": PARAM, "SubSp": PARAM, "Setcc": PARAM,
  112. "Call": COMP, "Jcc": COMP, "JmpNear": COMP,
  113. "BpDisp": COMP, "LoadVar": COMP, "StoreVar": COMP,
  114. "PushVarAddr": COMP, "MovAxi": COMP, "CmpAxi": COMP,
  115. "MovCxi": COMP, "LeaDi": COMP,
  116. # PushStrAddr is PushVarAddr's sibling one level up: it pushes the
  117. # ADDRESS of a whole-string operand, and a string LITERAL has no storage
  118. # to take the address of, so it emits the literal INLINE - a jump over
  119. # it, then <len><chars> - and pushes the address of those bytes. The
  120. # emitted sequence therefore depends on the operand: a variable is a
  121. # single EmPushVarAddr, a literal is two bytes of jump plus the string
  122. # itself. That is COMP, not PARAM (whose immediate varies but whose
  123. # opcode does not). Listed here rather than left out: an emitter the
  124. # name grammar cannot describe has to be written down, or the report is
  125. # green and the helper unchecked.
  126. "PushStrAddr": COMP,
  127. # SetBlob is PushStrAddr's sibling for sets: a set cannot fold to a
  128. # word, so its bytes travel inline - a JMP over them, then the bytes -
  129. # and the address of those bytes is the value. Like PushStrAddr the
  130. # emitted sequence depends on the operand (its length), so there is no
  131. # single sequence to disassemble; the LAYOUT it produces is pinned by
  132. # t61_setconst's .out instead.
  133. "SetBlob": COMP,
  134. # StrBlob is the string-literal blob on its own (factored out of
  135. # PushStrAddr so a string value parameter can copy from it); PushStrSlot
  136. # reserves a slot and copies a string value into it. Both emit data
  137. # behind a jump / a data-dependent byte loop, so there is no single
  138. # sequence to disassemble. The layouts they produce are pinned by the
  139. # string and string-parameter fixtures instead.
  140. "StrBlob": COMP,
  141. "PushStrSlot": COMP,
  142. # The CRT sequences (INT 10h) are one parameterised helper: several
  143. # instructions each, with no single name grammar. Listed, not left
  144. # implicit.
  145. "CrtSeq": COMP,
  146. # A VAR parameter's slot holds an ADDRESS. These three read and write
  147. # one level down: LoadVarInd fetches the pointer into BX and then the
  148. # byte/word at [BX]; StoreVarIndPrep parks the pointer in DI across the
  149. # right-hand side and StoreInd stores through it. The sequence depends
  150. # on the width (one byte or a word), so there is no single opcode to
  151. # name; t73_varparam's execution pins both widths, the local/global
  152. # address forms, passing one on, and READ through one.
  153. "LoadVarInd": COMP,
  154. "StoreVarIndPrep": COMP,
  155. "StoreInd": COMP,
  156. # The WITH/2-D field helpers. AddDiImm adds a record field's fixed
  157. # offset to an address already in DI; LdDiGlob and StDiGlob reload and
  158. # park that address in a data slot across a WITH body. Each is a fixed
  159. # 3-4 byte sequence whose operand is a data/field offset, and t77_with
  160. # pins the behaviour (2-D element, constant element, field offsets).
  161. "AddDiImm": COMP,
  162. "LdDiGlob": COMP,
  163. "StDiGlob": COMP,
  164. # Set-field helpers: store one constructor byte through DI
  165. # (StDiImm), and test one element's bit in a DI-addressed set
  166. # (LdAlDiOff + AndAlImm). t78_setfield pins both.
  167. "StDiImm": COMP,
  168. "LdAlDiOff": COMP,
  169. "AndAlImm": COMP,
  170. # A real copy through DI: one byte of the six.
  171. "StDiAlOff": COMP,
  172. # UpCase's byte tests and the 'a'..'z' fold.
  173. "CmpAlImm": COMP,
  174. "SubAlImm": COMP,
  175. # The run-time set bit test (`e in set'): shift/mask/index helpers.
  176. "MovClAl": COMP, "AndClImm": COMP,
  177. "MovBxi": COMP, "LeaBx": COMP, "MovBxDi": COMP,
  178. "AddBxAx": COMP, "LdAlBx": COMP, "MovDlImm": COMP,
  179. "AndAlDl": COMP,
  180. },
  181. }
  182. # The Vx family is NOT here: those four are audited by audit_vx_helpers,
  183. # because their shape is fixed even though their immediate is a data offset.
  184. # They are named explicitly rather than filtered out of VX_KINDS so that this
  185. # table stands on its own and a name added to VX_KINDS later cannot silently
  186. # become unchecked - it would instead be reported, which is the point.
  187. _NONLINE_VX = ("LdBxVx", "MovBxVx", "MovDxVx", "StVxBx")
  188. def scan_emitters(src):
  189. """Yield (name, text) for every PROCEDURE, however it is written.
  190. The INVENTORY side of the coverage check, and deliberately as dumb as it
  191. can be: a name at the start of a line, and everything up to the next one.
  192. It does not check the parameter list, does not look for BEGIN, does not
  193. care where comments are, and cannot be defeated by the same mistake twice.
  194. The reason it exists at all is that the other list - the one the audit
  195. actually reads - is produced by find_helpers, which is a real parser and
  196. therefore fails on real inputs (a parameter list it did not expect, a
  197. comment in the wrong place). Comparing a parser against itself finds
  198. nothing; comparing it against a scan that cannot parse anything finds
  199. exactly the cases where the parser is the thing that is wrong.
  200. """
  201. heads = list(RE_ANY_PROC.finditer(src))
  202. for k, h in enumerate(heads):
  203. stop = heads[k + 1].start() if k + 1 < len(heads) else len(src)
  204. yield h.group(1), src[h.end():stop]
  205. def find_helpers(src):
  206. """Yield (name, body) for every PROCEDURE, in source order.
  207. A documented emitter must be auditable, so text between the header and
  208. BEGIN - which is where the explanatory comment belongs, and the only place
  209. it can be read next to the code it describes - is allowed through. A
  210. regex that permitted it had nested quantifiers and took exponential time
  211. on these files, so this splits on procedure HEADERS instead and takes the
  212. text up to the next header. That is linear, and it also cannot read one
  213. procedure's bytes as another's.
  214. The header regex is still strict about the parameter list (empty only),
  215. because that is what distinguishes an emitter from a real routine.
  216. """
  217. heads = list(RE_PROC_HEAD.finditer(src))
  218. for k, h in enumerate(heads):
  219. stop = heads[k + 1].start() if k + 1 < len(heads) else len(src)
  220. chunk = src[h.end():stop]
  221. m = re.search(r"\bBEGIN\b(.*)\bEND\s+%s\s*;" % re.escape(h.group(1)),
  222. chunk, re.DOTALL)
  223. if not m:
  224. continue
  225. yield h.group(1), m.group(1)
  226. # The H suffix is optional: the source mixes B (8AH) and B (0) for the same
  227. # kind of literal, and a byte written without H used to be silently dropped
  228. # from the audit, which made three helpers look like truncated prefixes.
  229. #
  230. # BOTH B (...) and Ebyte (...) are accepted. Runtime.mod spells a byte B(v)
  231. # and Compiler.mod spells the same thing Ebyte(v); the Em prefix is on the
  232. # *procedure* names there, not on the byte call, so the byte regex has to cover
  233. # both spellings or Compiler.mod's 22 emitters are silently skipped - which is
  234. # what happened, and how EmXchgAxCx stayed wrong for its whole life with
  235. # correct byte counts and a green compile matrix.
  236. RE_BYTE = re.compile(r"(?:\bB|\bEbyte)\s*\(\s*([0-9A-Fa-f]+)H?\s*\)")
  237. # Both modules emit bytes, so both must be swept. Compiler.mod is where the
  238. # procedure-skip jump, the FOR test ordering and EmXchgAxCx (93h = XCHG BX,AX
  239. # where the name says XCHG AX,CX) all live: bugs that break every two-variable
  240. # arithmetic and comparison in every program, invisible because the audit
  241. # looked at Runtime.mod and found nothing wrong there.
  242. MODULES = ["Runtime.mod", "Compiler.mod"]
  243. # --- name grammar -----------------------------------------------------
  244. #
  245. # An ordered table of (name regex, mnemonic, operand specs, allowed opcodes).
  246. #
  247. # Why the opcode column exists
  248. # ----------------------------
  249. # "MovDlSi" and "MovBpSp" are ambiguous from the name alone: SI and BP are
  250. # both in the register list and in the memory-base list. What settles it is
  251. # the opcode, and the rule is the asymmetry that makes 16-bit hand-assembly
  252. # so error-prone:
  253. #
  254. # 88 /r MOV r/m8, r8 reg is the SOURCE (store)
  255. # 89 /r MOV r/m16, r16 reg is the SOURCE (store)
  256. # 8A /r MOV r8, r/m8 reg is the DESTINATION (load)
  257. # 8B /r MOV r16, r/m16 reg is the DESTINATION (load)
  258. #
  259. # A byte move (8A/88) to or from a bare base register can only be a memory
  260. # access, because "mov dl, si" is not an instruction. A word move (8B/89)
  261. # could be either, so for word moves the register reading is tried first.
  262. # Encoding that rule in the table, rather than in pattern order alone, is what
  263. # stops a helper being "verified" against the wrong reading of its own name.
  264. #
  265. # Operand spec tokens:
  266. #
  267. # "r:Name" a bare register spelled Name (Ax, Al, Dx, Si, Ds, ...)
  268. # "i:N" the immediate N; the name spells it in DECIMAL
  269. # "ih:N" like i: but the name spells it in HEX (only Int, whose
  270. # vector 21h FCML prints as "21h" and which reads as decimal
  271. # 33 if you do not notice)
  272. # "m:Base" memory at [Base], with no displacement
  273. # "m:Base+D" memory at [Base + D]
  274. #
  275. # "Arg" is the emitter's alias for BP: on entry BP points at the return
  276. # address, so a word argument starts at [BP+2]. Naming it "Arg" rather than
  277. # "Bp" is what stops these being misread as [BP] accesses, and it is also
  278. # how the grammar knows to expect the +2.
  279. #
  280. # "Arg" is the frame in which the entry did NOT push BP, and it is the only
  281. # one where the argument is two bytes up: an entry that pushes BP first has
  282. # moved it to +4, which is a different name and a different byte
  283. # (MovAlArg2 / MovAlArg4, CmpArg2W0 / CmpArg4W0). A name that says only
  284. # "Arg" always means +2, with no exceptions -- the alternatives to spelling
  285. # the displacement out were a Modula-2 parameter, which this audit cannot
  286. # see, and a comment, which nothing checks.
  287. #
  288. # A name matching NO pattern is reported UNPARSED, which counts as a failure
  289. # on purpose: a helper the audit cannot read is a helper nobody is checking.
  290. REGS = ["Ax", "Bx", "Cx", "Dx", "Si", "Di", "Bp", "Sp",
  291. "Al", "Bl", "Cl", "Dl", "Ah", "Bh", "Ch", "Dh"]
  292. # Segment registers: the 8086 can only PUSH/POP them, never MOV to or from
  293. # one, so they need names of their own.
  294. SEGS = ["Ds", "Es", "Cs", "Ss"]
  295. JCCS = ["Je", "Jne", "Jz", "Jnz", "Jge", "Jnl", "Jle", "Jl", "Ja", "Jae",
  296. "Jb", "Jbe", "Jg", "Jns", "Js", "Jo", "Jno", "Jp", "Jnp", "Jcxz",
  297. "Jecxz", "Jrcxz", "Loop", "Loope", "Loopne"]
  298. _ALT = "|".join(REGS)
  299. _ALT16 = "Ax|Bx|Cx|Dx|Si|Di|Bp|Sp"
  300. _BASE = "Si|Di|Bx|Bp|Sp|Arg|Data"
  301. _STORE = (0x88, 0x89) # reg is the source
  302. _LOAD = (0x8A, 0x8B) # reg is the destination
  303. PATTERNS = [
  304. # --- no-operand and fixed-operand forms -------------------------
  305. (re.compile(r"^RetR?$"), "ret", [], None),
  306. (re.compile(r"^LeaveR?$"), "leave", [], None),
  307. (re.compile(r"^Int([0-9A-Fa-f]+)$"), "int", ["ih:%(1)s"], {0xCD}),
  308. (re.compile(r"^(Push|Pop)(%s)$" % "|".join(SEGS)),
  309. None, ["r:%(2)s"], None),
  310. (re.compile(r"^(Push|Pop)(%s)$" % _ALT), None, ["r:%(2)s"], None),
  311. # --- jumps: the target is a fixup, so only the mnemonic is checked
  312. (re.compile(r"^(%s)(8|16)?$" % "|".join(JCCS)), None, [], None),
  313. (re.compile(r"^Jmp(%s)$" % _ALT), "jmp", ["r:%(1)s"], {0xFF}),
  314. # --- MUL names one operand; 16-bit DIV names two because it always
  315. # divides DX:AX, so the decode has an AX the name has no room for
  316. (re.compile(r"^Mul(%s)$" % _ALT), "mul", ["r:%(1)s"], {0xF7}),
  317. (re.compile(r"^Div(%s)$" % _ALT), "div", ["r:Ax", "r:%(1)s"], {0xF7}),
  318. # --- store: name is St<base><reg>, decode puts the register last --
  319. (re.compile(r"^St(%s)(%s)$" % (_BASE, _ALT)),
  320. "mov", ["m:%(1)s", "r:%(2)s"], _STORE),
  321. # --- load through a bare base register: Ld<reg><base> ------------
  322. # The mirror of the St pattern above, and spelled Ld rather than Mov on
  323. # purpose. `8A 07` and `8A C3` are one byte apart and do opposite
  324. # things: LdAlBx reads the byte AT the pointer in BX, MovAlBl takes the
  325. # low byte OF BX. Naming the first "MovAlBx" put the two one letter
  326. # apart, and getch's pushback path used the wrong one - it read memory
  327. # 010Ah, the address of the character, instead of the character. So the
  328. # grammar is part of the safety: Ld* may only be spelled for a memory
  329. # source, which is what distinguishes it from Mov*<reg><lowreg>.
  330. (re.compile(r"^Ld(%s)(%s)$" % (_ALT, _BASE)),
  331. "mov", ["r:%(1)s", "m:%(2)s"], (0x8A, 0x8B)),
  332. # --- XCHG is symmetric, so the name's operand order carries no
  333. # information and must not be checked positionally. 87 /r is
  334. # XCHG r/m16, r16, so FCML always prints the r/m operand first:
  335. # 87 C7 - reg=AX, rm=DI - decodes as "xchg di,ax" whichever way the
  336. # author thought about it. What the audit is really for here is the
  337. # PAIR: XchgAxDi must not come out as XCHG AX,CX. So the two
  338. # registers are compared as a set (see the "rx:" handling below).
  339. (re.compile(r"^Xchg(%s)(%s)$" % (_ALT, _ALT)),
  340. "xchg", ["rx:%(1)s|%(2)s"], {0x87, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96,
  341. 0x97}),
  342. # 8-bit XCHG, 86 /r. The MBF intreal swaps DL and DH to move the low byte
  343. # of a 16-bit value into the exponent position.
  344. (re.compile(r"^Xchg(Al|Cl|Dl|Bl|Ah|Ch|Dh|Bh)(Al|Cl|Dl|Bl|Ah|Ch|Dh|Bh)$"),
  345. "xchg", ["rx:%(1)s|%(2)s"], {0x86}),
  346. # --- the accumulator-implicit XCHGs, 91h..97h, join the /r form -----
  347. # One byte, no ModRM, both registers fixed by the opcode. They are in
  348. # the same opcode set because the same `rx:` set comparison decides them,
  349. # and that comparison is exactly what catches 93h (XCHG BX,AX) under the
  350. # name XchgAxCx - the fault that broke every two-variable arithmetic
  351. # operation in every program. 90h is deliberately absent: it is NOP, and
  352. # FCML decodes it as "nop", not as an XCHG AX,AX.
  353. #
  354. # --- MOV r8, imm8 : B0+reg, and the AH-specific B4 form -----------
  355. # MOV AH,imm8 is B4 imm8, which is not B0+reg. The name says which
  356. # register, so the opcode is checked against it: B4 must be AH and B0+4
  357. # must not be. (The runtime's wrchar used to load the character with the
  358. # wrong register, so the store landed in AL-adjacent memory.)
  359. # B4 imm8 is MOV AH,imm8 and is NOT B0+reg, so AH and AL are listed
  360. # separately with their own opcodes rather than sharing one pattern that
  361. # would accept B4 under the name MovAl0.
  362. # --- shifts: by one, and by CL. The set membership test uses both:
  363. # D1 E8 (SHR AX,1) three times, and D2 E2 (SHL DL,CL) for the mask.
  364. # Shifts and rotates by one, byte and word. The MBF core shifts a 40-bit
  365. # mantissa with SHR/RCR and normalises with SHL/RCL.
  366. (re.compile(r"^(Shl|Shr|Rcl|Rcr|Rol|Ror|Sar)(%s)1$" % _ALT),
  367. None, ["r:%(2)s", "i:1"], {0xD0, 0xD1}),
  368. (re.compile(r"^Stc$"), "stc", [], {0xF9}),
  369. (re.compile(r"^PushBp4$"), "push", ["m:Bp+4"], {0xFF}),
  370. (re.compile(r"^Adc(%s)(%s)$" % (_ALT16, _ALT16)),
  371. "adc", ["r:%(1)s", "r:%(2)s"], {0x11, 0x13}),
  372. (re.compile(r"^Sbb(%s)(%s)$" % (_ALT16, _ALT16)),
  373. "sbb", ["r:%(1)s", "r:%(2)s"], {0x18, 0x19, 0x1A, 0x1B}),
  374. (re.compile(r"^ShlDlCl$"), "shl", ["r:Dl", "r:Cl"], {0xD2}),
  375. # --- CMP reg,0 as a flag test (the 8086 has no OR reg,reg FCML can
  376. # decode here; CMP sets the same ZF/SF/CF for a non-negative reg) ---
  377. (re.compile(r"^CmpAl0$"), "cmp", ["r:Al", "i:0"], {0x3C}),
  378. (re.compile(r"^Cmp(Cl|Dl|Bl|Ah|Ch|Dh|Bh)0$"),
  379. "cmp", ["r:%(1)s", "i:0"], {0x80}),
  380. (re.compile(r"^MovAh(0|1)$"),
  381. "mov", ["r:Ah", "i:%(1)s"], {0xB4}),
  382. (re.compile(r"^MovAl(0|1)$"),
  383. "mov", ["r:Al", "i:%(1)s"], {0xB0}),
  384. # --- MUL/IMUL r/m16, accumulator implicit, /5 and /4 -----------------
  385. # The destination is AX:DX and the operand is the named register, so
  386. # "MulAxCx" means CX := CX, i.e. AX:DX := AX * CX. FCML prints only the
  387. # one explicit operand ("imul cx"), which is why the spec has one token.
  388. (re.compile(r"^Mul(%s)(%s)$" % (_ALT, _ALT)),
  389. "imul", ["r:%(2)s"], {0xF7}),
  390. # --- CmpArg<N>W0 : CMP WORD [BP+N],0 between MOV BP,SP / MOV SP,BP -----
  391. # An odd but exact shape: [SP] is not encodable, so the frame pointer is
  392. # borrowed for the one comparison and handed back untouched. The pair of
  393. # saves at the ends is what makes it correct, so the whole sequence is
  394. # checked - a missing MOV SP,BP would leave BP clobbered for the caller.
  395. #
  396. # The displacement is a CAPTURED GROUP, not a constant, and the name has to
  397. # spell it out. That is the rule that keeps CmpArg2W0 and CmpArg4W0
  398. # honest: they differ by one byte, and one byte is the difference between
  399. # reading a BOOLEAN and reading a character, so the name -- which is the
  400. # only thing this audit can read -- has to carry it. A single helper
  401. # taking the displacement as a Modula-2 parameter would decode correctly
  402. # under either name and be unchecked under both.
  403. (re.compile(r"^CmpArg(2|4)W0$"),
  404. None, [["mov", ["r:Bp", "r:Sp"]],
  405. ["cmp", ["m:Bp+%(1)s", "i:0"]],
  406. ["mov", ["r:Sp", "r:Bp"]]], {0x8B}),
  407. # --- a "Mov<reg>Sp" that is a POP/PUSH pair, not a memory access ----
  408. # MOV AX,[SP] does not exist on the 8086 at all, so the stack top is read
  409. # with POP and given back with PUSH: an observational no-op that leaves SP
  410. # where it found it. Named MovAxSp because that is the OPERATION, and the
  411. # two-step shape is recorded here because it is the only correct encoding.
  412. (re.compile(r"^Mov(%s)Sp$" % _ALT),
  413. None, [["pop", ["r:%(1)s"]], ["push", ["r:%(1)s"]]], None),
  414. # --- IDIV: CWD then IDIV r/m16, sign-extending into DX:AX ---------
  415. # A two-instruction shape, so the specs are a list of one list per
  416. # instruction. The CWD is not incidental: F7 /7 divides the 32-bit value
  417. # in DX:AX, and a signed dividend is only in DX:AX if CWD ran. Drop the
  418. # 99h and the divisor goes to the same place, so the pair is checked
  419. # together - which is the reason the sequence form exists.
  420. # FCML prints BOTH ends of a /r divide ("idiv ax, cx"), because the
  421. # accumulator is not implicit in the mnemonic the way it is for MUL, so
  422. # the spec needs two operands and the AX comes first.
  423. (re.compile(r"^I?Div(%s)(%s)$" % (_ALT, _ALT)),
  424. None, [["cwd", []], ["idiv", ["r:%(1)s", "r:%(2)s"]]], {0xF7}),
  425. # --- load a register from memory, with a displacement ------------
  426. # The displacement makes the name unambiguous, so any load opcode works.
  427. (re.compile(r"^Mov(%s)(%s)(\d+)$" % (_ALT, _BASE)),
  428. "mov", ["r:%(1)s", "m:%(2)s+%(3)s"], _LOAD),
  429. # --- two registers: MOV BP,SP / CMP SI,BX / XOR AX,AX / ADD DI,AX -
  430. # Tried before the bare-base reading below, because for a word move
  431. # "MovBpSp" means BP := SP and only "MovDlSi" means DL := [SI]. The two
  432. # readings cannot both match, because one demands a register operand
  433. # where the other demands a memory operand.
  434. # Xchg is NOT in this list. It has its own pattern above, which compares
  435. # the two registers as a SET because 87 /r is symmetric and FCML always
  436. # prints the r/m operand first - so the name's order carries no
  437. # information and must not be checked positionally. Listing Xchg here as
  438. # well gave it a second, ordered reading, and the audit then reported
  439. # "2 name readings fit the same bytes" for XchgAxCx: not a real ambiguity
  440. # in the code, but two overlapping rows in the grammar saying the same
  441. # thing. A grammar that can be satisfied two ways for one name is a
  442. # grammar that can be satisfied the wrong way.
  443. (re.compile(r"^(Mov|Cmp|Add|Sub|Xor|And|Or)(%s)(%s)$" % (_ALT, _ALT)),
  444. None, ["r:%(2)s", "r:%(3)s"], None),
  445. # --- byte move from a bare base register: necessarily memory ------
  446. # Restricted to 8A/88 on purpose: if this ever matched an 8B/89 it would
  447. # be a register move already claimed by the pattern above.
  448. (re.compile(r"^Mov(%s)(%s)$" % (_ALT, _BASE)),
  449. "mov", ["r:%(1)s", "m:%(2)s"], (0x8A, 0x88)),
  450. # --- immediate against a register --------------------------------
  451. (re.compile(r"^(Add|Sub|Cmp|Xor|And|Or)(%s)(\d+)$" % _ALT),
  452. None, ["r:%(2)s", "i:%(3)s"], {0x83}),
  453. # --- XOR AL,#imm8 is `34 ib': no ModRM byte at all, and only the
  454. # accumulator. The row above is pinned to 83 /r, which is the word
  455. # `op r, imm8' encoding, so `XorAl01' matched it and was rejected on
  456. # the opcode - leaving the helper with NO reading rather than a wrong
  457. # one, which is how the audit reported it. Narrow on purpose: 34 is
  458. # XOR AL, nothing else, so a name claiming `xor al, 1h' against any
  459. # other byte fails here. This is TPSRC9 neglevel's boolean NOT.
  460. (re.compile(r"^XorAl(\d+)$"),
  461. "xor", ["r:Al", "i:%(1)s"], {0x34}),
  462. # --- one register: INC CX / DEC SI / NOT DX / NEG AX ------------
  463. (re.compile(r"^(Inc|Dec|Not|Neg|Shl|Shr|Sar)(%s)$" % _ALT),
  464. None, ["r:%(2)s"], None),
  465. ]
  466. def register_word(tok):
  467. """Map an FCML operand token to a REGS/SEGS word, or None."""
  468. t = tok.strip().lower()
  469. if not t or t.startswith("word ptr ") or t.startswith("byte ptr "):
  470. return None
  471. t = t.split()[-1]
  472. for r in REGS + SEGS:
  473. if t == r.lower():
  474. return r
  475. return None
  476. def match_operand(spec, tok):
  477. """Does one FCML operand token satisfy one token spec? Returns None on
  478. success, or a human-readable reason on failure."""
  479. kind, arg = spec.split(":", 1)
  480. t = tok.strip()
  481. if kind in ("i", "ih"):
  482. # FCML prints an immediate as hex with an h suffix ("0h", "2h", "21h")
  483. raw = t.lower().rstrip("h")
  484. try:
  485. v = int(raw, 16)
  486. except ValueError:
  487. return "%r is not an immediate" % t
  488. want = int(arg, 16) if kind == "ih" else int(arg, 10)
  489. if v != want:
  490. return "immediate is %d, name says %d" % (v, want)
  491. return None
  492. if kind == "r":
  493. got = register_word(t)
  494. if got != arg:
  495. return "%r is not the register %s" % (t, arg)
  496. return None
  497. if kind == "m":
  498. base, _, disp = arg.partition("+")
  499. real = {"Arg": "bp", "Data": "si"}.get(base, base.lower())
  500. if base == "Arg" and not disp:
  501. disp = "2" # the first word argument lives at [BP+2]
  502. u = t.lower()
  503. u = re.sub(r"^(word|byte) ptr ", "", u)
  504. if not (u.startswith("[") and u.endswith("]")):
  505. return "%r is not a memory reference" % t
  506. body = u[1:-1]
  507. if not body.startswith(real):
  508. return "memory base is %r, name says [%s]" % (body, real)
  509. if not disp:
  510. if body != real:
  511. return ("memory is %r, name says [%s] with no displacement"
  512. % (body, real))
  513. return None
  514. got = body[len(real):].strip()
  515. m = re.fullmatch(r"\+\s*([0-9a-f]+)h?", got)
  516. if not m:
  517. return "displacement %r is not a number" % got
  518. if int(m.group(1), 16) != int(disp):
  519. return "displacement is %d, name says %s" \
  520. % (int(m.group(1), 16), disp)
  521. return None
  522. return "bad spec %r" % spec
  523. def operands_of(decode):
  524. """Split an FCML decode like "mov si,bx" or "cmp word ptr [bp+4h],0h"
  525. into operand tokens. Returns (mnemonic, [tokens])."""
  526. i = decode.find(" ")
  527. if i < 0:
  528. return decode.strip(), []
  529. return decode[:i].strip(), decode[i + 1:].split(",")
  530. # --- the Vx helpers, which the one-line grammar cannot see ---------------
  531. #
  532. # find_helpers takes a body spanning any number of lines, so a helper written
  533. # as
  534. #
  535. # PROCEDURE LdBxVx (delta : CARDINAL ) ;
  536. # BEGIN
  537. # B (8BH) ; B (01EH) ; Dd (delta)
  538. # END LdBxVx ;
  539. #
  540. # does not reach the one-line pass. That is a gap rather than a documented
  541. # exclusion, because these are exactly the helpers that are hard
  542. # to get right and easy to get subtly wrong:
  543. #
  544. # - the operand is a D_ data-block offset, so the immediate is a FIXUP.
  545. # Emitting W (delta) instead of Dd (delta) produces a byte-identical
  546. # instruction with the raw offset in it: `MOV BX,001Ch` for
  547. # `MOV BX,<D_PUSH>`, pointing into the entry JMP. That happened, and
  548. # the result was a hang rather than a crash.
  549. # - in 16-bit mode there is no [BX] memory form, so a data address has to
  550. # go through mod=00 / rm=110, which is ModRM 1E. rm=111 is [BX+SI], and
  551. # it decodes cleanly, so a wrong ModRM here is invisible to every other
  552. # check in the tree.
  553. # - LdBxVx and StVxBx are a load/store pair on the same word and MUST use
  554. # the same ModRM and the same fixup kind, or one of them silently
  555. # addresses something else.
  556. # - LdBxVx and MovBxVx differ only in the ModRM's mod field, i.e. "the word
  557. # at the address" versus "the address". Both mistakes above have been
  558. # made here, in this pair.
  559. #
  560. # So they get their own check, keyed on "Vx" - the operand word Runtime.mod
  561. # reserves for "a 16-bit absolute address into the data block" - and every one
  562. # of them is declared explicitly, so ADDING a Vx helper is a deliberate act:
  563. # an undeclared one is reported rather than ignored.
  564. #
  565. # The header, the body and the closing name are separated by `(.*?)` rather
  566. # than by `\s*`, because two of these helpers carry an explanatory comment
  567. # between the parameter list and BEGIN. A `\s*` there skips them silently -
  568. # which is the failure this whole check exists to prevent - so the pattern is
  569. # deliberately loose about whitespace and strict about the `END <name>`.
  570. RE_VX = re.compile(
  571. r"^PROCEDURE\s+(\w*Vx\w*)\s*\([^)]*\)\s*;.*?\bBEGIN\b(.*?)\bEND\s+\1\s*;",
  572. re.MULTILINE | re.DOTALL)
  573. # (opcode, shape). "address" = MOV reg,imm16: the register is in the opcode
  574. # and there is no ModRM. "load"/"store" = opcode + ModRM 1E + the
  575. # Dd-supplied immediate.
  576. VX_KINDS = {
  577. "MovBxVx": (0xBB, "address"), # MOV BX, Vx
  578. "MovDxVx": (0xBA, "address"), # MOV DX, Vx
  579. "LdBxVx": (0x8B, "load"), # MOV BX, WORD PTR [Vx]
  580. "StVxBx": (0x89, "store"), # MOV WORD PTR [Vx], BX
  581. }
  582. def audit_vx_helpers(src, mod, verbose):
  583. """Check the multi-line *Vx helpers. Returns (n_checked, problems)."""
  584. found = {}
  585. for m in RE_VX.finditer(src):
  586. found[m.group(1)] = m.group(2)
  587. problems = []
  588. # The count reported is the number of helpers ACTUALLY examined, not the
  589. # size of VX_KINDS. Those were the same number when only Runtime.mod was
  590. # swept, and printing the constant for Compiler.mod - which defines no Vx
  591. # emitter at all - made the report claim four subjects in a module that
  592. # has none. A count that does not change when the module does is a
  593. # cosmetic lie, and this report exists to be believed.
  594. checked = 0
  595. for name in sorted(set(found) | set(VX_KINDS)):
  596. # A *Vx helper is a Runtime.mod concept (the V* data word at a fixed
  597. # offset). Compiler.mod has none, so VX_KINDS is only a floor for the
  598. # module that is supposed to define them; asking Compiler.mod for four
  599. # Vx emitters it should not have produces four phantom failures.
  600. if mod != "Runtime.mod" and name in VX_KINDS and name not in found:
  601. continue
  602. if name not in VX_KINDS:
  603. problems.append("%s: a Vx helper with no declared kind - add it "
  604. "to VX_KINDS with its opcode and shape" % name)
  605. continue
  606. if name not in found:
  607. # The module is named rather than hardcoded: these four live in
  608. # Runtime.mod, and reporting them as missing from Compiler.mod
  609. # would be a false alarm about a module that has no business
  610. # defining them.
  611. problems.append("%s: declared in VX_KINDS but not defined in %s"
  612. % (name, mod))
  613. continue
  614. checked += 1
  615. body = found[name]
  616. want_op, shape = VX_KINDS[name]
  617. by = [int(b, 16) for b in RE_BYTE.findall(body)]
  618. hexs = " ".join("%02X" % b for b in by[:2])
  619. if not by or by[0] != want_op:
  620. problems.append("%s: starts %s, expected opcode %02X (%s)"
  621. % (name, hexs or "<nothing>", want_op, shape))
  622. continue
  623. # The two shapes need different numbers of literal bytes, and asking
  624. # for a ModRM on an address move is a false alarm: MOV reg,imm16
  625. # carries the register in the opcode and has no ModRM at all.
  626. if shape == "address":
  627. if len(by) != 1:
  628. problems.append("%s: emits %s, expected just the opcode - an "
  629. "address move is `MOV reg,imm16` and has no "
  630. "ModRM" % (name, hexs))
  631. continue
  632. elif len(by) < 2:
  633. problems.append("%s: emits %s, expected opcode and ModRM"
  634. % (name, hexs))
  635. continue
  636. elif by[1] != 0x1E:
  637. problems.append(
  638. "%s: ModRM is %02X, expected 1E. 16-bit mode has no [BX] "
  639. "form, so a data address must use mod=00 / rm=110; rm=111 "
  640. "is [BX+SI] and decodes cleanly, so nothing else would "
  641. "notice" % (name, by[1]))
  642. continue
  643. if "Dd" not in body:
  644. problems.append(
  645. "%s: does not call Dd, so the D_ offset is emitted raw "
  646. "instead of as a fixup - the instruction will be "
  647. "byte-correct and point at the wrong place" % name)
  648. continue
  649. if re.search(r"\bW\s*\(", body):
  650. problems.append("%s: calls W as well as Dd - the offset must be "
  651. "placed by Dd alone" % name)
  652. continue
  653. if verbose:
  654. print("%-20s %-16s %s (%s, Dd fixup)"
  655. % (name, hexs, shape,
  656. "no ModRM" if shape == "address" else "mod=00 rm=110"))
  657. return checked, problems
  658. def sweep(src, mod, verbose):
  659. """Audit every one-line emitter helper in one module. Returns
  660. (n_ok, n_unparsed, n_bad, problems)."""
  661. n_ok = n_unparsed = n_bad = 0
  662. problems = []
  663. seen = set()
  664. # EmitData is a documented EXCLUSION, not a gap: its B(...) calls are DATA
  665. # (the D_ block), not instructions, so running the code buffer through a
  666. # disassembler produces nonsense like "add [bx+si],al" and a DECODE ERROR.
  667. # It is listed here rather than left implicit, because an exclusion that is
  668. # not written down is indistinguishable from a helper nobody audited.
  669. #
  670. # Its D_ offsets are asserted against this block by check_runtime.py's
  671. # golden, so "excluded from the name audit" is not "unchecked": the bytes
  672. # are pinned, only the name-to-opcode correspondence does not apply.
  673. excluded = {"EmitData"}
  674. if verbose:
  675. print("-- %s" % mod)
  676. print("name bytes FCML decode")
  677. for name, body in find_helpers(src):
  678. # Compiler.mod prefixes its emitters with Em. The name grammar is
  679. # shared with Runtime.mod and has no Em entries, so the prefix is
  680. # stripped here and the module name only ever appears in output. It
  681. # is stripped for PATTERNS matching and for the Vx pass alike, which
  682. # is why a single RE_BYTE and a single PATTERNS table can serve both
  683. # modules instead of two drifting copies.
  684. if mod != "Runtime.mod" and name.startswith("Em"):
  685. name = name[2:]
  686. seen.add(name)
  687. if name in excluded:
  688. continue
  689. by = [int(b, 16) for b in RE_BYTE.findall(body)]
  690. if not by:
  691. continue
  692. raw = bytes(by)
  693. # Disassemble; helpers are 1-2 bytes so one instruction is the norm,
  694. # but StCx / Shl / Div style helpers can be longer, so decode all.
  695. decodes = []
  696. off = 0
  697. while off < len(raw):
  698. text, length = disasm16.decode(raw[off:], off)
  699. if length == 0:
  700. decodes.append(("<DECODE ERROR>", ""))
  701. break
  702. decodes.append(operands_of(text or "?"))
  703. off += length
  704. hexs = " ".join("%02X" % b for b in raw)
  705. dec_text = "; ".join(d[0] + " " + ", ".join(d[1]) for d in decodes)
  706. if verbose:
  707. print("%-20s %-16s %s" % (name, hexs, dec_text))
  708. # A name can have more than one reading (MovBpSp: BP := SP, or
  709. # BP := [SP]). Take the first reading whose operand specs the decode
  710. # actually satisfies; if two readings both fit, the name is genuinely
  711. # ambiguous and that is itself reported, because it would mean the
  712. # audit could be satisfied by the wrong one.
  713. reads = []
  714. nopattern = True
  715. for rx, pmnem, specs, ops in PATTERNS:
  716. mm = rx.match(name)
  717. if not mm:
  718. continue
  719. # The opcode gate applies to the FIRST byte. That is right for a
  720. # one-instruction helper, and wrong for a sequence, where the first
  721. # byte belongs to the setup step: IDivAxCx starts with 99h (CWD)
  722. # and its F7h /7 is the second instruction, so a first-byte gate
  723. # rejected the one helper whose opcode matters most. Skipping the
  724. # gate for a sequence loses nothing, because the sequence check
  725. # below compares every decoded mnemonic against its own spec.
  726. is_seq = (isinstance(specs, list) and specs
  727. and isinstance(specs[0], list))
  728. if ops is not None and not is_seq and by[0] not in ops:
  729. continue
  730. nopattern = False
  731. gmap = {str(i): g for i, g in enumerate(mm.groups(), 1)}
  732. mn = pmnem
  733. if mn is None:
  734. # A pattern with no capture group has no mnemonic to infer
  735. # from the name, and a pattern with none either names the
  736. # mnemonic outright (a sequence) or is checked by operands
  737. # alone. A group-less pattern with pmnem None is a grammar
  738. # bug, not a helper bug, so say so instead of crashing.
  739. if not mm.re.groups:
  740. reads.append((mm, None, specs,
  741. "pattern %r has no capture group and names "
  742. "no mnemonic" % rx.pattern))
  743. continue
  744. mn = mm.group(1).lower()
  745. elif "%(" in mn:
  746. mn = mn % gmap
  747. if len(decodes) != 1:
  748. # A pattern may describe a fixed SEQUENCE of instructions by
  749. # passing a list of specs, one per instruction - IDIV is the
  750. # only user (CWD, then IDIV r/m16), and without this it was
  751. # reported as "emits 2 instructions, name describes one",
  752. # which is a true observation and a useless one: the two
  753. # instructions are one operation and the pair is the thing
  754. # that has to be checked, because a lone F7 /7 divides by an
  755. # un-sign-extended dividend.
  756. if not is_seq or len(specs) != len(decodes):
  757. reads.append((mm, mn, specs,
  758. "emits %d instructions, name describes %s"
  759. % (len(decodes),
  760. len(specs) if is_seq else "one")))
  761. continue
  762. why = []
  763. for k, sub in enumerate(specs):
  764. smn, sspecs = sub
  765. gmnem, got = decodes[k]
  766. if gmnem != smn:
  767. why.append("step %d is %r, name says %s"
  768. % (k + 1, gmnem, smn))
  769. break
  770. if len(got) != len(sspecs):
  771. why.append("step %d: name asserts %d operand(s), "
  772. "FCML decoded %d"
  773. % (k + 1, len(sspecs), len(got)))
  774. break
  775. w = [x for x in
  776. (match_operand(s % gmap if "%(" in s else s, t)
  777. for s, t in zip(sspecs, got)) if x]
  778. if w:
  779. why.append("step %d: %s" % (k + 1, "; ".join(w)))
  780. break
  781. reads.append((mm, mn, specs, "; ".join(why) if why else None))
  782. continue
  783. gmnem, got = decodes[0]
  784. if gmnem != mn:
  785. reads.append((mm, mn, specs,
  786. "FCML decodes %r, name says %s" % (gmnem, mn)))
  787. continue
  788. if len(specs) == 1 and specs[0].startswith("rx:"):
  789. # XCHG: compare the two registers as a set, not in order.
  790. # This has to come before the operand-COUNT check, because
  791. # the "rx:" spec packs both registers into one token while
  792. # the decode has two operands - which is the whole point of
  793. # it.
  794. want = set((specs[0] % gmap)[3:].split("|"))
  795. got_regs = [register_word(t) for t in got]
  796. if len(got_regs) != 2 or None in got_regs:
  797. reads.append((mm, mn, specs,
  798. "XCHG needs two bare registers, decoded %s"
  799. % ", ".join(got)))
  800. continue
  801. if set(got_regs) != want:
  802. reads.append((mm, mn, specs,
  803. "exchanges %s, name says %s"
  804. % (" and ".join(sorted(got_regs)),
  805. " and ".join(sorted(want)))))
  806. continue
  807. reads.append((mm, mn, specs, None))
  808. continue
  809. if len(got) != len(specs):
  810. reads.append((mm, mn, specs,
  811. "name asserts %d operand(s), FCML decoded %d"
  812. % (len(specs), len(got))))
  813. continue
  814. why = [w for w in
  815. (match_operand(sp % gmap if "%(" in sp else sp, tok)
  816. for sp, tok in zip(specs, got)) if w]
  817. if why:
  818. reads.append((mm, mn, specs, "; ".join(why)))
  819. continue
  820. reads.append((mm, mn, specs, None))
  821. if nopattern:
  822. n_unparsed += 1
  823. problems.append("%s: no name pattern accepts it (emits %s %r) - "
  824. "extend PATTERNS or fix the name"
  825. % (name, hexs, dec_text))
  826. continue
  827. fitting = [r for r in reads if r[3] is None]
  828. if len(fitting) > 1:
  829. n_bad += 1
  830. problems.append("%s: %d name readings fit the same bytes - the "
  831. "name is ambiguous" % (name, len(fitting)))
  832. continue
  833. if not fitting:
  834. n_bad += 1
  835. problems.append("%s: %s" % (name, reads[0][3]))
  836. continue
  837. n_ok += 1
  838. n_vx, vx_problems = audit_vx_helpers(src, mod, verbose)
  839. problems.extend(vx_problems)
  840. n_vx_ok = n_vx - len(vx_problems)
  841. # Every PROCEDURE that EMITS BYTES must have been looked at. A helper
  842. # whose body the parser skipped is the failure mode that has bitten this
  843. # file three times now: the Em prefix, the missing parameter list, and a
  844. # comment in the wrong place - each time the suite stayed green while the
  845. # helper went unexamined and the reported subject count quietly dropped.
  846. #
  847. # The inventory is scanned INDEPENDENTLY of find_helpers, and that is the
  848. # whole design of this check. Deriving both lists from the same parser
  849. # makes it vacuous: any input that stops find_helpers from matching also
  850. # removes the helper from the inventory, so the two lists agree and the
  851. # check reports nothing - which is exactly the failure it exists to catch.
  852. # So the inventory is a separate, deliberately dumb scan: every
  853. # `PROCEDURE <name>` at the start of a line, with everything up to the
  854. # next one as its text. It makes no attempt to parse Modula-2, so it
  855. # cannot fail in the same way twice. A helper the audit cannot reach is
  856. # therefore in `declared` and not in `seen`, which is a finding.
  857. #
  858. # The filter is the byte-call regex, so a parser, a lexer or a symbol
  859. # lookup that emits nothing is not expected to be audited - only helpers
  860. # that actually put bytes in the code buffer. A helper that emitted bytes
  861. # under some spelling RE_BYTE does not know would be missed here too, which
  862. # is why RE_BYTE accepts both B(...) and Ebyte(...) and why a new spelling
  863. # has to be added there rather than relied on to be found by accident.
  864. strip = (lambda s: s[2:]) if mod != "Runtime.mod" else (lambda s: s)
  865. declared = {strip(nm) for nm, txt in scan_emitters(src)
  866. if RE_BYTE.search(txt)}
  867. # EmitData is a documented EXCLUSION, not a gap: its B(...) calls are DATA
  868. # (the D_ block), not instructions, so running the code buffer through a
  869. # disassembler produces nonsense like "add [bx+si],al" and a DECODE ERROR.
  870. # It is listed here rather than left implicit, because an exclusion that is
  871. # not written down is indistinguishable from a helper nobody audited.
  872. #
  873. # Its D_ offsets are asserted against this block by check_runtime.py's
  874. # golden, so "excluded from the name audit" is not "unchecked": the bytes
  875. # are pinned, only the name-to-opcode correspondence does not apply.
  876. excluded = {"EmitData"}
  877. strip = (lambda s: s[2:]) if mod != "Runtime.mod" else (lambda s: s)
  878. # The table is written in the names the audit uses (Runtime.mod's own
  879. # names, Compiler.mod's with the Em prefix already removed), so it is NOT
  880. # stripped again here. Stripping it twice turns "AddSp" into "dSp", which
  881. # matches nothing - and a wrong match here is the dangerous direction: the
  882. # entries would stop suppressing their helpers and the report would look
  883. # like a genuine finding rather than like a broken table.
  884. documented = set(NON_ONE_LINE.get(mod, {}))
  885. audited_elsewhere = ({strip(n) for n in _NONLINE_VX}
  886. if mod == "Runtime.mod" else set())
  887. unexamined = (declared - seen - {strip(e) for e in excluded}
  888. - documented - audited_elsewhere)
  889. for n in sorted(unexamined):
  890. problems.append("%s: emits bytes but the audit never examined it - "
  891. "the report would be green and the helper unchecked"
  892. % n)
  893. print("\n %s: %d one-line helpers audited, %d agree with their name, "
  894. "%d disagree, %d outside the grammar"
  895. % (mod, n_ok + n_unparsed + n_bad, n_ok, n_bad, n_unparsed))
  896. print(" %s: %d multi-line *Vx helpers audited, %d agree, %d problem(s)"
  897. % (mod, n_vx, n_vx_ok, len(vx_problems)))
  898. return n_ok + n_vx_ok, n_unparsed, n_bad, problems
  899. def main(argv):
  900. verbose = "-v" in argv
  901. files = [a for a in argv[1:] if not a.startswith("-")]
  902. # No argument means "audit every module that emits bytes", not
  903. # "audit Runtime.mod". The single-module default is why the audit was
  904. # green while EmXchgAxCx encoded XCHG BX,AX under a name that says
  905. # XCHG AX,CX - a fault that broke every two-variable arithmetic operation
  906. # in every program the compiler could produce.
  907. mods = [os.path.basename(f) for f in files] if files else list(MODULES)
  908. tot_ok = tot_unparsed = tot_bad = 0
  909. problems = []
  910. for mod in mods:
  911. path = os.path.join(SHELL, mod)
  912. if not os.path.exists(path):
  913. print("FAIL: no such module: %s" % path)
  914. return 1
  915. with open(path) as f:
  916. src = f.read()
  917. n_ok, n_unp, n_bad, pr = sweep(src, mod, verbose)
  918. tot_ok += n_ok
  919. tot_unparsed += n_unp
  920. tot_bad += n_bad
  921. problems.extend("%s: %s" % (mod, p) for p in pr)
  922. total = tot_ok + tot_unparsed + tot_bad
  923. print("\nAUDITED %d emitter helpers across %d module(s): %d agree with "
  924. "their name, %d disagree, %d outside the grammar"
  925. % (total, len(mods), tot_ok, tot_bad, tot_unparsed))
  926. if problems:
  927. print("FAIL: %d problem(s)" % len(problems))
  928. for p in problems:
  929. print(" - %s" % p)
  930. return 1
  931. print("PASS: every audited helper emits what its name says")
  932. return 0
  933. if __name__ == "__main__":
  934. sys.exit(main(sys.argv))