unicode inc.2: Utf8 codec + TextIO U* output; UORD->INTEGER, UCHR accepts ints

- runtime/syslib/Utf8.def/.mod: strict Decode(one scalar from s[lo])
  + Encode(UCHAR -> bytes); MaxUTF8Length=4, MaxCodePoint=1114111.
- stdlib/textio: UWriteChar/UWriteString(VAR ARRAY OF UCHAR)/UWriteLn
  encode to UTF-8 and emit via the shim; U"..." literals pass
  directly (Assignable/VarParamOk accept ClUStr -> open UCHAR array).
- M2.atg: UORD now returns INTEGER (V3 has no LONGCARD name / 64->32
  conversion; codepoints fit in 32 bits), UCHR accepts CHAR or an
  integer family operand (the UCHAR constructor).
- tests: utf8_prog (127), utext_prog (stdout 'héllo') -> 120/120.
- fixpoint OK (2,062,392 bytes).
- docs updated.

compiler/run_tests.sh

@@ -228,6 +228,8 @@ expect_run_files_out Hello 0 "Hello, Modula-2!" ../stdlib/sysio.def ../stdlib/sy
 expect_run_files_out StringsProg 0 "Hello World" ../stdlib/sysio.def ../stdlib/sysio.mod ../stdlib/strings.def ../stdlib/strings.mod strings_prog.mod
 expect_run_files_out Stdlib2Prog 42 "stdlib2 n=42" ../stdlib/sysio.def ../stdlib/sysio.mod ../stdlib/math.def ../stdlib/math.mod ../stdlib/strings.def ../stdlib/strings.mod ../stdlib/charclass.def ../stdlib/charclass.mod stdlib2_prog.mod
 expect_run_files MathProg 12 ../stdlib/sysio.def ../stdlib/sysio.mod ../stdlib/math.def ../stdlib/math.mod math_prog.mod
+expect_run_files Utf8Prog 127 ../runtime/syslib/Utf8.def ../runtime/syslib/Utf8.mod utf8_prog.mod
+expect_run_files_out UTextProg 42 "héllo" ../runtime/syslib/Utf8.def ../runtime/syslib/Utf8.mod ../stdlib/textio.def ../stdlib/textio.mod utext_prog.mod
 
 expect_run_files_in() {
   # $1 prog, $2 exit code, $3 expected first line, $4 stdin, then files
@@ -251,6 +253,7 @@ expect_run_files TBadFile 42 ../stdlib/files.def ../stdlib/files.mod t_badfile.m
 rm -f gen_ssa/_files_tmp.txt
 expect_run_files_in Showcase11 80 "Modula-2" "42" \
   ../stdlib/sysio.def ../stdlib/sysio.mod \
+  ../runtime/syslib/Utf8.def ../runtime/syslib/Utf8.mod \
   ../stdlib/textio.def ../stdlib/textio.mod \
   ../stdlib/wholeio.def ../stdlib/wholeio.mod \
   ../stdlib/strings.def ../stdlib/strings.mod \
@@ -262,7 +265,7 @@ rm -f gen_ssa/_showcase11.txt
 # statements, 1-char strings, qualified type names.
 expect_run_files Showcase12 42 showcase12lib.def showcase12lib.mod showcase12.mod
 expect_run_files_out Showcase13 42 "Even(8)=T Odd(7)=T total=42" ../stdlib/sysio.def ../stdlib/sysio.mod showcase13.mod
-expect_run_files_out MixProg 0 "7" ../stdlib/textio.def ../stdlib/textio.mod ../stdlib/sysio.def ../stdlib/sysio.mod ../stdlib/files.def ../stdlib/files.mod d_mix_prog.mod
+expect_run_files_out MixProg 0 "7" ../runtime/syslib/Utf8.def ../runtime/syslib/Utf8.mod ../stdlib/textio.def ../stdlib/textio.mod ../stdlib/sysio.def ../stdlib/sysio.mod ../stdlib/files.def ../stdlib/files.mod d_mix_prog.mod
 rm -f gen_ssa/_mix.txt
 
 expect_fail_files() {

compiler/src/M2.atg

@@ -2141,8 +2141,13 @@ PRODUCTIONS
                                            QbeGen.CopyOp(qa, q);
                                            t := SymTab.CharType(); .)
     | "UCHR" "(" Expr<et, q> ")"
-                                        (. IF (et # SymTab.InvalidType)
-  AND (SymTab.ClassOf(et) # SymTab.ClChar) THEN
+                                        (. (* UCHR: the UCHAR constructor.
+                                              CHAR -> UCHAR (identity);
+                                              INTEGER familly -> UCHAR
+                                              (codepoint value). *)
+                                           IF (et # SymTab.InvalidType)
+  AND (SymTab.ClassOf(et) # SymTab.ClChar)
+  AND NOT SymTab.IsIntFamily(et) THEN
                                              SemError(211) END;
                                            t := SymTab.UCharType(); .)
     | "CHR8" "(" Expr<et, q> ")"
@@ -2153,12 +2158,13 @@ PRODUCTIONS
                                            QbeGen.CheckRange(qa, "0", "255");
                                            t := SymTab.CharType(); .)
     | "UORD" "(" Expr<et, q> ")"
-                                        (. IF (et # SymTab.InvalidType)
+                                        (. (* UORD(u): the codepoint as a
+                                              32-bit ordinal (INTEGER),
+                                              cf. ORD for CHAR. *)
+                                           IF (et # SymTab.InvalidType)
   AND (SymTab.ClassOf(et) # SymTab.ClUChar) THEN
                                              SemError(211) END;
-                                           QbeGen.WidenLong(q, qa);
-                                           QbeGen.CopyOp(qa, q);
-                                           t := SymTab.LongType(); .)
+                                           t := SymTab.IntType(); .)
     | "ABS" "(" Expr<et, q> ")"
                                         (. IF (et # SymTab.InvalidType)
  AND NOT SymTab.IsIntFamily(et)

compiler/src/M2.lst

@@ -2158,174 +2158,180 @@ Listing:
  2141                                             QbeGen.CopyOp(qa, q);
  2142                                             t := SymTab.CharType(); .)
  2143      | "UCHR" "(" Expr<et, q> ")"
- 2144                                          (. IF (et # SymTab.InvalidType)
- 2145    AND (SymTab.ClassOf(et) # SymTab.ClChar) THEN
- 2146                                               SemError(211) END;
- 2147                                             t := SymTab.UCharType(); .)
- 2148      | "CHR8" "(" Expr<et, q> ")"
- 2149                                          (. IF (et # SymTab.InvalidType)
- 2150    AND (SymTab.ClassOf(et) # SymTab.ClUChar) THEN
+ 2144                                          (. (* UCHR: the UCHAR constructor.
+ 2145                                                CHAR -> UCHAR (identity);
+ 2146                                                INTEGER familly -> UCHAR
+ 2147                                                (codepoint value). *)
+ 2148                                             IF (et # SymTab.InvalidType)
+ 2149    AND (SymTab.ClassOf(et) # SymTab.ClChar)
+ 2150    AND NOT SymTab.IsIntFamily(et) THEN
  2151                                               SemError(211) END;
- 2152                                             QbeGen.WidenLong(q, qa);
- 2153                                             QbeGen.CheckRange(qa, "0", "255");
- 2154                                             t := SymTab.CharType(); .)
- 2155      | "UORD" "(" Expr<et, q> ")"
- 2156                                          (. IF (et # SymTab.InvalidType)
- 2157    AND (SymTab.ClassOf(et) # SymTab.ClUChar) THEN
- 2158                                               SemError(211) END;
- 2159                                             QbeGen.WidenLong(q, qa);
- 2160                                             QbeGen.CopyOp(qa, q);
- 2161                                             t := SymTab.LongType(); .)
- 2162      | "ABS" "(" Expr<et, q> ")"
- 2163                                          (. IF (et # SymTab.InvalidType)
- 2164   AND NOT SymTab.IsIntFamily(et)
- 2165   AND (SymTab.ClassOf(et) #
- 2166                                                  SymTab.ClReal) THEN
- 2167                                               SemError(211)
- 2168                                             ELSE QbeGen.AbsQ(q, qa,
- 2169                                                    SymTab.ClassOf(et) =
- 2170                                                      SymTab.ClReal);
- 2171                                               QbeGen.CopyOp(qa, q)
- 2172                                             END;
- 2173                                             t := et; .)
- 2174      | "VAL" "(" GetIdent<vn> "," Expr<et, q> ")"
- 2175                                          (. IF NOT SymTab.Lookup(vn) THEN
- 2176                                               SemError(201);
- 2177                                               t := SymTab.InvalidType
- 2178                                             ELSE vt := SymTab.SymType(vn);
- 2179                                               IF vt = SymTab.InvalidType THEN
- 2180                                                 t := SymTab.InvalidType
- 2181                                               ELSIF et =
- 2182                                                  SymTab.InvalidType THEN
- 2183                                                 t := vt
- 2184                                               ELSE
- 2185                                                 c1 := SymTab.ClassOf(et);
- 2186                                                 c2 := SymTab.ClassOf(vt);
- 2187                                                 IF ((c1 = SymTab.ClInt)
- 2188                                                     OR (c1 =
- 2189                                                        SymTab.ClChar)
- 2190                                                     OR (c1 =
- 2191                                                        SymTab.ClBool)
- 2192                                                     OR (c1 =
- 2193                                                        SymTab.ClEnum))
- 2194   AND ((c2 = SymTab.ClInt)
- 2195                                                     OR (c2 =
- 2196                                                        SymTab.ClChar)
- 2197                                                     OR (c2 =
- 2198                                                        SymTab.ClBool)
- 2199                                                     OR (c2 =
- 2200                                                        SymTab.ClEnum)) THEN
- 2201                                                   t := vt
- 2202                                                 ELSIF (c1 = SymTab.ClPtr)
- 2203   AND (c2 = SymTab.ClPtr) THEN
- 2204                                                   t := vt
- 2205                                                 ELSIF (c1 = SymTab.ClReal)
- 2206   AND (c2 = SymTab.ClReal) THEN
+ 2152                                             t := SymTab.UCharType(); .)
+ 2153      | "CHR8" "(" Expr<et, q> ")"
+ 2154                                          (. IF (et # SymTab.InvalidType)
+ 2155    AND (SymTab.ClassOf(et) # SymTab.ClUChar) THEN
+ 2156                                               SemError(211) END;
+ 2157                                             QbeGen.WidenLong(q, qa);
+ 2158                                             QbeGen.CheckRange(qa, "0", "255");
+ 2159                                             t := SymTab.CharType(); .)
+ 2160      | "UORD" "(" Expr<et, q> ")"
+ 2161                                          (. (* UORD(u): the codepoint as a
+ 2162                                                32-bit ordinal (INTEGER),
+ 2163                                                cf. ORD for CHAR. *)
+ 2164                                             IF (et # SymTab.InvalidType)
+ 2165    AND (SymTab.ClassOf(et) # SymTab.ClUChar) THEN
+ 2166                                               SemError(211) END;
+ 2167                                             t := SymTab.IntType(); .)
+ 2168      | "ABS" "(" Expr<et, q> ")"
+ 2169                                          (. IF (et # SymTab.InvalidType)
+ 2170   AND NOT SymTab.IsIntFamily(et)
+ 2171   AND (SymTab.ClassOf(et) #
+ 2172                                                  SymTab.ClReal) THEN
+ 2173                                               SemError(211)
+ 2174                                             ELSE QbeGen.AbsQ(q, qa,
+ 2175                                                    SymTab.ClassOf(et) =
+ 2176                                                      SymTab.ClReal);
+ 2177                                               QbeGen.CopyOp(qa, q)
+ 2178                                             END;
+ 2179                                             t := et; .)
+ 2180      | "VAL" "(" GetIdent<vn> "," Expr<et, q> ")"
+ 2181                                          (. IF NOT SymTab.Lookup(vn) THEN
+ 2182                                               SemError(201);
+ 2183                                               t := SymTab.InvalidType
+ 2184                                             ELSE vt := SymTab.SymType(vn);
+ 2185                                               IF vt = SymTab.InvalidType THEN
+ 2186                                                 t := SymTab.InvalidType
+ 2187                                               ELSIF et =
+ 2188                                                  SymTab.InvalidType THEN
+ 2189                                                 t := vt
+ 2190                                               ELSE
+ 2191                                                 c1 := SymTab.ClassOf(et);
+ 2192                                                 c2 := SymTab.ClassOf(vt);
+ 2193                                                 IF ((c1 = SymTab.ClInt)
+ 2194                                                     OR (c1 =
+ 2195                                                        SymTab.ClChar)
+ 2196                                                     OR (c1 =
+ 2197                                                        SymTab.ClBool)
+ 2198                                                     OR (c1 =
+ 2199                                                        SymTab.ClEnum))
+ 2200   AND ((c2 = SymTab.ClInt)
+ 2201                                                     OR (c2 =
+ 2202                                                        SymTab.ClChar)
+ 2203                                                     OR (c2 =
+ 2204                                                        SymTab.ClBool)
+ 2205                                                     OR (c2 =
+ 2206                                                        SymTab.ClEnum)) THEN
  2207                                                   t := vt
- 2208                                                 ELSE SemError(230);
- 2209                                                   t := SymTab.InvalidType
- 2210                                                 END
- 2211                                               END
- 2212                                             END; .)
- 2213      | "(" Expr<et, q> ")"               (. t := et; .)
- 2214      | SetLit<st, sq>                    (. t := st;
- 2215                                             QbeGen.CopyOp(sq, q); .)
- 2216      | ( "NOT" | "~" ) Fact<t2, q2>      (. IF SymTab.BoolCheck(t2) THEN
- 2217                                               t := SymTab.BoolType()
- 2218                                             ELSE SemError(212);
- 2219                                               t := SymTab.InvalidType END;
- 2220                                             IF t # SymTab.InvalidType THEN
- 2221                                               QbeGen.NotQ(q2, q)
- 2222                                             ELSE QbeGen.CopyOp("0", q)
- 2223                                             END; .) .
- 2224    (* Set literals are SET OF [0..255] (8 words); elements validated
- 2225       0..255 statically when foldable (222 otherwise), runtime trap
- 2226       for computed elements. Ranges always lower via SetRange. *)
- 2227    SetLit<VAR t: SymTab.TypeIndex; VAR q: QbeGen.QVal>
- 2228      = "{"                               (. t := SymTab.NewSet(
- 2229                                               SymTab.NewSubR(0, 255));
- 2230                                             QbeGen.NewSetTemp(8, q);
- 2231                                             QbeGen.SetZero(q, 8); .)
- 2232        [ SetElem<t, q> { "," SetElem<t, q> } ]
- 2233        "}" .
- 2234    (* Typed set constructor: TypeName{ elems } — e.g. BITSET{0},
- 2235       BITSET{}.  The declared type (not SET OF [0..255]) sets the
- 2236       width and element span. *)
- 2237    TypedSetLit<vt: SymTab.TypeIndex; VAR q: QbeGen.QVal>
- 2238                                          (. VAR nw: CARDINAL; .)
- 2239      = "{"                               (. IF SymTab.ClassOf(vt) #
- 2240                                                SymTab.ClSet THEN
- 2241                                               SemError(230); nw := 8
- 2242                                             ELSE nw := SymTab.SetWords(vt);
- 2243                                               IF nw = 0 THEN nw := 8 END
- 2244                                             END;
- 2245                                             QbeGen.NewSetTemp(nw, q);
- 2246                                             QbeGen.SetZero(q, nw); .)
- 2247        [ SetElem<vt, q> { "," SetElem<vt, q> } ]
- 2248        "}" .
- 2249    SetElem<st: SymTab.TypeIndex; sq: QbeGen.QVal>         (. VAR et, et2: SymTab.TypeIndex;
- 2250                                               qe, q2: QbeGen.QVal;
- 2251                                               v, v2: INTEGER;
- 2252                                               lo: INTEGER;
- 2253                                               span: CARDINAL;
- 2254                                               cl, cl2: INTEGER;
- 2255                                               hasR: BOOLEAN; .)
- 2256      =                                   (. hasR := FALSE; .)
- 2257        Expr<et, qe>
- 2258        [ ".." Expr<et2, q2>              (. hasR := TRUE; .) ]
- 2259                                          (. lo := SymTab.SetBaseLo(st);
- 2260                                             span := SymTab.SetCount(st);
- 2261                                             IF (et = SymTab.InvalidType)
- 2262                                                OR (hasR AND (et2 =
- 2263                                                   SymTab.InvalidType)) THEN
- 2264                                             ELSE cl :=
- 2265                                                    SymTab.ClassOf(et);
- 2266                                               IF hasR THEN
- 2267                                                 cl2 :=
- 2268                                                   SymTab.ClassOf(et2)
- 2269                                               ELSE cl2 := SymTab.ClInt
- 2270                                               END;
- 2271                                               IF ((cl # SymTab.ClInt)
- 2272   AND (cl # SymTab.ClChar)
- 2273   AND (cl # SymTab.ClBool))
- 2274                                                  OR (hasR AND 
- 2275                                                     ((cl2
- 2276                                                       # SymTab.ClInt)
- 2277   AND (cl2
- 2278                                                        # SymTab.ClChar)
- 2279   AND (cl2
- 2280                                                        # SymTab.ClBool))) THEN
- 2281                                                 SemError(222)
- 2282                                               ELSIF hasR
- 2283   AND SymTab.ConstInt(qe, v)
- 2284   AND SymTab.ConstInt(q2,
- 2285                                                     v2)
- 2286   AND ((v < lo)
- 2287                                                     OR (v2 < lo)
- 2288                                                     OR (v >= lo +
- 2289                                                        VAL(INTEGER, span))
- 2290                                                     OR (v2 >= lo +
- 2291                                                        VAL(INTEGER, span))
- 2292                                                     OR (v > v2)) THEN
- 2293                                                 SemError(222)
- 2294                                                ELSIF hasR THEN
- 2295                                                  QbeGen.SetRange(sq, qe, q2,
- 2296                                                    lo, span)
- 2297                                                ELSIF SymTab.ConstInt(qe,
- 2298                                                        v)
- 2299   AND ((v < lo)
- 2300                                                      OR (v >= lo +
- 2301                                                         VAL(INTEGER,
- 2302                                                           span))) THEN
- 2303                                                  SemError(222)
- 2304                                                ELSE QbeGen.SetBit(sq, qe,
- 2305                                                  lo, span)
- 2306                                               END
- 2307                                             END; .) .
- 2308    GetIdent<VAR n: SymTab.Name>
- 2309      = ident                             (. LexName(n); .) .
- 2310  
- 2311  END M2.
+ 2208                                                 ELSIF (c1 = SymTab.ClPtr)
+ 2209   AND (c2 = SymTab.ClPtr) THEN
+ 2210                                                   t := vt
+ 2211                                                 ELSIF (c1 = SymTab.ClReal)
+ 2212   AND (c2 = SymTab.ClReal) THEN
+ 2213                                                   t := vt
+ 2214                                                 ELSE SemError(230);
+ 2215                                                   t := SymTab.InvalidType
+ 2216                                                 END
+ 2217                                               END
+ 2218                                             END; .)
+ 2219      | "(" Expr<et, q> ")"               (. t := et; .)
+ 2220      | SetLit<st, sq>                    (. t := st;
+ 2221                                             QbeGen.CopyOp(sq, q); .)
+ 2222      | ( "NOT" | "~" ) Fact<t2, q2>      (. IF SymTab.BoolCheck(t2) THEN
+ 2223                                               t := SymTab.BoolType()
+ 2224                                             ELSE SemError(212);
+ 2225                                               t := SymTab.InvalidType END;
+ 2226                                             IF t # SymTab.InvalidType THEN
+ 2227                                               QbeGen.NotQ(q2, q)
+ 2228                                             ELSE QbeGen.CopyOp("0", q)
+ 2229                                             END; .) .
+ 2230    (* Set literals are SET OF [0..255] (8 words); elements validated
+ 2231       0..255 statically when foldable (222 otherwise), runtime trap
+ 2232       for computed elements. Ranges always lower via SetRange. *)
+ 2233    SetLit<VAR t: SymTab.TypeIndex; VAR q: QbeGen.QVal>
+ 2234      = "{"                               (. t := SymTab.NewSet(
+ 2235                                               SymTab.NewSubR(0, 255));
+ 2236                                             QbeGen.NewSetTemp(8, q);
+ 2237                                             QbeGen.SetZero(q, 8); .)
+ 2238        [ SetElem<t, q> { "," SetElem<t, q> } ]
+ 2239        "}" .
+ 2240    (* Typed set constructor: TypeName{ elems } — e.g. BITSET{0},
+ 2241       BITSET{}.  The declared type (not SET OF [0..255]) sets the
+ 2242       width and element span. *)
+ 2243    TypedSetLit<vt: SymTab.TypeIndex; VAR q: QbeGen.QVal>
+ 2244                                          (. VAR nw: CARDINAL; .)
+ 2245      = "{"                               (. IF SymTab.ClassOf(vt) #
+ 2246                                                SymTab.ClSet THEN
+ 2247                                               SemError(230); nw := 8
+ 2248                                             ELSE nw := SymTab.SetWords(vt);
+ 2249                                               IF nw = 0 THEN nw := 8 END
+ 2250                                             END;
+ 2251                                             QbeGen.NewSetTemp(nw, q);
+ 2252                                             QbeGen.SetZero(q, nw); .)
+ 2253        [ SetElem<vt, q> { "," SetElem<vt, q> } ]
+ 2254        "}" .
+ 2255    SetElem<st: SymTab.TypeIndex; sq: QbeGen.QVal>         (. VAR et, et2: SymTab.TypeIndex;
+ 2256                                               qe, q2: QbeGen.QVal;
+ 2257                                               v, v2: INTEGER;
+ 2258                                               lo: INTEGER;
+ 2259                                               span: CARDINAL;
+ 2260                                               cl, cl2: INTEGER;
+ 2261                                               hasR: BOOLEAN; .)
+ 2262      =                                   (. hasR := FALSE; .)
+ 2263        Expr<et, qe>
+ 2264        [ ".." Expr<et2, q2>              (. hasR := TRUE; .) ]
+ 2265                                          (. lo := SymTab.SetBaseLo(st);
+ 2266                                             span := SymTab.SetCount(st);
+ 2267                                             IF (et = SymTab.InvalidType)
+ 2268                                                OR (hasR AND (et2 =
+ 2269                                                   SymTab.InvalidType)) THEN
+ 2270                                             ELSE cl :=
+ 2271                                                    SymTab.ClassOf(et);
+ 2272                                               IF hasR THEN
+ 2273                                                 cl2 :=
+ 2274                                                   SymTab.ClassOf(et2)
+ 2275                                               ELSE cl2 := SymTab.ClInt
+ 2276                                               END;
+ 2277                                               IF ((cl # SymTab.ClInt)
+ 2278   AND (cl # SymTab.ClChar)
+ 2279   AND (cl # SymTab.ClBool))
+ 2280                                                  OR (hasR AND 
+ 2281                                                     ((cl2
+ 2282                                                       # SymTab.ClInt)
+ 2283   AND (cl2
+ 2284                                                        # SymTab.ClChar)
+ 2285   AND (cl2
+ 2286                                                        # SymTab.ClBool))) THEN
+ 2287                                                 SemError(222)
+ 2288                                               ELSIF hasR
+ 2289   AND SymTab.ConstInt(qe, v)
+ 2290   AND SymTab.ConstInt(q2,
+ 2291                                                     v2)
+ 2292   AND ((v < lo)
+ 2293                                                     OR (v2 < lo)
+ 2294                                                     OR (v >= lo +
+ 2295                                                        VAL(INTEGER, span))
+ 2296                                                     OR (v2 >= lo +
+ 2297                                                        VAL(INTEGER, span))
+ 2298                                                     OR (v > v2)) THEN
+ 2299                                                 SemError(222)
+ 2300                                                ELSIF hasR THEN
+ 2301                                                  QbeGen.SetRange(sq, qe, q2,
+ 2302                                                    lo, span)
+ 2303                                                ELSIF SymTab.ConstInt(qe,
+ 2304                                                        v)
+ 2305   AND ((v < lo)
+ 2306                                                      OR (v >= lo +
+ 2307                                                         VAL(INTEGER,
+ 2308                                                           span))) THEN
+ 2309                                                  SemError(222)
+ 2310                                                ELSE QbeGen.SetBit(sq, qe,
+ 2311                                                  lo, span)
+ 2312                                               END
+ 2313                                             END; .) .
+ 2314    GetIdent<VAR n: SymTab.Name>
+ 2315      = ident                             (. LexName(n); .) .
+ 2316  
+ 2317  END M2.
 
     0 errors
 

compiler/src/SymTab.mod

@@ -1361,6 +1361,10 @@ PROCEDURE VarParamOk (actual, formal: TypeIndex): BOOLEAN;
       fe := ArrayElem(formal);
       IF ClassOf(fe) = ClChar THEN RETURN TRUE END
     END;
+    IF IsOpenArray(formal) AND (ClassOf(actual) = ClUStr) THEN
+      fe := ArrayElem(formal);
+      IF ClassOf(fe) = ClUChar THEN RETURN TRUE END
+    END;
     RETURN FALSE
   END VarParamOk;
 
@@ -1818,6 +1822,12 @@ PROCEDURE Assignable (src, dst: TypeIndex): BOOLEAN;
     IF (ClassOf(src) = ClStr) AND (ClassOf(dst) = ClArray) THEN
       RETURN (ArrayDepth(dst) = 1)
  AND (ClassOf(ArrayElem(dst)) = ClChar)
+    END;
+    IF (ClassOf(src) = ClUStr) AND (ClassOf(dst) = ClArray) THEN
+      (* a U"..." literal is a valid actual for an open ARRAY OF UCHAR
+         formal: the descriptor { l count, w cp... } is a V3 array. *)
+      RETURN (ArrayDepth(dst) = 1)
+ AND (ClassOf(ArrayElem(dst)) = ClUChar)
     END;
     IF ClassOf(src) = ClNil THEN
       RETURN ClassOf(dst) = ClPtr

compiler/tests/utext_prog.mod

@@ -0,0 +1,11 @@
+MODULE UTextProg;
+(* TextIO Unicode output: U"héllo" is written as UTF-8. *)
+IMPORT TextIO;
+VAR ExitCode : INTEGER;
+BEGIN
+  TextIO.UWriteString(U"héllo");
+  TextIO.UWriteLn;
+  TextIO.UWriteChar(U'😀');
+  TextIO.UWriteLn;
+  ExitCode := 42
+END UTextProg.

compiler/tests/utf8_prog.mod

@@ -0,0 +1,21 @@
+MODULE Utf8Prog;
+(* Utf8 codec: Encode/Decode round-trips.  Exit 127. *)
+IMPORT Utf8;
+VAR ExitCode : INTEGER;
+VAR buf : ARRAY [0 .. 3] OF CHAR;
+VAR n : CARDINAL;
+VAR cp : UCHAR;
+VAR next : CARDINAL;
+BEGIN
+  n := Utf8.Encode(U'é', buf);                    (* C3 A9 *)
+  IF n = 2 THEN ExitCode := ExitCode + 1 ELSE ExitCode := 100 END;
+  IF ORD(buf[0]) = 195 THEN ExitCode := ExitCode + 2 ELSE ExitCode := 100 END;
+  IF ORD(buf[1]) = 169 THEN ExitCode := ExitCode + 4 ELSE ExitCode := 100 END;
+  n := Utf8.Encode(U'😀', buf);                    (* F0 9F 98 80 *)
+  IF n = 4 THEN ExitCode := ExitCode + 8 ELSE ExitCode := 100 END;
+  IF ORD(buf[0]) = 240 THEN ExitCode := ExitCode + 16 ELSE ExitCode := 100 END;
+  IF Utf8.Decode("é", 0, cp, next) THEN
+    IF UORD(cp) = 233 THEN ExitCode := ExitCode + 32 ELSE ExitCode := 100 END;
+    IF next = 2 THEN ExitCode := ExitCode + 64 ELSE ExitCode := 100 END
+  ELSE ExitCode := 100 END
+END Utf8Prog.

docs/features.md

@@ -80,9 +80,9 @@ Legend: ✅ done · 🔄 partial · ⏸ not started / deferred.
 
 ## Not started
 - ⏸ Clarion `CLASS` lowering (declared + checked only); Unicode
-  `UString` backing/assignment, `runtime/syslib/Utf8`, `TextIO.U*`
-  output. The TopSpeed legacy grammar (`TopSpeed-V3-M2.atg`) is a
-  separate sidecar, not merged.
+  `UString` whole-string assignment (`=`/`+`). `runtime/syslib/Utf8`
+  and `TextIO.U*` output are in. The TopSpeed legacy grammar
+  (`TopSpeed-V3-M2.atg`) is a separate sidecar, not merged.
 
 (Self-hosting is reached: `bootstrap/fixpoint.sh` builds a
 byte-identical compiler image; `stdlib/` + `runtime/syslib` are in

docs/plan.md

@@ -104,9 +104,9 @@ self-hosting is the last phase. Hygiene from step 1: deterministic
 
 ## 6. Unicode — UCHAR / UString (LOCKED)
 
-Status: **increment 1 landed** (see `docs/summary_unicode.md`).
-Remaining: `UString` backing/assignment, `runtime/syslib/Utf8`,
-`TextIO.U*` output.
+Status: **increments 1–2 landed** (see `docs/summary_unicode.md`).
+Remaining: `UString` whole-string assignment to a fixed array; UString
+`=`/`+`/concat.
 
 - `UCHAR`: 32-bit codepoint `0..1114111` (QBE `w`). `CHAR` unchanged
   (8-bit, `0..255`). Distinct types, no implicit mixing. ✅
@@ -120,23 +120,26 @@ Remaining: `UString` backing/assignment, `runtime/syslib/Utf8`,
   longer/empty → UString. `'a'` ≠ `U'a'`. ✅
 - Invalid UTF-8 in literal → hard lex error `234`. ✅ (reported in
   `.LST` + console; note the image is still written, gate pending)
-- Conversions: `UCHR(c:CHAR):UCHAR` safe; `CHR8(u:UCHAR):CHAR` traps
-  unless `u ≤ 255`; `ORD` 8-bit; new `UORD(u):LONGCARD`. ✅
-- `runtime/syslib/Utf8` (`decode/Encode`, `MaxUTF8Length=4`,
-  `MaxCodePoint=1114111`, cf. Benjamin `Utf8.def`): shared by the
-  lexer (decode) and output libs (encode). ⏸ (decode lives in
-  `QbeGen.DeclUStr` for now)
+- Conversions: `UCHR(x:CHAR|INTEGER):UCHAR` (constructor); `CHR8(u:UCHAR):CHAR`
+  traps unless `u ≤ 255`; `ORD` 8-bit; `UORD(u):INTEGER`.
+  ⚠ deviation: `UORD` returns `INTEGER`, not `LONGCARD` (V3 has no
+  `LONGCARD` name or 64→32-bit conversion; a codepoint fits in 32
+  bits). ⚠ `UCHR` also accepts an integer family operand.
+- `runtime/syslib/Utf8` (`Decode`/`Encode`, `MaxUTF8Length=4`,
+  `MaxCodePoint=1114111`, cf. Benjamin `Utf8.def`): used by the output
+  libs. ⚠ the *front-end* decode lives in `QbeGen.DeclUStr` (stage 1 is
+  gm2-built and does not know `UCHAR`, so it cannot import `Utf8`).
 - `TextIO.UWriteChar/UWriteString/UWriteLn`: encode to UTF-8, emit via
-  `SysIO` byte descriptors. Raw `WriteChar` stays byte-oriented. ⏸
+  the shim. Raw `WriteChar` stays byte-oriented. ✅
 
-## 7. Interactive ATG order (done through §6, increment 1)
+## 7. Interactive ATG order (done through §6)
 
 1. `COMPILER M2` header + `CHARACTERS/IGNORE/COMMENTS/TOKENS` ✅
    (`0x`, `E`-exponents, `//` decided interactively; `ustring`
-   token spelling locked, implementation pending with step 6).
+   token implemented, see §6).
 2. Module / Definition / Implementation / Import (+ opaque hook) ✅
 3. `CONST/TYPE/VAR/ProcHeading` + error codes (200–234) ✅
-4. Types (scalars ✅, UCHAR pending, composites 🔄 arrays+sets ✅,
+4. Types (scalars ✅, UCHAR ✅, composites 🔄 arrays+sets ✅,
    records/pointers next, proc types with step 4).
 5. Statements ✅ (all but `WITH`, which rides with records).
 6. Expressions ✅ (relations, `IN`, set ops; pointer ops with 3.4).

docs/summary_unicode.md

@@ -1,9 +1,9 @@
-# Step: Unicode — increment 1 (UCHAR + U-literals)
+# Step: Unicode — UCHAR / UString
 
-Tags `v3-unicode-1`. Suite **118/118**; self-hosting fixpoint still
-**OK** (`bootstrap/fixpoint.sh`, image 2,060,379 bytes).
+Tags `v3-unicode-1`, `v3-unicode-2`. Suite **120/120**; self-hosting
+fixpoint **OK** (`bootstrap/fixpoint.sh`, image 2,062,392 bytes).
 
-## Landed
+## Increment 1 — types, literals, conversions
 
 - **`UCHAR`** — 32-bit codepoint (`0..1114111`), QBE `w`, distinct
   from `CHAR`/`INTEGER` (class `ClUChar`, form `FUChar`). Usable in
@@ -12,43 +12,65 @@ Tags `v3-unicode-1`. Suite **118/118**; self-hosting fixpoint still
   action strict-RFC3629-decodes the bytes between the quotes
   (`QbeGen.DeclUStr`). One codepoint → `UCHAR` immediate; otherwise a
   **UString** descriptor `data $ustrN = { l count, w cp0, … }`
-  (`QbeGen.FlushUStrings`, emitted at `EndModule`).
+  (`QbeGen.FlushUStrings`, emitted at `EndModule`). A UString
+  descriptor *is* a V3 length-prefixed array, so a `U"…"` literal is a
+  valid actual for an open `ARRAY OF UCHAR` formal
+  (`SymTab.Assignable`/`VarParamOk`).
 - **Strict decoding** — rejects overlongs, surrogates (`D800..DFFF`),
   `>10FFFF`, and truncated sequences → semantic error
-  **234 `invalid UTF-8 in U-literal`** (new message in
-  `compiler.frm`).
-- **Conversions** — `UCHR(c:CHAR):UCHAR` (widening, identity);
-  `CHR8(u:UCHAR):CHAR` with an `l`-domain `0..255` range check
-  (traps via `$abort`, rc 134); `UORD(u):LONGCARD`.
-- **Comparisons** — `OrdCheck`/`EqCheck` accept `UCHAR` vs `UCHAR`,
-  and (generally useful) `LONGCARD` vs `INTEGER`; the existing
-  long-family comparison path widens the int side.
+  **234 `invalid UTF-8 in U-literal`** (message in `compiler.frm`).
+- **Conversions** — `UCHR(x)`: `CHAR`→`UCHAR` (identity) *or*
+  integer→`UCHAR` (codepoint constructor, cf. `CHR`);
+  `CHR8(u:UCHAR):CHAR` with an `l`-domain `0..255` range check (traps
+  via `$abort`, rc 134); `UORD(u):INTEGER`.
+
+## Increment 2 — codec + output
+
+- **`runtime/syslib/Utf8`** — `Decode` (one scalar from
+  `s[lo]` → `cp`,`next`, `FALSE` on malformed input) and `Encode`
+  (`UCHAR` → bytes, returns the count). `MaxUTF8Length = 4`,
+  `MaxCodePoint = 1114111`.
+- **`TextIO.UWriteChar` / `UWriteString(VAR u: ARRAY OF UCHAR)` /
+  `UWriteLn`** — encode codepoints to UTF-8 and emit the bytes via the
+  shim; raw `WriteChar` stays byte-oriented. A `U"héllo"` literal can
+  be passed directly to `UWriteString`.
+
+## Deviations from the locked design (flagged)
+
+- **`UORD` returns `INTEGER`, not `LONGCARD`.** V3 has no `LONGCARD`
+  type name and no 64→32-bit conversion, so a `LONGCARD` result would
+  be unusable in codegen (`CHR`, `DIV`/`MOD`, comparisons). A
+  codepoint fits in 32 bits, so `INTEGER` is lossless and consistent
+  with `ORD` for `CHAR`.
+- **`UCHR` accepts an integer family operand** as well as `CHAR`, so a
+  computed codepoint can be built without a literal.
+- `.ssa` is still written when error 234 fires (image gate pending).
 
 ## Tests (`compiler/run_tests.sh`)
 
-- `t_unicode.mod` (exit 31): `U'é'`, `U'😀'`, `U'€'`, `UCHR`,
-  `CHR8`, `UORD`.
+- `t_unicode.mod` (exit 31): `U'é'`, `U'😀'`, `U'€'`, `UCHR`, `CHR8`,
+  `UORD`.
 - `t_trap_chr8.mod`: `CHR8(U'😀')` → abort (rc 134).
 - `t_bad_overlong.mod`, `t_bad_surrogate.mod`, `t_bad_utf8big.mod`,
-  `t_bad_truncutf.mod`: expect `invalid UTF-8 in U-literal` (234).
-  These carry raw invalid bytes; `.LST` output is git-ignored.
+  `t_bad_truncutf.mod`: expect `invalid UTF-8 in U-literal` (234);
+  raw invalid bytes in the sources, `.LST` output git-ignored.
+- `utf8_prog.mod` (exit 127): `Utf8.Encode`/`Decode` round-trips.
+- `utext_prog.mod`: `TextIO.UWriteString(U"héllo")` → stdout `héllo`
+  (UTF-8 bytes verified).
 
-## Deferred to increment 2
+## Deferred
 
-- `UString` backing/assignment (array-of-UCHAR view, whole-string
-  assignment, index/`HIGH`).
-- `runtime/syslib/Utf8` (`Decode`/`Encode`, `MaxUTF8Length=4`,
-  `MaxCodePoint=1114111`), shared by lexer and output libs.
-- `TextIO.UWriteChar/UWriteString/UWriteLn` (UTF-8 output via
-  `SysIO`); raw `WriteChar` stays byte-oriented.
-- Suppressing `.ssa` on error 234 (the image is still written).
+- `UString` whole-string assignment to a *fixed* array (`a := U"…"`):
+  the runtime count check traps, exactly as `a := "…"` does for
+  `CHAR` arrays. Passing a literal to an open `ARRAY OF UCHAR` formal
+  works.
+- Concatenation/`=`/`+` for UStrings, `UCHAR` `CASE` labels.
 
 ## Files
 
 `compiler/src/{SymTab.def,SymTab.mod}` (UCHAR type, ordinal/equality
-rules), `compiler/src/{QbeGen.def,QbeGen.mod}`
-(`DeclUStr`/`FlushUStrings`, pool + descriptor emission),
-`compiler/src/M2.atg` (`ustring` token + `Fact` branch +
-`UCHR`/`CHR8`/`UORD`), `compiler/src/compiler.frm` (message 234),
-`compiler/run_tests.sh`, `compiler/tests/t_unicode.mod`,
-`compiler/tests/t_trap_chr8.mod`, `compiler/tests/t_bad_*.mod`.
+rules, UString↔open-array compat), `compiler/src/{QbeGen.def,QbeGen.mod}`
+(`DeclUStr`/`FlushUStrings`), `compiler/src/M2.atg` (`ustring` token +
+`Fact` branch + `UCHR`/`CHR8`/`UORD`), `compiler/src/compiler.frm`
+(message 234), `runtime/syslib/Utf8.def`/`.mod`,
+`stdlib/textio.def`/`.mod`, `compiler/run_tests.sh`, `compiler/tests/`.

runtime/syslib/Utf8.def

@@ -0,0 +1,19 @@
+DEFINITION MODULE Utf8;
+(* Strict RFC3629 UTF-8 codec, shared by the front end and the text
+   output libraries.  Codepoints are UCHAR scalars. *)
+
+CONST
+  MaxUTF8Length = 4;
+  MaxCodePoint  = 1114111;
+
+PROCEDURE Decode (s : ARRAY OF CHAR; lo : CARDINAL;
+                  VAR cp : UCHAR; VAR next : CARDINAL) : BOOLEAN;
+(* Decode one sequence starting at s[lo]: cp := the codepoint, next :=
+   the index just past it.  FALSE on malformed or truncated input
+   (overlong, surrogate, > MaxCodePoint, bad continuation byte). *)
+
+PROCEDURE Encode (cp : UCHAR; VAR buf : ARRAY OF CHAR) : CARDINAL;
+(* Encode cp as UTF-8 into buf[0..]; returns the byte count (0 when cp
+   is not a Unicode scalar value).  buf needs MaxUTF8Length slots. *)
+
+END Utf8.

runtime/syslib/Utf8.mod

@@ -0,0 +1,75 @@
+IMPLEMENTATION MODULE Utf8;
+
+PROCEDURE Decode (s : ARRAY OF CHAR; lo : CARDINAL;
+                  VAR cp : UCHAR; VAR next : CARDINAL) : BOOLEAN;
+  VAR b0, b1, b2, b3, v : INTEGER;
+    i : CARDINAL;
+
+  PROCEDURE Cont (VAR b : INTEGER) : BOOLEAN;
+  (* consume a continuation byte (0x80..0xBF), or fail *)
+    BEGIN
+      IF i > HIGH(s) THEN RETURN FALSE END;
+      b := ORD(s[i]); INC(i);
+      RETURN (b >= 128) AND (b <= 191)
+    END Cont;
+
+  BEGIN
+    cp := UCHR(0);
+    next := lo;
+    IF lo > HIGH(s) THEN RETURN FALSE END;
+    b0 := ORD(s[lo]); i := lo + 1; v := -1;
+    IF b0 <= 127 THEN
+      v := b0
+    ELSIF (b0 >= 194) AND (b0 <= 223) THEN
+      IF Cont(b1) THEN v := ((b0 - 192) * 64) + (b1 - 128) END
+    ELSIF (b0 >= 224) AND (b0 <= 239) THEN
+      IF Cont(b1) AND Cont(b2) THEN
+        v := ((b0 - 224) * 4096) + ((b1 - 128) * 64) + (b2 - 128);
+        IF v < 2048 THEN v := -1 END
+      END
+    ELSIF (b0 >= 240) AND (b0 <= 244) THEN
+      IF Cont(b1) AND Cont(b2) AND Cont(b3) THEN
+        v := ((b0 - 240) * 262144) + ((b1 - 128) * 4096)
+             + ((b2 - 128) * 64) + (b3 - 128);
+        IF v < 65536 THEN v := -1 END
+      END
+    ELSE
+      v := -1
+    END;
+    IF v < 0 THEN RETURN FALSE END;
+    IF (v >= 55296) AND (v <= 57343) THEN RETURN FALSE END;
+    IF v > 1114111 THEN RETURN FALSE END;
+    cp := UCHR(v);
+    next := i;
+    RETURN TRUE
+  END Decode;
+
+PROCEDURE Encode (cp : UCHAR; VAR buf : ARRAY OF CHAR) : CARDINAL;
+  VAR v : INTEGER;
+    n : CARDINAL;
+  BEGIN
+    v := UORD(cp);
+    IF (v >= 0) AND (v <= 127) THEN
+      buf[0] := CHR(v); n := 1
+    ELSIF (v >= 128) AND (v <= 2047) THEN
+      buf[0] := CHR(192 + (v DIV 64));
+      buf[1] := CHR(128 + (v MOD 64));
+      n := 2
+    ELSIF (v >= 2048) AND (v <= 65535) THEN
+      buf[0] := CHR(224 + (v DIV 4096));
+      buf[1] := CHR(128 + ((v DIV 64) MOD 64));
+      buf[2] := CHR(128 + (v MOD 64));
+      n := 3
+    ELSIF (v >= 65536) AND (v <= 1114111) THEN
+      buf[0] := CHR(240 + (v DIV 262144));
+      buf[1] := CHR(128 + ((v DIV 4096) MOD 64));
+      buf[2] := CHR(128 + ((v DIV 64) MOD 64));
+      buf[3] := CHR(128 + (v MOD 64));
+      n := 4
+    ELSE
+      n := 0
+    END;
+    RETURN n
+  END Encode;
+
+END Utf8.

stdlib/textio.def

@@ -7,4 +7,9 @@ PROCEDURE WriteInt(n : INTEGER);
 PROCEDURE WriteCard(n : CARDINAL);
 PROCEDURE WriteChar(c : CHAR);
 
+(* Unicode output: codepoints are encoded as UTF-8 bytes. *)
+PROCEDURE UWriteChar(cp : UCHAR);
+PROCEDURE UWriteString(VAR u : ARRAY OF UCHAR);
+PROCEDURE UWriteLn;
+
 END TextIO.

stdlib/textio.mod

@@ -2,6 +2,8 @@ IMPLEMENTATION MODULE TextIO;
 (* Binds the shim externals directly (avoids name shadowing with
    SysIO's wrappers). *)
 
+IMPORT Utf8;
+
 PROCEDURE m2write(VAR s : ARRAY OF CHAR);
   EXTERNAL;
 
@@ -39,4 +41,25 @@ BEGIN
   m2writechar(c)
 END WriteChar;
 
+PROCEDURE UWriteChar(cp : UCHAR);
+  VAR buf : ARRAY [0 .. 3] OF CHAR;
+    n, i : CARDINAL;
+BEGIN
+  n := Utf8.Encode(cp, buf);
+  i := 0;
+  WHILE i < n DO m2writechar(buf[i]); INC(i) END
+END UWriteChar;
+
+PROCEDURE UWriteString(VAR u : ARRAY OF UCHAR);
+  VAR i : CARDINAL;
+BEGIN
+  i := 0;
+  WHILE i <= HIGH(u) DO UWriteChar(u[i]); INC(i) END
+END UWriteString;
+
+PROCEDURE UWriteLn;
+BEGIN
+  m2writeln
+END UWriteLn;
+
 END TextIO.