M2compiler-V3/tools/v3-corpus/corpus.sh

#!/bin/sh
# Runs the V3 compiler (compiler/M2) over a corpus of standard Modula-2
# sources and reports how many compile cleanly.
#
# Each source is copied into a scratch tree (so its .LST listing never
# touches the corpus) and compiled alone.  A file with 0 errors is a
# pass.  Failures are split into:
#   CRASH  -- M2 died (rc >= 128): a real compiler bug
#   LIB    -- the file imports a module outside V3's known stdlib set
#             (gm2's StrIO/NumberIO/STextIO/...), so "undeclared" is
#             expected and not a language gap
#   LANG   -- otherwise: a genuine parse/semantic gap in V3
#
# Usage: ./corpus.sh [corpus-dir ...]
cd "$(dirname "$0")"
ROOT="$(cd ../.. && pwd)"
M2="$ROOT/compiler/M2"
WORK=/tmp/opencode/v3corpus
KNOWN="SYSTEM SysIO TextIO Strings Math Conversions RealIO ProgramArgs IOChan Files Storage WholeIO CharClass SysClock"
[ -x "$M2" ] || { echo "compiler/M2 missing -- run compiler/build.sh"; exit 1; }
[ "$#" -gt 0 ] || set -- \
  "$HOME/Projets/Projets-Modula2/MyWork/gcc-git/gcc/testsuite/gm2/pim/pass" \
  "$HOME/Projets/Projets-Modula2/MyWork/gcc-git/gcc/testsuite/gm2/iso/run/pass"

known() { for k in $KNOWN; do [ "$1" = "$k" ] && return 0; done; return 1; }

rm -rf "$WORK"; mkdir -p "$WORK"; : > "$WORK/results.txt"
n=0
for dir in "$@"; do
  find "$dir" -maxdepth 1 \( -iname '*.mod' -o -iname '*.def' \) 2>/dev/null | sort |
  while read -r f; do
    n=$((n+1))
    w="$WORK/f$n"; mkdir -p "$w"; base=$(basename "$f")
    cp "$f" "$w/$base"
    "$M2" "$w/$base" >/dev/null 2>&1
    rc=$?
    lst="$w/$(echo "$base" | sed 's/\.[^.]*$//').LST"
    if [ "$rc" -ge 128 ]; then echo "CRASH||$f" >> "$WORK/results.txt"; continue; fi
    if [ ! -f "$lst" ]; then echo "LANG|no listing|$f" >> "$WORK/results.txt"; continue; fi
    e=$(grep -oE '[0-9]+ errors?' "$lst" | head -1 | grep -oE '[0-9]+')
    [ -z "$e" ] && e=-1
    if [ "$e" = 0 ]; then echo "OK||$f" >> "$WORK/results.txt"; continue; fi
    # classify: does it import a module outside the known set?
    names=$( { sed -nE 's/^[[:space:]]*FROM[[:space:]]+([A-Za-z][A-Za-z0-9_]*).*/\1/p' "$w/$base"
               sed -nE 's/^[[:space:]]*IMPORT[[:space:]]+(.*)/\1/p' "$w/$base" | sed 's/;.*//' | tr ',' '\n'
             } | sed -E 's/[[:space:]]//g' | grep -E '^[A-Za-z][A-Za-z0-9_]*$' )
    lib=0
    for m in $names; do known "$m" || lib=1; done
    m=$(grep -E '^\*\*\*\*\*' "$lst" | head -1 | sed -E 's/^\*\*\*\*\* *\^? *//; s/^\*\*\*\*\*//')
    if [ "$lib" = 1 ]; then echo "LIB|$m|$f" >> "$WORK/results.txt"
    else echo "LANG|$m|$f" >> "$WORK/results.txt"; fi
  done
done
total=$(wc -l < "$WORK/results.txt")
cnt() { grep -c "^$1" "$WORK/results.txt" 2>/dev/null || true; }
echo "total      : $total"
echo "compile-OK : $(cnt OK)"
echo "CRASH      : $(cnt CRASH)"
echo "LIB (needs a gm2 library): $(cnt LIB)"
echo "LANG (V3 gap)            : $(cnt LANG)"
echo "=== LANG error kinds (top) ==="
grep '^LANG' "$WORK/results.txt" | cut -d'|' -f2 | sed -E 's/[0-9]+/N/g' | sort | uniq -c | sort -rn | head -15
echo "=== LANG files (first 40) ==="
grep '^LANG' "$WORK/results.txt" | head -40 | sed -E 's/^LANG\|([^|]*)\|/  [\1] /'