vibe coded more

init.sh

@@ -23,12 +23,12 @@ cd /tmp
   echo $? >/root/results/build.status
 )
 
-for TEST in /root/project/tests/*.sh ; do
+for TEST in /root/project/tests/* ; do
   (
     ulimit -t 2 -u 256 -n 256 -f 2048
     timeout -s 9 2 \
       setpriv --reuid=1 --regid=1 --clear-groups --no-new-privs --inh-caps=-all \
-      "$TEST" >"/root/results/$(basename "$TEST" .sh)" 2>/dev/null
+      "$TEST" >"/root/results/$(basename "$TEST")" 2>/dev/null
-    echo $? >"/root/results/$(basename "$TEST" .sh).status"
+    echo $? >"/root/results/$(basename "$TEST").status"
   )
 done

pigeon.sh

@@ -15,16 +15,8 @@ PIGEON_DATABASE="$(realpath -m "$PIGEON_STATE/db")"
 PIGEON_PROJECTS="$(realpath -m "$PIGEON_PROJECTS")"
 RELEASE_TIME="$(date -d "$RELEASE_TIME" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || true)"
 
-report() {
-  curl -s --show-error --out-null --request PUT \
-    --header "PRIVATE-TOKEN: $PIGEON_GITLAB_TOKEN" \
-    --header 'Content-Type: application/json' \
-    --data "$(jq -n --arg d "$1" '{"description": $d}')" \
-    --url "https://gitlab.com/api/v4/projects/$PROJECT_ID/releases/$RELEASE_TAG"
-}
-# shellcheck disable=SC2120,SC2119
 db() {
-  sqlite3 -bail -cmd '.timeout 10000' "$PIGEON_DATABASE" "$@"
+  sqlite3 -bail -cmd '.timeout 10000' "$PIGEON_DATABASE"
 }
 q() {
   printf "'%s'" "$(printf %s "$1" | sed "s/'/''/g")"
@@ -52,12 +44,11 @@ rm -rf student/.git
 
 TESTS="$PIGEON_PROJECTS/$PROJECT_NAME/tests"
 mkdir -p project/tests project/build
-cp -a "$TESTS"/*.sh project/tests/
+for DIR in "$TESTS"/*/ ; do
+  cp -a "$DIR/input.sh" "project/tests/$(basename "$DIR")"
+done
 cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/build/. project/build/
 
-# A project may ship an answer-key solution. It is built and run only here,
-# on the host, and never copied onto any drive the guest can reach -- tests
-# compare the student's reported output against it after the VM has exited.
 CANONICAL=""
 if [ -d "$PIGEON_PROJECTS/$PROJECT_NAME/canonical" ] ; then
   mkdir -p canonical
@@ -89,91 +80,51 @@ EOF
 
 firecracker --no-api --level Error --config-file vm.json
 
-PUBLIC_SCORE=0; PUBLIC_TOTAL=0
+SCORE=0
-RELEASE_SCORE=0; RELEASE_TOTAL=0
+PUBLIC_PASS=1
-SECRET_SCORE=0; SECRET_TOTAL=0
+: >public
-NAMES=()
+: >release
-STATUSES=()
+
-ANSWERS=()
+for DIR in "$TESTS"/*/ ; do
-QUESTIONS=()
+  NAME="$(basename "$DIR")"
-TIERS=()
 
-for TEST in "$TESTS"/*.sh ; do
-  NAME="$(basename "$TEST" .sh)"
-  TIER="${NAME%%-*}"
-  QUESTION="$(cat "$TESTS/$NAME.question")"
   debugfs -R "cat /$NAME" results.img >scratch 2>/dev/null
   EXIT="$(debugfs -R "cat /$NAME.status" results.img 2>/dev/null)"
+  EXPECTED_EXIT="$(cat "$DIR/status.txt" 2>/dev/null)"
   case "$EXIT" in '' | *[!0-9]*) EXIT=0 ;; esac
+  case "$EXPECTED_EXIT" in '' | *[!0-9]*) EXPECTED_EXIT=0 ;; esac
 
-  # A test may expect the student's program to exit with a specific status
+  if [ -f "$DIR/output.txt" ] ; then
-  # (e.g. 134 for a SIGABRT from a correctly-triggered assert()) instead of
+    cmp -s scratch "$DIR/output.txt" && [ "$EXIT" -eq "$EXPECTED_EXIT" ] && PASS=1 || PASS=0
-  # the default 0.
-  EXPECTED_EXIT=0
-  [ -f "$TESTS/$NAME.exit" ] && EXPECTED_EXIT="$(cat "$TESTS/$NAME.exit")"
-
-  if [ -f "$TESTS/$NAME.expected" ] ; then
-    if cmp -s scratch "$TESTS/$NAME.expected" && [ "$EXIT" -eq "$EXPECTED_EXIT" ] ; then
-      MATCH=1
-    else
-      MATCH=0
-    fi
   elif [ -n "$CANONICAL" ] ; then
-    # No static .expected: the test reported its own "ARG<TAB>OUTPUT" pairs,
+    PASS=1
-    # so check each one against the canonical solution run here on the host.
-    MATCH=1
     while IFS="$(printf '\t')" read -r ARG OUT ; do
       [ -n "$ARG" ] || continue
-      EXPECTED="$("$CANONICAL" "$ARG" 2>/dev/null)"
+      [ "$OUT" = "$("$CANONICAL" "$ARG" 2>/dev/null)" ] || PASS=0
-      [ "$OUT" = "$EXPECTED" ] || MATCH=0
     done <scratch
   else
-    MATCH=0
+    PASS=0
   fi
 
-  if [ "$MATCH" = 1 ] ; then
+  if [ "$PASS" = 1 ] ; then
-    STATUS=PASS
     ANSWER=YES
+    SCORE=$((SCORE + 1))
+  elif [ "$EXIT" -eq 124 ] || [ "$EXIT" -eq 137 ] ; then
+    ANSWER="NO (timed out)"
+  elif [ "$EXIT" -eq "$EXPECTED_EXIT" ] ; then
+    ANSWER="NO (wrong output)"
+  elif [ "$EXIT" -gt 128 ] ; then
+    ANSWER="NO (crashed, signal $((EXIT - 128)))"
   else
-    STATUS=FAIL
+    ANSWER="NO (exited with status $EXIT, expected $EXPECTED_EXIT)"
-    # timeout -s 9 (used in init.sh) exits 137 (128+SIGKILL) on a timeout,
-    # not the 124 it uses for its default signal (SIGTERM). Timing out is
-    # never the expected outcome, so it's checked before EXPECTED_EXIT.
-    if [ "$EXIT" -eq 124 ] || [ "$EXIT" -eq 137 ] ; then
-      ANSWER="NO (timed out)"
-    elif [ "$EXIT" -eq "$EXPECTED_EXIT" ] ; then
-      ANSWER="NO (wrong output)"
-    elif [ "$EXIT" -gt 128 ] ; then
-      ANSWER="NO (crashed, signal $((EXIT - 128)))"
-    else
-      ANSWER="NO (exited with status $EXIT, expected $EXPECTED_EXIT)"
-    fi
   fi
-  NAMES+=("$NAME")
+
-  STATUSES+=("$STATUS")
+  case "$NAME" in
-  ANSWERS+=("$ANSWER")
+    p*) [ "$PASS" = 1 ] || PUBLIC_PASS=0; echo "$(cat "$DIR/test.txt") $ANSWER  " >>public ;;
-  QUESTIONS+=("$QUESTION")
+    r*) echo "$(cat "$DIR/test.txt") $ANSWER  " >>release ;;
-  TIERS+=("$TIER")
-  case "$TIER" in
-    public)
-      PUBLIC_TOTAL=$((PUBLIC_TOTAL + 1))
-      [ "$STATUS" = PASS ] && PUBLIC_SCORE=$((PUBLIC_SCORE + 1))
-      ;;
-    release)
-      RELEASE_TOTAL=$((RELEASE_TOTAL + 1))
-      [ "$STATUS" = PASS ] && RELEASE_SCORE=$((RELEASE_SCORE + 1))
-      ;;
-    secret)
-      SECRET_TOTAL=$((SECRET_TOTAL + 1))
-      [ "$STATUS" = PASS ] && SECRET_SCORE=$((SECRET_SCORE + 1))
-      ;;
   esac
 done
 
-PUBLIC_PASS=0
-[ "$PUBLIC_SCORE" -eq "$PUBLIC_TOTAL" ] && PUBLIC_PASS=1
-SCORE=$((PUBLIC_SCORE + RELEASE_SCORE + SECRET_SCORE))
-
 TOKEN="$(db <<EOF
 INSERT INTO results (student, project, hash, time, score, token)
 SELECT $(q "$PROJECT_NAMESPACE"), $(q "$PROJECT_NAME"), $(q "$RELEASE_COMMIT"),
@@ -188,25 +139,15 @@ RETURNING token;
 EOF
 )"
 
-SHOWN_QUESTIONS=()
+if [ "$TOKEN" = 1 ] ; then
-SHOWN_ANSWERS=()
+  MESSAGE="$(cat public release)"
-for i in "${!NAMES[@]}" ; do
+else
-  case "${TIERS[$i]}" in
+  MESSAGE="$(cat public)"
-    public) ;;
+  [ "$PUBLIC_PASS" = 1 ] && MESSAGE="${MESSAGE}"$'\n'"All public tests passed, but you have used all $RATE_LIMIT submissions for today. Release test results are withheld until tomorrow."
-    release) [ "$TOKEN" = 1 ] || continue ;;
-    *) continue ;;
-  esac
-  SHOWN_QUESTIONS+=("${QUESTIONS[$i]}")
-  SHOWN_ANSWERS+=("${ANSWERS[$i]}")
-done
-
-MESSAGE=""
-for i in "${!SHOWN_QUESTIONS[@]}" ; do
-  MESSAGE="${MESSAGE}${SHOWN_QUESTIONS[$i]} ${SHOWN_ANSWERS[$i]}  "$'\n'
-done
-
-if [ "$TOKEN" != 1 ] && [ "$PUBLIC_PASS" = 1 ] ; then
-  MESSAGE="${MESSAGE}"$'\n'"All public tests passed, but you have used all $RATE_LIMIT submissions for today. Release test results are withheld until tomorrow."
 fi
 
-report "$MESSAGE"
+curl -s --show-error --out-null --request PUT \
+  --header "PRIVATE-TOKEN: $PIGEON_GITLAB_TOKEN" \
+  --header 'Content-Type: application/json' \
+  --data "$(jq -n --arg d "$MESSAGE" '{"description": $d}')" \
+  --url "https://gitlab.com/api/v4/projects/$PROJECT_ID/releases/$RELEASE_TAG"