vibe coded a little

flake.nix

@@ -9,7 +9,7 @@
     # rootfs will contain all binaries given by toolchain
     toolchain = pkgs.buildEnv {
       name = "pigeon-toolchain";
-      paths = with pkgs; [ bash coreutils gcc gnumake util-linux ];
+      paths = with pkgs; [ binutils bash coreutils gcc gnumake util-linux ];
     };
     closure = pkgs.closureInfo {
       rootPaths = [ toolchain ];
@@ -77,7 +77,7 @@
 
     pigeon = pkgs.writeShellApplication {
       name = "pigeon";
-      runtimeInputs = with pkgs; [ coreutils curl e2fsprogs firecracker git ];
+      runtimeInputs = with pkgs; [ coreutils curl e2fsprogs firecracker gcc git gnumake ];
       text = builtins.replaceStrings
         [ "@kernel@" "@rootfs@" ] [ "${kernel}" "${rootfs}"]
         (builtins.readFile ./pigeon.sh);

init.sh

@@ -22,7 +22,6 @@ cd /tmp
     make >/root/results/build.log 2>&1
   echo $? >/root/results/build.status
 )
-[ "$(cat /root/results/build.status)" != 0 ] && exit 1
 
 for TEST in /root/project/tests/*.sh ; do
   (
@@ -30,5 +29,6 @@ for TEST in /root/project/tests/*.sh ; do
     timeout -s 9 2 \
       setpriv --reuid=1 --regid=1 --clear-groups --no-new-privs --inh-caps=-all \
       "$TEST" >"/root/results/$(basename "$TEST" .sh)" 2>/dev/null
+    echo $? >"/root/results/$(basename "$TEST" .sh).status"
   )
 done

pigeon.sh

@@ -12,6 +12,7 @@ RELEASE_TIME="$6"
 RATE_LIMIT=3
 
 PIGEON_DATABASE="$(realpath -m "$PIGEON_STATE/db")"
+PIGEON_PROJECTS="$(realpath -m "$PIGEON_PROJECTS")"
 RELEASE_TIME="$(date -d "$RELEASE_TIME" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || true)"
 
 report() {
@@ -21,6 +22,7 @@ report() {
     --data "$(jq -n --arg d "$1" '{"description": $d}')" \
     --url "https://gitlab.com/api/v4/projects/$PROJECT_ID/releases/$RELEASE_TAG"
 }
+# shellcheck disable=SC2120,SC2119
 db() {
   sqlite3 -bail -cmd '.timeout 10000' "$PIGEON_DATABASE" "$@"
 }
@@ -35,26 +37,11 @@ CREATE TABLE IF NOT EXISTS results (
     project TEXT NOT NULL,
     hash    TEXT NOT NULL,
     time    TEXT NOT NULL DEFAULT (datetime('now')),
-    score   INTEGER CHECK (score >= 0)
+    score   INTEGER CHECK (score >= 0),
+    token   INTEGER NOT NULL
 );
 EOF
 
-ROW="$(db <<EOF
-INSERT INTO results (student, project, hash, time)
-SELECT $(q "$PROJECT_NAMESPACE"), $(q "$PROJECT_NAME"), $(q "$RELEASE_COMMIT"),
-  COALESCE(NULLIF($(q "$RELEASE_TIME"), ''), datetime('now'))
-WHERE (SELECT COUNT(*) FROM results
-  WHERE student = $(q "$PROJECT_NAMESPACE") AND project = $(q "$PROJECT_NAME")
-  AND time > datetime('now', '-1 day')) < $RATE_LIMIT
-RETURNING id;
-EOF
-)"
-
-if [ -z "$ROW" ] ; then
-  report "Rate limit exceeded!"
-  exit
-fi
-
 DIRECTORY="$(mktemp -d /tmp/pigeon.XXXXXXXXXX)"
 trap 'rm -rf "$DIRECTORY"' EXIT
 cd "$DIRECTORY"
@@ -63,10 +50,23 @@ git -c advice.detachedHead=false clone -q --rev "$RELEASE_COMMIT" --depth 1 \
   "https://oauth2:$PIGEON_GITLAB_TOKEN@gitlab.com/$PROJECT_NAMESPACE/$PROJECT_NAME.git" student/
 rm -rf student/.git
 
+TESTS="$PIGEON_PROJECTS/$PROJECT_NAME/tests"
 mkdir -p project/tests project/build
-cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/tests/*.sh project/tests/
+cp -a "$TESTS"/*.sh project/tests/
 cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/build/. project/build/
 
+# A project may ship an answer-key solution. It is built and run only here,
+# on the host, and never copied onto any drive the guest can reach -- tests
+# compare the student's reported output against it after the VM has exited.
+CANONICAL=""
+if [ -d "$PIGEON_PROJECTS/$PROJECT_NAME/canonical" ] ; then
+  mkdir -p canonical
+  cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/build/. canonical/
+  cp -a --remove-destination "$PIGEON_PROJECTS/$PROJECT_NAME"/canonical/. canonical/
+  ( cd canonical && timeout 30 make >/dev/null 2>&1 ) || true
+  [ -x canonical/program ] && CANONICAL="$(realpath canonical/program)"
+fi
+
 mkfs.ext4 -q -F -d student student.img 16M
 mkfs.ext4 -q -F -d project project.img 16M
 mkfs.ext4 -q -F results.img 16M
@@ -89,21 +89,124 @@ EOF
 
 firecracker --no-api --level Error --config-file vm.json
 
-RESULT=""
+PUBLIC_SCORE=0; PUBLIC_TOTAL=0
-SCORE=0
+RELEASE_SCORE=0; RELEASE_TOTAL=0
-TESTS="$PIGEON_PROJECTS/$PROJECT_NAME/tests"
+SECRET_SCORE=0; SECRET_TOTAL=0
+NAMES=()
+STATUSES=()
+ANSWERS=()
+QUESTIONS=()
+TIERS=()
+
 for TEST in "$TESTS"/*.sh ; do
-  debugfs -R "cat /$(basename "$TEST" .sh)" results.img >scratch 2>/dev/null
+  NAME="$(basename "$TEST" .sh)"
-  if cmp -s scratch "$TESTS/$(basename "$TEST" .sh).expected" ; then
+  TIER="${NAME%%-*}"
-    RESULT="${RESULT}PASS\n"
+  QUESTION="$(cat "$TESTS/$NAME.question")"
-    SCORE=$((SCORE + 1))
+  debugfs -R "cat /$NAME" results.img >scratch 2>/dev/null
+  EXIT="$(debugfs -R "cat /$NAME.status" results.img 2>/dev/null)"
+  case "$EXIT" in '' | *[!0-9]*) EXIT=0 ;; esac
+
+  # A test may expect the student's program to exit with a specific status
+  # (e.g. 134 for a SIGABRT from a correctly-triggered assert()) instead of
+  # the default 0.
+  EXPECTED_EXIT=0
+  [ -f "$TESTS/$NAME.exit" ] && EXPECTED_EXIT="$(cat "$TESTS/$NAME.exit")"
+
+  if [ -f "$TESTS/$NAME.expected" ] ; then
+    if cmp -s scratch "$TESTS/$NAME.expected" && [ "$EXIT" -eq "$EXPECTED_EXIT" ] ; then
+      MATCH=1
+    else
+      MATCH=0
+    fi
+  elif [ -n "$CANONICAL" ] ; then
+    # No static .expected: the test reported its own "ARG<TAB>OUTPUT" pairs,
+    # so check each one against the canonical solution run here on the host.
+    MATCH=1
+    while IFS="$(printf '\t')" read -r ARG OUT ; do
+      [ -n "$ARG" ] || continue
+      EXPECTED="$("$CANONICAL" "$ARG" 2>/dev/null)"
+      [ "$OUT" = "$EXPECTED" ] || MATCH=0
+    done <scratch
   else
-    RESULT="${RESULT}FAIL: $(cat "$TESTS/$(basename "$TEST" .sh).hint")\n"
+    MATCH=0
   fi
+
+  if [ "$MATCH" = 1 ] ; then
+    STATUS=PASS
+    ANSWER=YES
+  else
+    STATUS=FAIL
+    # timeout -s 9 (used in init.sh) exits 137 (128+SIGKILL) on a timeout,
+    # not the 124 it uses for its default signal (SIGTERM). Timing out is
+    # never the expected outcome, so it's checked before EXPECTED_EXIT.
+    if [ "$EXIT" -eq 124 ] || [ "$EXIT" -eq 137 ] ; then
+      ANSWER="NO (timed out)"
+    elif [ "$EXIT" -eq "$EXPECTED_EXIT" ] ; then
+      ANSWER="NO (wrong output)"
+    elif [ "$EXIT" -gt 128 ] ; then
+      ANSWER="NO (crashed, signal $((EXIT - 128)))"
+    else
+      ANSWER="NO (exited with status $EXIT, expected $EXPECTED_EXIT)"
+    fi
+  fi
+  NAMES+=("$NAME")
+  STATUSES+=("$STATUS")
+  ANSWERS+=("$ANSWER")
+  QUESTIONS+=("$QUESTION")
+  TIERS+=("$TIER")
+  case "$TIER" in
+    public)
+      PUBLIC_TOTAL=$((PUBLIC_TOTAL + 1))
+      [ "$STATUS" = PASS ] && PUBLIC_SCORE=$((PUBLIC_SCORE + 1))
+      ;;
+    release)
+      RELEASE_TOTAL=$((RELEASE_TOTAL + 1))
+      [ "$STATUS" = PASS ] && RELEASE_SCORE=$((RELEASE_SCORE + 1))
+      ;;
+    secret)
+      SECRET_TOTAL=$((SECRET_TOTAL + 1))
+      [ "$STATUS" = PASS ] && SECRET_SCORE=$((SECRET_SCORE + 1))
+      ;;
+  esac
 done
 
-# Record before reporting, so a GitLab outage doesn't lose the result.
+PUBLIC_PASS=0
-db "UPDATE results SET score = $SCORE WHERE id = $ROW;" ||
+[ "$PUBLIC_SCORE" -eq "$PUBLIC_TOTAL" ] && PUBLIC_PASS=1
-  echo "pigeon: failed to record score for row $ROW in $PIGEON_DB" >&2
+SCORE=$((PUBLIC_SCORE + RELEASE_SCORE + SECRET_SCORE))
 
-report "$RESULT"
+TOKEN="$(db <<EOF
+INSERT INTO results (student, project, hash, time, score, token)
+SELECT $(q "$PROJECT_NAMESPACE"), $(q "$PROJECT_NAME"), $(q "$RELEASE_COMMIT"),
+  COALESCE(NULLIF($(q "$RELEASE_TIME"), ''), datetime('now')),
+  $SCORE,
+  CASE WHEN $PUBLIC_PASS = 1 AND (
+    SELECT COUNT(*) FROM results
+    WHERE student = $(q "$PROJECT_NAMESPACE") AND project = $(q "$PROJECT_NAME")
+      AND token = 1 AND time > datetime('now', '-1 day')
+  ) < $RATE_LIMIT THEN 1 ELSE 0 END
+RETURNING token;
+EOF
+)"
+
+SHOWN_QUESTIONS=()
+SHOWN_ANSWERS=()
+for i in "${!NAMES[@]}" ; do
+  case "${TIERS[$i]}" in
+    public) ;;
+    release) [ "$TOKEN" = 1 ] || continue ;;
+    *) continue ;;
+  esac
+  SHOWN_QUESTIONS+=("${QUESTIONS[$i]}")
+  SHOWN_ANSWERS+=("${ANSWERS[$i]}")
+done
+
+MESSAGE=""
+for i in "${!SHOWN_QUESTIONS[@]}" ; do
+  MESSAGE="${MESSAGE}${SHOWN_QUESTIONS[$i]} ${SHOWN_ANSWERS[$i]}  "$'\n'
+done
+
+if [ "$TOKEN" != 1 ] && [ "$PUBLIC_PASS" = 1 ] ; then
+  MESSAGE="${MESSAGE}"$'\n'"All public tests passed, but you have used all $RATE_LIMIT submissions for today. Release test results are withheld until tomorrow."
+fi
+
+report "$MESSAGE"