tidied more

flake.nix

@@ -9,7 +9,7 @@
     # rootfs will contain all binaries given by toolchain
     toolchain = pkgs.buildEnv {
       name = "pigeon-toolchain";
-      paths = with pkgs; [ bash coreutils gcc gnumake util-linux ];
+      paths = with pkgs; [ binutils bash coreutils gcc gnumake util-linux ];
     };
     closure = pkgs.closureInfo {
       rootPaths = [ toolchain ];
@@ -21,6 +21,7 @@
       umask 022
       mkdir -p dev proc sys tmp usr nix/store root/student root/project root/results
       chmod 777 tmp
+      chmod 700 root
 
       for FILE in $(cat ${closure}/store-paths) ; do
         cp -a "$FILE" nix/store
@@ -77,9 +78,10 @@
 
     pigeon = pkgs.writeShellApplication {
       name = "pigeon";
-      runtimeInputs = with pkgs; [ coreutils curl e2fsprogs firecracker git ];
+      runtimeInputs = with pkgs; [ coreutils curl diffutils e2fsprogs firecracker git jq sqlite ];
+      excludeShellChecks = [ "SC2016" "SC2030" "SC2031" ];
       text = builtins.replaceStrings
-        [ "@kernel@" "@rootfs@" ] [ "${kernel}" "${rootfs}"]
+        [ "@kernel@" "@rootfs@" "@toolchain@" ] [ "${kernel}" "${rootfs}" "${toolchain}" ]
         (builtins.readFile ./pigeon.sh);
     };
 
@@ -92,8 +94,7 @@
         pass-arguments-to-command = [
           { source = "string";  name = "${pigeon}/bin/pigeon"; }
           { source = "payload"; name = "project.id"; }
-          { source = "payload"; name = "project.namespace"; }
-          { source = "payload"; name = "project.name"; }
+          { source = "payload"; name = "project.path_with_namespace"; }
           { source = "payload"; name = "commit.id"; }
           { source = "payload"; name = "tag"; }
           { source = "payload"; name = "created_at"; }
@@ -117,7 +118,7 @@
             match = {
               type = "value";
               parameter = { source = "header"; name = "X-Gitlab-Token"; };
-              value = "{{ getenv `PIGEON_GITLAB_SECRET` | js }}";
+              value = "{{ getenv `SECRET` | js }}";
             };
           }
         ];
@@ -127,13 +128,13 @@
       name = "pigeon-webhook";
       runtimeInputs = with pkgs; [ coreutils jq nq sqlite webhook ];
       text = ''
-        : "''${PIGEON_GITLAB_SECRET:?}"
-        : "''${PIGEON_GITLAB_TOKEN:?}"
-        : "''${PIGEON_PROJECTS:?}"
-        : "''${PIGEON_STATE}"
+        : "''${SECRET:?}"
+        : "''${TOKEN:?}"
+        : "''${GRADER:?}"
+        : "''${STATE:?}"
 
-        mkdir -p "$PIGEON_STATE"
-        export NQDIR="$PIGEON_STATE/nqdir"
+        mkdir -p "$STATE"
+        export NQDIR="$STATE/nqdir"
         exec webhook -hooks ${hooks} -template -port 3000 -verbose -debug
       '';
     };

init.sh

@@ -14,21 +14,29 @@ cp -a /root/student/. /tmp/
 cp -a --remove-destination /root/project/build/. /tmp/
 chmod -R 777 /tmp
 cd /tmp
+rm -f program
 
 (
   ulimit -t 30 -u 256 -n 256 -f 2048
   timeout -s 9 30 \
     setpriv --reuid=1 --regid=1 --clear-groups --no-new-privs --inh-caps=-all \
-    make >/root/results/build.log 2>&1
+    make -B >/root/results/build.log 2>&1
   echo $? >/root/results/build.status
 )
-[ "$(cat /root/results/build.status)" != 0 ] && exit 1
 
-for TEST in /root/project/tests/*.sh ; do
+for TEST in /root/project/tests/* ; do
+  NAME="$(basename "$TEST")"
+
+  ARGS="/root/project/args/$NAME"
+  [ -f "$ARGS" ] || ARGS=/dev/null
+
+  cp -a --remove-destination "$TEST" /tmp/test
+  chmod 755 /tmp/test
   (
     ulimit -t 2 -u 256 -n 256 -f 2048
     timeout -s 9 2 \
       setpriv --reuid=1 --regid=1 --clear-groups --no-new-privs --inh-caps=-all \
-      "$TEST" >"/root/results/$(basename "$TEST" .sh)" 2>/dev/null
+      /tmp/test <"$ARGS" >"/root/results/$NAME" 2>/dev/null
+    echo $? >"/root/results/$NAME.status"
   )
 done

pigeon.sh

@@ -1,76 +1,38 @@
-PROJECT_ID="$1"
-PROJECT_NAMESPACE="$2"
-PROJECT_NAME="$3"
-RELEASE_COMMIT="$4"
-RELEASE_TAG="$5"
-RELEASE_TIME="$6"
-
-: "${PIGEON_GITLAB_TOKEN:?}"
-: "${PIGEON_PROJECTS:?}"
-: "${PIGEON_STATE:?}"
-
-RATE_LIMIT=3
-
-PIGEON_DATABASE="$(realpath -m "$PIGEON_STATE/db")"
-RELEASE_TIME="$(date -d "$RELEASE_TIME" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || true)"
-
-report() {
-  curl -s --show-error --out-null --request PUT \
-    --header "PRIVATE-TOKEN: $PIGEON_GITLAB_TOKEN" \
-    --header 'Content-Type: application/json' \
-    --data "$(jq -n --arg d "$1" '{"description": $d}')" \
-    --url "https://gitlab.com/api/v4/projects/$PROJECT_ID/releases/$RELEASE_TAG"
-}
-db() {
-  sqlite3 -bail -cmd '.timeout 10000' "$PIGEON_DATABASE" "$@"
-}
-q() {
-  printf "'%s'" "$(printf %s "$1" | sed "s/'/''/g")"
-}
-
-db << 'EOF'
-CREATE TABLE IF NOT EXISTS results (
-    id      INTEGER PRIMARY KEY,
-    student TEXT NOT NULL,
-    project TEXT NOT NULL,
-    hash    TEXT NOT NULL,
-    time    TEXT NOT NULL DEFAULT (datetime('now')),
-    score   INTEGER CHECK (score >= 0)
-);
-EOF
-
-ROW="$(db <<EOF
-INSERT INTO results (student, project, hash, time)
-SELECT $(q "$PROJECT_NAMESPACE"), $(q "$PROJECT_NAME"), $(q "$RELEASE_COMMIT"),
-  COALESCE(NULLIF($(q "$RELEASE_TIME"), ''), datetime('now'))
-WHERE (SELECT COUNT(*) FROM results
-  WHERE student = $(q "$PROJECT_NAMESPACE") AND project = $(q "$PROJECT_NAME")
-  AND time > datetime('now', '-1 day')) < $RATE_LIMIT
-RETURNING id;
-EOF
-)"
-
-if [ -z "$ROW" ] ; then
-  report "Rate limit exceeded!"
-  exit
-fi
-
+PROJECT="${2##*/}"
+STUDENT="${2%/*}"
+GRADER="$(realpath -m "${GRADER:?}")/$PROJECT"
+DATABASE="$(realpath -m "${STATE:?}")/db"
 DIRECTORY="$(mktemp -d /tmp/pigeon.XXXXXXXXXX)"
 trap 'rm -rf "$DIRECTORY"' EXIT
 cd "$DIRECTORY"
 
-git -c advice.detachedHead=false clone -q --rev "$RELEASE_COMMIT" --depth 1 \
-  "https://oauth2:$PIGEON_GITLAB_TOKEN@gitlab.com/$PROJECT_NAMESPACE/$PROJECT_NAME.git" student/
+git -c advice.detachedHead=false clone -q --rev "$3" --depth 1 \
+  "https://oauth2:${TOKEN:?}@gitlab.com/$2.git" student/
 rm -rf student/.git
 
-mkdir -p project/tests project/build
-cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/tests/*.sh project/tests/
-cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/build/. project/build/
+mkdir -p project/tests project/build project/args
+for DIR in "$GRADER"/tests/*/ ; do
+  NAME="$(basename "$DIR")"
+  cp -a "$DIR/input.sh" "project/tests/$NAME"
+  [ -f "$DIR/output.txt" ] ||
+    head -c 4096 /dev/urandom | LC_ALL=C tr -dc ' -~' >"project/args/$NAME"
+done
+cp -a "$GRADER"/build/. project/build/
 
 mkfs.ext4 -q -F -d student student.img 16M
 mkfs.ext4 -q -F -d project project.img 16M
 mkfs.ext4 -q -F results.img 16M
 
+if [ -d "$GRADER/canonical" ] ; then
+  mkdir -p canonical
+  cp -a "$GRADER"/canonical/. canonical/
+  cp -a --remove-destination "$GRADER"/build/. canonical/
+  (
+    export PATH=@toolchain@/bin
+    cd canonical && timeout 30 make
+  ) >/dev/null 2>&1 || exit 1
+fi
+
 cat >vm.json <<'EOF'
 {
   "boot-source": {
@@ -87,23 +49,87 @@ cat >vm.json <<'EOF'
 }
 EOF
 
-firecracker --no-api --level Error --config-file vm.json
+timeout 60 firecracker --no-api --level Error --config-file vm.json
 
-RESULT=""
 SCORE=0
-TESTS="$PIGEON_PROJECTS/$PROJECT_NAME/tests"
-for TEST in "$TESTS"/*.sh ; do
-  debugfs -R "cat /$(basename "$TEST" .sh)" results.img >scratch 2>/dev/null
-  if cmp -s scratch "$TESTS/$(basename "$TEST" .sh).expected" ; then
-    RESULT="${RESULT}PASS\n"
+PUBLIC=1
+for DIR in "$GRADER"/tests/*/ ; do
+  NAME="$(basename "$DIR")"
+
+  debugfs -R "cat /$NAME" results.img >scratch 2>/dev/null
+  EXIT="$(debugfs -R "cat /$NAME.status" results.img 2>/dev/null)"
+  EXPECTED="$(cat "$DIR/status.txt" 2>/dev/null || true)"
+  case "$EXPECTED" in '' | *[!0-9]*) EXPECTED=0 ;; esac
+
+  if [ -f "$DIR/output.txt" ] ; then
+    cat "$DIR/output.txt"
+  elif [ -d canonical ] ; then
+    (
+      export PATH=@toolchain@/bin
+      cd canonical && timeout 2 sh "$DIRECTORY/project/tests/$NAME"
+    ) <"project/args/$NAME" 2>/dev/null || true
+  fi >expected
+
+  if cmp -s scratch expected && [ "$EXIT" -eq "$EXPECTED" ] ; then
+    RESULT=YES
     SCORE=$((SCORE + 1))
+  elif [ "$EXIT" -eq 124 ] || [ "$EXIT" -eq 137 ] ; then
+    RESULT="NO (timed out)"
+  elif [ "$EXIT" -gt 128 ] ; then
+    RESULT="NO (crashed with signal $((EXIT - 128)))"
+  elif [ "$EXIT" -ne "$EXPECTED" ] ; then
+    RESULT="NO (exited with status $EXIT, expected $EXPECTED)"
   else
-    RESULT="${RESULT}FAIL: $(cat "$TESTS/$(basename "$TEST" .sh).hint")\n"
+    RESULT="NO (wrong output)"
   fi
+
+  case "$NAME" in
+    p*)
+      if [ "$RESULT" != YES ] ; then PUBLIC=0 ; fi
+      echo "$(cat "$DIR/test.txt") $RESULT  " >>public ;;
+    r*)
+      echo "$(cat "$DIR/test.txt") $RESULT  " >>release ;;
+  esac
 done
 
-# Record before reporting, so a GitLab outage doesn't lose the result.
-db "UPDATE results SET score = $SCORE WHERE id = $ROW;" ||
-  echo "pigeon: failed to record score for row $ROW in $PIGEON_DB" >&2
+q() {
+  printf "'%s'" "${1//\'/\'\'}"
+}
 
-report "$RESULT"
+MESSAGE="$(cat public || true)"
+[ "$(sqlite3 -bail -cmd '.timeout 10000' "$DATABASE" <<EOF
+CREATE TABLE IF NOT EXISTS results (
+    id      INTEGER PRIMARY KEY,
+    student TEXT NOT NULL,
+    project TEXT NOT NULL,
+    hash    TEXT NOT NULL,
+    time    TEXT NOT NULL,
+    score   INTEGER NOT NULL,
+    token   BOOLEAN NOT NULL
+);
+INSERT INTO results (student, project, hash, time, score, token)
+SELECT
+  $(q "$STUDENT"),
+  $(q "$PROJECT"),
+  $(q "$3"),
+  $(q "$(date -u -d "$5" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -u '+%Y-%m-%d %H:%M:%S')"),
+  $SCORE,
+  ($PUBLIC = 1) AND (
+    SELECT COUNT(*) FROM results
+    WHERE student = $(q "$STUDENT") AND project = $(q "$PROJECT")
+      AND token AND time > datetime('now', '-1 day')
+  ) < 3
+RETURNING token;
+EOF
+)" = 1 ] && MESSAGE="$(cat public release || true)"
+
+if [ "$(debugfs -R "cat /build.status" results.img 2>/dev/null)" -ne 0 ] ; then
+  debugfs -R "cat /build.log" results.img >build.log 2>/dev/null
+  MESSAGE="$(printf '````\n%s\n````' "$(head -c 8192 build.log)")"
+fi
+
+curl -s --show-error --out-null --request PUT \
+  --header "PRIVATE-TOKEN: ${TOKEN:?}" \
+  --header 'Content-Type: application/json' \
+  --data "$(jq -n --arg d "$MESSAGE" '{"description": $d}')" \
+  --url "https://gitlab.com/api/v4/projects/$1/releases/$(jq -rn --arg t "$4" '$t|@uri')"