Pigeon/pigeon.sh

PROJECT_ID="$1"
PROJECT_NAMESPACE="$2"
PROJECT_NAME="$3"
RELEASE_COMMIT="$4"
RELEASE_TAG="$5"
RELEASE_TIME="$6"

: "${PIGEON_GITLAB_TOKEN:?}"
: "${PIGEON_PROJECTS:?}"
: "${PIGEON_STATE:?}"

RATE_LIMIT=3

PIGEON_DATABASE="$(realpath -m "$PIGEON_STATE/db")"
PIGEON_PROJECTS="$(realpath -m "$PIGEON_PROJECTS")"
RELEASE_TIME="$(date -d "$RELEASE_TIME" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || true)"

report() {
  curl -s --show-error --out-null --request PUT \
    --header "PRIVATE-TOKEN: $PIGEON_GITLAB_TOKEN" \
    --header 'Content-Type: application/json' \
    --data "$(jq -n --arg d "$1" '{"description": $d}')" \
    --url "https://gitlab.com/api/v4/projects/$PROJECT_ID/releases/$RELEASE_TAG"
}
# shellcheck disable=SC2120,SC2119
db() {
  sqlite3 -bail -cmd '.timeout 10000' "$PIGEON_DATABASE" "$@"
}
q() {
  printf "'%s'" "$(printf %s "$1" | sed "s/'/''/g")"
}

db << 'EOF'
CREATE TABLE IF NOT EXISTS results (
    id      INTEGER PRIMARY KEY,
    student TEXT NOT NULL,
    project TEXT NOT NULL,
    hash    TEXT NOT NULL,
    time    TEXT NOT NULL DEFAULT (datetime('now')),
    score   INTEGER CHECK (score >= 0),
    token   INTEGER NOT NULL
);
EOF

DIRECTORY="$(mktemp -d /tmp/pigeon.XXXXXXXXXX)"
trap 'rm -rf "$DIRECTORY"' EXIT
cd "$DIRECTORY"

git -c advice.detachedHead=false clone -q --rev "$RELEASE_COMMIT" --depth 1 \
  "https://oauth2:$PIGEON_GITLAB_TOKEN@gitlab.com/$PROJECT_NAMESPACE/$PROJECT_NAME.git" student/
rm -rf student/.git

TESTS="$PIGEON_PROJECTS/$PROJECT_NAME/tests"
mkdir -p project/tests project/build
cp -a "$TESTS"/*.sh project/tests/
cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/build/. project/build/

# A project may ship an answer-key solution. It is built and run only here,
# on the host, and never copied onto any drive the guest can reach -- tests
# compare the student's reported output against it after the VM has exited.
CANONICAL=""
if [ -d "$PIGEON_PROJECTS/$PROJECT_NAME/canonical" ] ; then
  mkdir -p canonical
  cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/build/. canonical/
  cp -a --remove-destination "$PIGEON_PROJECTS/$PROJECT_NAME"/canonical/. canonical/
  ( cd canonical && timeout 30 make >/dev/null 2>&1 ) || true
  [ -x canonical/program ] && CANONICAL="$(realpath canonical/program)"
fi

mkfs.ext4 -q -F -d student student.img 16M
mkfs.ext4 -q -F -d project project.img 16M
mkfs.ext4 -q -F results.img 16M

cat >vm.json <<'EOF'
{
  "boot-source": {
    "kernel_image_path": "@kernel@",
    "boot_args": "reboot=k panic=1 init=/init quiet nomodule i8042.noaux i8042.nomux i8042.dumbkbd swiotlb=noforce console=ttyS0"
  },
  "drives": [
    { "drive_id": "rootfs",  "path_on_host": "@rootfs@",    "is_root_device": true,  "is_read_only": true },
    { "drive_id": "student", "path_on_host": "student.img", "is_root_device": false, "is_read_only": true },
    { "drive_id": "project", "path_on_host": "project.img", "is_root_device": false, "is_read_only": true },
    { "drive_id": "results", "path_on_host": "results.img", "is_root_device": false, "is_read_only": false }
  ],
  "machine-config": { "vcpu_count": 1, "mem_size_mib": 512 }
}
EOF

firecracker --no-api --level Error --config-file vm.json

PUBLIC_SCORE=0; PUBLIC_TOTAL=0
RELEASE_SCORE=0; RELEASE_TOTAL=0
SECRET_SCORE=0; SECRET_TOTAL=0
NAMES=()
STATUSES=()
ANSWERS=()
QUESTIONS=()
TIERS=()

for TEST in "$TESTS"/*.sh ; do
  NAME="$(basename "$TEST" .sh)"
  TIER="${NAME%%-*}"
  QUESTION="$(cat "$TESTS/$NAME.question")"
  debugfs -R "cat /$NAME" results.img >scratch 2>/dev/null
  EXIT="$(debugfs -R "cat /$NAME.status" results.img 2>/dev/null)"
  case "$EXIT" in '' | *[!0-9]*) EXIT=0 ;; esac

  # A test may expect the student's program to exit with a specific status
  # (e.g. 134 for a SIGABRT from a correctly-triggered assert()) instead of
  # the default 0.
  EXPECTED_EXIT=0
  [ -f "$TESTS/$NAME.exit" ] && EXPECTED_EXIT="$(cat "$TESTS/$NAME.exit")"

  if [ -f "$TESTS/$NAME.expected" ] ; then
    if cmp -s scratch "$TESTS/$NAME.expected" && [ "$EXIT" -eq "$EXPECTED_EXIT" ] ; then
      MATCH=1
    else
      MATCH=0
    fi
  elif [ -n "$CANONICAL" ] ; then
    # No static .expected: the test reported its own "ARG<TAB>OUTPUT" pairs,
    # so check each one against the canonical solution run here on the host.
    MATCH=1
    while IFS="$(printf '\t')" read -r ARG OUT ; do
      [ -n "$ARG" ] || continue
      EXPECTED="$("$CANONICAL" "$ARG" 2>/dev/null)"
      [ "$OUT" = "$EXPECTED" ] || MATCH=0
    done <scratch
  else
    MATCH=0
  fi

  if [ "$MATCH" = 1 ] ; then
    STATUS=PASS
    ANSWER=YES
  else
    STATUS=FAIL
    # timeout -s 9 (used in init.sh) exits 137 (128+SIGKILL) on a timeout,
    # not the 124 it uses for its default signal (SIGTERM). Timing out is
    # never the expected outcome, so it's checked before EXPECTED_EXIT.
    if [ "$EXIT" -eq 124 ] || [ "$EXIT" -eq 137 ] ; then
      ANSWER="NO (timed out)"
    elif [ "$EXIT" -eq "$EXPECTED_EXIT" ] ; then
      ANSWER="NO (wrong output)"
    elif [ "$EXIT" -gt 128 ] ; then
      ANSWER="NO (crashed, signal $((EXIT - 128)))"
    else
      ANSWER="NO (exited with status $EXIT, expected $EXPECTED_EXIT)"
    fi
  fi
  NAMES+=("$NAME")
  STATUSES+=("$STATUS")
  ANSWERS+=("$ANSWER")
  QUESTIONS+=("$QUESTION")
  TIERS+=("$TIER")
  case "$TIER" in
    public)
      PUBLIC_TOTAL=$((PUBLIC_TOTAL + 1))
      [ "$STATUS" = PASS ] && PUBLIC_SCORE=$((PUBLIC_SCORE + 1))
      ;;
    release)
      RELEASE_TOTAL=$((RELEASE_TOTAL + 1))
      [ "$STATUS" = PASS ] && RELEASE_SCORE=$((RELEASE_SCORE + 1))
      ;;
    secret)
      SECRET_TOTAL=$((SECRET_TOTAL + 1))
      [ "$STATUS" = PASS ] && SECRET_SCORE=$((SECRET_SCORE + 1))
      ;;
  esac
done

PUBLIC_PASS=0
[ "$PUBLIC_SCORE" -eq "$PUBLIC_TOTAL" ] && PUBLIC_PASS=1
SCORE=$((PUBLIC_SCORE + RELEASE_SCORE + SECRET_SCORE))

TOKEN="$(db <<EOF
INSERT INTO results (student, project, hash, time, score, token)
SELECT $(q "$PROJECT_NAMESPACE"), $(q "$PROJECT_NAME"), $(q "$RELEASE_COMMIT"),
  COALESCE(NULLIF($(q "$RELEASE_TIME"), ''), datetime('now')),
  $SCORE,
  CASE WHEN $PUBLIC_PASS = 1 AND (
    SELECT COUNT(*) FROM results
    WHERE student = $(q "$PROJECT_NAMESPACE") AND project = $(q "$PROJECT_NAME")
      AND token = 1 AND time > datetime('now', '-1 day')
  ) < $RATE_LIMIT THEN 1 ELSE 0 END
RETURNING token;
EOF
)"

SHOWN_QUESTIONS=()
SHOWN_ANSWERS=()
for i in "${!NAMES[@]}" ; do
  case "${TIERS[$i]}" in
    public) ;;
    release) [ "$TOKEN" = 1 ] || continue ;;
    *) continue ;;
  esac
  SHOWN_QUESTIONS+=("${QUESTIONS[$i]}")
  SHOWN_ANSWERS+=("${ANSWERS[$i]}")
done

MESSAGE=""
for i in "${!SHOWN_QUESTIONS[@]}" ; do
  MESSAGE="${MESSAGE}${SHOWN_QUESTIONS[$i]} ${SHOWN_ANSWERS[$i]}  "$'\n'
done

if [ "$TOKEN" != 1 ] && [ "$PUBLIC_PASS" = 1 ] ; then
  MESSAGE="${MESSAGE}"$'\n'"All public tests passed, but you have used all $RATE_LIMIT submissions for today. Release test results are withheld until tomorrow."
fi

report "$MESSAGE"