PROJECT_ID="$1" PROJECT_NAMESPACE="$2" PROJECT_NAME="$3" RELEASE_COMMIT="$4" RELEASE_TAG="$5" RELEASE_TIME="$6" : "${PIGEON_GITLAB_TOKEN:?}" : "${PIGEON_PROJECTS:?}" : "${PIGEON_STATE:?}" RATE_LIMIT=3 PIGEON_DATABASE="$(realpath -m "$PIGEON_STATE/db")" PIGEON_PROJECTS="$(realpath -m "$PIGEON_PROJECTS")" RELEASE_TIME="$(date -d "$RELEASE_TIME" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || true)" report() { curl -s --show-error --out-null --request PUT \ --header "PRIVATE-TOKEN: $PIGEON_GITLAB_TOKEN" \ --header 'Content-Type: application/json' \ --data "$(jq -n --arg d "$1" '{"description": $d}')" \ --url "https://gitlab.com/api/v4/projects/$PROJECT_ID/releases/$RELEASE_TAG" } # shellcheck disable=SC2120,SC2119 db() { sqlite3 -bail -cmd '.timeout 10000' "$PIGEON_DATABASE" "$@" } q() { printf "'%s'" "$(printf %s "$1" | sed "s/'/''/g")" } db << 'EOF' CREATE TABLE IF NOT EXISTS results ( id INTEGER PRIMARY KEY, student TEXT NOT NULL, project TEXT NOT NULL, hash TEXT NOT NULL, time TEXT NOT NULL DEFAULT (datetime('now')), score INTEGER CHECK (score >= 0), token INTEGER NOT NULL ); EOF DIRECTORY="$(mktemp -d /tmp/pigeon.XXXXXXXXXX)" trap 'rm -rf "$DIRECTORY"' EXIT cd "$DIRECTORY" git -c advice.detachedHead=false clone -q --rev "$RELEASE_COMMIT" --depth 1 \ "https://oauth2:$PIGEON_GITLAB_TOKEN@gitlab.com/$PROJECT_NAMESPACE/$PROJECT_NAME.git" student/ rm -rf student/.git TESTS="$PIGEON_PROJECTS/$PROJECT_NAME/tests" mkdir -p project/tests project/build cp -a "$TESTS"/*.sh project/tests/ cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/build/. project/build/ # A project may ship an answer-key solution. It is built and run only here, # on the host, and never copied onto any drive the guest can reach -- tests # compare the student's reported output against it after the VM has exited. CANONICAL="" if [ -d "$PIGEON_PROJECTS/$PROJECT_NAME/canonical" ] ; then mkdir -p canonical cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/build/. canonical/ cp -a --remove-destination "$PIGEON_PROJECTS/$PROJECT_NAME"/canonical/. canonical/ ( cd canonical && timeout 30 make >/dev/null 2>&1 ) || true [ -x canonical/program ] && CANONICAL="$(realpath canonical/program)" fi mkfs.ext4 -q -F -d student student.img 16M mkfs.ext4 -q -F -d project project.img 16M mkfs.ext4 -q -F results.img 16M cat >vm.json <<'EOF' { "boot-source": { "kernel_image_path": "@kernel@", "boot_args": "reboot=k panic=1 init=/init quiet nomodule i8042.noaux i8042.nomux i8042.dumbkbd swiotlb=noforce console=ttyS0" }, "drives": [ { "drive_id": "rootfs", "path_on_host": "@rootfs@", "is_root_device": true, "is_read_only": true }, { "drive_id": "student", "path_on_host": "student.img", "is_root_device": false, "is_read_only": true }, { "drive_id": "project", "path_on_host": "project.img", "is_root_device": false, "is_read_only": true }, { "drive_id": "results", "path_on_host": "results.img", "is_root_device": false, "is_read_only": false } ], "machine-config": { "vcpu_count": 1, "mem_size_mib": 512 } } EOF firecracker --no-api --level Error --config-file vm.json PUBLIC_SCORE=0; PUBLIC_TOTAL=0 RELEASE_SCORE=0; RELEASE_TOTAL=0 SECRET_SCORE=0; SECRET_TOTAL=0 NAMES=() STATUSES=() ANSWERS=() QUESTIONS=() TIERS=() for TEST in "$TESTS"/*.sh ; do NAME="$(basename "$TEST" .sh)" TIER="${NAME%%-*}" QUESTION="$(cat "$TESTS/$NAME.question")" debugfs -R "cat /$NAME" results.img >scratch 2>/dev/null EXIT="$(debugfs -R "cat /$NAME.status" results.img 2>/dev/null)" case "$EXIT" in '' | *[!0-9]*) EXIT=0 ;; esac # A test may expect the student's program to exit with a specific status # (e.g. 134 for a SIGABRT from a correctly-triggered assert()) instead of # the default 0. EXPECTED_EXIT=0 [ -f "$TESTS/$NAME.exit" ] && EXPECTED_EXIT="$(cat "$TESTS/$NAME.exit")" if [ -f "$TESTS/$NAME.expected" ] ; then if cmp -s scratch "$TESTS/$NAME.expected" && [ "$EXIT" -eq "$EXPECTED_EXIT" ] ; then MATCH=1 else MATCH=0 fi elif [ -n "$CANONICAL" ] ; then # No static .expected: the test reported its own "ARGOUTPUT" pairs, # so check each one against the canonical solution run here on the host. MATCH=1 while IFS="$(printf '\t')" read -r ARG OUT ; do [ -n "$ARG" ] || continue EXPECTED="$("$CANONICAL" "$ARG" 2>/dev/null)" [ "$OUT" = "$EXPECTED" ] || MATCH=0 done datetime('now', '-1 day') ) < $RATE_LIMIT THEN 1 ELSE 0 END RETURNING token; EOF )" SHOWN_QUESTIONS=() SHOWN_ANSWERS=() for i in "${!NAMES[@]}" ; do case "${TIERS[$i]}" in public) ;; release) [ "$TOKEN" = 1 ] || continue ;; *) continue ;; esac SHOWN_QUESTIONS+=("${QUESTIONS[$i]}") SHOWN_ANSWERS+=("${ANSWERS[$i]}") done MESSAGE="" for i in "${!SHOWN_QUESTIONS[@]}" ; do MESSAGE="${MESSAGE}${SHOWN_QUESTIONS[$i]} ${SHOWN_ANSWERS[$i]} "$'\n' done if [ "$TOKEN" != 1 ] && [ "$PUBLIC_PASS" = 1 ] ; then MESSAGE="${MESSAGE}"$'\n'"All public tests passed, but you have used all $RATE_LIMIT submissions for today. Release test results are withheld until tomorrow." fi report "$MESSAGE"