diff --git a/init.sh b/init.sh index 12abceb..523e3fd 100755 --- a/init.sh +++ b/init.sh @@ -23,12 +23,12 @@ cd /tmp echo $? >/root/results/build.status ) -for TEST in /root/project/tests/*.sh ; do +for TEST in /root/project/tests/* ; do ( ulimit -t 2 -u 256 -n 256 -f 2048 timeout -s 9 2 \ setpriv --reuid=1 --regid=1 --clear-groups --no-new-privs --inh-caps=-all \ - "$TEST" >"/root/results/$(basename "$TEST" .sh)" 2>/dev/null - echo $? >"/root/results/$(basename "$TEST" .sh).status" + "$TEST" >"/root/results/$(basename "$TEST")" 2>/dev/null + echo $? >"/root/results/$(basename "$TEST").status" ) done diff --git a/pigeon.sh b/pigeon.sh index 850c20c..3dc2168 100755 --- a/pigeon.sh +++ b/pigeon.sh @@ -15,16 +15,8 @@ PIGEON_DATABASE="$(realpath -m "$PIGEON_STATE/db")" PIGEON_PROJECTS="$(realpath -m "$PIGEON_PROJECTS")" RELEASE_TIME="$(date -d "$RELEASE_TIME" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || true)" -report() { - curl -s --show-error --out-null --request PUT \ - --header "PRIVATE-TOKEN: $PIGEON_GITLAB_TOKEN" \ - --header 'Content-Type: application/json' \ - --data "$(jq -n --arg d "$1" '{"description": $d}')" \ - --url "https://gitlab.com/api/v4/projects/$PROJECT_ID/releases/$RELEASE_TAG" -} -# shellcheck disable=SC2120,SC2119 db() { - sqlite3 -bail -cmd '.timeout 10000' "$PIGEON_DATABASE" "$@" + sqlite3 -bail -cmd '.timeout 10000' "$PIGEON_DATABASE" } q() { printf "'%s'" "$(printf %s "$1" | sed "s/'/''/g")" @@ -52,12 +44,11 @@ rm -rf student/.git TESTS="$PIGEON_PROJECTS/$PROJECT_NAME/tests" mkdir -p project/tests project/build -cp -a "$TESTS"/*.sh project/tests/ +for DIR in "$TESTS"/*/ ; do + cp -a "$DIR/input.sh" "project/tests/$(basename "$DIR")" +done cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/build/. project/build/ -# A project may ship an answer-key solution. It is built and run only here, -# on the host, and never copied onto any drive the guest can reach -- tests -# compare the student's reported output against it after the VM has exited. CANONICAL="" if [ -d "$PIGEON_PROJECTS/$PROJECT_NAME/canonical" ] ; then mkdir -p canonical @@ -89,91 +80,51 @@ EOF firecracker --no-api --level Error --config-file vm.json -PUBLIC_SCORE=0; PUBLIC_TOTAL=0 -RELEASE_SCORE=0; RELEASE_TOTAL=0 -SECRET_SCORE=0; SECRET_TOTAL=0 -NAMES=() -STATUSES=() -ANSWERS=() -QUESTIONS=() -TIERS=() +SCORE=0 +PUBLIC_PASS=1 +: >public +: >release + +for DIR in "$TESTS"/*/ ; do + NAME="$(basename "$DIR")" -for TEST in "$TESTS"/*.sh ; do - NAME="$(basename "$TEST" .sh)" - TIER="${NAME%%-*}" - QUESTION="$(cat "$TESTS/$NAME.question")" debugfs -R "cat /$NAME" results.img >scratch 2>/dev/null EXIT="$(debugfs -R "cat /$NAME.status" results.img 2>/dev/null)" + EXPECTED_EXIT="$(cat "$DIR/status.txt" 2>/dev/null)" case "$EXIT" in '' | *[!0-9]*) EXIT=0 ;; esac + case "$EXPECTED_EXIT" in '' | *[!0-9]*) EXPECTED_EXIT=0 ;; esac - # A test may expect the student's program to exit with a specific status - # (e.g. 134 for a SIGABRT from a correctly-triggered assert()) instead of - # the default 0. - EXPECTED_EXIT=0 - [ -f "$TESTS/$NAME.exit" ] && EXPECTED_EXIT="$(cat "$TESTS/$NAME.exit")" - - if [ -f "$TESTS/$NAME.expected" ] ; then - if cmp -s scratch "$TESTS/$NAME.expected" && [ "$EXIT" -eq "$EXPECTED_EXIT" ] ; then - MATCH=1 - else - MATCH=0 - fi + if [ -f "$DIR/output.txt" ] ; then + cmp -s scratch "$DIR/output.txt" && [ "$EXIT" -eq "$EXPECTED_EXIT" ] && PASS=1 || PASS=0 elif [ -n "$CANONICAL" ] ; then - # No static .expected: the test reported its own "ARGOUTPUT" pairs, - # so check each one against the canonical solution run here on the host. - MATCH=1 + PASS=1 while IFS="$(printf '\t')" read -r ARG OUT ; do [ -n "$ARG" ] || continue - EXPECTED="$("$CANONICAL" "$ARG" 2>/dev/null)" - [ "$OUT" = "$EXPECTED" ] || MATCH=0 + [ "$OUT" = "$("$CANONICAL" "$ARG" 2>/dev/null)" ] || PASS=0 done >public ;; + r*) echo "$(cat "$DIR/test.txt") $ANSWER " >>release ;; esac done -PUBLIC_PASS=0 -[ "$PUBLIC_SCORE" -eq "$PUBLIC_TOTAL" ] && PUBLIC_PASS=1 -SCORE=$((PUBLIC_SCORE + RELEASE_SCORE + SECRET_SCORE)) - TOKEN="$(db <