vibe coded more
This commit is contained in:
@@ -15,16 +15,8 @@ PIGEON_DATABASE="$(realpath -m "$PIGEON_STATE/db")"
|
||||
PIGEON_PROJECTS="$(realpath -m "$PIGEON_PROJECTS")"
|
||||
RELEASE_TIME="$(date -d "$RELEASE_TIME" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || true)"
|
||||
|
||||
report() {
|
||||
curl -s --show-error --out-null --request PUT \
|
||||
--header "PRIVATE-TOKEN: $PIGEON_GITLAB_TOKEN" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "$(jq -n --arg d "$1" '{"description": $d}')" \
|
||||
--url "https://gitlab.com/api/v4/projects/$PROJECT_ID/releases/$RELEASE_TAG"
|
||||
}
|
||||
# shellcheck disable=SC2120,SC2119
|
||||
db() {
|
||||
sqlite3 -bail -cmd '.timeout 10000' "$PIGEON_DATABASE" "$@"
|
||||
sqlite3 -bail -cmd '.timeout 10000' "$PIGEON_DATABASE"
|
||||
}
|
||||
q() {
|
||||
printf "'%s'" "$(printf %s "$1" | sed "s/'/''/g")"
|
||||
@@ -52,12 +44,11 @@ rm -rf student/.git
|
||||
|
||||
TESTS="$PIGEON_PROJECTS/$PROJECT_NAME/tests"
|
||||
mkdir -p project/tests project/build
|
||||
cp -a "$TESTS"/*.sh project/tests/
|
||||
for DIR in "$TESTS"/*/ ; do
|
||||
cp -a "$DIR/input.sh" "project/tests/$(basename "$DIR")"
|
||||
done
|
||||
cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/build/. project/build/
|
||||
|
||||
# A project may ship an answer-key solution. It is built and run only here,
|
||||
# on the host, and never copied onto any drive the guest can reach -- tests
|
||||
# compare the student's reported output against it after the VM has exited.
|
||||
CANONICAL=""
|
||||
if [ -d "$PIGEON_PROJECTS/$PROJECT_NAME/canonical" ] ; then
|
||||
mkdir -p canonical
|
||||
@@ -89,91 +80,51 @@ EOF
|
||||
|
||||
firecracker --no-api --level Error --config-file vm.json
|
||||
|
||||
PUBLIC_SCORE=0; PUBLIC_TOTAL=0
|
||||
RELEASE_SCORE=0; RELEASE_TOTAL=0
|
||||
SECRET_SCORE=0; SECRET_TOTAL=0
|
||||
NAMES=()
|
||||
STATUSES=()
|
||||
ANSWERS=()
|
||||
QUESTIONS=()
|
||||
TIERS=()
|
||||
SCORE=0
|
||||
PUBLIC_PASS=1
|
||||
: >public
|
||||
: >release
|
||||
|
||||
for DIR in "$TESTS"/*/ ; do
|
||||
NAME="$(basename "$DIR")"
|
||||
|
||||
for TEST in "$TESTS"/*.sh ; do
|
||||
NAME="$(basename "$TEST" .sh)"
|
||||
TIER="${NAME%%-*}"
|
||||
QUESTION="$(cat "$TESTS/$NAME.question")"
|
||||
debugfs -R "cat /$NAME" results.img >scratch 2>/dev/null
|
||||
EXIT="$(debugfs -R "cat /$NAME.status" results.img 2>/dev/null)"
|
||||
EXPECTED_EXIT="$(cat "$DIR/status.txt" 2>/dev/null)"
|
||||
case "$EXIT" in '' | *[!0-9]*) EXIT=0 ;; esac
|
||||
case "$EXPECTED_EXIT" in '' | *[!0-9]*) EXPECTED_EXIT=0 ;; esac
|
||||
|
||||
# A test may expect the student's program to exit with a specific status
|
||||
# (e.g. 134 for a SIGABRT from a correctly-triggered assert()) instead of
|
||||
# the default 0.
|
||||
EXPECTED_EXIT=0
|
||||
[ -f "$TESTS/$NAME.exit" ] && EXPECTED_EXIT="$(cat "$TESTS/$NAME.exit")"
|
||||
|
||||
if [ -f "$TESTS/$NAME.expected" ] ; then
|
||||
if cmp -s scratch "$TESTS/$NAME.expected" && [ "$EXIT" -eq "$EXPECTED_EXIT" ] ; then
|
||||
MATCH=1
|
||||
else
|
||||
MATCH=0
|
||||
fi
|
||||
if [ -f "$DIR/output.txt" ] ; then
|
||||
cmp -s scratch "$DIR/output.txt" && [ "$EXIT" -eq "$EXPECTED_EXIT" ] && PASS=1 || PASS=0
|
||||
elif [ -n "$CANONICAL" ] ; then
|
||||
# No static .expected: the test reported its own "ARG<TAB>OUTPUT" pairs,
|
||||
# so check each one against the canonical solution run here on the host.
|
||||
MATCH=1
|
||||
PASS=1
|
||||
while IFS="$(printf '\t')" read -r ARG OUT ; do
|
||||
[ -n "$ARG" ] || continue
|
||||
EXPECTED="$("$CANONICAL" "$ARG" 2>/dev/null)"
|
||||
[ "$OUT" = "$EXPECTED" ] || MATCH=0
|
||||
[ "$OUT" = "$("$CANONICAL" "$ARG" 2>/dev/null)" ] || PASS=0
|
||||
done <scratch
|
||||
else
|
||||
MATCH=0
|
||||
PASS=0
|
||||
fi
|
||||
|
||||
if [ "$MATCH" = 1 ] ; then
|
||||
STATUS=PASS
|
||||
if [ "$PASS" = 1 ] ; then
|
||||
ANSWER=YES
|
||||
SCORE=$((SCORE + 1))
|
||||
elif [ "$EXIT" -eq 124 ] || [ "$EXIT" -eq 137 ] ; then
|
||||
ANSWER="NO (timed out)"
|
||||
elif [ "$EXIT" -eq "$EXPECTED_EXIT" ] ; then
|
||||
ANSWER="NO (wrong output)"
|
||||
elif [ "$EXIT" -gt 128 ] ; then
|
||||
ANSWER="NO (crashed, signal $((EXIT - 128)))"
|
||||
else
|
||||
STATUS=FAIL
|
||||
# timeout -s 9 (used in init.sh) exits 137 (128+SIGKILL) on a timeout,
|
||||
# not the 124 it uses for its default signal (SIGTERM). Timing out is
|
||||
# never the expected outcome, so it's checked before EXPECTED_EXIT.
|
||||
if [ "$EXIT" -eq 124 ] || [ "$EXIT" -eq 137 ] ; then
|
||||
ANSWER="NO (timed out)"
|
||||
elif [ "$EXIT" -eq "$EXPECTED_EXIT" ] ; then
|
||||
ANSWER="NO (wrong output)"
|
||||
elif [ "$EXIT" -gt 128 ] ; then
|
||||
ANSWER="NO (crashed, signal $((EXIT - 128)))"
|
||||
else
|
||||
ANSWER="NO (exited with status $EXIT, expected $EXPECTED_EXIT)"
|
||||
fi
|
||||
ANSWER="NO (exited with status $EXIT, expected $EXPECTED_EXIT)"
|
||||
fi
|
||||
NAMES+=("$NAME")
|
||||
STATUSES+=("$STATUS")
|
||||
ANSWERS+=("$ANSWER")
|
||||
QUESTIONS+=("$QUESTION")
|
||||
TIERS+=("$TIER")
|
||||
case "$TIER" in
|
||||
public)
|
||||
PUBLIC_TOTAL=$((PUBLIC_TOTAL + 1))
|
||||
[ "$STATUS" = PASS ] && PUBLIC_SCORE=$((PUBLIC_SCORE + 1))
|
||||
;;
|
||||
release)
|
||||
RELEASE_TOTAL=$((RELEASE_TOTAL + 1))
|
||||
[ "$STATUS" = PASS ] && RELEASE_SCORE=$((RELEASE_SCORE + 1))
|
||||
;;
|
||||
secret)
|
||||
SECRET_TOTAL=$((SECRET_TOTAL + 1))
|
||||
[ "$STATUS" = PASS ] && SECRET_SCORE=$((SECRET_SCORE + 1))
|
||||
;;
|
||||
|
||||
case "$NAME" in
|
||||
p*) [ "$PASS" = 1 ] || PUBLIC_PASS=0; echo "$(cat "$DIR/test.txt") $ANSWER " >>public ;;
|
||||
r*) echo "$(cat "$DIR/test.txt") $ANSWER " >>release ;;
|
||||
esac
|
||||
done
|
||||
|
||||
PUBLIC_PASS=0
|
||||
[ "$PUBLIC_SCORE" -eq "$PUBLIC_TOTAL" ] && PUBLIC_PASS=1
|
||||
SCORE=$((PUBLIC_SCORE + RELEASE_SCORE + SECRET_SCORE))
|
||||
|
||||
TOKEN="$(db <<EOF
|
||||
INSERT INTO results (student, project, hash, time, score, token)
|
||||
SELECT $(q "$PROJECT_NAMESPACE"), $(q "$PROJECT_NAME"), $(q "$RELEASE_COMMIT"),
|
||||
@@ -188,25 +139,15 @@ RETURNING token;
|
||||
EOF
|
||||
)"
|
||||
|
||||
SHOWN_QUESTIONS=()
|
||||
SHOWN_ANSWERS=()
|
||||
for i in "${!NAMES[@]}" ; do
|
||||
case "${TIERS[$i]}" in
|
||||
public) ;;
|
||||
release) [ "$TOKEN" = 1 ] || continue ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
SHOWN_QUESTIONS+=("${QUESTIONS[$i]}")
|
||||
SHOWN_ANSWERS+=("${ANSWERS[$i]}")
|
||||
done
|
||||
|
||||
MESSAGE=""
|
||||
for i in "${!SHOWN_QUESTIONS[@]}" ; do
|
||||
MESSAGE="${MESSAGE}${SHOWN_QUESTIONS[$i]} ${SHOWN_ANSWERS[$i]} "$'\n'
|
||||
done
|
||||
|
||||
if [ "$TOKEN" != 1 ] && [ "$PUBLIC_PASS" = 1 ] ; then
|
||||
MESSAGE="${MESSAGE}"$'\n'"All public tests passed, but you have used all $RATE_LIMIT submissions for today. Release test results are withheld until tomorrow."
|
||||
if [ "$TOKEN" = 1 ] ; then
|
||||
MESSAGE="$(cat public release)"
|
||||
else
|
||||
MESSAGE="$(cat public)"
|
||||
[ "$PUBLIC_PASS" = 1 ] && MESSAGE="${MESSAGE}"$'\n'"All public tests passed, but you have used all $RATE_LIMIT submissions for today. Release test results are withheld until tomorrow."
|
||||
fi
|
||||
|
||||
report "$MESSAGE"
|
||||
curl -s --show-error --out-null --request PUT \
|
||||
--header "PRIVATE-TOKEN: $PIGEON_GITLAB_TOKEN" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "$(jq -n --arg d "$MESSAGE" '{"description": $d}')" \
|
||||
--url "https://gitlab.com/api/v4/projects/$PROJECT_ID/releases/$RELEASE_TAG"
|
||||
|
||||
Reference in New Issue
Block a user