vibe coded a little
This commit is contained in:
@@ -12,6 +12,7 @@ RELEASE_TIME="$6"
|
||||
RATE_LIMIT=3
|
||||
|
||||
PIGEON_DATABASE="$(realpath -m "$PIGEON_STATE/db")"
|
||||
PIGEON_PROJECTS="$(realpath -m "$PIGEON_PROJECTS")"
|
||||
RELEASE_TIME="$(date -d "$RELEASE_TIME" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || true)"
|
||||
|
||||
report() {
|
||||
@@ -21,6 +22,7 @@ report() {
|
||||
--data "$(jq -n --arg d "$1" '{"description": $d}')" \
|
||||
--url "https://gitlab.com/api/v4/projects/$PROJECT_ID/releases/$RELEASE_TAG"
|
||||
}
|
||||
# shellcheck disable=SC2120,SC2119
|
||||
db() {
|
||||
sqlite3 -bail -cmd '.timeout 10000' "$PIGEON_DATABASE" "$@"
|
||||
}
|
||||
@@ -35,26 +37,11 @@ CREATE TABLE IF NOT EXISTS results (
|
||||
project TEXT NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
time TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
score INTEGER CHECK (score >= 0)
|
||||
score INTEGER CHECK (score >= 0),
|
||||
token INTEGER NOT NULL
|
||||
);
|
||||
EOF
|
||||
|
||||
ROW="$(db <<EOF
|
||||
INSERT INTO results (student, project, hash, time)
|
||||
SELECT $(q "$PROJECT_NAMESPACE"), $(q "$PROJECT_NAME"), $(q "$RELEASE_COMMIT"),
|
||||
COALESCE(NULLIF($(q "$RELEASE_TIME"), ''), datetime('now'))
|
||||
WHERE (SELECT COUNT(*) FROM results
|
||||
WHERE student = $(q "$PROJECT_NAMESPACE") AND project = $(q "$PROJECT_NAME")
|
||||
AND time > datetime('now', '-1 day')) < $RATE_LIMIT
|
||||
RETURNING id;
|
||||
EOF
|
||||
)"
|
||||
|
||||
if [ -z "$ROW" ] ; then
|
||||
report "Rate limit exceeded!"
|
||||
exit
|
||||
fi
|
||||
|
||||
DIRECTORY="$(mktemp -d /tmp/pigeon.XXXXXXXXXX)"
|
||||
trap 'rm -rf "$DIRECTORY"' EXIT
|
||||
cd "$DIRECTORY"
|
||||
@@ -63,10 +50,23 @@ git -c advice.detachedHead=false clone -q --rev "$RELEASE_COMMIT" --depth 1 \
|
||||
"https://oauth2:$PIGEON_GITLAB_TOKEN@gitlab.com/$PROJECT_NAMESPACE/$PROJECT_NAME.git" student/
|
||||
rm -rf student/.git
|
||||
|
||||
TESTS="$PIGEON_PROJECTS/$PROJECT_NAME/tests"
|
||||
mkdir -p project/tests project/build
|
||||
cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/tests/*.sh project/tests/
|
||||
cp -a "$TESTS"/*.sh project/tests/
|
||||
cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/build/. project/build/
|
||||
|
||||
# A project may ship an answer-key solution. It is built and run only here,
|
||||
# on the host, and never copied onto any drive the guest can reach -- tests
|
||||
# compare the student's reported output against it after the VM has exited.
|
||||
CANONICAL=""
|
||||
if [ -d "$PIGEON_PROJECTS/$PROJECT_NAME/canonical" ] ; then
|
||||
mkdir -p canonical
|
||||
cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/build/. canonical/
|
||||
cp -a --remove-destination "$PIGEON_PROJECTS/$PROJECT_NAME"/canonical/. canonical/
|
||||
( cd canonical && timeout 30 make >/dev/null 2>&1 ) || true
|
||||
[ -x canonical/program ] && CANONICAL="$(realpath canonical/program)"
|
||||
fi
|
||||
|
||||
mkfs.ext4 -q -F -d student student.img 16M
|
||||
mkfs.ext4 -q -F -d project project.img 16M
|
||||
mkfs.ext4 -q -F results.img 16M
|
||||
@@ -89,21 +89,124 @@ EOF
|
||||
|
||||
firecracker --no-api --level Error --config-file vm.json
|
||||
|
||||
RESULT=""
|
||||
SCORE=0
|
||||
TESTS="$PIGEON_PROJECTS/$PROJECT_NAME/tests"
|
||||
PUBLIC_SCORE=0; PUBLIC_TOTAL=0
|
||||
RELEASE_SCORE=0; RELEASE_TOTAL=0
|
||||
SECRET_SCORE=0; SECRET_TOTAL=0
|
||||
NAMES=()
|
||||
STATUSES=()
|
||||
ANSWERS=()
|
||||
QUESTIONS=()
|
||||
TIERS=()
|
||||
|
||||
for TEST in "$TESTS"/*.sh ; do
|
||||
debugfs -R "cat /$(basename "$TEST" .sh)" results.img >scratch 2>/dev/null
|
||||
if cmp -s scratch "$TESTS/$(basename "$TEST" .sh).expected" ; then
|
||||
RESULT="${RESULT}PASS\n"
|
||||
SCORE=$((SCORE + 1))
|
||||
NAME="$(basename "$TEST" .sh)"
|
||||
TIER="${NAME%%-*}"
|
||||
QUESTION="$(cat "$TESTS/$NAME.question")"
|
||||
debugfs -R "cat /$NAME" results.img >scratch 2>/dev/null
|
||||
EXIT="$(debugfs -R "cat /$NAME.status" results.img 2>/dev/null)"
|
||||
case "$EXIT" in '' | *[!0-9]*) EXIT=0 ;; esac
|
||||
|
||||
# A test may expect the student's program to exit with a specific status
|
||||
# (e.g. 134 for a SIGABRT from a correctly-triggered assert()) instead of
|
||||
# the default 0.
|
||||
EXPECTED_EXIT=0
|
||||
[ -f "$TESTS/$NAME.exit" ] && EXPECTED_EXIT="$(cat "$TESTS/$NAME.exit")"
|
||||
|
||||
if [ -f "$TESTS/$NAME.expected" ] ; then
|
||||
if cmp -s scratch "$TESTS/$NAME.expected" && [ "$EXIT" -eq "$EXPECTED_EXIT" ] ; then
|
||||
MATCH=1
|
||||
else
|
||||
MATCH=0
|
||||
fi
|
||||
elif [ -n "$CANONICAL" ] ; then
|
||||
# No static .expected: the test reported its own "ARG<TAB>OUTPUT" pairs,
|
||||
# so check each one against the canonical solution run here on the host.
|
||||
MATCH=1
|
||||
while IFS="$(printf '\t')" read -r ARG OUT ; do
|
||||
[ -n "$ARG" ] || continue
|
||||
EXPECTED="$("$CANONICAL" "$ARG" 2>/dev/null)"
|
||||
[ "$OUT" = "$EXPECTED" ] || MATCH=0
|
||||
done <scratch
|
||||
else
|
||||
RESULT="${RESULT}FAIL: $(cat "$TESTS/$(basename "$TEST" .sh).hint")\n"
|
||||
MATCH=0
|
||||
fi
|
||||
|
||||
if [ "$MATCH" = 1 ] ; then
|
||||
STATUS=PASS
|
||||
ANSWER=YES
|
||||
else
|
||||
STATUS=FAIL
|
||||
# timeout -s 9 (used in init.sh) exits 137 (128+SIGKILL) on a timeout,
|
||||
# not the 124 it uses for its default signal (SIGTERM). Timing out is
|
||||
# never the expected outcome, so it's checked before EXPECTED_EXIT.
|
||||
if [ "$EXIT" -eq 124 ] || [ "$EXIT" -eq 137 ] ; then
|
||||
ANSWER="NO (timed out)"
|
||||
elif [ "$EXIT" -eq "$EXPECTED_EXIT" ] ; then
|
||||
ANSWER="NO (wrong output)"
|
||||
elif [ "$EXIT" -gt 128 ] ; then
|
||||
ANSWER="NO (crashed, signal $((EXIT - 128)))"
|
||||
else
|
||||
ANSWER="NO (exited with status $EXIT, expected $EXPECTED_EXIT)"
|
||||
fi
|
||||
fi
|
||||
NAMES+=("$NAME")
|
||||
STATUSES+=("$STATUS")
|
||||
ANSWERS+=("$ANSWER")
|
||||
QUESTIONS+=("$QUESTION")
|
||||
TIERS+=("$TIER")
|
||||
case "$TIER" in
|
||||
public)
|
||||
PUBLIC_TOTAL=$((PUBLIC_TOTAL + 1))
|
||||
[ "$STATUS" = PASS ] && PUBLIC_SCORE=$((PUBLIC_SCORE + 1))
|
||||
;;
|
||||
release)
|
||||
RELEASE_TOTAL=$((RELEASE_TOTAL + 1))
|
||||
[ "$STATUS" = PASS ] && RELEASE_SCORE=$((RELEASE_SCORE + 1))
|
||||
;;
|
||||
secret)
|
||||
SECRET_TOTAL=$((SECRET_TOTAL + 1))
|
||||
[ "$STATUS" = PASS ] && SECRET_SCORE=$((SECRET_SCORE + 1))
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Record before reporting, so a GitLab outage doesn't lose the result.
|
||||
db "UPDATE results SET score = $SCORE WHERE id = $ROW;" ||
|
||||
echo "pigeon: failed to record score for row $ROW in $PIGEON_DB" >&2
|
||||
PUBLIC_PASS=0
|
||||
[ "$PUBLIC_SCORE" -eq "$PUBLIC_TOTAL" ] && PUBLIC_PASS=1
|
||||
SCORE=$((PUBLIC_SCORE + RELEASE_SCORE + SECRET_SCORE))
|
||||
|
||||
report "$RESULT"
|
||||
TOKEN="$(db <<EOF
|
||||
INSERT INTO results (student, project, hash, time, score, token)
|
||||
SELECT $(q "$PROJECT_NAMESPACE"), $(q "$PROJECT_NAME"), $(q "$RELEASE_COMMIT"),
|
||||
COALESCE(NULLIF($(q "$RELEASE_TIME"), ''), datetime('now')),
|
||||
$SCORE,
|
||||
CASE WHEN $PUBLIC_PASS = 1 AND (
|
||||
SELECT COUNT(*) FROM results
|
||||
WHERE student = $(q "$PROJECT_NAMESPACE") AND project = $(q "$PROJECT_NAME")
|
||||
AND token = 1 AND time > datetime('now', '-1 day')
|
||||
) < $RATE_LIMIT THEN 1 ELSE 0 END
|
||||
RETURNING token;
|
||||
EOF
|
||||
)"
|
||||
|
||||
SHOWN_QUESTIONS=()
|
||||
SHOWN_ANSWERS=()
|
||||
for i in "${!NAMES[@]}" ; do
|
||||
case "${TIERS[$i]}" in
|
||||
public) ;;
|
||||
release) [ "$TOKEN" = 1 ] || continue ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
SHOWN_QUESTIONS+=("${QUESTIONS[$i]}")
|
||||
SHOWN_ANSWERS+=("${ANSWERS[$i]}")
|
||||
done
|
||||
|
||||
MESSAGE=""
|
||||
for i in "${!SHOWN_QUESTIONS[@]}" ; do
|
||||
MESSAGE="${MESSAGE}${SHOWN_QUESTIONS[$i]} ${SHOWN_ANSWERS[$i]} "$'\n'
|
||||
done
|
||||
|
||||
if [ "$TOKEN" != 1 ] && [ "$PUBLIC_PASS" = 1 ] ; then
|
||||
MESSAGE="${MESSAGE}"$'\n'"All public tests passed, but you have used all $RATE_LIMIT submissions for today. Release test results are withheld until tomorrow."
|
||||
fi
|
||||
|
||||
report "$MESSAGE"
|
||||
|
||||
Reference in New Issue
Block a user