Pigeon/pigeon.sh

PROJECT_ID="$1"
PROJECT_NAMESPACE="$2"
PROJECT_NAME="$3"
RELEASE_COMMIT="$4"
RELEASE_TAG="$5"
RELEASE_TIME="$6"

: "${PIGEON_GITLAB_TOKEN:?}"
: "${PIGEON_PROJECTS:?}"
: "${PIGEON_STATE:?}"

RATE_LIMIT=3

PIGEON_DATABASE="$(realpath -m "$PIGEON_STATE/db")"
PIGEON_PROJECTS="$(realpath -m "$PIGEON_PROJECTS")"
RELEASE_TIME="$(date -d "$RELEASE_TIME" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || true)"

db() {
  sqlite3 -bail -cmd '.timeout 10000' "$PIGEON_DATABASE"
}
q() {
  printf "'%s'" "$(printf %s "$1" | sed "s/'/''/g")"
}

db << 'EOF'
CREATE TABLE IF NOT EXISTS results (
    id      INTEGER PRIMARY KEY,
    student TEXT NOT NULL,
    project TEXT NOT NULL,
    hash    TEXT NOT NULL,
    time    TEXT NOT NULL DEFAULT (datetime('now')),
    score   INTEGER CHECK (score >= 0),
    token   INTEGER NOT NULL
);
EOF

DIRECTORY="$(mktemp -d /tmp/pigeon.XXXXXXXXXX)"
trap 'rm -rf "$DIRECTORY"' EXIT
cd "$DIRECTORY"

git -c advice.detachedHead=false clone -q --rev "$RELEASE_COMMIT" --depth 1 \
  "https://oauth2:$PIGEON_GITLAB_TOKEN@gitlab.com/$PROJECT_NAMESPACE/$PROJECT_NAME.git" student/
rm -rf student/.git

TESTS="$PIGEON_PROJECTS/$PROJECT_NAME/tests"
mkdir -p project/tests project/build
for DIR in "$TESTS"/*/ ; do
  cp -a "$DIR/input.sh" "project/tests/$(basename "$DIR")"
done
cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/build/. project/build/

CANONICAL=""
if [ -d "$PIGEON_PROJECTS/$PROJECT_NAME/canonical" ] ; then
  mkdir -p canonical
  cp -a "$PIGEON_PROJECTS/$PROJECT_NAME"/build/. canonical/
  cp -a --remove-destination "$PIGEON_PROJECTS/$PROJECT_NAME"/canonical/. canonical/
  ( cd canonical && timeout 30 make >/dev/null 2>&1 ) || true
  [ -x canonical/program ] && CANONICAL="$(realpath canonical/program)"
fi

mkfs.ext4 -q -F -d student student.img 16M
mkfs.ext4 -q -F -d project project.img 16M
mkfs.ext4 -q -F results.img 16M

cat >vm.json <<'EOF'
{
  "boot-source": {
    "kernel_image_path": "@kernel@",
    "boot_args": "reboot=k panic=1 init=/init quiet nomodule i8042.noaux i8042.nomux i8042.dumbkbd swiotlb=noforce console=ttyS0"
  },
  "drives": [
    { "drive_id": "rootfs",  "path_on_host": "@rootfs@",    "is_root_device": true,  "is_read_only": true },
    { "drive_id": "student", "path_on_host": "student.img", "is_root_device": false, "is_read_only": true },
    { "drive_id": "project", "path_on_host": "project.img", "is_root_device": false, "is_read_only": true },
    { "drive_id": "results", "path_on_host": "results.img", "is_root_device": false, "is_read_only": false }
  ],
  "machine-config": { "vcpu_count": 1, "mem_size_mib": 512 }
}
EOF

firecracker --no-api --level Error --config-file vm.json

SCORE=0
PUBLIC_PASS=1
: >public
: >release

for DIR in "$TESTS"/*/ ; do
  NAME="$(basename "$DIR")"

  debugfs -R "cat /$NAME" results.img >scratch 2>/dev/null
  EXIT="$(debugfs -R "cat /$NAME.status" results.img 2>/dev/null)"
  EXPECTED_EXIT="$(cat "$DIR/status.txt" 2>/dev/null)"
  case "$EXIT" in '' | *[!0-9]*) EXIT=0 ;; esac
  case "$EXPECTED_EXIT" in '' | *[!0-9]*) EXPECTED_EXIT=0 ;; esac

  if [ -f "$DIR/output.txt" ] ; then
    cmp -s scratch "$DIR/output.txt" && [ "$EXIT" -eq "$EXPECTED_EXIT" ] && PASS=1 || PASS=0
  elif [ -n "$CANONICAL" ] ; then
    PASS=1
    while IFS="$(printf '\t')" read -r ARG OUT ; do
      [ -n "$ARG" ] || continue
      [ "$OUT" = "$("$CANONICAL" "$ARG" 2>/dev/null)" ] || PASS=0
    done <scratch
  else
    PASS=0
  fi

  if [ "$PASS" = 1 ] ; then
    ANSWER=YES
    SCORE=$((SCORE + 1))
  elif [ "$EXIT" -eq 124 ] || [ "$EXIT" -eq 137 ] ; then
    ANSWER="NO (timed out)"
  elif [ "$EXIT" -eq "$EXPECTED_EXIT" ] ; then
    ANSWER="NO (wrong output)"
  elif [ "$EXIT" -gt 128 ] ; then
    ANSWER="NO (crashed, signal $((EXIT - 128)))"
  else
    ANSWER="NO (exited with status $EXIT, expected $EXPECTED_EXIT)"
  fi

  case "$NAME" in
    p*) [ "$PASS" = 1 ] || PUBLIC_PASS=0; echo "$(cat "$DIR/test.txt") $ANSWER  " >>public ;;
    r*) echo "$(cat "$DIR/test.txt") $ANSWER  " >>release ;;
  esac
done

TOKEN="$(db <<EOF
INSERT INTO results (student, project, hash, time, score, token)
SELECT $(q "$PROJECT_NAMESPACE"), $(q "$PROJECT_NAME"), $(q "$RELEASE_COMMIT"),
  COALESCE(NULLIF($(q "$RELEASE_TIME"), ''), datetime('now')),
  $SCORE,
  CASE WHEN $PUBLIC_PASS = 1 AND (
    SELECT COUNT(*) FROM results
    WHERE student = $(q "$PROJECT_NAMESPACE") AND project = $(q "$PROJECT_NAME")
      AND token = 1 AND time > datetime('now', '-1 day')
  ) < $RATE_LIMIT THEN 1 ELSE 0 END
RETURNING token;
EOF
)"

if [ "$TOKEN" = 1 ] ; then
  MESSAGE="$(cat public release)"
else
  MESSAGE="$(cat public)"
  [ "$PUBLIC_PASS" = 1 ] && MESSAGE="${MESSAGE}"$'\n'"All public tests passed, but you have used all $RATE_LIMIT submissions for today. Release test results are withheld until tomorrow."
fi

curl -s --show-error --out-null --request PUT \
  --header "PRIVATE-TOKEN: $PIGEON_GITLAB_TOKEN" \
  --header 'Content-Type: application/json' \
  --data "$(jq -n --arg d "$MESSAGE" '{"description": $d}')" \
  --url "https://gitlab.com/api/v4/projects/$PROJECT_ID/releases/$RELEASE_TAG"