fix: dind image readiness, exit codes and shutdown order (#1265)

Fixes https://gitea.com/gitea/runner/issues/1263

- Wait for dockerd without a check limit. s6 readiness gave up after 7 checks, so the runner never started on slow hosts, also seen in https://gitea.com/gitea/runner/issues/712, https://gitea.com/gitea/runner/issues/652 and https://gitea.com/gitea/runner/issues/758.
- Let the runner run the docker service's readiness check itself instead of reading s6 status, which can be missing at startup or stale after an unclean stop.
- Exit the container with the first failing service's exit code instead of always 0, so `restart: on-failure` works. A requested stop still exits 0.
- Stop the runner before dockerd, so running jobs can finish within `shutdown_timeout`. The stop grace period has to cover that drain.
- Check the supervision exit codes in `make test-dind`.

---------

Co-authored-by: bircni <[email protected]>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1265
Reviewed-by: bircni <[email protected]>

scripts/s6/.s6-svscan/SIGINT

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+exec /etc/s6/.s6-svscan/SIGTERM

scripts/s6/.s6-svscan/SIGTERM

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+(set -C; echo 0 > "$(/etc/s6/.s6-svscan/exitcode-path)") 2> /dev/null
+s6-svc -wD -d /etc/s6/gitea-runner
+exec s6-svscanctl -t /etc/s6

scripts/s6/.s6-svscan/exitcode-path

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo "/etc/s6/.s6-svscan/exitcode.$(cat /proc/sys/kernel/random/boot_id).$(cut -d ' ' -f 22 /proc/1/stat)" # unique per container start, so a killed run's record is ignored

scripts/s6/.s6-svscan/finish

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+exitcode=$(cat "$(/etc/s6/.s6-svscan/exitcode-path)" 2> /dev/null)
+rm -f /etc/s6/.s6-svscan/exitcode.*
+exit "${exitcode:-0}"

scripts/s6/docker/finish

@@ -1,4 +1,8 @@
 #!/usr/bin/env bash
 
-exec s6-svscanctl -t /etc/s6
+if [ "$1" -eq 256 ]; then
+  set -- "$((128 + $2))"
+fi
+(set -C; echo "$1" > "$(/etc/s6/.s6-svscan/exitcode-path)") 2> /dev/null
 
+s6-svscanctl -t /etc/s6 && exit 125 # 125 keeps s6 from restarting the service during teardown

scripts/s6/docker/run

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-exec s6-notifyoncheck dockerd-entrypoint.sh
+exec s6-notifyoncheck -n 0 dockerd-entrypoint.sh

scripts/s6/gitea-runner/finish

@@ -1,3 +1,8 @@
 #!/usr/bin/env bash
 
-exec s6-svscanctl -t /etc/s6
+if [ "$1" -eq 256 ]; then
+  set -- "$((128 + $2))"
+fi
+(set -C; echo "$1" > "$(/etc/s6/.s6-svscan/exitcode-path)") 2> /dev/null
+
+s6-svscanctl -t /etc/s6 && exit 125 # 125 keeps s6 from restarting the service during teardown

scripts/s6/gitea-runner/run

@@ -1,9 +1,7 @@
 #!/usr/bin/env bash
 
-while [ ! -d /etc/s6/docker/supervise ]; do
-    sleep 0.1
+until /etc/s6/docker/data/check &> /dev/null; do # s6-svwait can read missing or stale readiness status
+  sleep 1
 done
 
-s6-svwait -U /etc/s6/docker
-
 exec run.sh