enhance: export the Prometheus metrics over OpenTelemetry (#1248)

Push the runner's Prometheus metrics to the OTLP endpoint next to the job traces, so both land in the same backend.

The `/metrics` registry is converted as the [Prometheus compatibility spec](https://opentelemetry.io/docs/specs/otel/compatibility/prometheus_and_openmetrics/) describes and sent through the existing gRPC-free client, so names stay unchanged and no second metrics stack is linked. Metrics are on by default and pushed every interval and once on shutdown, so ephemeral runners report too.

New standard variables, each `OTEL_EXPORTER_OTLP_*` one with `TRACES` and `METRICS` variants:

- `OTEL_METRICS_EXPORTER`, `none` turns metrics off
- `OTEL_METRIC_EXPORT_INTERVAL`, `OTEL_METRIC_EXPORT_TIMEOUT`
- `OTEL_EXPORTER_OTLP_TIMEOUT`, `OTEL_EXPORTER_OTLP_COMPRESSION`
- `OTEL_EXPORTER_OTLP_CERTIFICATE`, `OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE`, `OTEL_EXPORTER_OTLP_CLIENT_KEY`

---------

Co-authored-by: silverwind <[email protected]>
Co-authored-by: silverwind <[email protected]>
Co-authored-by: bircni <[email protected]>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1248
Reviewed-by: silverwind <[email protected]>
Reviewed-by: bircni <[email protected]>
Co-authored-by: wmTJc9IK0Q <[email protected]>

internal/pkg/metrics/metrics.go

@@ -176,7 +176,7 @@ func Init() {
 
 // RegisterUptimeFunc registers a GaugeFunc that reports seconds since startTime.
 func RegisterUptimeFunc(startTime time.Time) {
-	Registry.MustRegister(prometheus.NewGaugeFunc(
+	register(prometheus.NewGaugeFunc(
 		prometheus.GaugeOpts{
 			Namespace: Namespace,
 			Name:      "uptime_seconds",
@@ -190,7 +190,7 @@ func RegisterUptimeFunc(startTime time.Time) {
 // capacity utilisation ratio, evaluated lazily at Prometheus scrape time.
 func RegisterRunningJobsFunc(countFn func() int64, capacity int) {
 	capF := float64(capacity)
-	Registry.MustRegister(prometheus.NewGaugeFunc(
+	register(prometheus.NewGaugeFunc(
 		prometheus.GaugeOpts{
 			Namespace: Namespace,
 			Subsystem: "job",
@@ -199,7 +199,7 @@ func RegisterRunningJobsFunc(countFn func() int64, capacity int) {
 		},
 		func() float64 { return float64(countFn()) },
 	))
-	Registry.MustRegister(prometheus.NewGaugeFunc(
+	register(prometheus.NewGaugeFunc(
 		prometheus.GaugeOpts{
 			Namespace: Namespace,
 			Subsystem: "job",
@@ -214,3 +214,9 @@ func RegisterRunningJobsFunc(countFn func() int64, capacity int) {
 		},
 	))
 }
+
+// register replaces an earlier registration, as the daemon can run more than once per process.
+func register(collector prometheus.Collector) {
+	Registry.Unregister(collector)
+	Registry.MustRegister(collector)
+}