enhance!: send OTLP metrics under OpenTelemetry semantic conventions, add runner state (#1282)

Sends the OTLP metrics under [OpenTelemetry semantic convention](https://opentelemetry.io/docs/specs/semconv/) names where the runner's data matches the definition, like `cicd.worker.count`, `cicd.system.errors` and the `go.*` and `process.*` runtime metrics, and under `gitea.runner.*` otherwise. OTel dashboards then work without remapping, as they already do for the traces from https://gitea.com/gitea/runner/pulls/1206.

Adds a `gitea_runner_state` gauge (`busy`, `idle`, `unavailable`), and `/readyz` now fails once the runner shuts down.

**Breaking**

1. OTLP metric names, units, types and attribute values change, and the info and start time metrics move to resource attributes.
1. Prometheus `gitea_runner_client_errors_total` gains a `code` label, and `gitea_runner_job_total` counts jobs stopped by `runner.timeout` as `timeout` instead of `failure`.

---------

Co-authored-by: silverwind <[email protected]>
Co-authored-by: silverwind <[email protected]>
Co-authored-by: bircni <[email protected]>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1282
Reviewed-by: silverwind <[email protected]>
Co-authored-by: wmTJc9IK0Q <[email protected]>

internal/pkg/metrics/metrics.go

@@ -4,6 +4,8 @@
 package metrics
 
 import (
+	"context"
+	"errors"
 	"sync"
 	"time"
 
@@ -20,6 +22,7 @@ const Namespace = "gitea_runner"
 //
 // LabelResult* values are used on metrics with label key "result" (RPC outcomes).
 // LabelStatus* values are used on metrics with label key "status" (job outcomes).
+// LabelState* values are used on metrics with label key "state" (runner state).
 const (
 	LabelResultTask    = "task"
 	LabelResultEmpty   = "empty"
@@ -35,6 +38,11 @@ const (
 	LabelStatusCancelled = "cancelled"
 	LabelStatusSkipped   = "skipped"
 	LabelStatusUnknown   = "unknown"
+	LabelStatusTimeout   = "timeout"
+
+	LabelStateBusy        = "busy"
+	LabelStateIdle        = "idle"
+	LabelStateUnavailable = "unavailable"
 )
 
 // rpcDurationBuckets covers the expected latency range for short-running
@@ -57,6 +65,15 @@ func ResultToStatusLabel(r runnerv1.Result) string {
 	}
 }
 
+// JobStatus reports an unsuccessful job past its deadline as timeout, like the job's trace span.
+func JobStatus(ctx context.Context, r runnerv1.Result) string {
+	status := ResultToStatusLabel(r)
+	if status != LabelStatusSuccess && status != LabelStatusSkipped && errors.Is(ctx.Err(), context.DeadlineExceeded) {
+		return LabelStatusTimeout
+	}
+	return status
+}
+
 var (
 	RunnerInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: Namespace,
@@ -96,7 +113,7 @@ var (
 		Namespace: Namespace,
 		Subsystem: "job",
 		Name:      "total",
-		Help:      "Total jobs processed by status (success, failure, cancelled, skipped, unknown).",
+		Help:      "Total jobs processed by status (success, failure, cancelled, skipped, timeout, unknown).",
 	}, []string{"status"})
 
 	JobDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
@@ -148,8 +165,8 @@ var (
 		Namespace: Namespace,
 		Subsystem: "client",
 		Name:      "errors_total",
-		Help:      "Total client RPC errors by method.",
-	}, []string{"method"})
+		Help:      "Total client RPC errors by method and connect code.",
+	}, []string{"method", "code"})
 )
 
 // Registry is the custom Prometheus registry used by the runner.
@@ -220,3 +237,28 @@ func register(collector prometheus.Collector) {
 	Registry.Unregister(collector)
 	Registry.MustRegister(collector)
 }
+
+// RegisterStateFunc registers gitea_runner_state, 1 for the state stateFn reports and 0 for the others.
+func RegisterStateFunc(stateFn func() string) {
+	register(stateCollector{prometheus.NewDesc(prometheus.BuildFQName(Namespace, "", "state"),
+		"Runner state (busy, idle, unavailable). 1 for the current state.", []string{"state"}, nil), stateFn})
+}
+
+// stateCollector reads the state once per collection, so a scrape never reports two states.
+type stateCollector struct {
+	desc  *prometheus.Desc
+	state func() string
+}
+
+func (c stateCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.desc }
+
+func (c stateCollector) Collect(ch chan<- prometheus.Metric) {
+	current := c.state()
+	for _, state := range []string{LabelStateBusy, LabelStateIdle, LabelStateUnavailable} {
+		value := 0.0
+		if state == current {
+			value = 1
+		}
+		ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, value, state)
+	}
+}

internal/pkg/metrics/metrics_test.go

@@ -34,6 +34,11 @@ func TestResultToStatusLabel(t *testing.T) {
 			require.Equal(t, tt.want, ResultToStatusLabel(tt.result))
 		})
 	}
+	expired, cancel := context.WithDeadline(t.Context(), time.Time{})
+	defer cancel()
+	require.Equal(t, LabelStatusTimeout, JobStatus(expired, runnerv1.Result_RESULT_FAILURE))
+	require.Equal(t, LabelStatusSuccess, JobStatus(expired, runnerv1.Result_RESULT_SUCCESS))
+	require.Equal(t, LabelStatusCancelled, JobStatus(t.Context(), runnerv1.Result_RESULT_CANCELLED))
 }
 
 func TestInitAndDynamicMetricRegistration(t *testing.T) {
@@ -50,6 +55,8 @@ func TestInitAndDynamicMetricRegistration(t *testing.T) {
 	RunnerInfo.WithLabelValues("test", "runner").Set(1)
 	RegisterUptimeFunc(time.Now().Add(-time.Second))
 	RegisterRunningJobsFunc(func() int64 { return 2 }, 4)
+	RegisterStateFunc(func() string { return LabelStateIdle })
+	RegisterStateFunc(func() string { return LabelStateBusy })
 
 	metrics, err := Registry.Gather()
 	require.NoError(t, err)
@@ -58,6 +65,15 @@ func TestInitAndDynamicMetricRegistration(t *testing.T) {
 	require.True(t, hasMetric(metrics, "gitea_runner_uptime_seconds"))
 	require.True(t, hasMetric(metrics, "gitea_runner_job_running"))
 	require.True(t, hasMetric(metrics, "gitea_runner_job_capacity_utilization_ratio"))
+	states := map[string]float64{}
+	for _, mf := range metrics {
+		if mf.GetName() == "gitea_runner_state" {
+			for _, m := range mf.GetMetric() {
+				states[m.GetLabel()[0].GetValue()] = m.GetGauge().GetValue()
+			}
+		}
+	}
+	require.Equal(t, map[string]float64{LabelStateBusy: 1, LabelStateIdle: 0, LabelStateUnavailable: 0}, states)
 }
 
 func TestRegisterRunningJobsFuncZeroCapacity(t *testing.T) {