enhance!: send OTLP metrics under OpenTelemetry semantic conventions, add runner state (#1282)

Sends the OTLP metrics under [OpenTelemetry semantic convention](https://opentelemetry.io/docs/specs/semconv/) names where the runner's data matches the definition, like `cicd.worker.count`, `cicd.system.errors` and the `go.*` and `process.*` runtime metrics, and under `gitea.runner.*` otherwise. OTel dashboards then work without remapping, as they already do for the traces from https://gitea.com/gitea/runner/pulls/1206.

Adds a `gitea_runner_state` gauge (`busy`, `idle`, `unavailable`), and `/readyz` now fails once the runner shuts down.

**Breaking**

1. OTLP metric names, units, types and attribute values change, and the info and start time metrics move to resource attributes.
1. Prometheus `gitea_runner_client_errors_total` gains a `code` label, and `gitea_runner_job_total` counts jobs stopped by `runner.timeout` as `timeout` instead of `failure`.

---------

Co-authored-by: silverwind <[email protected]>
Co-authored-by: silverwind <[email protected]>
Co-authored-by: bircni <[email protected]>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1282
Reviewed-by: silverwind <[email protected]>
Co-authored-by: wmTJc9IK0Q <[email protected]>

internal/pkg/metrics/metrics.go

@@ -4,6 +4,8 @@
 package metrics
 
 import (
+	"context"
+	"errors"
 	"sync"
 	"time"
 
@@ -20,6 +22,7 @@ const Namespace = "gitea_runner"
 //
 // LabelResult* values are used on metrics with label key "result" (RPC outcomes).
 // LabelStatus* values are used on metrics with label key "status" (job outcomes).
+// LabelState* values are used on metrics with label key "state" (runner state).
 const (
 	LabelResultTask    = "task"
 	LabelResultEmpty   = "empty"
@@ -35,6 +38,11 @@ const (
 	LabelStatusCancelled = "cancelled"
 	LabelStatusSkipped   = "skipped"
 	LabelStatusUnknown   = "unknown"
+	LabelStatusTimeout   = "timeout"
+
+	LabelStateBusy        = "busy"
+	LabelStateIdle        = "idle"
+	LabelStateUnavailable = "unavailable"
 )
 
 // rpcDurationBuckets covers the expected latency range for short-running
@@ -57,6 +65,15 @@ func ResultToStatusLabel(r runnerv1.Result) string {
 	}
 }
 
+// JobStatus reports an unsuccessful job past its deadline as timeout, like the job's trace span.
+func JobStatus(ctx context.Context, r runnerv1.Result) string {
+	status := ResultToStatusLabel(r)
+	if status != LabelStatusSuccess && status != LabelStatusSkipped && errors.Is(ctx.Err(), context.DeadlineExceeded) {
+		return LabelStatusTimeout
+	}
+	return status
+}
+
 var (
 	RunnerInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: Namespace,
@@ -96,7 +113,7 @@ var (
 		Namespace: Namespace,
 		Subsystem: "job",
 		Name:      "total",
-		Help:      "Total jobs processed by status (success, failure, cancelled, skipped, unknown).",
+		Help:      "Total jobs processed by status (success, failure, cancelled, skipped, timeout, unknown).",
 	}, []string{"status"})
 
 	JobDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
@@ -148,8 +165,8 @@ var (
 		Namespace: Namespace,
 		Subsystem: "client",
 		Name:      "errors_total",
-		Help:      "Total client RPC errors by method.",
-	}, []string{"method"})
+		Help:      "Total client RPC errors by method and connect code.",
+	}, []string{"method", "code"})
 )
 
 // Registry is the custom Prometheus registry used by the runner.
@@ -220,3 +237,28 @@ func register(collector prometheus.Collector) {
 	Registry.Unregister(collector)
 	Registry.MustRegister(collector)
 }
+
+// RegisterStateFunc registers gitea_runner_state, 1 for the state stateFn reports and 0 for the others.
+func RegisterStateFunc(stateFn func() string) {
+	register(stateCollector{prometheus.NewDesc(prometheus.BuildFQName(Namespace, "", "state"),
+		"Runner state (busy, idle, unavailable). 1 for the current state.", []string{"state"}, nil), stateFn})
+}
+
+// stateCollector reads the state once per collection, so a scrape never reports two states.
+type stateCollector struct {
+	desc  *prometheus.Desc
+	state func() string
+}
+
+func (c stateCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.desc }
+
+func (c stateCollector) Collect(ch chan<- prometheus.Metric) {
+	current := c.state()
+	for _, state := range []string{LabelStateBusy, LabelStateIdle, LabelStateUnavailable} {
+		value := 0.0
+		if state == current {
+			value = 1
+		}
+		ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, value, state)
+	}
+}

internal/pkg/metrics/metrics_test.go

@@ -34,6 +34,11 @@ func TestResultToStatusLabel(t *testing.T) {
 			require.Equal(t, tt.want, ResultToStatusLabel(tt.result))
 		})
 	}
+	expired, cancel := context.WithDeadline(t.Context(), time.Time{})
+	defer cancel()
+	require.Equal(t, LabelStatusTimeout, JobStatus(expired, runnerv1.Result_RESULT_FAILURE))
+	require.Equal(t, LabelStatusSuccess, JobStatus(expired, runnerv1.Result_RESULT_SUCCESS))
+	require.Equal(t, LabelStatusCancelled, JobStatus(t.Context(), runnerv1.Result_RESULT_CANCELLED))
 }
 
 func TestInitAndDynamicMetricRegistration(t *testing.T) {
@@ -50,6 +55,8 @@ func TestInitAndDynamicMetricRegistration(t *testing.T) {
 	RunnerInfo.WithLabelValues("test", "runner").Set(1)
 	RegisterUptimeFunc(time.Now().Add(-time.Second))
 	RegisterRunningJobsFunc(func() int64 { return 2 }, 4)
+	RegisterStateFunc(func() string { return LabelStateIdle })
+	RegisterStateFunc(func() string { return LabelStateBusy })
 
 	metrics, err := Registry.Gather()
 	require.NoError(t, err)
@@ -58,6 +65,15 @@ func TestInitAndDynamicMetricRegistration(t *testing.T) {
 	require.True(t, hasMetric(metrics, "gitea_runner_uptime_seconds"))
 	require.True(t, hasMetric(metrics, "gitea_runner_job_running"))
 	require.True(t, hasMetric(metrics, "gitea_runner_job_capacity_utilization_ratio"))
+	states := map[string]float64{}
+	for _, mf := range metrics {
+		if mf.GetName() == "gitea_runner_state" {
+			for _, m := range mf.GetMetric() {
+				states[m.GetLabel()[0].GetValue()] = m.GetGauge().GetValue()
+			}
+		}
+	}
+	require.Equal(t, map[string]float64{LabelStateBusy: 1, LabelStateIdle: 0, LabelStateUnavailable: 0}, states)
 }
 
 func TestRegisterRunningJobsFuncZeroCapacity(t *testing.T) {

internal/pkg/report/reporter.go

@@ -624,7 +624,7 @@ func (r *Reporter) reportLog(noMore bool) (bool, error) {
 	metrics.ReportLogDuration.Observe(time.Since(start).Seconds())
 	if err != nil {
 		metrics.ReportLogTotal.WithLabelValues(metrics.LabelResultError).Inc()
-		metrics.ClientErrors.WithLabelValues(metrics.LabelMethodUpdateLog).Inc()
+		metrics.ClientErrors.WithLabelValues(metrics.LabelMethodUpdateLog, connect.CodeOf(err).String()).Inc()
 		return false, err
 	}
 	metrics.ReportLogTotal.WithLabelValues(metrics.LabelResultSuccess).Inc()
@@ -701,7 +701,7 @@ func (r *Reporter) ReportState(reportResult bool) error {
 	metrics.ReportStateDuration.Observe(time.Since(start).Seconds())
 	if err != nil {
 		metrics.ReportStateTotal.WithLabelValues(metrics.LabelResultError).Inc()
-		metrics.ClientErrors.WithLabelValues(metrics.LabelMethodUpdateTask).Inc()
+		metrics.ClientErrors.WithLabelValues(metrics.LabelMethodUpdateTask, connect.CodeOf(err).String()).Inc()
 		r.stateMu.Lock()
 		r.stateChanged = true
 		r.stateMu.Unlock()

internal/pkg/telemetry/metrics.go

@@ -7,12 +7,18 @@ import (
 	"cmp"
 	"context"
 	"errors"
+	"math"
+	"runtime"
 	"strings"
 	"time"
 
+	"gitea.com/gitea/runner/internal/pkg/metrics"
+
+	"gitea.dev/actionslib/runner/v1/runnerv1connect"
 	"github.com/prometheus/client_golang/prometheus"
 	dto "github.com/prometheus/client_model/go"
 	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/sdk/resource"
 	commonpb "go.opentelemetry.io/proto/otlp/common/v1"
 	metricpb "go.opentelemetry.io/proto/otlp/metrics/v1"
@@ -26,7 +32,11 @@ const cumulative = metricpb.AggregationTemporality_AGGREGATION_TEMPORALITY_CUMUL
 func pushMetrics(exporter *client, res *resource.Resource, gatherer prometheus.Gatherer) func(context.Context) error {
 	attrs := make([]*commonpb.KeyValue, 0, res.Len())
 	for _, kv := range res.Attributes() {
-		attrs = append(attrs, keyValue(string(kv.Key), kv.Value.String()))
+		attr := keyValue(string(kv.Key), kv.Value.String())
+		if kv.Value.Type() == attribute.INT64 {
+			attr.Value.Value = &commonpb.AnyValue_IntValue{IntValue: kv.Value.AsInt64()}
+		}
+		attrs = append(attrs, attr)
 	}
 	interval, timeout := millis("OTEL_METRIC_EXPORT_INTERVAL", time.Minute, time.Minute), millis("OTEL_METRIC_EXPORT_TIMEOUT", 30*time.Second, noLimit)
 	push := func(ctx context.Context) error {
@@ -70,39 +80,132 @@ var ucum = map[string]string{
 	"meters": "m", "volts": "V", "amperes": "A", "joules": "J", "watts": "W", "grams": "g", "celsius": "Cel", "hertz": "Hz", "percent": "%",
 }
 
-// convert follows https://opentelemetry.io/docs/specs/otel/compatibility/prometheus_and_openmetrics/#prometheus-metric-points-to-otlp
+type otlpMetric struct {
+	name, unit string
+	upDown     bool              // a gauge semconv defines as an UpDownCounter
+	labels     map[string]string // label → attribute
+	values     map[string]string // label value → attribute value
+	extra      []*commonpb.KeyValue
+	unlimited  float64 // gauge value meaning no limit, not sent
+}
+
+var rpcMethods = map[string]string{
+	metrics.LabelMethodFetchTask:  strings.TrimPrefix(runnerv1connect.RunnerServiceFetchTaskProcedure, "/"),
+	metrics.LabelMethodUpdateLog:  strings.TrimPrefix(runnerv1connect.RunnerServiceUpdateLogProcedure, "/"),
+	metrics.LabelMethodUpdateTask: strings.TrimPrefix(runnerv1connect.RunnerServiceUpdateTaskProcedure, "/"),
+}
+
+// openFiles maps process_open_fds, which client_golang fills with the handle count on Windows.
+func openFiles() otlpMetric {
+	if runtime.GOOS == "windows" {
+		return otlpMetric{name: "process.windows.handle.count", unit: "{handle}", upDown: true}
+	}
+	return otlpMetric{name: "process.unix.file_descriptor.count", unit: "{file_descriptor}", upDown: true}
+}
+
+// otlpMetrics maps to semconv names, unlisted metrics keep theirs and an empty name drops one the resource already carries.
+var otlpMetrics = map[string]otlpMetric{
+	"gitea_runner_info":                           {},
+	"go_info":                                     {},
+	"process_start_time_seconds":                  {},
+	"gitea_runner_capacity":                       {name: "gitea.runner.job.limit", unit: "{job}", upDown: true},
+	"gitea_runner_uptime_seconds":                 {name: "gitea.runner.uptime", unit: "s"},
+	"gitea_runner_job_running":                    {name: "gitea.runner.job.active", unit: "{job}", upDown: true},
+	"gitea_runner_job_capacity_utilization_ratio": {name: "gitea.runner.job.utilization", unit: "1"},
+	"gitea_runner_job_duration_seconds":           {name: "gitea.runner.job.duration", unit: "s"},
+	"gitea_runner_job_total": {
+		name: "gitea.runner.jobs", unit: "{job}",
+		labels: map[string]string{"status": "cicd.pipeline.result"},
+		values: map[string]string{metrics.LabelStatusCancelled: "cancellation", metrics.LabelStatusSkipped: "skip", metrics.LabelStatusUnknown: "error"},
+	},
+	"gitea_runner_poll_fetch_total": {
+		name: "gitea.runner.poll.fetches", unit: "{fetch}",
+		labels: map[string]string{"result": "gitea.runner.poll.result"},
+	},
+	"gitea_runner_poll_fetch_duration_seconds":   {name: "gitea.runner.poll.fetch.duration", unit: "s"},
+	"gitea_runner_poll_backoff_seconds":          {name: "gitea.runner.poll.backoff", unit: "s"},
+	"gitea_runner_report_log_duration_seconds":   {name: "gitea.runner.report.log.duration", unit: "s"},
+	"gitea_runner_report_state_duration_seconds": {name: "gitea.runner.report.state.duration", unit: "s"},
+	"gitea_runner_report_log_buffer_rows":        {name: "gitea.runner.report.log.buffer", unit: "{row}", upDown: true},
+	"gitea_runner_report_log_total": {
+		name: "gitea.runner.report.log.requests", unit: "{request}",
+		labels: map[string]string{"result": "gitea.runner.report.result"},
+	},
+	"gitea_runner_report_state_total": {
+		name: "gitea.runner.report.state.requests", unit: "{request}",
+		labels: map[string]string{"result": "gitea.runner.report.result"},
+	},
+	"gitea_runner_client_errors_total": {
+		name: "cicd.system.errors", unit: "{error}",
+		labels: map[string]string{"method": "rpc.method", "code": "error.type"}, values: rpcMethods,
+		extra: []*commonpb.KeyValue{keyValue("cicd.system.component", "runner")},
+	},
+	"gitea_runner_state": {
+		name: "cicd.worker.count", unit: "{worker}", upDown: true,
+		labels: map[string]string{"state": "cicd.worker.state"},
+		values: map[string]string{metrics.LabelStateIdle: "available", metrics.LabelStateUnavailable: "offline"},
+	},
+	"go_goroutines":                 {name: "go.goroutine.count", unit: "{goroutine}", upDown: true},
+	"go_memstats_alloc_bytes_total": {name: "go.memory.allocated", unit: "By"},
+	"go_memstats_next_gc_bytes":     {name: "go.memory.gc.goal", unit: "By", upDown: true},
+	"go_gc_gomemlimit_bytes":        {name: "go.memory.limit", unit: "By", upDown: true, unlimited: math.MaxInt64},
+	"go_gc_gogc_percent":            {name: "go.config.gogc", unit: "%", upDown: true},
+	"go_sched_gomaxprocs_threads":   {name: "go.processor.limit", unit: "{thread}", upDown: true},
+	"process_resident_memory_bytes": {name: "process.memory.usage", unit: "By", upDown: true},
+	"process_virtual_memory_bytes":  {name: "process.memory.virtual", unit: "By", upDown: true},
+	"process_open_fds":              openFiles(),
+}
+
+// convert follows https://opentelemetry.io/docs/specs/otel/compatibility/prometheus_and_openmetrics/#prometheus-metric-points-to-otlp except for otlpMetrics.
 func convert(families []*dto.MetricFamily) []*metricpb.Metric {
 	now := uint64(time.Now().UnixNano())
 	metrics := make([]*metricpb.Metric, 0, len(families))
+	byFamily := make(map[string]*dto.MetricFamily, len(families))
 	for _, family := range families {
+		byFamily[family.GetName()] = family
+		mapping, ok := otlpMetrics[family.GetName()]
+		if !ok {
+			mapping.name = family.GetName()
+		} else if mapping.name == "" {
+			continue
+		}
 		metric := &metricpb.Metric{
-			Name: family.GetName(), Description: family.GetHelp(), Unit: cmp.Or(ucum[family.GetUnit()], family.GetUnit()),
+			Name: mapping.name, Description: family.GetHelp(), Unit: cmp.Or(mapping.unit, ucum[family.GetUnit()], family.GetUnit()),
 			Metadata: []*commonpb.KeyValue{keyValue("prometheus.type", strings.ToLower(family.GetType().String()))},
 		}
 		switch family.GetType() {
 		case dto.MetricType_COUNTER:
 			sum := &metricpb.Sum{AggregationTemporality: cumulative, IsMonotonic: true}
 			for _, series := range family.GetMetric() {
-				sum.DataPoints = append(sum.DataPoints, number(series, unixNano(series.GetCounter().GetCreatedTimestamp()), now, series.GetCounter().GetValue()))
+				sum.DataPoints = append(sum.DataPoints, number(labels(series, mapping), unixNano(series.GetCounter().GetCreatedTimestamp()), now, series.GetCounter().GetValue()))
 			}
 			metric.Data = &metricpb.Metric_Sum{Sum: sum}
 		case dto.MetricType_GAUGE:
-			gauge := &metricpb.Gauge{}
+			var points []*metricpb.NumberDataPoint
 			for _, series := range family.GetMetric() {
-				gauge.DataPoints = append(gauge.DataPoints, number(series, 0, now, series.GetGauge().GetValue()))
+				if value := series.GetGauge().GetValue(); mapping.unlimited == 0 || value != mapping.unlimited {
+					points = append(points, number(labels(series, mapping), 0, now, value))
+				}
+			}
+			if len(points) == 0 {
+				continue
+			}
+			if mapping.upDown {
+				metric.Data = &metricpb.Metric_Sum{Sum: &metricpb.Sum{AggregationTemporality: cumulative, DataPoints: points}}
+			} else {
+				metric.Data = &metricpb.Metric_Gauge{Gauge: &metricpb.Gauge{DataPoints: points}}
 			}
-			metric.Data = &metricpb.Metric_Gauge{Gauge: gauge}
 		case dto.MetricType_HISTOGRAM:
 			histogram := &metricpb.Histogram{AggregationTemporality: cumulative}
 			for _, series := range family.GetMetric() {
-				histogram.DataPoints = append(histogram.DataPoints, histogramPoint(series, now))
+				histogram.DataPoints = append(histogram.DataPoints, histogramPoint(series, labels(series, mapping), now))
 			}
 			metric.Data = &metricpb.Metric_Histogram{Histogram: histogram}
 		case dto.MetricType_SUMMARY:
 			summary := &metricpb.Summary{}
 			for _, series := range family.GetMetric() {
 				point := &metricpb.SummaryDataPoint{
-					Attributes: labels(series), StartTimeUnixNano: unixNano(series.GetSummary().GetCreatedTimestamp()), TimeUnixNano: now,
+					Attributes: labels(series, mapping), StartTimeUnixNano: unixNano(series.GetSummary().GetCreatedTimestamp()), TimeUnixNano: now,
 					Count: series.GetSummary().GetSampleCount(), Sum: series.GetSummary().GetSampleSum(),
 				}
 				for _, quantile := range series.GetSummary().GetQuantile() {
@@ -116,11 +219,44 @@ func convert(families []*dto.MetricFamily) []*metricpb.Metric {
 		}
 		metrics = append(metrics, metric)
 	}
+	return append(metrics, derived(byFamily, now)...)
+}
+
+// derived adds the semconv Go metrics that are computed from other metrics rather than renamed.
+func derived(families map[string]*dto.MetricFamily, now uint64) []*metricpb.Metric {
+	first := func(name string) *dto.Metric {
+		if series := families[name].GetMetric(); len(series) > 0 {
+			return series[0]
+		}
+		return nil
+	}
+	gauge := func(name string) (float64, bool) {
+		series := first(name)
+		return series.GetGauge().GetValue(), series != nil
+	}
+	sum := func(name, unit, description string, monotonic bool, points ...*metricpb.NumberDataPoint) *metricpb.Metric {
+		return &metricpb.Metric{Name: name, Unit: unit, Description: description, Data: &metricpb.Metric_Sum{Sum: &metricpb.Sum{
+			AggregationTemporality: cumulative, IsMonotonic: monotonic, DataPoints: points,
+		}}}
+	}
+	var metrics []*metricpb.Metric
+	total, ok1 := gauge("go_memstats_sys_bytes")
+	released, ok2 := gauge("go_memstats_heap_released_bytes")
+	stack, ok3 := gauge("go_memstats_stack_inuse_bytes")
+	if ok1 && ok2 && ok3 {
+		metrics = append(metrics, sum("go.memory.used", "By", "Memory used by the Go runtime.", false,
+			number([]*commonpb.KeyValue{keyValue("go.memory.type", "stack")}, 0, now, stack),
+			number([]*commonpb.KeyValue{keyValue("go.memory.type", "other")}, 0, now, total-released-stack)))
+	}
+	if gc := first("go_gc_duration_seconds"); gc != nil {
+		metrics = append(metrics, sum("go.memory.gc.cycles", "{gc_cycle}", "Number of completed GC cycles.", true,
+			number(nil, unixNano(gc.GetSummary().GetCreatedTimestamp()), now, float64(gc.GetSummary().GetSampleCount()))))
+	}
 	return metrics
 }
 
-func number(series *dto.Metric, start, now uint64, value float64) *metricpb.NumberDataPoint {
-	return &metricpb.NumberDataPoint{Attributes: labels(series), StartTimeUnixNano: start, TimeUnixNano: now, Value: &metricpb.NumberDataPoint_AsDouble{AsDouble: value}}
+func number(attrs []*commonpb.KeyValue, start, now uint64, value float64) *metricpb.NumberDataPoint {
+	return &metricpb.NumberDataPoint{Attributes: attrs, StartTimeUnixNano: start, TimeUnixNano: now, Value: &metricpb.NumberDataPoint_AsDouble{AsDouble: value}}
 }
 
 // unixNano maps a nil timestamp to 0, leaving the start time unset as the spec asks.
@@ -128,10 +264,10 @@ func unixNano(timestamp *timestamppb.Timestamp) uint64 {
 	return uint64(timestamp.AsTime().UnixNano())
 }
 
-func histogramPoint(series *dto.Metric, now uint64) *metricpb.HistogramDataPoint {
+func histogramPoint(series *dto.Metric, attrs []*commonpb.KeyValue, now uint64) *metricpb.HistogramDataPoint {
 	histogram := series.GetHistogram()
 	point := &metricpb.HistogramDataPoint{
-		Attributes: labels(series), StartTimeUnixNano: unixNano(histogram.GetCreatedTimestamp()), TimeUnixNano: now,
+		Attributes: attrs, StartTimeUnixNano: unixNano(histogram.GetCreatedTimestamp()), TimeUnixNano: now,
 		Count: histogram.GetSampleCount(), Sum: histogram.SampleSum,
 		BucketCounts: make([]uint64, len(histogram.GetBucket())+1),
 	}
@@ -144,12 +280,12 @@ func histogramPoint(series *dto.Metric, now uint64) *metricpb.HistogramDataPoint
 	return point
 }
 
-func labels(series *dto.Metric) []*commonpb.KeyValue {
-	attrs := make([]*commonpb.KeyValue, 0, len(series.GetLabel()))
+func labels(series *dto.Metric, mapping otlpMetric) []*commonpb.KeyValue {
+	attrs := make([]*commonpb.KeyValue, 0, len(series.GetLabel())+len(mapping.extra))
 	for _, label := range series.GetLabel() {
-		attrs = append(attrs, keyValue(label.GetName(), label.GetValue()))
+		attrs = append(attrs, keyValue(cmp.Or(mapping.labels[label.GetName()], label.GetName()), cmp.Or(mapping.values[label.GetValue()], label.GetValue())))
 	}
-	return attrs
+	return append(attrs, mapping.extra...)
 }
 
 func keyValue(key, value string) *commonpb.KeyValue {

internal/pkg/telemetry/telemetry.go

@@ -45,6 +45,8 @@ import (
 
 const scope = "gitea.com/gitea/runner"
 
+var processCreation = time.Now() // package init, within milliseconds of the process start
+
 type taskRunKey struct{}
 
 type taskRun struct {
@@ -102,12 +104,13 @@ func Setup(ctx context.Context, uuid, name string, gatherer prometheus.Gatherer)
 	if len(clients) == 0 {
 		return noop, nil
 	}
-	res, err := resource.New(ctx, resource.WithTelemetrySDK(), resource.WithAttributes(
+	res, err := resource.New(ctx, resource.WithTelemetrySDK(), resource.WithProcessPID(), resource.WithProcessRuntimeName(), resource.WithProcessRuntimeVersion(), resource.WithAttributes(
 		semconv.ServiceName("gitea-runner"),
 		semconv.ServiceVersion(ver.Version()),
 		semconv.ServiceInstanceID(uuid),
 		semconv.CICDWorkerID(uuid),
 		semconv.CICDWorkerName(name),
+		semconv.ProcessCreationTime(processCreation.UTC().Format(time.RFC3339Nano)),
 	), resource.WithFromEnv())
 	if err != nil {
 		return noop, err

internal/pkg/telemetry/telemetry_test.go

@@ -12,6 +12,8 @@ import (
 	"encoding/pem"
 	"fmt"
 	"io"
+	"maps"
+	"math"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -154,9 +156,10 @@ func TestExportRetriesAndMatchesOTLP(t *testing.T) {
 	assert.Equal(t, "op", tracesData.GetResourceSpans()[0].GetScopeSpans()[0].GetSpans()[0].GetName())
 
 	resourceMetrics := metricsData.GetResourceMetrics()[0]
-	assert.True(t, slices.ContainsFunc(resourceMetrics.GetResource().GetAttributes(), func(kv *commonpb.KeyValue) bool {
-		return proto.Equal(kv, keyValue("service.name", "gitea-runner"))
-	}))
+	pid := &commonpb.KeyValue{Key: "process.pid", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_IntValue{IntValue: int64(os.Getpid())}}}
+	for _, want := range []*commonpb.KeyValue{keyValue("service.name", "gitea-runner"), keyValue("process.creation.time", processCreation.UTC().Format(time.RFC3339Nano)), pid} {
+		assert.True(t, slices.ContainsFunc(resourceMetrics.GetResource().GetAttributes(), func(kv *commonpb.KeyValue) bool { return proto.Equal(kv, want) }), want.GetKey())
+	}
 	assert.InDelta(t, 2, resourceMetrics.GetScopeMetrics()[0].GetMetrics()[0].GetGauge().GetDataPoints()[0].GetAsDouble(), 0)
 }
 
@@ -165,31 +168,76 @@ func TestConvert(t *testing.T) {
 	counter := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "jobs_total", Help: "Jobs."}, []string{"status"})
 	histogram := prometheus.NewHistogram(prometheus.HistogramOpts{Name: "duration_seconds", Unit: "seconds", Buckets: []float64{1, 2}})
 	summary := prometheus.NewSummary(prometheus.SummaryOpts{Name: "gc_seconds", Objectives: map[float64]float64{0.5: 0.05}})
-	registry.MustRegister(counter, histogram, summary)
+	jobs := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "gitea_runner_job_total"}, []string{"status"})
+	running := prometheus.NewGauge(prometheus.GaugeOpts{Name: "gitea_runner_job_running"})
+	gauges := map[string]float64{
+		"go_info": 1, "go_gc_gomemlimit_bytes": math.MaxInt64,
+		"go_memstats_sys_bytes": 100, "go_memstats_heap_released_bytes": 30, "go_memstats_stack_inuse_bytes": 20,
+	}
+	for name, value := range gauges {
+		registry.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name: name}, func() float64 { return value }))
+	}
+	state := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "gitea_runner_state"}, []string{"state"})
+	clientErrors := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "gitea_runner_client_errors_total"}, []string{"method", "code"})
+	registry.MustRegister(counter, histogram, summary, jobs, running, state, clientErrors)
 	counter.WithLabelValues("success").Add(3)
 	for _, value := range []float64{0.5, 1.5, 3} {
 		histogram.Observe(value)
 	}
 	summary.Observe(4)
+	jobs.WithLabelValues("cancelled").Inc()
+	running.Set(2)
+	state.WithLabelValues("idle").Set(1)
+	state.WithLabelValues("unavailable").Set(0)
+	clientErrors.WithLabelValues("FetchTask", "unavailable").Inc()
 	families, err := registry.Gather()
 	require.NoError(t, err)
 
-	exported := convert(families)
-	assert.Equal(t, "s", exported[0].GetUnit())
-	buckets := exported[0].GetHistogram().GetDataPoints()[0]
+	exported := map[string]*metricpb.Metric{}
+	for _, metric := range convert(families) {
+		exported[metric.GetName()] = metric
+	}
+	assert.ElementsMatch(t, []string{
+		"duration_seconds", "gc_seconds", "jobs_total", "gitea.runner.jobs", "gitea.runner.job.active", "cicd.system.errors",
+		"go_memstats_sys_bytes", "go_memstats_heap_released_bytes", "go_memstats_stack_inuse_bytes", "go.memory.used", "cicd.worker.count",
+	}, slices.Collect(maps.Keys(exported)))
+	assert.Equal(t, "s", exported["duration_seconds"].GetUnit())
+	buckets := exported["duration_seconds"].GetHistogram().GetDataPoints()[0]
 	assert.Equal(t, []float64{1, 2}, buckets.GetExplicitBounds())
 	assert.Equal(t, []uint64{1, 1, 1}, buckets.GetBucketCounts())
 	assert.Equal(t, uint64(3), buckets.GetCount())
 	assert.InDelta(t, 5, buckets.GetSum(), 0)
-	assert.InDelta(t, 4, exported[1].GetSummary().GetDataPoints()[0].GetQuantileValues()[0].GetValue(), 0)
-	sum := exported[2].GetSum()
-	assert.Equal(t, "Jobs.", exported[2].GetDescription())
-	assert.True(t, proto.Equal(keyValue("prometheus.type", "counter"), exported[2].GetMetadata()[0]))
+	assert.InDelta(t, 4, exported["gc_seconds"].GetSummary().GetDataPoints()[0].GetQuantileValues()[0].GetValue(), 0)
+	sum := exported["jobs_total"].GetSum()
+	assert.Equal(t, "Jobs.", exported["jobs_total"].GetDescription())
+	assert.True(t, proto.Equal(keyValue("prometheus.type", "counter"), exported["jobs_total"].GetMetadata()[0]))
 	assert.NotZero(t, sum.GetDataPoints()[0].GetStartTimeUnixNano())
 	assert.True(t, sum.GetIsMonotonic())
 	assert.Equal(t, cumulative, sum.GetAggregationTemporality())
 	assert.InDelta(t, 3, sum.GetDataPoints()[0].GetAsDouble(), 0)
 	assert.True(t, proto.Equal(keyValue("status", "success"), sum.GetDataPoints()[0].GetAttributes()[0]))
+
+	assert.True(t, proto.Equal(keyValue("cicd.pipeline.result", "cancellation"), exported["gitea.runner.jobs"].GetSum().GetDataPoints()[0].GetAttributes()[0]))
+	active := exported["gitea.runner.job.active"]
+	assert.Equal(t, "{job}", active.GetUnit())
+	assert.False(t, active.GetSum().GetIsMonotonic())
+	assert.InDelta(t, 2, active.GetSum().GetDataPoints()[0].GetAsDouble(), 0)
+	attributes := func(points []*metricpb.NumberDataPoint, key string) map[string]float64 {
+		values := map[string]float64{}
+		for _, point := range points {
+			for _, kv := range point.GetAttributes() {
+				if kv.GetKey() == key {
+					values[kv.GetValue().GetStringValue()] = point.GetAsDouble()
+				}
+			}
+		}
+		return values
+	}
+	assert.Equal(t, map[string]float64{"stack": 20, "other": 50}, attributes(exported["go.memory.used"].GetSum().GetDataPoints(), "go.memory.type"))
+	assert.Equal(t, map[string]float64{"available": 1, "offline": 0}, attributes(exported["cicd.worker.count"].GetSum().GetDataPoints(), "cicd.worker.state"))
+	assert.True(t, proto.Equal(&commonpb.KeyValueList{Values: []*commonpb.KeyValue{
+		keyValue("error.type", "unavailable"), keyValue("rpc.method", "runner.v1.RunnerService/FetchTask"), keyValue("cicd.system.component", "runner"),
+	}}, &commonpb.KeyValueList{Values: exported["cicd.system.errors"].GetSum().GetDataPoints()[0].GetAttributes()}))
 }
 
 func TestExportRefusesRedirects(t *testing.T) {