enhance!: send OTLP metrics under OpenTelemetry semantic conventions, add runner state (#1282)

Sends the OTLP metrics under [OpenTelemetry semantic convention](https://opentelemetry.io/docs/specs/semconv/) names where the runner's data matches the definition, like `cicd.worker.count`, `cicd.system.errors` and the `go.*` and `process.*` runtime metrics, and under `gitea.runner.*` otherwise. OTel dashboards then work without remapping, as they already do for the traces from https://gitea.com/gitea/runner/pulls/1206.

Adds a `gitea_runner_state` gauge (`busy`, `idle`, `unavailable`), and `/readyz` now fails once the runner shuts down.

**Breaking**

1. OTLP metric names, units, types and attribute values change, and the info and start time metrics move to resource attributes.
1. Prometheus `gitea_runner_client_errors_total` gains a `code` label, and `gitea_runner_job_total` counts jobs stopped by `runner.timeout` as `timeout` instead of `failure`.

---------

Co-authored-by: silverwind <[email protected]>
Co-authored-by: silverwind <[email protected]>
Co-authored-by: bircni <[email protected]>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1282
Reviewed-by: silverwind <[email protected]>
Co-authored-by: wmTJc9IK0Q <[email protected]>

docs/telemetry.md

@@ -31,7 +31,7 @@ Except for the endpoint, each `OTEL_EXPORTER_OTLP_*` variable has `TRACES` and `
 
 - A span per job, named `RUN {workflow} {job}`, with the repository, ref, commit, run URL, run number, attempt and result.
 - A span per step, named as in Gitea's step list, including `Pre` and `Post` stages and the steps of composite actions, with the step's result.
-- The runner's name, UUID and version.
+- The runner's name, UUID, version, process ID and start time, and the Go version.
 
 Logs, step output and secrets are never sent. Step names are taken from the workflow file as written, without expanding `${{ }}` expressions.
 
@@ -41,7 +41,23 @@ Jobs receive `TRACEPARENT`, so tools in a job that use OpenTelemetry can add the
 
 ## Metrics
 
-The metrics of the Prometheus endpoint are also exported, under the same names and whether or not `metrics.enabled` is set. They are sent every export interval and once more on shutdown. If your collector only accepts traces, set `OTEL_METRICS_EXPORTER=none`.
+The metrics of the Prometheus endpoint are also exported, whether or not `metrics.enabled` is set, except `gitea_runner_info`, `go_info` and `process_start_time_seconds`, which are sent as resource attributes. They are sent every export interval and once more on shutdown. To turn them off, set `OTEL_METRICS_EXPORTER=none`.
+
+Metrics are named after the [OpenTelemetry semantic conventions](https://opentelemetry.io/docs/specs/semconv/) where one exists, and under `gitea.runner.*` otherwise. Other Go and process metrics, such as `go_memstats_heap_alloc_bytes`, keep their Prometheus names.
+
+| Metric | Attributes |
+| --- | --- |
+| `cicd.worker.count` | `cicd.worker.state` |
+| `cicd.system.errors` | `cicd.system.component`, `rpc.method`, `error.type` |
+| `gitea.runner.jobs` | `cicd.pipeline.result` |
+| `gitea.runner.job.active`, `gitea.runner.job.limit`, `gitea.runner.job.utilization`, `gitea.runner.job.duration` | |
+| `gitea.runner.poll.fetches` | `gitea.runner.poll.result` |
+| `gitea.runner.uptime`, `gitea.runner.poll.backoff`, `gitea.runner.poll.fetch.duration` | |
+| `gitea.runner.report.log.requests`, `gitea.runner.report.state.requests` | `gitea.runner.report.result` |
+| `gitea.runner.report.log.buffer`, `gitea.runner.report.log.duration`, `gitea.runner.report.state.duration` | |
+| `go.memory.used` | `go.memory.type` |
+| `go.goroutine.count`, `go.memory.allocated`, `go.memory.gc.goal`, `go.memory.gc.cycles`, `go.memory.limit`, `go.config.gogc`, `go.processor.limit` | |
+| `process.memory.usage`, `process.memory.virtual`, `process.unix.file_descriptor.count` (`process.windows.handle.count` on Windows) | |
 
 ## Security
 

internal/app/cmd/daemon.go

@@ -209,6 +209,15 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
 		metrics.RunnerCapacity.Set(float64(cfg.Runner.Capacity))
 		metrics.RegisterUptimeFunc(time.Now())
 		metrics.RegisterRunningJobsFunc(runner.RunningCount, cfg.Runner.Capacity)
+		metrics.RegisterStateFunc(func() string {
+			if runner.RunningCount() > 0 {
+				return metrics.LabelStateBusy
+			}
+			if ready, _ := poller.Ready(cfg.Metrics.ReadinessGrace); !ready {
+				return metrics.LabelStateUnavailable
+			}
+			return metrics.LabelStateIdle
+		})
 		if cfg.Metrics.Enabled {
 			metrics.StartServer(ctx, cfg.Metrics.Addr, func() (bool, string) {
 				return poller.Ready(cfg.Metrics.ReadinessGrace)

internal/app/poll/poller.go

@@ -206,6 +206,9 @@ func (p *Poller) Ready(grace time.Duration) (bool, string) {
 	if p.unregistered.Load() {
 		return false, "runner is no longer registered"
 	}
+	if p.pollingCtx.Err() != nil {
+		return false, "runner is shutting down"
+	}
 	p.availabilityMu.Lock()
 	ready, reason := p.availabilityReady, p.availabilityReason
 	p.availabilityMu.Unlock()
@@ -394,7 +397,7 @@ func (p *Poller) fetchTask(ctx context.Context, s *workerState) (*runnerv1.Task,
 		p.lastPollFailed.Store(true)
 		s.consecutiveErrors++
 		metrics.PollFetchTotal.WithLabelValues(metrics.LabelResultError).Inc()
-		metrics.ClientErrors.WithLabelValues(metrics.LabelMethodFetchTask).Inc()
+		metrics.ClientErrors.WithLabelValues(metrics.LabelMethodFetchTask, connect.CodeOf(err).String()).Inc()
 		return nil, false
 	}
 	p.markHealthyPoll()

internal/app/poll/poller_test.go

@@ -227,6 +227,11 @@ func TestPollerReady(t *testing.T) {
 	assert.False(t, ready)
 	assert.Equal(t, "unable to poll Gitea", reason)
 
+	poller.shutdownPolling()
+	ready, reason = poller.Ready(time.Second)
+	assert.False(t, ready)
+	assert.Equal(t, "runner is shutting down", reason)
+
 	poller.unregistered.Store(true)
 	ready, reason = poller.Ready(time.Second)
 	assert.False(t, ready)

internal/app/run/runner.go

@@ -334,7 +334,7 @@ func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) error {
 		}
 
 		metrics.JobDuration.Observe(time.Since(start).Seconds())
-		metrics.JobsTotal.WithLabelValues(metrics.ResultToStatusLabel(reporter.Result())).Inc()
+		metrics.JobsTotal.WithLabelValues(metrics.JobStatus(ctx, reporter.Result())).Inc()
 		endJob(reporter.Result())
 	}()
 	reporter.RunDaemon()

internal/pkg/metrics/metrics.go

@@ -4,6 +4,8 @@
 package metrics
 
 import (
+	"context"
+	"errors"
 	"sync"
 	"time"
 
@@ -20,6 +22,7 @@ const Namespace = "gitea_runner"
 //
 // LabelResult* values are used on metrics with label key "result" (RPC outcomes).
 // LabelStatus* values are used on metrics with label key "status" (job outcomes).
+// LabelState* values are used on metrics with label key "state" (runner state).
 const (
 	LabelResultTask    = "task"
 	LabelResultEmpty   = "empty"
@@ -35,6 +38,11 @@ const (
 	LabelStatusCancelled = "cancelled"
 	LabelStatusSkipped   = "skipped"
 	LabelStatusUnknown   = "unknown"
+	LabelStatusTimeout   = "timeout"
+
+	LabelStateBusy        = "busy"
+	LabelStateIdle        = "idle"
+	LabelStateUnavailable = "unavailable"
 )
 
 // rpcDurationBuckets covers the expected latency range for short-running
@@ -57,6 +65,15 @@ func ResultToStatusLabel(r runnerv1.Result) string {
 	}
 }
 
+// JobStatus reports an unsuccessful job past its deadline as timeout, like the job's trace span.
+func JobStatus(ctx context.Context, r runnerv1.Result) string {
+	status := ResultToStatusLabel(r)
+	if status != LabelStatusSuccess && status != LabelStatusSkipped && errors.Is(ctx.Err(), context.DeadlineExceeded) {
+		return LabelStatusTimeout
+	}
+	return status
+}
+
 var (
 	RunnerInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: Namespace,
@@ -96,7 +113,7 @@ var (
 		Namespace: Namespace,
 		Subsystem: "job",
 		Name:      "total",
-		Help:      "Total jobs processed by status (success, failure, cancelled, skipped, unknown).",
+		Help:      "Total jobs processed by status (success, failure, cancelled, skipped, timeout, unknown).",
 	}, []string{"status"})
 
 	JobDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
@@ -148,8 +165,8 @@ var (
 		Namespace: Namespace,
 		Subsystem: "client",
 		Name:      "errors_total",
-		Help:      "Total client RPC errors by method.",
+		Help:      "Total client RPC errors by method and connect code.",
-	}, []string{"method"})
+	}, []string{"method", "code"})
 )
 
 // Registry is the custom Prometheus registry used by the runner.
@@ -220,3 +237,28 @@ func register(collector prometheus.Collector) {
 	Registry.Unregister(collector)
 	Registry.MustRegister(collector)
 }
+
+// RegisterStateFunc registers gitea_runner_state, 1 for the state stateFn reports and 0 for the others.
+func RegisterStateFunc(stateFn func() string) {
+	register(stateCollector{prometheus.NewDesc(prometheus.BuildFQName(Namespace, "", "state"),
+		"Runner state (busy, idle, unavailable). 1 for the current state.", []string{"state"}, nil), stateFn})
+}
+
+// stateCollector reads the state once per collection, so a scrape never reports two states.
+type stateCollector struct {
+	desc  *prometheus.Desc
+	state func() string
+}
+
+func (c stateCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.desc }
+
+func (c stateCollector) Collect(ch chan<- prometheus.Metric) {
+	current := c.state()
+	for _, state := range []string{LabelStateBusy, LabelStateIdle, LabelStateUnavailable} {
+		value := 0.0
+		if state == current {
+			value = 1
+		}
+		ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, value, state)
+	}
+}

internal/pkg/metrics/metrics_test.go

@@ -34,6 +34,11 @@ func TestResultToStatusLabel(t *testing.T) {
 			require.Equal(t, tt.want, ResultToStatusLabel(tt.result))
 		})
 	}
+	expired, cancel := context.WithDeadline(t.Context(), time.Time{})
+	defer cancel()
+	require.Equal(t, LabelStatusTimeout, JobStatus(expired, runnerv1.Result_RESULT_FAILURE))
+	require.Equal(t, LabelStatusSuccess, JobStatus(expired, runnerv1.Result_RESULT_SUCCESS))
+	require.Equal(t, LabelStatusCancelled, JobStatus(t.Context(), runnerv1.Result_RESULT_CANCELLED))
 }
 
 func TestInitAndDynamicMetricRegistration(t *testing.T) {
@@ -50,6 +55,8 @@ func TestInitAndDynamicMetricRegistration(t *testing.T) {
 	RunnerInfo.WithLabelValues("test", "runner").Set(1)
 	RegisterUptimeFunc(time.Now().Add(-time.Second))
 	RegisterRunningJobsFunc(func() int64 { return 2 }, 4)
+	RegisterStateFunc(func() string { return LabelStateIdle })
+	RegisterStateFunc(func() string { return LabelStateBusy })
 
 	metrics, err := Registry.Gather()
 	require.NoError(t, err)
@@ -58,6 +65,15 @@ func TestInitAndDynamicMetricRegistration(t *testing.T) {
 	require.True(t, hasMetric(metrics, "gitea_runner_uptime_seconds"))
 	require.True(t, hasMetric(metrics, "gitea_runner_job_running"))
 	require.True(t, hasMetric(metrics, "gitea_runner_job_capacity_utilization_ratio"))
+	states := map[string]float64{}
+	for _, mf := range metrics {
+		if mf.GetName() == "gitea_runner_state" {
+			for _, m := range mf.GetMetric() {
+				states[m.GetLabel()[0].GetValue()] = m.GetGauge().GetValue()
+			}
+		}
+	}
+	require.Equal(t, map[string]float64{LabelStateBusy: 1, LabelStateIdle: 0, LabelStateUnavailable: 0}, states)
 }
 
 func TestRegisterRunningJobsFuncZeroCapacity(t *testing.T) {

internal/pkg/report/reporter.go

@@ -624,7 +624,7 @@ func (r *Reporter) reportLog(noMore bool) (bool, error) {
 	metrics.ReportLogDuration.Observe(time.Since(start).Seconds())
 	if err != nil {
 		metrics.ReportLogTotal.WithLabelValues(metrics.LabelResultError).Inc()
-		metrics.ClientErrors.WithLabelValues(metrics.LabelMethodUpdateLog).Inc()
+		metrics.ClientErrors.WithLabelValues(metrics.LabelMethodUpdateLog, connect.CodeOf(err).String()).Inc()
 		return false, err
 	}
 	metrics.ReportLogTotal.WithLabelValues(metrics.LabelResultSuccess).Inc()
@@ -701,7 +701,7 @@ func (r *Reporter) ReportState(reportResult bool) error {
 	metrics.ReportStateDuration.Observe(time.Since(start).Seconds())
 	if err != nil {
 		metrics.ReportStateTotal.WithLabelValues(metrics.LabelResultError).Inc()
-		metrics.ClientErrors.WithLabelValues(metrics.LabelMethodUpdateTask).Inc()
+		metrics.ClientErrors.WithLabelValues(metrics.LabelMethodUpdateTask, connect.CodeOf(err).String()).Inc()
 		r.stateMu.Lock()
 		r.stateChanged = true
 		r.stateMu.Unlock()

internal/pkg/telemetry/metrics.go

@@ -7,12 +7,18 @@ import (
 	"cmp"
 	"context"
 	"errors"
+	"math"
+	"runtime"
 	"strings"
 	"time"
 
+	"gitea.com/gitea/runner/internal/pkg/metrics"
+
+	"gitea.dev/actionslib/runner/v1/runnerv1connect"
 	"github.com/prometheus/client_golang/prometheus"
 	dto "github.com/prometheus/client_model/go"
 	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/sdk/resource"
 	commonpb "go.opentelemetry.io/proto/otlp/common/v1"
 	metricpb "go.opentelemetry.io/proto/otlp/metrics/v1"
@@ -26,7 +32,11 @@ const cumulative = metricpb.AggregationTemporality_AGGREGATION_TEMPORALITY_CUMUL
 func pushMetrics(exporter *client, res *resource.Resource, gatherer prometheus.Gatherer) func(context.Context) error {
 	attrs := make([]*commonpb.KeyValue, 0, res.Len())
 	for _, kv := range res.Attributes() {
-		attrs = append(attrs, keyValue(string(kv.Key), kv.Value.String()))
+		attr := keyValue(string(kv.Key), kv.Value.String())
+		if kv.Value.Type() == attribute.INT64 {
+			attr.Value.Value = &commonpb.AnyValue_IntValue{IntValue: kv.Value.AsInt64()}
+		}
+		attrs = append(attrs, attr)
 	}
 	interval, timeout := millis("OTEL_METRIC_EXPORT_INTERVAL", time.Minute, time.Minute), millis("OTEL_METRIC_EXPORT_TIMEOUT", 30*time.Second, noLimit)
 	push := func(ctx context.Context) error {
@@ -70,39 +80,132 @@ var ucum = map[string]string{
 	"meters": "m", "volts": "V", "amperes": "A", "joules": "J", "watts": "W", "grams": "g", "celsius": "Cel", "hertz": "Hz", "percent": "%",
 }
 
-// convert follows https://opentelemetry.io/docs/specs/otel/compatibility/prometheus_and_openmetrics/#prometheus-metric-points-to-otlp
+type otlpMetric struct {
+	name, unit string
+	upDown     bool              // a gauge semconv defines as an UpDownCounter
+	labels     map[string]string // label → attribute
+	values     map[string]string // label value → attribute value
+	extra      []*commonpb.KeyValue
+	unlimited  float64 // gauge value meaning no limit, not sent
+}
+
+var rpcMethods = map[string]string{
+	metrics.LabelMethodFetchTask:  strings.TrimPrefix(runnerv1connect.RunnerServiceFetchTaskProcedure, "/"),
+	metrics.LabelMethodUpdateLog:  strings.TrimPrefix(runnerv1connect.RunnerServiceUpdateLogProcedure, "/"),
+	metrics.LabelMethodUpdateTask: strings.TrimPrefix(runnerv1connect.RunnerServiceUpdateTaskProcedure, "/"),
+}
+
+// openFiles maps process_open_fds, which client_golang fills with the handle count on Windows.
+func openFiles() otlpMetric {
+	if runtime.GOOS == "windows" {
+		return otlpMetric{name: "process.windows.handle.count", unit: "{handle}", upDown: true}
+	}
+	return otlpMetric{name: "process.unix.file_descriptor.count", unit: "{file_descriptor}", upDown: true}
+}
+
+// otlpMetrics maps to semconv names, unlisted metrics keep theirs and an empty name drops one the resource already carries.
+var otlpMetrics = map[string]otlpMetric{
+	"gitea_runner_info":                           {},
+	"go_info":                                     {},
+	"process_start_time_seconds":                  {},
+	"gitea_runner_capacity":                       {name: "gitea.runner.job.limit", unit: "{job}", upDown: true},
+	"gitea_runner_uptime_seconds":                 {name: "gitea.runner.uptime", unit: "s"},
+	"gitea_runner_job_running":                    {name: "gitea.runner.job.active", unit: "{job}", upDown: true},
+	"gitea_runner_job_capacity_utilization_ratio": {name: "gitea.runner.job.utilization", unit: "1"},
+	"gitea_runner_job_duration_seconds":           {name: "gitea.runner.job.duration", unit: "s"},
+	"gitea_runner_job_total": {
+		name: "gitea.runner.jobs", unit: "{job}",
+		labels: map[string]string{"status": "cicd.pipeline.result"},
+		values: map[string]string{metrics.LabelStatusCancelled: "cancellation", metrics.LabelStatusSkipped: "skip", metrics.LabelStatusUnknown: "error"},
+	},
+	"gitea_runner_poll_fetch_total": {
+		name: "gitea.runner.poll.fetches", unit: "{fetch}",
+		labels: map[string]string{"result": "gitea.runner.poll.result"},
+	},
+	"gitea_runner_poll_fetch_duration_seconds":   {name: "gitea.runner.poll.fetch.duration", unit: "s"},
+	"gitea_runner_poll_backoff_seconds":          {name: "gitea.runner.poll.backoff", unit: "s"},
+	"gitea_runner_report_log_duration_seconds":   {name: "gitea.runner.report.log.duration", unit: "s"},
+	"gitea_runner_report_state_duration_seconds": {name: "gitea.runner.report.state.duration", unit: "s"},
+	"gitea_runner_report_log_buffer_rows":        {name: "gitea.runner.report.log.buffer", unit: "{row}", upDown: true},
+	"gitea_runner_report_log_total": {
+		name: "gitea.runner.report.log.requests", unit: "{request}",
+		labels: map[string]string{"result": "gitea.runner.report.result"},
+	},
+	"gitea_runner_report_state_total": {
+		name: "gitea.runner.report.state.requests", unit: "{request}",
+		labels: map[string]string{"result": "gitea.runner.report.result"},
+	},
+	"gitea_runner_client_errors_total": {
+		name: "cicd.system.errors", unit: "{error}",
+		labels: map[string]string{"method": "rpc.method", "code": "error.type"}, values: rpcMethods,
+		extra: []*commonpb.KeyValue{keyValue("cicd.system.component", "runner")},
+	},
+	"gitea_runner_state": {
+		name: "cicd.worker.count", unit: "{worker}", upDown: true,
+		labels: map[string]string{"state": "cicd.worker.state"},
+		values: map[string]string{metrics.LabelStateIdle: "available", metrics.LabelStateUnavailable: "offline"},
+	},
+	"go_goroutines":                 {name: "go.goroutine.count", unit: "{goroutine}", upDown: true},
+	"go_memstats_alloc_bytes_total": {name: "go.memory.allocated", unit: "By"},
+	"go_memstats_next_gc_bytes":     {name: "go.memory.gc.goal", unit: "By", upDown: true},
+	"go_gc_gomemlimit_bytes":        {name: "go.memory.limit", unit: "By", upDown: true, unlimited: math.MaxInt64},
+	"go_gc_gogc_percent":            {name: "go.config.gogc", unit: "%", upDown: true},
+	"go_sched_gomaxprocs_threads":   {name: "go.processor.limit", unit: "{thread}", upDown: true},
+	"process_resident_memory_bytes": {name: "process.memory.usage", unit: "By", upDown: true},
+	"process_virtual_memory_bytes":  {name: "process.memory.virtual", unit: "By", upDown: true},
+	"process_open_fds":              openFiles(),
+}
+
+// convert follows https://opentelemetry.io/docs/specs/otel/compatibility/prometheus_and_openmetrics/#prometheus-metric-points-to-otlp except for otlpMetrics.
 func convert(families []*dto.MetricFamily) []*metricpb.Metric {
 	now := uint64(time.Now().UnixNano())
 	metrics := make([]*metricpb.Metric, 0, len(families))
+	byFamily := make(map[string]*dto.MetricFamily, len(families))
 	for _, family := range families {
+		byFamily[family.GetName()] = family
+		mapping, ok := otlpMetrics[family.GetName()]
+		if !ok {
+			mapping.name = family.GetName()
+		} else if mapping.name == "" {
+			continue
+		}
 		metric := &metricpb.Metric{
-			Name: family.GetName(), Description: family.GetHelp(), Unit: cmp.Or(ucum[family.GetUnit()], family.GetUnit()),
+			Name: mapping.name, Description: family.GetHelp(), Unit: cmp.Or(mapping.unit, ucum[family.GetUnit()], family.GetUnit()),
 			Metadata: []*commonpb.KeyValue{keyValue("prometheus.type", strings.ToLower(family.GetType().String()))},
 		}
 		switch family.GetType() {
 		case dto.MetricType_COUNTER:
 			sum := &metricpb.Sum{AggregationTemporality: cumulative, IsMonotonic: true}
 			for _, series := range family.GetMetric() {
-				sum.DataPoints = append(sum.DataPoints, number(series, unixNano(series.GetCounter().GetCreatedTimestamp()), now, series.GetCounter().GetValue()))
+				sum.DataPoints = append(sum.DataPoints, number(labels(series, mapping), unixNano(series.GetCounter().GetCreatedTimestamp()), now, series.GetCounter().GetValue()))
 			}
 			metric.Data = &metricpb.Metric_Sum{Sum: sum}
 		case dto.MetricType_GAUGE:
-			gauge := &metricpb.Gauge{}
+			var points []*metricpb.NumberDataPoint
 			for _, series := range family.GetMetric() {
-				gauge.DataPoints = append(gauge.DataPoints, number(series, 0, now, series.GetGauge().GetValue()))
+				if value := series.GetGauge().GetValue(); mapping.unlimited == 0 || value != mapping.unlimited {
+					points = append(points, number(labels(series, mapping), 0, now, value))
+				}
+			}
+			if len(points) == 0 {
+				continue
+			}
+			if mapping.upDown {
+				metric.Data = &metricpb.Metric_Sum{Sum: &metricpb.Sum{AggregationTemporality: cumulative, DataPoints: points}}
+			} else {
+				metric.Data = &metricpb.Metric_Gauge{Gauge: &metricpb.Gauge{DataPoints: points}}
 			}
-			metric.Data = &metricpb.Metric_Gauge{Gauge: gauge}
 		case dto.MetricType_HISTOGRAM:
 			histogram := &metricpb.Histogram{AggregationTemporality: cumulative}
 			for _, series := range family.GetMetric() {
-				histogram.DataPoints = append(histogram.DataPoints, histogramPoint(series, now))
+				histogram.DataPoints = append(histogram.DataPoints, histogramPoint(series, labels(series, mapping), now))
 			}
 			metric.Data = &metricpb.Metric_Histogram{Histogram: histogram}
 		case dto.MetricType_SUMMARY:
 			summary := &metricpb.Summary{}
 			for _, series := range family.GetMetric() {
 				point := &metricpb.SummaryDataPoint{
-					Attributes: labels(series), StartTimeUnixNano: unixNano(series.GetSummary().GetCreatedTimestamp()), TimeUnixNano: now,
+					Attributes: labels(series, mapping), StartTimeUnixNano: unixNano(series.GetSummary().GetCreatedTimestamp()), TimeUnixNano: now,
 					Count: series.GetSummary().GetSampleCount(), Sum: series.GetSummary().GetSampleSum(),
 				}
 				for _, quantile := range series.GetSummary().GetQuantile() {
@@ -116,11 +219,44 @@ func convert(families []*dto.MetricFamily) []*metricpb.Metric {
 		}
 		metrics = append(metrics, metric)
 	}
+	return append(metrics, derived(byFamily, now)...)
+}
+
+// derived adds the semconv Go metrics that are computed from other metrics rather than renamed.
+func derived(families map[string]*dto.MetricFamily, now uint64) []*metricpb.Metric {
+	first := func(name string) *dto.Metric {
+		if series := families[name].GetMetric(); len(series) > 0 {
+			return series[0]
+		}
+		return nil
+	}
+	gauge := func(name string) (float64, bool) {
+		series := first(name)
+		return series.GetGauge().GetValue(), series != nil
+	}
+	sum := func(name, unit, description string, monotonic bool, points ...*metricpb.NumberDataPoint) *metricpb.Metric {
+		return &metricpb.Metric{Name: name, Unit: unit, Description: description, Data: &metricpb.Metric_Sum{Sum: &metricpb.Sum{
+			AggregationTemporality: cumulative, IsMonotonic: monotonic, DataPoints: points,
+		}}}
+	}
+	var metrics []*metricpb.Metric
+	total, ok1 := gauge("go_memstats_sys_bytes")
+	released, ok2 := gauge("go_memstats_heap_released_bytes")
+	stack, ok3 := gauge("go_memstats_stack_inuse_bytes")
+	if ok1 && ok2 && ok3 {
+		metrics = append(metrics, sum("go.memory.used", "By", "Memory used by the Go runtime.", false,
+			number([]*commonpb.KeyValue{keyValue("go.memory.type", "stack")}, 0, now, stack),
+			number([]*commonpb.KeyValue{keyValue("go.memory.type", "other")}, 0, now, total-released-stack)))
+	}
+	if gc := first("go_gc_duration_seconds"); gc != nil {
+		metrics = append(metrics, sum("go.memory.gc.cycles", "{gc_cycle}", "Number of completed GC cycles.", true,
+			number(nil, unixNano(gc.GetSummary().GetCreatedTimestamp()), now, float64(gc.GetSummary().GetSampleCount()))))
+	}
 	return metrics
 }
 
-func number(series *dto.Metric, start, now uint64, value float64) *metricpb.NumberDataPoint {
+func number(attrs []*commonpb.KeyValue, start, now uint64, value float64) *metricpb.NumberDataPoint {
-	return &metricpb.NumberDataPoint{Attributes: labels(series), StartTimeUnixNano: start, TimeUnixNano: now, Value: &metricpb.NumberDataPoint_AsDouble{AsDouble: value}}
+	return &metricpb.NumberDataPoint{Attributes: attrs, StartTimeUnixNano: start, TimeUnixNano: now, Value: &metricpb.NumberDataPoint_AsDouble{AsDouble: value}}
 }
 
 // unixNano maps a nil timestamp to 0, leaving the start time unset as the spec asks.
@@ -128,10 +264,10 @@ func unixNano(timestamp *timestamppb.Timestamp) uint64 {
 	return uint64(timestamp.AsTime().UnixNano())
 }
 
-func histogramPoint(series *dto.Metric, now uint64) *metricpb.HistogramDataPoint {
+func histogramPoint(series *dto.Metric, attrs []*commonpb.KeyValue, now uint64) *metricpb.HistogramDataPoint {
 	histogram := series.GetHistogram()
 	point := &metricpb.HistogramDataPoint{
-		Attributes: labels(series), StartTimeUnixNano: unixNano(histogram.GetCreatedTimestamp()), TimeUnixNano: now,
+		Attributes: attrs, StartTimeUnixNano: unixNano(histogram.GetCreatedTimestamp()), TimeUnixNano: now,
 		Count: histogram.GetSampleCount(), Sum: histogram.SampleSum,
 		BucketCounts: make([]uint64, len(histogram.GetBucket())+1),
 	}
@@ -144,12 +280,12 @@ func histogramPoint(series *dto.Metric, now uint64) *metricpb.HistogramDataPoint
 	return point
 }
 
-func labels(series *dto.Metric) []*commonpb.KeyValue {
+func labels(series *dto.Metric, mapping otlpMetric) []*commonpb.KeyValue {
-	attrs := make([]*commonpb.KeyValue, 0, len(series.GetLabel()))
+	attrs := make([]*commonpb.KeyValue, 0, len(series.GetLabel())+len(mapping.extra))
 	for _, label := range series.GetLabel() {
-		attrs = append(attrs, keyValue(label.GetName(), label.GetValue()))
+		attrs = append(attrs, keyValue(cmp.Or(mapping.labels[label.GetName()], label.GetName()), cmp.Or(mapping.values[label.GetValue()], label.GetValue())))
 	}
-	return attrs
+	return append(attrs, mapping.extra...)
 }
 
 func keyValue(key, value string) *commonpb.KeyValue {

internal/pkg/telemetry/telemetry.go

@@ -45,6 +45,8 @@ import (
 
 const scope = "gitea.com/gitea/runner"
 
+var processCreation = time.Now() // package init, within milliseconds of the process start
+
 type taskRunKey struct{}
 
 type taskRun struct {
@@ -102,12 +104,13 @@ func Setup(ctx context.Context, uuid, name string, gatherer prometheus.Gatherer)
 	if len(clients) == 0 {
 		return noop, nil
 	}
-	res, err := resource.New(ctx, resource.WithTelemetrySDK(), resource.WithAttributes(
+	res, err := resource.New(ctx, resource.WithTelemetrySDK(), resource.WithProcessPID(), resource.WithProcessRuntimeName(), resource.WithProcessRuntimeVersion(), resource.WithAttributes(
 		semconv.ServiceName("gitea-runner"),
 		semconv.ServiceVersion(ver.Version()),
 		semconv.ServiceInstanceID(uuid),
 		semconv.CICDWorkerID(uuid),
 		semconv.CICDWorkerName(name),
+		semconv.ProcessCreationTime(processCreation.UTC().Format(time.RFC3339Nano)),
 	), resource.WithFromEnv())
 	if err != nil {
 		return noop, err

internal/pkg/telemetry/telemetry_test.go

@@ -12,6 +12,8 @@ import (
 	"encoding/pem"
 	"fmt"
 	"io"
+	"maps"
+	"math"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -154,9 +156,10 @@ func TestExportRetriesAndMatchesOTLP(t *testing.T) {
 	assert.Equal(t, "op", tracesData.GetResourceSpans()[0].GetScopeSpans()[0].GetSpans()[0].GetName())
 
 	resourceMetrics := metricsData.GetResourceMetrics()[0]
-	assert.True(t, slices.ContainsFunc(resourceMetrics.GetResource().GetAttributes(), func(kv *commonpb.KeyValue) bool {
+	pid := &commonpb.KeyValue{Key: "process.pid", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_IntValue{IntValue: int64(os.Getpid())}}}
-		return proto.Equal(kv, keyValue("service.name", "gitea-runner"))
+	for _, want := range []*commonpb.KeyValue{keyValue("service.name", "gitea-runner"), keyValue("process.creation.time", processCreation.UTC().Format(time.RFC3339Nano)), pid} {
-	}))
+		assert.True(t, slices.ContainsFunc(resourceMetrics.GetResource().GetAttributes(), func(kv *commonpb.KeyValue) bool { return proto.Equal(kv, want) }), want.GetKey())
+	}
 	assert.InDelta(t, 2, resourceMetrics.GetScopeMetrics()[0].GetMetrics()[0].GetGauge().GetDataPoints()[0].GetAsDouble(), 0)
 }
 
@@ -165,31 +168,76 @@ func TestConvert(t *testing.T) {
 	counter := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "jobs_total", Help: "Jobs."}, []string{"status"})
 	histogram := prometheus.NewHistogram(prometheus.HistogramOpts{Name: "duration_seconds", Unit: "seconds", Buckets: []float64{1, 2}})
 	summary := prometheus.NewSummary(prometheus.SummaryOpts{Name: "gc_seconds", Objectives: map[float64]float64{0.5: 0.05}})
-	registry.MustRegister(counter, histogram, summary)
+	jobs := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "gitea_runner_job_total"}, []string{"status"})
+	running := prometheus.NewGauge(prometheus.GaugeOpts{Name: "gitea_runner_job_running"})
+	gauges := map[string]float64{
+		"go_info": 1, "go_gc_gomemlimit_bytes": math.MaxInt64,
+		"go_memstats_sys_bytes": 100, "go_memstats_heap_released_bytes": 30, "go_memstats_stack_inuse_bytes": 20,
+	}
+	for name, value := range gauges {
+		registry.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name: name}, func() float64 { return value }))
+	}
+	state := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "gitea_runner_state"}, []string{"state"})
+	clientErrors := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "gitea_runner_client_errors_total"}, []string{"method", "code"})
+	registry.MustRegister(counter, histogram, summary, jobs, running, state, clientErrors)
 	counter.WithLabelValues("success").Add(3)
 	for _, value := range []float64{0.5, 1.5, 3} {
 		histogram.Observe(value)
 	}
 	summary.Observe(4)
+	jobs.WithLabelValues("cancelled").Inc()
+	running.Set(2)
+	state.WithLabelValues("idle").Set(1)
+	state.WithLabelValues("unavailable").Set(0)
+	clientErrors.WithLabelValues("FetchTask", "unavailable").Inc()
 	families, err := registry.Gather()
 	require.NoError(t, err)
 
-	exported := convert(families)
+	exported := map[string]*metricpb.Metric{}
-	assert.Equal(t, "s", exported[0].GetUnit())
+	for _, metric := range convert(families) {
-	buckets := exported[0].GetHistogram().GetDataPoints()[0]
+		exported[metric.GetName()] = metric
+	}
+	assert.ElementsMatch(t, []string{
+		"duration_seconds", "gc_seconds", "jobs_total", "gitea.runner.jobs", "gitea.runner.job.active", "cicd.system.errors",
+		"go_memstats_sys_bytes", "go_memstats_heap_released_bytes", "go_memstats_stack_inuse_bytes", "go.memory.used", "cicd.worker.count",
+	}, slices.Collect(maps.Keys(exported)))
+	assert.Equal(t, "s", exported["duration_seconds"].GetUnit())
+	buckets := exported["duration_seconds"].GetHistogram().GetDataPoints()[0]
 	assert.Equal(t, []float64{1, 2}, buckets.GetExplicitBounds())
 	assert.Equal(t, []uint64{1, 1, 1}, buckets.GetBucketCounts())
 	assert.Equal(t, uint64(3), buckets.GetCount())
 	assert.InDelta(t, 5, buckets.GetSum(), 0)
-	assert.InDelta(t, 4, exported[1].GetSummary().GetDataPoints()[0].GetQuantileValues()[0].GetValue(), 0)
+	assert.InDelta(t, 4, exported["gc_seconds"].GetSummary().GetDataPoints()[0].GetQuantileValues()[0].GetValue(), 0)
-	sum := exported[2].GetSum()
+	sum := exported["jobs_total"].GetSum()
-	assert.Equal(t, "Jobs.", exported[2].GetDescription())
+	assert.Equal(t, "Jobs.", exported["jobs_total"].GetDescription())
-	assert.True(t, proto.Equal(keyValue("prometheus.type", "counter"), exported[2].GetMetadata()[0]))
+	assert.True(t, proto.Equal(keyValue("prometheus.type", "counter"), exported["jobs_total"].GetMetadata()[0]))
 	assert.NotZero(t, sum.GetDataPoints()[0].GetStartTimeUnixNano())
 	assert.True(t, sum.GetIsMonotonic())
 	assert.Equal(t, cumulative, sum.GetAggregationTemporality())
 	assert.InDelta(t, 3, sum.GetDataPoints()[0].GetAsDouble(), 0)
 	assert.True(t, proto.Equal(keyValue("status", "success"), sum.GetDataPoints()[0].GetAttributes()[0]))
+
+	assert.True(t, proto.Equal(keyValue("cicd.pipeline.result", "cancellation"), exported["gitea.runner.jobs"].GetSum().GetDataPoints()[0].GetAttributes()[0]))
+	active := exported["gitea.runner.job.active"]
+	assert.Equal(t, "{job}", active.GetUnit())
+	assert.False(t, active.GetSum().GetIsMonotonic())
+	assert.InDelta(t, 2, active.GetSum().GetDataPoints()[0].GetAsDouble(), 0)
+	attributes := func(points []*metricpb.NumberDataPoint, key string) map[string]float64 {
+		values := map[string]float64{}
+		for _, point := range points {
+			for _, kv := range point.GetAttributes() {
+				if kv.GetKey() == key {
+					values[kv.GetValue().GetStringValue()] = point.GetAsDouble()
+				}
+			}
+		}
+		return values
+	}
+	assert.Equal(t, map[string]float64{"stack": 20, "other": 50}, attributes(exported["go.memory.used"].GetSum().GetDataPoints(), "go.memory.type"))
+	assert.Equal(t, map[string]float64{"available": 1, "offline": 0}, attributes(exported["cicd.worker.count"].GetSum().GetDataPoints(), "cicd.worker.state"))
+	assert.True(t, proto.Equal(&commonpb.KeyValueList{Values: []*commonpb.KeyValue{
+		keyValue("error.type", "unavailable"), keyValue("rpc.method", "runner.v1.RunnerService/FetchTask"), keyValue("cicd.system.component", "runner"),
+	}}, &commonpb.KeyValueList{Values: exported["cicd.system.errors"].GetSum().GetDataPoints()[0].GetAttributes()}))
 }
 
 func TestExportRefusesRedirects(t *testing.T) {