enhance!: send OTLP metrics under OpenTelemetry semantic conventions, add runner state (#1282)

Sends the OTLP metrics under [OpenTelemetry semantic convention](https://opentelemetry.io/docs/specs/semconv/) names where the runner's data matches the definition, like `cicd.worker.count`, `cicd.system.errors` and the `go.*` and `process.*` runtime metrics, and under `gitea.runner.*` otherwise. OTel dashboards then work without remapping, as they already do for the traces from https://gitea.com/gitea/runner/pulls/1206.

Adds a `gitea_runner_state` gauge (`busy`, `idle`, `unavailable`), and `/readyz` now fails once the runner shuts down.

**Breaking**

1. OTLP metric names, units, types and attribute values change, and the info and start time metrics move to resource attributes.
1. Prometheus `gitea_runner_client_errors_total` gains a `code` label, and `gitea_runner_job_total` counts jobs stopped by `runner.timeout` as `timeout` instead of `failure`.

---------

Co-authored-by: silverwind <[email protected]>
Co-authored-by: silverwind <[email protected]>
Co-authored-by: bircni <[email protected]>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1282
Reviewed-by: silverwind <[email protected]>
Co-authored-by: wmTJc9IK0Q <[email protected]>

internal/pkg/telemetry/telemetry.go

@@ -45,6 +45,8 @@ import (
 
 const scope = "gitea.com/gitea/runner"
 
+var processCreation = time.Now() // package init, within milliseconds of the process start
+
 type taskRunKey struct{}
 
 type taskRun struct {
@@ -102,12 +104,13 @@ func Setup(ctx context.Context, uuid, name string, gatherer prometheus.Gatherer)
 	if len(clients) == 0 {
 		return noop, nil
 	}
-	res, err := resource.New(ctx, resource.WithTelemetrySDK(), resource.WithAttributes(
+	res, err := resource.New(ctx, resource.WithTelemetrySDK(), resource.WithProcessPID(), resource.WithProcessRuntimeName(), resource.WithProcessRuntimeVersion(), resource.WithAttributes(
 		semconv.ServiceName("gitea-runner"),
 		semconv.ServiceVersion(ver.Version()),
 		semconv.ServiceInstanceID(uuid),
 		semconv.CICDWorkerID(uuid),
 		semconv.CICDWorkerName(name),
+		semconv.ProcessCreationTime(processCreation.UTC().Format(time.RFC3339Nano)),
 	), resource.WithFromEnv())
 	if err != nil {
 		return noop, err