feat: OpenTelemetry tracing (#1206)

Adds OpenTelemetry tracing, configured through the standard `OTEL_*` environment variables and off until an OTLP endpoint is set.

![image.png](https://gitea.com/attachments/c4b49cdf-a32d-403c-94e9-30cd06ee6477)

Each job becomes a trace with a span per step, so operators can see where CI time goes in their existing observability backend. Jobs receive only `TRACEPARENT`, so instrumented tools inside them join the trace without seeing the collector's credentials. Metrics stay on the Prometheus endpoint, which a collector can scrape. See [docs/telemetry.md](https://gitea.com/wmTJc9IK0Q/runner/src/branch/otel/docs/telemetry.md) for setup and security notes.

Also keeps `GITHUB_SHA` and `GITHUB_REF` fixed for a whole run when HEAD moves during `exec` runs and the act tests.

[image.png](https://gitea.com/attachments/c4b49cdf-a32d-403c-94e9-30cd06ee6477)

[image.png](https://gitea.com/attachments/c4b49cdf-a32d-403c-94e9-30cd06ee6477)

---------

Co-authored-by: silverwind <[email protected]>
Co-authored-by: silverwind <[email protected]>
Co-authored-by: bircni <[email protected]>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1206
Reviewed-by: bircni <[email protected]>
Reviewed-by: silverwind <[email protected]>
Co-authored-by: wmTJc9IK0Q <[email protected]>
This commit is contained in:
authored and silverwind committed 2026-09-24 15:57:18 +00:00
1 parent 7ad12038d1
commit 1952e5cdb6
15 files changed
+603 -28

No files matched your search

+4
View File
@@ -384,6 +384,10 @@ Both hooks are synchronous and block the job while they run. Either one exiting
See **[docs/job-hooks.md](docs/job-hooks.md)** for the execution order, environment, and platform notes.
#### OpenTelemetry (`OTEL_*`)
Set `OTEL_EXPORTER_OTLP_ENDPOINT` in the runner's environment to send a trace of every job to an OpenTelemetry collector. See **[docs/telemetry.md](docs/telemetry.md)**.
#### Local job logs (`log.job.dir`)
Set `log.job.dir` to a path and the runner writes a copy of every task's log there as `<start time>-task-<id>.log`: the rows exactly as Gitea received them, with the same secrets masked and the job's result on the last line. Off by default, and what Gitea shows does not change.
+26 -2
View File
@@ -12,6 +12,7 @@ import (
"context"
"fmt"
"strings"
"sync"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/git"
@@ -25,6 +26,26 @@ var (
findGithubRepo = git.FindGithubRepo
)
type pinnedKey struct{}
// WithPinnedCheckout keeps a run's sha and ref fixed when HEAD moves.
func WithPinnedCheckout(ctx context.Context) context.Context {
if ctx.Value(pinnedKey{}) != nil {
return ctx
}
return context.WithValue(ctx, pinnedKey{}, &sync.Map{})
}
func pinned(ctx context.Context, key string, lookup func() (string, error)) (string, error) {
cache, ok := ctx.Value(pinnedKey{}).(*sync.Map)
if !ok {
return lookup()
}
cached, _ := cache.LoadOrStore(key, sync.OnceValues(lookup))
lookupOnce, _ := cached.(func() (string, error))
return lookupOnce()
}
// SetRef resolves the ref of the context from its event payload, falling back
// to the ref checked out in repoPath.
func SetRef(ctx context.Context, ghc *model.GithubContext, repoPath string) {
@@ -51,7 +72,7 @@ func SetRef(ctx context.Context, ghc *model.GithubContext, repoPath string) {
}
if ghc.Ref == "" {
ref, err := findGitRef(ctx, repoPath)
ref, err := pinned(ctx, "ref:"+repoPath, func() (string, error) { return findGitRef(ctx, repoPath) })
if err != nil {
logger.Warningf("unable to get git ref: %v", err)
} else {
@@ -95,7 +116,10 @@ func SetSha(ctx context.Context, ghc *model.GithubContext, repoPath string) {
}
if ghc.Sha == "" {
_, sha, err := findGitRevision(ctx, repoPath)
sha, err := pinned(ctx, "sha:"+repoPath, func() (string, error) {
_, revision, err := findGitRevision(ctx, repoPath)
return revision, err
})
if err != nil {
logger.Warningf("unable to get git revision: %v", err)
} else {
+13
View File
@@ -214,4 +214,17 @@ func TestSetSha(t *testing.T) {
assert.Equal(t, table.sha, ghc.Sha)
})
}
t.Run("pinned checkout keeps the first sha and ref", func(t *testing.T) {
ctx := WithPinnedCheckout(context.Background())
for _, head := range []string{"first", "moved"} {
findGitRevision = func(context.Context, string) (string, string, error) { return "", head, nil }
findGitRef = func(context.Context, string) (string, error) { return "refs/heads/" + head, nil }
ghc := &model.GithubContext{Event: map[string]any{}}
SetSha(ctx, ghc, "/some/dir")
SetRef(ctx, ghc, "/some/dir")
assert.Equal(t, "first", ghc.Sha)
assert.Equal(t, "refs/heads/first", ghc.Ref)
}
})
}
+4 -1
View File
@@ -14,6 +14,7 @@ import (
"gitea.dev/actionslib/pkg/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/trace"
"go.yaml.in/yaml/v4"
)
@@ -249,7 +250,8 @@ func TestPreStepFailureAffectsMainStepIfStatus(t *testing.T) {
// SetJobError dereferences a nil map and panics. The post context must therefore be detached
// from cancellation (so the steps run) yet still carry a usable error container.
func TestPostStepsContextCancelledIsUsableForFailingStep(t *testing.T) {
cancelled, cancel := context.WithCancel(common.WithJobErrorContainer(context.Background()))
jobSpan := trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{1}})
cancelled, cancel := context.WithCancel(trace.ContextWithSpanContext(common.WithJobErrorContainer(context.Background()), jobSpan))
cancel()
require.ErrorIs(t, cancelled.Err(), context.Canceled)
@@ -258,6 +260,7 @@ func TestPostStepsContextCancelledIsUsableForFailingStep(t *testing.T) {
// Detached from cancellation, so the post steps actually run.
require.NoError(t, postCtx.Err(), "post context must not be cancelled")
assert.Equal(t, jobSpan, trace.SpanContextFromContext(postCtx), "post steps must stay in the job's trace")
// A failing post step records its error instead of panicking.
require.NotPanics(t, func() {
+3 -6
View File
@@ -281,8 +281,8 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo
// finished main-pipeline context. Cleanup has to run even when the run was interrupted, so the
// returned context always carries a fresh bounded deadline and is never itself cancelled.
//
// - context.Canceled (server cancel): detach from the cancelled context via a fresh root so
// the post steps can run.
// - context.Canceled (server cancel): detach with WithoutCancel under a fresh job-error
// container, so a failing post step cannot turn the cancellation into a failure.
// - context.DeadlineExceeded (job timeout): detach the deadline with WithoutCancel, which
// keeps the original values — including the job-error container — so the timeout failure and
// any post-step error are preserved and the job is still reported as failed.
@@ -290,10 +290,7 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo
func postStepsContext(ctx context.Context) (context.Context, context.CancelFunc) {
switch ctx.Err() {
case context.Canceled:
// The cancelled context is abandoned for a fresh root, which drops the job-error
// container installed at the job root. Re-attach a fresh one so a failing post step
// records its error via SetJobError instead of panicking on a nil container.
return context.WithTimeout(common.WithJobErrorContainer(common.WithLogger(context.Background(), common.Logger(ctx))), 5*time.Minute)
return context.WithTimeout(common.WithJobErrorContainer(context.WithoutCancel(ctx)), 5*time.Minute)
case context.DeadlineExceeded:
return context.WithTimeout(context.WithoutCancel(ctx), 5*time.Minute)
default:
+3 -1
View File
@@ -14,6 +14,7 @@ import (
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/container"
"gitea.com/gitea/runner/act/ghcontext"
"gitea.dev/actionslib/pkg/model"
docker_container "github.com/moby/moby/api/types/container"
@@ -321,7 +322,8 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor {
})
}
return common.NewPipelineExecutor(stagePipeline...).Then(handleFailure(plan))
executor := common.NewPipelineExecutor(stagePipeline...).Then(handleFailure(plan))
return func(ctx context.Context) error { return executor(ghcontext.WithPinnedCheckout(ctx)) }
}
func handleFailure(plan *model.Plan) common.Executor {
+9 -5
View File
@@ -15,6 +15,7 @@ import (
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/container"
"gitea.com/gitea/runner/internal/pkg/telemetry"
"gitea.dev/actionslib/pkg/exprparser"
"gitea.dev/actionslib/pkg/model"
@@ -65,24 +66,27 @@ func processRunnerEnvFileCommand(ctx context.Context, fileName string, rc *RunCo
}
func runStepExecutor(step step, stage stepStage, executor common.Executor) common.Executor {
return func(ctx context.Context) error {
return func(ctx context.Context) (err error) {
logger := common.Logger(ctx)
rc := step.getRunContext()
stepModel := step.getStepModel()
ifExpression := step.getIfExpression(ctx, stage)
rc.CurrentStep = stepModel.ID
stepResult := &model.StepResult{
Outcome: model.StepStatusSuccess,
Conclusion: model.StepStatusSuccess,
Outputs: make(map[string]string),
}
ctx, endStep := telemetry.StartStep(ctx, stepModel, stage.String())
defer func() { endStep(err, stepResult) }()
ifExpression := step.getIfExpression(ctx, stage)
rc.CurrentStep = stepModel.ID
if stage == stepStageMain {
rc.StepResults[rc.CurrentStep] = stepResult
}
err := setupEnv(ctx, step)
err = setupEnv(ctx, step)
var runStep bool
if err == nil {
runStep, err = isStepEnabled(ctx, ifExpression, step, stage)
+29
View File
@@ -13,13 +13,19 @@ import (
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/container"
"gitea.com/gitea/runner/internal/pkg/telemetry"
"gitea.dev/actionslib/pkg/model"
runnerv1 "gitea.dev/actionslib/runner/v1"
log "github.com/sirupsen/logrus"
logrustest "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
semconv "go.opentelemetry.io/otel/semconv/v1.43.0"
tracenoop "go.opentelemetry.io/otel/trace/noop"
yaml "go.yaml.in/yaml/v4"
)
@@ -547,6 +553,29 @@ func TestRunStepExecutorParity(t *testing.T) {
assert.Equal(t, "child", childRC.IntraActionState[child.Step.ID]["saved"])
})
t.Run("stages and composite steps are traced under Gitea's step names", func(t *testing.T) {
spans := tracetest.NewSpanRecorder()
telemetry.SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(spans)))
t.Cleanup(func() { telemetry.SetTracerProvider(tracenoop.NewTracerProvider()) })
ctx, _ := telemetry.StartJob(t.Context(), &runnerv1.Task{Id: 7})
outer := newStep(t, &model.Step{ID: "outer", Uses: "actions/cache@v4", Number: 3})
child := newStep(t, &model.Step{ID: "child", Name: "Restore", Number: 1})
skipped := newStep(t, &model.Step{ID: "skipped", Name: "Deploy", Number: 4, If: yaml.Node{Value: "false"}})
broken := newStep(t, &model.Step{ID: "broken", Name: "Broken", Number: 5, If: yaml.Node{Value: badExpression}})
require.NoError(t, runStepExecutor(outer, stepStagePost, runStepExecutor(child, stepStageMain, noopExecutor))(ctx))
require.NoError(t, runStepExecutor(skipped, stepStageMain, noopExecutor)(ctx))
require.Error(t, runStepExecutor(broken, stepStageMain, noopExecutor)(ctx))
ended := spans.Ended()
require.Len(t, ended, 4)
assert.Equal(t, []string{"Restore", "Post Run actions/cache@v4", "Deploy", "Broken"}, []string{ended[0].Name(), ended[1].Name(), ended[2].Name(), ended[3].Name()})
assert.Equal(t, ended[1].SpanContext().SpanID(), ended[0].Parent().SpanID())
assert.Contains(t, ended[0].Attributes(), semconv.CICDPipelineTaskRunID("7.3.post.1"))
assert.Contains(t, ended[2].Attributes(), semconv.CICDPipelineTaskRunResultSkip)
assert.Contains(t, ended[3].Attributes(), semconv.CICDPipelineTaskRunResultFailure)
})
t.Run("timeout must be positive", func(t *testing.T) {
exprEval := createRunContext(t).NewExpressionEvaluator(context.Background())
for timeout, wantDeadline := range map[string]bool{"-1": false, "0": false, "1": true} {
+46
View File
@@ -0,0 +1,46 @@
# OpenTelemetry tracing
The runner can send a trace of every job it runs to an OpenTelemetry collector or any backend that accepts OTLP. Each job becomes a trace with a span per step, so you can see where CI time goes and which steps fail. Tracing stays off until you set an endpoint.
## Setup
Set the endpoint in the runner's environment, for example in its systemd unit, with `docker run -e` or in the Kubernetes pod spec:
```sh
OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
```
| Variable | Purpose |
| --- | --- |
| `OTEL_EXPORTER_OTLP_ENDPOINT` | Collector base URL, traces go to `/v1/traces` under it |
| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | Full traces URL, takes precedence over the base URL |
| `OTEL_EXPORTER_OTLP_HEADERS`, `OTEL_EXPORTER_OTLP_TRACES_HEADERS` | Request headers such as an API key, as comma-separated `key=value` pairs with URL-encoded values |
| `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES` | Resource attributes, the service name defaults to `gitea-runner` |
| `OTEL_TRACES_SAMPLER`, `OTEL_BSP_*` | Sampling and batching, see the [SDK variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/) |
| `OTEL_SDK_DISABLED=true`, `OTEL_TRACES_EXPORTER=none` | Turn tracing off |
Only the `http/protobuf` protocol is supported.
## What is sent
- A span per job, named `RUN {workflow} {job}`, with the repository, ref, commit, run URL, run number, attempt and result.
- A span per step, named as in Gitea's step list, including `Pre` and `Post` stages and the steps of composite actions, with the step's result.
- The runner's name, UUID and version.
Logs, step output and secrets are never sent. Step names are taken from the workflow file as written, without expanding `${{ }}` expressions.
## Tracing inside jobs
Jobs receive `TRACEPARENT`, so tools in a job that use OpenTelemetry can add their spans to the job's trace. To let them export, set their `OTEL_*` variables in `runner.envs`, see [Security](#security).
## Metrics
Metrics stay on the Prometheus endpoint enabled by `metrics.enabled`. To get them into OpenTelemetry, scrape `/metrics` with the collector's `prometheus` receiver.
## Security
- OTLP has no authentication of its own. Send to a collector on the runner's host that holds the backend credentials, as the [OpenTelemetry security guide](https://opentelemetry.io/docs/security/config-best-practices/) recommends, or use `https` with a token in `OTEL_EXPORTER_OTLP_HEADERS`. Over plain `http`, headers travel unencrypted.
- TLS certificates are verified against the system CA bundle, set `SSL_CERT_FILE` to use your own. Client certificates are not supported, use a local collector for mTLS. Redirects are not followed, so headers only reach the configured endpoint.
- Jobs in containers never see the runner's `OTEL_*` variables. Jobs on the host inherit the runner's whole environment, headers included, and can read its files anyway since they run as the same user.
- Every workflow can read `runner.envs`, so never put credentials there. Point jobs at a collector that holds them instead, keeping in mind that any workflow can then send data through it.
- Traces carry the repository and step names of every job, private repositories included, so give the collector the same access controls as the runner's logs.
+7 -3
View File
@@ -37,6 +37,11 @@ require (
github.com/stretchr/testify v1.12.1
github.com/timshannon/bolthold v0.0.0-20240314194003-30aac6950928
go.etcd.io/bbolt v1.5.0
go.opentelemetry.io/otel v1.46.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0
go.opentelemetry.io/otel/sdk v1.46.0
go.opentelemetry.io/otel/trace v1.46.0
go.opentelemetry.io/proto/otlp v1.11.0
go.yaml.in/yaml/v4 v4.0.0-rc.6
golang.org/x/net v0.59.0
golang.org/x/sync v0.23.0
@@ -68,6 +73,7 @@ require (
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kevinburke/ssh_config v1.6.0 // indirect
@@ -89,9 +95,7 @@ require (
github.com/xanzy/ssh-agent v0.3.3 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect
go.opentelemetry.io/otel v1.45.0 // indirect
go.opentelemetry.io/otel/metric v1.45.0 // indirect
go.opentelemetry.io/otel/trace v1.45.0 // indirect
go.opentelemetry.io/otel/metric v1.46.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/crypto v0.57.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
+14 -10
View File
@@ -196,16 +196,20 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=
go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g=
go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8=
go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o=
go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI=
go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM=
go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE=
go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4=
go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c=
go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI=
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
+13
View File
@@ -25,6 +25,7 @@ import (
"gitea.com/gitea/runner/internal/pkg/lock"
"gitea.com/gitea/runner/internal/pkg/metrics"
"gitea.com/gitea/runner/internal/pkg/report"
"gitea.com/gitea/runner/internal/pkg/telemetry"
"gitea.com/gitea/runner/internal/pkg/ver"
"connectrpc.com/connect"
@@ -71,6 +72,18 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
defer func() { _ = releaseLock() }()
}
shutdownTelemetry, err := telemetry.Setup(ctx, reg.UUID, reg.Name)
if err != nil {
log.WithError(err).Warn("OpenTelemetry export failed to start")
}
defer func() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := shutdownTelemetry(ctx); err != nil {
log.WithError(err).Warn("OpenTelemetry export failed to flush")
}
}()
lbls := resolveLabels(daemArgs.Labels, cfg.Runner.Labels, reg.Labels)
ls := labels.Labels{}
+6
View File
@@ -5,6 +5,7 @@ package run
import (
"bytes"
"cmp"
"context"
"encoding/json/v2"
"errors"
@@ -34,6 +35,7 @@ import (
"gitea.com/gitea/runner/internal/pkg/labels"
"gitea.com/gitea/runner/internal/pkg/metrics"
"gitea.com/gitea/runner/internal/pkg/report"
"gitea.com/gitea/runner/internal/pkg/telemetry"
"gitea.com/gitea/runner/internal/pkg/ver"
"connectrpc.com/connect"
@@ -298,6 +300,7 @@ func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) error {
ctx, cancel := context.WithTimeout(ctx, r.cfg.Runner.Timeout)
defer cancel()
ctx, endJob := telemetry.StartJob(ctx, task)
// A proxy URL may carry credentials, and every job is given it; keep them out of the log.
reporter := report.NewReporter(ctx, cancel, r.client, task, r.cfg, proxyPasswords()...)
var volumeCleanup []common.Executor
@@ -322,6 +325,7 @@ func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) error {
metrics.JobDuration.Observe(time.Since(start).Seconds())
metrics.JobsTotal.WithLabelValues(metrics.ResultToStatusLabel(reporter.Result())).Inc()
endJob(reporter.Result())
}()
reporter.RunDaemon()
runErr = r.run(ctx, task, reporter)
@@ -409,6 +413,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
}
job := workflow.GetJob(jobID)
reporter.ResetSteps(len(job.Steps))
telemetry.SetJobName(ctx, cmp.Or(job.Name, jobID))
taskContext := task.Context.Fields
envs := r.cloneEnvs()
@@ -426,6 +431,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
if v := taskContext["run_id"].GetStringValue(); v != "" {
envs["GITEA_RUN_ID"] = v
}
telemetry.AddJobEnv(ctx, envs)
log.Infof("task %v repo is %v %v %v", task.Id, taskContext["repository"].GetStringValue(),
r.getDefaultActionsURL(task),
+240
View File
@@ -0,0 +1,240 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package telemetry
import (
"bytes"
"cmp"
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"slices"
"strconv"
"strings"
"gitea.com/gitea/runner/internal/pkg/ver"
"gitea.dev/actionslib/pkg/model"
runnerv1 "gitea.dev/actionslib/runner/v1"
"github.com/avast/retry-go/v5"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.43.0"
"go.opentelemetry.io/otel/trace"
tracenoop "go.opentelemetry.io/otel/trace/noop"
tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
"google.golang.org/protobuf/proto"
)
const scope = "gitea.com/gitea/runner"
type taskRunKey struct{}
type taskRun struct {
id, url, pipeline string
span trace.Span
}
// tracer stays off the global provider and ctx carries only span contexts, so the Docker client's otelhttp records nothing.
var tracer trace.Tracer = tracenoop.Tracer{}
func SetTracerProvider(provider trace.TracerProvider) {
tracer = provider.Tracer(scope)
}
func Setup(ctx context.Context, uuid, name string) (func(context.Context) error, error) {
noop := func(context.Context) error { return nil }
endpoint := os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
if base := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); endpoint == "" && base != "" {
endpoint = strings.TrimSuffix(base, "/") + "/v1/traces"
}
if endpoint == "" || strings.EqualFold(os.Getenv("OTEL_SDK_DISABLED"), "true") || strings.EqualFold(os.Getenv("OTEL_TRACES_EXPORTER"), "none") {
return noop, nil
}
if protocol := cmp.Or(os.Getenv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"), os.Getenv("OTEL_EXPORTER_OTLP_PROTOCOL")); protocol != "" && !strings.EqualFold(protocol, "http/protobuf") {
return noop, fmt.Errorf("OTLP protocol %q is not supported, only http/protobuf is", protocol)
}
res, err := resource.New(ctx, resource.WithTelemetrySDK(), resource.WithAttributes(
semconv.ServiceName("gitea-runner"),
semconv.ServiceVersion(ver.Version()),
semconv.ServiceInstanceID(uuid),
semconv.CICDWorkerID(uuid),
semconv.CICDWorkerName(name),
), resource.WithFromEnv())
if err != nil {
return noop, err
}
headers := http.Header{}
for _, key := range []string{"OTEL_EXPORTER_OTLP_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS"} {
for pair := range strings.SplitSeq(os.Getenv(key), ",") {
key, value, found := strings.Cut(pair, "=")
if decoded, err := url.PathUnescape(strings.TrimSpace(value)); found && err == nil {
headers.Set(strings.TrimSpace(key), decoded)
}
}
}
provider := sdktrace.NewTracerProvider(sdktrace.WithBatcher(otlptrace.NewUnstarted(&client{endpoint, headers})), sdktrace.WithResource(res))
SetTracerProvider(provider)
return provider.Shutdown, nil
}
// client replaces otlptracehttp, which links gRPC (+7 MB): https://github.com/open-telemetry/opentelemetry-go/issues/2579
type client struct {
endpoint string
headers http.Header
}
// exportClient refuses redirects, which would carry the collector's header credentials to another host.
var exportClient = &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
func (*client) Start(context.Context) error { return nil }
func (*client) Stop(context.Context) error { return nil }
func (c *client) UploadTraces(ctx context.Context, spans []*tracepb.ResourceSpans) error {
body, err := proto.Marshal(&tracepb.TracesData{ResourceSpans: spans})
if err != nil {
return err
}
return retry.New(retry.Context(ctx), retry.Attempts(5), retry.LastErrorOnly(true)).Do(func() error {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint, bytes.NewReader(body))
if err != nil {
return retry.Unrecoverable(err)
}
req.Header = c.headers.Clone()
req.Header.Set("Content-Type", "application/x-protobuf")
resp, err := exportClient.Do(req)
if err != nil {
return err
}
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
if resp.StatusCode < http.StatusMultipleChoices {
return nil
}
err = fmt.Errorf("OTLP export failed: %s", resp.Status)
if slices.Contains([]int{http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout}, resp.StatusCode) {
return err
}
return retry.Unrecoverable(err)
})
}
func StartJob(ctx context.Context, task *runnerv1.Task) (context.Context, func(runnerv1.Result)) {
field := func(key string) string { return task.GetContext().GetFields()[key].GetStringValue() }
pipeline, owner, repo := field("workflow"), field("repository_owner"), field("repository")
repoURL := strings.TrimSuffix(field("server_url"), "/") + "/" + repo
runURL := repoURL + "/actions/runs/" + field("run_id")
_, span := tracer.Start(ctx, "RUN "+pipeline, trace.WithSpanKind(trace.SpanKindServer), trace.WithAttributes(
semconv.CICDPipelineName(pipeline),
semconv.CICDPipelineActionNameRun,
semconv.CICDPipelineRunID(field("run_id")),
semconv.CICDPipelineRunURLFull(runURL),
semconv.VCSOwnerName(owner),
semconv.VCSRepositoryName(strings.TrimPrefix(repo, owner+"/")),
semconv.VCSRepositoryURLFull(repoURL),
semconv.VCSRefHeadName(field("ref_name")),
semconv.VCSRefHeadRevision(field("sha")),
attribute.String("gitea.job.key", field("job")),
attribute.String("gitea.run.number", field("run_number")),
attribute.String("gitea.run.attempt", field("run_attempt")),
attribute.Int64("gitea.task.id", task.Id),
))
if !span.SpanContext().IsValid() {
return ctx, func(runnerv1.Result) {}
}
ctx = context.WithValue(trace.ContextWithSpanContext(ctx, span.SpanContext()), taskRunKey{}, taskRun{strconv.FormatInt(task.Id, 10), runURL, pipeline, span})
return ctx, func(result runnerv1.Result) {
end(span, semconv.CICDPipelineResultKey, jobResult(ctx, result))
}
}
func SetJobName(ctx context.Context, job string) {
if run, ok := ctx.Value(taskRunKey{}).(taskRun); ok {
run.span.SetName("RUN " + run.pipeline + " " + job)
}
}
func AddJobEnv(ctx context.Context, envs map[string]string) {
carrier := propagation.MapCarrier{}
propagation.TraceContext{}.Inject(ctx, carrier)
for key, value := range carrier {
envs[strings.ToUpper(key)] = value
}
}
func StartStep(ctx context.Context, step *model.Step, stage string) (context.Context, func(error, *model.StepResult)) {
parent, ok := ctx.Value(taskRunKey{}).(taskRun)
if !ok {
return ctx, func(error, *model.StepResult) {}
}
name, id := step.Name, parent.id+"."+strconv.Itoa(step.Number)
if name == "" {
run, _, _ := strings.Cut(strings.TrimSpace(step.Run), "\n")
name = "Run " + cmp.Or(strings.TrimSpace(run), step.String())
}
if stage != "Main" {
name, id = stage+" "+name, id+"."+strings.ToLower(stage)
}
_, span := tracer.Start(ctx, name, trace.WithAttributes(
semconv.CICDPipelineTaskName(name),
semconv.CICDPipelineTaskRunID(id),
semconv.CICDPipelineTaskRunURLFull(parent.url),
))
ctx = context.WithValue(trace.ContextWithSpanContext(ctx, span.SpanContext()), taskRunKey{}, taskRun{id, parent.url, parent.pipeline, span})
return ctx, func(err error, result *model.StepResult) {
end(span, semconv.CICDPipelineTaskRunResultKey, stepResult(ctx, err, result))
}
}
func jobResult(ctx context.Context, result runnerv1.Result) string {
switch {
case result == runnerv1.Result_RESULT_SUCCESS:
return "success"
case result == runnerv1.Result_RESULT_SKIPPED:
return "skip"
case errors.Is(ctx.Err(), context.DeadlineExceeded):
return "timeout"
case result == runnerv1.Result_RESULT_CANCELLED:
return "cancellation"
case result == runnerv1.Result_RESULT_FAILURE:
return "failure"
}
return "error"
}
func stepResult(ctx context.Context, err error, result *model.StepResult) string {
switch {
case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded):
return "timeout"
case errors.Is(err, context.Canceled) || ctx.Err() != nil:
return "cancellation"
case err != nil || result.Conclusion == model.StepStatusFailure:
return "failure"
case result.Conclusion == model.StepStatusSkipped:
return "skip"
}
return "success"
}
func end(span trace.Span, key attribute.Key, result string) {
span.SetAttributes(key.String(result))
switch result {
case "timeout":
span.SetAttributes(semconv.ErrorTypeKey.String(result))
span.SetStatus(codes.Error, "")
case "failure", "error":
span.SetAttributes(semconv.ErrorTypeOther)
span.SetStatus(codes.Error, "")
}
span.End()
}
+186
View File
@@ -0,0 +1,186 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package telemetry
import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"testing"
"time"
"gitea.dev/actionslib/pkg/model"
runnerv1 "gitea.dev/actionslib/runner/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
semconv "go.opentelemetry.io/otel/semconv/v1.43.0"
"go.opentelemetry.io/otel/trace"
tracenoop "go.opentelemetry.io/otel/trace/noop"
tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/structpb"
)
func setEnv(t *testing.T, env map[string]string) {
t.Cleanup(func() { SetTracerProvider(tracenoop.NewTracerProvider()) })
for _, key := range []string{
"OTEL_SDK_DISABLED", "OTEL_TRACES_EXPORTER", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
"OTEL_EXPORTER_OTLP_PROTOCOL", "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "OTEL_EXPORTER_OTLP_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS",
} {
t.Setenv(key, env[key])
}
}
func TestSetup(t *testing.T) {
for _, tc := range []struct {
env map[string]string
enabled bool
err string
}{
{},
{env: map[string]string{"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://collector:4318/v1/traces"}, enabled: true},
{env: map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4318", "OTEL_TRACES_EXPORTER": "NONE"}},
{env: map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4318", "OTEL_SDK_DISABLED": "TRUE"}},
{env: map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4318", "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc"}, err: `OTLP protocol "grpc" is not supported, only http/protobuf is`},
} {
t.Run(fmt.Sprint(tc.env), func(t *testing.T) {
setEnv(t, tc.env)
shutdown, err := Setup(t.Context(), "uuid", "runner")
if tc.err != "" {
require.EqualError(t, err, tc.err)
} else {
require.NoError(t, err)
}
_, disabled := tracer.(tracenoop.Tracer)
assert.Equal(t, tc.enabled, !disabled)
require.NoError(t, shutdown(t.Context()))
})
}
}
func TestExportRetriesAndMatchesOTLP(t *testing.T) {
bodies := make(chan []byte, 2)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/otlp/v1/traces", r.URL.Path)
assert.Equal(t, "Bearer token", r.Header.Get("Authorization"))
body, _ := io.ReadAll(r.Body)
bodies <- body
if len(bodies) == 1 {
w.WriteHeader(http.StatusServiceUnavailable)
}
}))
defer server.Close()
setEnv(t, map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": server.URL + "/otlp/", "OTEL_EXPORTER_OTLP_HEADERS": "authorization=Bearer%20token"})
shutdown, err := Setup(t.Context(), "uuid", "runner")
require.NoError(t, err)
_, span := tracer.Start(t.Context(), "op")
span.End()
require.NoError(t, shutdown(t.Context()))
require.Len(t, bodies, 2)
<-bodies
var decoded tracepb.TracesData
require.NoError(t, proto.Unmarshal(<-bodies, &decoded))
assert.Equal(t, "op", decoded.GetResourceSpans()[0].GetScopeSpans()[0].GetSpans()[0].GetName())
}
func TestExportRefusesRedirects(t *testing.T) {
target := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("redirect followed") }))
defer target.Close()
origin := httptest.NewServer(http.RedirectHandler(target.URL, http.StatusTemporaryRedirect))
defer origin.Close()
require.EqualError(t, (&client{origin.URL, http.Header{}}).UploadTraces(t.Context(), nil), "OTLP export failed: 307 Temporary Redirect")
}
func TestJob(t *testing.T) {
fields, err := structpb.NewStruct(map[string]any{
"workflow": "ci.yml",
"repository": "owner/repo",
"repository_owner": "owner",
"server_url": "https://gitea.example/",
"run_id": "42",
"run_number": "40",
"run_attempt": "2",
"job": "build",
})
require.NoError(t, err)
task := &runnerv1.Task{Id: 7, Context: fields}
envs := map[string]string{}
ctx, endJob := StartJob(t.Context(), task)
AddJobEnv(ctx, envs)
stepCtx, endStep := StartStep(ctx, &model.Step{}, "Main")
endStep(nil, nil)
endJob(runnerv1.Result_RESULT_SUCCESS)
assert.Equal(t, ctx, stepCtx)
assert.Empty(t, envs)
spans := tracetest.NewSpanRecorder()
SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(spans)))
t.Cleanup(func() { SetTracerProvider(tracenoop.NewTracerProvider()) })
ctx, endJob = StartJob(t.Context(), task)
SetJobName(ctx, "Build and test")
AddJobEnv(ctx, envs)
stepCtx, endStep = StartStep(ctx, &model.Step{Run: "make\nmake test", Number: 1}, "Main")
assert.False(t, trace.SpanFromContext(ctx).IsRecording())
assert.False(t, trace.SpanFromContext(stepCtx).IsRecording())
endStep(assert.AnError, nil)
endJob(runnerv1.Result_RESULT_FAILURE)
jobSpan := trace.SpanContextFromContext(ctx)
assert.Equal(t, "00-"+jobSpan.TraceID().String()+"-"+jobSpan.SpanID().String()+"-01", envs["TRACEPARENT"])
ended := spans.Ended()
require.Len(t, ended, 2)
step, job := ended[0], ended[1]
assert.Equal(t, "Run make", step.Name())
assert.Equal(t, jobSpan.SpanID(), step.Parent().SpanID())
assert.Equal(t, codes.Error, step.Status().Code)
assert.Subset(t, step.Attributes(), []any{
semconv.CICDPipelineTaskRunID("7.1"),
semconv.CICDPipelineTaskRunURLFull("https://gitea.example/owner/repo/actions/runs/42"),
semconv.ErrorTypeOther,
})
assert.Equal(t, "RUN ci.yml Build and test", job.Name())
assert.Equal(t, trace.SpanKindServer, job.SpanKind())
assert.Subset(t, job.Attributes(), []any{
semconv.VCSOwnerName("owner"),
semconv.VCSRepositoryName("repo"),
semconv.CICDPipelineRunID("42"),
attribute.String("gitea.run.number", "40"),
attribute.String("gitea.run.attempt", "2"),
attribute.String("gitea.job.key", "build"),
attribute.Int64("gitea.task.id", 7),
})
}
func TestResults(t *testing.T) {
cancelled, cancel := context.WithCancel(t.Context())
cancel()
expired, expire := context.WithDeadline(t.Context(), time.Time{})
defer expire()
for want, tc := range map[string]struct {
ctx context.Context
err error
result *model.StepResult
}{
"timeout": {ctx: t.Context(), err: fmt.Errorf("exec: %w", context.DeadlineExceeded)},
"cancellation": {ctx: cancelled},
"failure": {ctx: t.Context(), result: &model.StepResult{Outcome: model.StepStatusFailure, Conclusion: model.StepStatusFailure}},
"success": {ctx: t.Context(), result: &model.StepResult{Outcome: model.StepStatusFailure, Conclusion: model.StepStatusSuccess}},
"skip": {ctx: t.Context(), result: &model.StepResult{Outcome: model.StepStatusSkipped, Conclusion: model.StepStatusSkipped}},
} {
assert.Equal(t, want, stepResult(tc.ctx, tc.err, tc.result))
}
assert.Equal(t, "timeout", jobResult(expired, runnerv1.Result_RESULT_FAILURE))
assert.Equal(t, "cancellation", jobResult(t.Context(), runnerv1.Result_RESULT_CANCELLED))
}