mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-13 03:13:42 +00:00
feat: OpenTelemetry tracing (#1206)
Adds OpenTelemetry tracing, configured through the standard `OTEL_*` environment variables and off until an OTLP endpoint is set.  Each job becomes a trace with a span per step, so operators can see where CI time goes in their existing observability backend. Jobs receive only `TRACEPARENT`, so instrumented tools inside them join the trace without seeing the collector's credentials. Metrics stay on the Prometheus endpoint, which a collector can scrape. See [docs/telemetry.md](https://gitea.com/wmTJc9IK0Q/runner/src/branch/otel/docs/telemetry.md) for setup and security notes. Also keeps `GITHUB_SHA` and `GITHUB_REF` fixed for a whole run when HEAD moves during `exec` runs and the act tests. [image.png](https://gitea.com/attachments/c4b49cdf-a32d-403c-94e9-30cd06ee6477) [image.png](https://gitea.com/attachments/c4b49cdf-a32d-403c-94e9-30cd06ee6477) --------- Co-authored-by: silverwind <[email protected]> Co-authored-by: silverwind <[email protected]> Co-authored-by: bircni <[email protected]> Reviewed-on: https://gitea.com/gitea/runner/pulls/1206 Reviewed-by: bircni <[email protected]> Reviewed-by: silverwind <[email protected]> Co-authored-by: wmTJc9IK0Q <[email protected]>
This commit is contained in:
authored and silverwind committed
1 parent
7ad12038d1
commit
1952e5cdb6
15 files changed
+603
-28
No files matched your search
@@ -0,0 +1,186 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
runnerv1 "gitea.dev/actionslib/runner/v1"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/sdk/trace/tracetest"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.43.0"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
tracenoop "go.opentelemetry.io/otel/trace/noop"
|
||||
tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/structpb"
|
||||
)
|
||||
|
||||
func setEnv(t *testing.T, env map[string]string) {
|
||||
t.Cleanup(func() { SetTracerProvider(tracenoop.NewTracerProvider()) })
|
||||
for _, key := range []string{
|
||||
"OTEL_SDK_DISABLED", "OTEL_TRACES_EXPORTER", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_PROTOCOL", "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "OTEL_EXPORTER_OTLP_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS",
|
||||
} {
|
||||
t.Setenv(key, env[key])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetup(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
env map[string]string
|
||||
enabled bool
|
||||
err string
|
||||
}{
|
||||
{},
|
||||
{env: map[string]string{"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://collector:4318/v1/traces"}, enabled: true},
|
||||
{env: map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4318", "OTEL_TRACES_EXPORTER": "NONE"}},
|
||||
{env: map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4318", "OTEL_SDK_DISABLED": "TRUE"}},
|
||||
{env: map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4318", "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc"}, err: `OTLP protocol "grpc" is not supported, only http/protobuf is`},
|
||||
} {
|
||||
t.Run(fmt.Sprint(tc.env), func(t *testing.T) {
|
||||
setEnv(t, tc.env)
|
||||
shutdown, err := Setup(t.Context(), "uuid", "runner")
|
||||
if tc.err != "" {
|
||||
require.EqualError(t, err, tc.err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
_, disabled := tracer.(tracenoop.Tracer)
|
||||
assert.Equal(t, tc.enabled, !disabled)
|
||||
require.NoError(t, shutdown(t.Context()))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportRetriesAndMatchesOTLP(t *testing.T) {
|
||||
bodies := make(chan []byte, 2)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/otlp/v1/traces", r.URL.Path)
|
||||
assert.Equal(t, "Bearer token", r.Header.Get("Authorization"))
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
bodies <- body
|
||||
if len(bodies) == 1 {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
setEnv(t, map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": server.URL + "/otlp/", "OTEL_EXPORTER_OTLP_HEADERS": "authorization=Bearer%20token"})
|
||||
|
||||
shutdown, err := Setup(t.Context(), "uuid", "runner")
|
||||
require.NoError(t, err)
|
||||
_, span := tracer.Start(t.Context(), "op")
|
||||
span.End()
|
||||
require.NoError(t, shutdown(t.Context()))
|
||||
|
||||
require.Len(t, bodies, 2)
|
||||
<-bodies
|
||||
var decoded tracepb.TracesData
|
||||
require.NoError(t, proto.Unmarshal(<-bodies, &decoded))
|
||||
assert.Equal(t, "op", decoded.GetResourceSpans()[0].GetScopeSpans()[0].GetSpans()[0].GetName())
|
||||
}
|
||||
|
||||
func TestExportRefusesRedirects(t *testing.T) {
|
||||
target := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("redirect followed") }))
|
||||
defer target.Close()
|
||||
origin := httptest.NewServer(http.RedirectHandler(target.URL, http.StatusTemporaryRedirect))
|
||||
defer origin.Close()
|
||||
require.EqualError(t, (&client{origin.URL, http.Header{}}).UploadTraces(t.Context(), nil), "OTLP export failed: 307 Temporary Redirect")
|
||||
}
|
||||
|
||||
func TestJob(t *testing.T) {
|
||||
fields, err := structpb.NewStruct(map[string]any{
|
||||
"workflow": "ci.yml",
|
||||
"repository": "owner/repo",
|
||||
"repository_owner": "owner",
|
||||
"server_url": "https://gitea.example/",
|
||||
"run_id": "42",
|
||||
"run_number": "40",
|
||||
"run_attempt": "2",
|
||||
"job": "build",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
task := &runnerv1.Task{Id: 7, Context: fields}
|
||||
|
||||
envs := map[string]string{}
|
||||
ctx, endJob := StartJob(t.Context(), task)
|
||||
AddJobEnv(ctx, envs)
|
||||
stepCtx, endStep := StartStep(ctx, &model.Step{}, "Main")
|
||||
endStep(nil, nil)
|
||||
endJob(runnerv1.Result_RESULT_SUCCESS)
|
||||
assert.Equal(t, ctx, stepCtx)
|
||||
assert.Empty(t, envs)
|
||||
|
||||
spans := tracetest.NewSpanRecorder()
|
||||
SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(spans)))
|
||||
t.Cleanup(func() { SetTracerProvider(tracenoop.NewTracerProvider()) })
|
||||
|
||||
ctx, endJob = StartJob(t.Context(), task)
|
||||
SetJobName(ctx, "Build and test")
|
||||
AddJobEnv(ctx, envs)
|
||||
stepCtx, endStep = StartStep(ctx, &model.Step{Run: "make\nmake test", Number: 1}, "Main")
|
||||
assert.False(t, trace.SpanFromContext(ctx).IsRecording())
|
||||
assert.False(t, trace.SpanFromContext(stepCtx).IsRecording())
|
||||
endStep(assert.AnError, nil)
|
||||
endJob(runnerv1.Result_RESULT_FAILURE)
|
||||
|
||||
jobSpan := trace.SpanContextFromContext(ctx)
|
||||
assert.Equal(t, "00-"+jobSpan.TraceID().String()+"-"+jobSpan.SpanID().String()+"-01", envs["TRACEPARENT"])
|
||||
|
||||
ended := spans.Ended()
|
||||
require.Len(t, ended, 2)
|
||||
step, job := ended[0], ended[1]
|
||||
assert.Equal(t, "Run make", step.Name())
|
||||
assert.Equal(t, jobSpan.SpanID(), step.Parent().SpanID())
|
||||
assert.Equal(t, codes.Error, step.Status().Code)
|
||||
assert.Subset(t, step.Attributes(), []any{
|
||||
semconv.CICDPipelineTaskRunID("7.1"),
|
||||
semconv.CICDPipelineTaskRunURLFull("https://gitea.example/owner/repo/actions/runs/42"),
|
||||
semconv.ErrorTypeOther,
|
||||
})
|
||||
assert.Equal(t, "RUN ci.yml Build and test", job.Name())
|
||||
assert.Equal(t, trace.SpanKindServer, job.SpanKind())
|
||||
assert.Subset(t, job.Attributes(), []any{
|
||||
semconv.VCSOwnerName("owner"),
|
||||
semconv.VCSRepositoryName("repo"),
|
||||
semconv.CICDPipelineRunID("42"),
|
||||
attribute.String("gitea.run.number", "40"),
|
||||
attribute.String("gitea.run.attempt", "2"),
|
||||
attribute.String("gitea.job.key", "build"),
|
||||
attribute.Int64("gitea.task.id", 7),
|
||||
})
|
||||
}
|
||||
|
||||
func TestResults(t *testing.T) {
|
||||
cancelled, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
expired, expire := context.WithDeadline(t.Context(), time.Time{})
|
||||
defer expire()
|
||||
for want, tc := range map[string]struct {
|
||||
ctx context.Context
|
||||
err error
|
||||
result *model.StepResult
|
||||
}{
|
||||
"timeout": {ctx: t.Context(), err: fmt.Errorf("exec: %w", context.DeadlineExceeded)},
|
||||
"cancellation": {ctx: cancelled},
|
||||
"failure": {ctx: t.Context(), result: &model.StepResult{Outcome: model.StepStatusFailure, Conclusion: model.StepStatusFailure}},
|
||||
"success": {ctx: t.Context(), result: &model.StepResult{Outcome: model.StepStatusFailure, Conclusion: model.StepStatusSuccess}},
|
||||
"skip": {ctx: t.Context(), result: &model.StepResult{Outcome: model.StepStatusSkipped, Conclusion: model.StepStatusSkipped}},
|
||||
} {
|
||||
assert.Equal(t, want, stepResult(tc.ctx, tc.err, tc.result))
|
||||
}
|
||||
assert.Equal(t, "timeout", jobResult(expired, runnerv1.Result_RESULT_FAILURE))
|
||||
assert.Equal(t, "cancellation", jobResult(t.Context(), runnerv1.Result_RESULT_CANCELLED))
|
||||
}
|
||||
Reference in new issue
Block a user