mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-17 13:23:43 +00:00
enhance(config): expand environment values (#1274)
Config values can reference the runner's environment as `${NAME}`, for example `capacity: ${RUNNER_CAPACITY}`, so container deployments can configure the runner without templating the file. An unset `NAME` fails the load instead of silently becoming empty, and `$${NAME}` writes a literal `${NAME}`. Keys are never expanded.
Breaking: an existing value containing `${NAME}`, such as a shell command in a Kubernetes `pod_template`, now needs `$${NAME}`.
Closes https://gitea.com/gitea/runner/issues/551
Co-authored-by: silverwind <[email protected]>
Co-authored-by: silverwind <[email protected]>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1274
Reviewed-by: silverwind <[email protected]>
Co-authored-by: bircni <[email protected]>
This commit is contained in:
1 parent
ff30f39b1b
commit
329115b148
5 files changed
+83
-21
No files matched your search
@@ -169,7 +169,7 @@ Of the [runner images](https://gitea.com/gitea/runner-images), the `-full` flavo
|
|||||||
|
|
||||||
#### Environment variables
|
#### Environment variables
|
||||||
|
|
||||||
Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
|
Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. Config values can read the runner's environment as `${NAME}`, for example `capacity: ${RUNNER_CAPACITY}`. An unset `NAME` fails the load, and `$${NAME}` is a literal `${NAME}`. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
|
||||||
|
|
||||||
### Labels
|
### Labels
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -45,7 +45,7 @@ kubernetes:
|
|||||||
nvidia.com/gpu.present: "true"
|
nvidia.com/gpu.present: "true"
|
||||||
```
|
```
|
||||||
|
|
||||||
A job with `runs-on: [linux, gpu]` gets the `linux` and then the `gpu` entry of `pod_templates` merged over `pod_template`, while its image comes from the first of its labels the runner has. A template volume named `workspace` or `act` replaces the `emptyDir` the runner mounts at the workspace or at `/var/run/act`, for example to limit its size. Images are pulled by the cluster's `imagePullPolicy` default, a template's, or always with `container.force_pull`.
|
A job with `runs-on: [linux, gpu]` gets the `linux` and then the `gpu` entry of `pod_templates` merged over `pod_template`, while its image comes from the first of its labels the runner has. A template volume named `workspace` or `act` replaces the `emptyDir` the runner mounts at the workspace or at `/var/run/act`, for example to limit its size. Images are pulled by the cluster's `imagePullPolicy` default, a template's, or always with `container.force_pull`. Like every config value, templates expand `${NAME}` from the runner's environment, so a shell command in a template writes `$${NAME}` for the pod's own variable.
|
||||||
|
|
||||||
Job pods mount no service account token unless the template sets `automountServiceAccountToken: true`. Job pods and their Secrets are labeled `app.kubernetes.io/managed-by: gitea-runner` and `com.gitea.runner.uuid: <runner UUID>`. The idle cleanup removes those the runner did not get to remove, once they are older than `runner.workdir_cleanup_age`.
|
Job pods mount no service account token unless the template sets `automountServiceAccountToken: true`. Job pods and their Secrets are labeled `app.kubernetes.io/managed-by: gitea-runner` and `com.gitea.runner.uuid: <runner UUID>`. The idle cleanup removes those the runner did not get to remove, once they are older than `runner.workdir_cleanup_age`.
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
# Every option with its default value, all commented out. Read this file, do not copy it.
|
# Every option with its default value, all commented out. Read this file, do not copy it.
|
||||||
# `./gitea-runner config init` writes a config file to copy the lines you change into.
|
# `./gitea-runner config init` writes a config file to copy the lines you change into.
|
||||||
|
# Values may read the runner's environment as ${NAME}, an unset NAME fails the load. $${NAME} is a literal ${NAME}.
|
||||||
|
|
||||||
# Logging for the runner process itself (messages printed to stderr), plus the copy of
|
# Logging for the runner process itself (messages printed to stderr), plus the copy of
|
||||||
# each task's log kept under log.job. Neither controls how workflow step output is streamed
|
# each task's log kept under log.job. Neither controls how workflow step output is streamed
|
||||||
|
|||||||
@@ -4,12 +4,12 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"maps"
|
"maps"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -33,6 +33,8 @@ const DefaultImage = "docker.gitea.com/runner-images:ubuntu-latest"
|
|||||||
// (so a programmatically built config still gets a sane bound).
|
// (so a programmatically built config still gets a sane bound).
|
||||||
const DefaultPostTaskScriptTimeout = 5 * time.Minute
|
const DefaultPostTaskScriptTimeout = 5 * time.Minute
|
||||||
|
|
||||||
|
var environmentReference = regexp.MustCompile(`\$?\$\{[A-Za-z_][A-Za-z0-9_]*\}`)
|
||||||
|
|
||||||
// Minimal is the smallest config file that runs the runner: options it does not
|
// Minimal is the smallest config file that runs the runner: options it does not
|
||||||
// name keep their default, and it names none.
|
// name keep their default, and it names none.
|
||||||
const Minimal = `# Minimal config file. Every option it does not set keeps its default.
|
const Minimal = `# Minimal config file. Every option it does not set keeps its default.
|
||||||
@@ -244,14 +246,18 @@ func LoadDefault(file string) (*Config, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("open config file %q: %w", file, err)
|
return nil, fmt.Errorf("open config file %q: %w", file, err)
|
||||||
}
|
}
|
||||||
if err := yaml.Unmarshal(content, cfg); err != nil {
|
var document yaml.Node
|
||||||
|
if err := yaml.Unmarshal(content, &document); err != nil {
|
||||||
return nil, fmt.Errorf("parse config file %q: %w", file, err)
|
return nil, fmt.Errorf("parse config file %q: %w", file, err)
|
||||||
}
|
}
|
||||||
warnUnknownKeys(file, content)
|
if err := expandEnvironment(&document); err != nil {
|
||||||
definedRunnerKeys, err = definedRunnerConfigKeys(content)
|
return nil, fmt.Errorf("parse config file %q: %w", file, err)
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("parse config file %q for defaults metadata: %w", file, err)
|
|
||||||
}
|
}
|
||||||
|
if err := document.Load(cfg, yaml.WithV3Defaults()); err != nil {
|
||||||
|
return nil, fmt.Errorf("parse config file %q: %w", file, err)
|
||||||
|
}
|
||||||
|
warnUnknownKeys(file, &document)
|
||||||
|
definedRunnerKeys = definedRunnerConfigKeys(&document)
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.Runner.EnvFile != "" {
|
if cfg.Runner.EnvFile != "" {
|
||||||
@@ -416,30 +422,53 @@ func LoadDefault(file string) (*Config, error) {
|
|||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// expandEnvironment replaces ${NAME} in values and retypes them, so ${CAPACITY} can be an int and an empty value is unset.
|
||||||
|
func expandEnvironment(node *yaml.Node) error {
|
||||||
|
var errs []error
|
||||||
|
if node.Kind == yaml.ScalarNode {
|
||||||
|
expanded := environmentReference.ReplaceAllStringFunc(node.Value, func(match string) string {
|
||||||
|
if strings.HasPrefix(match, "$$") {
|
||||||
|
return match[1:]
|
||||||
|
}
|
||||||
|
value, ok := os.LookupEnv(strings.Trim(match, "${}"))
|
||||||
|
if !ok {
|
||||||
|
errs = append(errs, fmt.Errorf("line %d: %s is not set", node.Line, match))
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
})
|
||||||
|
if expanded != node.Value && node.Style&yaml.TaggedStyle == 0 {
|
||||||
|
node.Tag = ""
|
||||||
|
if expanded == "" && node.Style == 0 {
|
||||||
|
node.Tag = "!!null"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
node.Value = expanded
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
|
for index, child := range node.Content {
|
||||||
|
if node.Kind != yaml.MappingNode || index%2 == 1 {
|
||||||
|
errs = append(errs, expandEnvironment(child))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
// warnUnknownKeys reports keys the config does not define, which are otherwise ignored
|
// warnUnknownKeys reports keys the config does not define, which are otherwise ignored
|
||||||
// without a trace. It only warns, so a config carrying keys from another runner version
|
// without a trace. It only warns, so a config carrying keys from another runner version
|
||||||
// still loads.
|
// still loads.
|
||||||
func warnUnknownKeys(file string, content []byte) {
|
func warnUnknownKeys(file string, document *yaml.Node) {
|
||||||
decoder := yaml.NewDecoder(bytes.NewReader(content))
|
|
||||||
decoder.KnownFields(true)
|
|
||||||
|
|
||||||
var loadErrs *yaml.LoadErrors
|
var loadErrs *yaml.LoadErrors
|
||||||
if err := decoder.Decode(&Config{}); errors.As(err, &loadErrs) {
|
if err := document.Load(&Config{}, yaml.WithV3Defaults(), yaml.WithKnownFields()); errors.As(err, &loadErrs) {
|
||||||
for _, loadErr := range loadErrs.Errors {
|
for _, loadErr := range loadErrs.Errors {
|
||||||
log.Warnf("config file %q: %s, it will be ignored", file, loadErr)
|
log.Warnf("config file %q: %s, it will be ignored", file, loadErr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func definedRunnerConfigKeys(content []byte) (map[string]bool, error) {
|
func definedRunnerConfigKeys(root *yaml.Node) map[string]bool {
|
||||||
var root yaml.Node
|
|
||||||
if err := yaml.Unmarshal(content, &root); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
defined := map[string]bool{}
|
defined := map[string]bool{}
|
||||||
if len(root.Content) == 0 {
|
if len(root.Content) == 0 {
|
||||||
return defined, nil
|
return defined
|
||||||
}
|
}
|
||||||
|
|
||||||
doc := root.Content[0]
|
doc := root.Content[0]
|
||||||
@@ -455,7 +484,7 @@ func definedRunnerConfigKeys(content []byte) (map[string]bool, error) {
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
return defined, nil
|
return defined
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateExtraHeaders(headers map[string]string) error {
|
func validateExtraHeaders(headers map[string]string) error {
|
||||||
|
|||||||
@@ -42,6 +42,38 @@ cache:
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadDefault_ExpandsEnvironmentValues(t *testing.T) {
|
||||||
|
t.Setenv("RUNNER_CAPACITY", "4")
|
||||||
|
t.Setenv("RUNNER_LABEL", "linux:host")
|
||||||
|
t.Setenv("RUNNER_KEY", "fetch_timeout")
|
||||||
|
t.Setenv("RUNNER_EMPTY", "")
|
||||||
|
t.Setenv("RUNNER_MISSING", "")
|
||||||
|
require.NoError(t, os.Unsetenv("RUNNER_MISSING"))
|
||||||
|
path := write(t, `runner:
|
||||||
|
capacity: ${RUNNER_CAPACITY}
|
||||||
|
labels:
|
||||||
|
- ${RUNNER_LABEL}
|
||||||
|
envs:
|
||||||
|
LITERAL: $${RUNNER_LABEL}
|
||||||
|
${RUNNER_KEY}: 1s
|
||||||
|
cache:
|
||||||
|
enabled: ${RUNNER_EMPTY}
|
||||||
|
`)
|
||||||
|
|
||||||
|
cfg, err := LoadDefault(path)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 4, cfg.Runner.Capacity)
|
||||||
|
assert.Equal(t, []string{"linux:host"}, cfg.Runner.Labels)
|
||||||
|
assert.Equal(t, map[string]string{"LITERAL": "${RUNNER_LABEL}"}, cfg.Runner.Envs)
|
||||||
|
assert.Equal(t, 5*time.Second, cfg.Runner.FetchTimeout)
|
||||||
|
assert.True(t, *cfg.Cache.Enabled)
|
||||||
|
|
||||||
|
_, err = LoadDefault(write(t, "runner:\n\n capacity: ${RUNNER_MISSING}\n file: ${RUNNER_MISSING}\n"))
|
||||||
|
require.ErrorContains(t, err, "line 3: ${RUNNER_MISSING} is not set\nline 4: ${RUNNER_MISSING} is not set")
|
||||||
|
_, err = LoadDefault(write(t, "runner:\n\n capacity: ${RUNNER_LABEL}\n"))
|
||||||
|
require.ErrorContains(t, err, "line 3:")
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadDefault_CacheS3(t *testing.T) {
|
func TestLoadDefault_CacheS3(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
accessPath := filepath.Join(dir, "access.key")
|
accessPath := filepath.Join(dir, "access.key")
|
||||||
|
|||||||
Reference in new issue
Block a user