// Copyright 2023 The Gitea Authors. All rights reserved. // Copyright 2023 The nektos/act Authors. All rights reserved. // SPDX-License-Identifier: MIT package artifactcache import ( "cmp" "context" "crypto/hmac" "crypto/rand" "crypto/sha256" "encoding/binary" "encoding/hex" "encoding/json/v2" "errors" "fmt" "io" "net" "net/http" "net/http/httputil" "net/url" "os" "path/filepath" "slices" "strconv" "strings" "sync" "sync/atomic" "time" "gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/act/common/logging" "gitea.com/gitea/runner/internal/pkg/disk" "go.etcd.io/bbolt" berrors "go.etcd.io/bbolt/errors" ) const ( apiPath = "/_apis/artifactcache" internalPath = "/_internal" // artifactURLTTL bounds how long a signed artifactLocation URL stays valid. // Short enough that a leaked URL is near-worthless; long enough to let the // @actions/cache client download a big blob that was returned from /cache. artifactURLTTL = 10 * time.Minute dbOpenTimeout = 5 * time.Second ) type credKey struct{} // JobCredential ties a per-job bearer token (ACTIONS_RUNTIME_TOKEN) to the // repository that owns it. Every cache entry is stamped with Repo on // reserve/commit and checked on read/write so one repo can never observe or // poison another repo's cache, even from inside a container that reaches the // cache server over the docker bridge network. type JobCredential struct { Repo string `json:"repo"` // Results is the instance whose artifact service this server forwards for the job, and // InsecureTLS how the runner reaches it; see results.go. Results string `json:"-"` InsecureTLS bool `json:"-"` // PublicURL is this server as a reverse proxy makes the job reach it, not the listen address. PublicURL string `json:"public_url"` } // credEntry holds a registered job's credential along with an active // registration count. RegisterJob is reference-counted so that if two tasks // briefly share an ACTIONS_RUNTIME_TOKEN — e.g. a runner that retries a task // after a crash before the old registration is revoked — the first task's // revoker does not cut the second task's auth out from under it. type credEntry struct { cred JobCredential refs int } type Handler struct { dir string storage cacheStorage s3 *s3Storage listener net.Listener port int server *http.Server logger *logging.Logger dbLock chan struct{} // queues store openers without polling bolt's file lock entries cacheEntries // guarded by dbLock gcing atomic.Bool gcAt time.Time stopSync context.CancelFunc syncing sync.WaitGroup outboundIP string // internalSecret guards /_internal/{register,revoke}. When set, a remote // runner can use these endpoints to pre-register per-job // ACTIONS_RUNTIME_TOKENs against this server, enabling the same // per-job auth and repo scoping as the embedded handler over the // network. Empty disables the control-plane entirely. internalSecret string // secret signs short-lived artifact download URLs. The @actions/cache // toolkit does not send Authorization on the download request, so blob // GETs authenticate via a per-URL HMAC signature with expiry rather than // via the bearer token used for management endpoints. secret []byte credMu sync.RWMutex creds map[string]*credEntry policy Policy // freeDisk is a field so tests can drive evictForFreeSpace without a full volume. freeDisk func(string) (uint64, error) } // Options configures a cache server started by StartHandler; the zero value is usable. type Options struct { Dir string OutboundIP string Port uint16 S3 *S3Options Upstream string // InternalSecret, when non-empty, enables a control-plane API at // /_internal/{register,revoke} that lets a remote runner pre-register the // per-job ACTIONS_RUNTIME_TOKENs it expects this server to honor. The // embedded in-process handler leaves it empty and registers tokens via the // in-process RegisterJob method directly. InternalSecret string Policy Policy Logger *logging.Logger } // StartHandler starts an embedded cache or forwards cache requests to Upstream. func StartHandler(opts Options) (*Handler, error) { dir, logger := opts.Dir, opts.Logger h := &Handler{ creds: make(map[string]*credEntry), internalSecret: opts.InternalSecret, policy: opts.Policy.withDefaults(), freeDisk: disk.FreeBytes, dbLock: make(chan struct{}, 1), } if logger == nil { logger = logging.New(logging.DiscardHandler) } logger = logger.With("module", "artifactcache") h.logger = logger if opts.OutboundIP != "" { h.outboundIP = opts.OutboundIP } else if ip := common.GetOutboundIP(); ip == nil { return nil, errors.New("unable to determine outbound IP address") } else { h.outboundIP = ip.String() } if opts.Upstream != "" { handler, err := h.cacheForwarder(opts.Upstream) if err != nil { return nil, err } if err := h.serve(opts.Port, handler); err != nil { return nil, err } return h, nil } if dir == "" { home, err := os.UserHomeDir() if err != nil { return nil, err } dir = filepath.Join(home, ".cache", "actcache") } if err := os.MkdirAll(dir, 0o755); err != nil { return nil, err } h.dir = dir storage, err := NewStorage(filepath.Join(dir, "cache")) if err != nil { return nil, err } h.storage = storage if opts.S3 != nil { if h.s3, err = newS3Storage(storage, *opts.S3); err != nil { return nil, err } h.storage = h.s3 } if h.secret, err = loadOrCreateSecret(dir); err != nil { return nil, err } router := http.NewServeMux() router.HandleFunc("GET "+apiPath+"/cache", h.bearerAuth(h.find)) router.HandleFunc("POST "+apiPath+"/caches", h.bearerAuth(h.reserve)) router.HandleFunc("PATCH "+apiPath+"/caches/{id}", h.bearerAuth(h.upload)) router.HandleFunc("POST "+apiPath+"/caches/{id}", h.bearerAuth(h.commit)) router.HandleFunc("POST "+apiPath+"/clean", h.bearerAuth(h.clean)) // Artifact GET is signed via query-string HMAC because @actions/cache // does not attach Authorization when downloading archiveLocation. router.HandleFunc("GET "+apiPath+"/artifacts/{id}", h.signedAuth("", h.get)) // Control-plane: a remote runner registers/revokes per-job tokens so the // cache API can authenticate them. Always wired so the routes exist; the // handlers themselves 401 when internalSecret is unset. router.HandleFunc("POST "+internalPath+"/register", h.internalAuth(h.internalRegister)) router.HandleFunc("POST "+internalPath+"/revoke", h.internalAuth(h.internalRevoke)) h.registerV2Routes(router) router.HandleFunc(artifactServicePath, h.forwardOrNotFound) h.gcCache() if err := h.serve(opts.Port, router); err != nil { return nil, err } if h.s3 != nil { ctx, cancel := context.WithCancel(context.Background()) h.stopSync = cancel h.syncing.Go(func() { h.syncMetadata(ctx) }) } return h, nil } func (h *Handler) serve(port uint16, handler http.Handler) error { // Listen on all interfaces. Binding to outboundIP only would give no real // security benefit (it is the LAN/internet-facing address either way) and // can break Docker Desktop variants where the host's outbound IP is not // routable from inside the container network. Authentication is enforced // by the bearer middleware and per-repo scoping, not by reachability. listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) if err != nil { return err } addr, ok := listener.Addr().(*net.TCPAddr) if !ok { listener.Close() return fmt.Errorf("cache server listens on %T, want a TCP address", listener.Addr()) } h.port = addr.Port server := &http.Server{ ReadHeaderTimeout: 2 * time.Second, Handler: handler, } go func() { if err := server.Serve(listener); err != nil && errors.Is(err, net.ErrClosed) { h.logger.Errorf("http serve: %v", err) } }() h.listener = listener h.server = server return nil } func (h *Handler) cacheForwarder(upstream string) (http.Handler, error) { target, err := url.Parse(strings.TrimRight(upstream, "/")) if err != nil { return nil, fmt.Errorf("parse cache upstream: %w", err) } transport, _ := http.DefaultTransport.(*http.Transport) transport = transport.Clone() transport.MaxIdleConnsPerHost = 64 // parallel block uploads would otherwise redial past the default 2 proxy := &httputil.ReverseProxy{ Transport: transport, Rewrite: func(r *httputil.ProxyRequest) { r.SetURL(target) r.Out.URL.RawQuery = r.In.URL.RawQuery // Rewrite drops unparsable query params, and signed URLs must pass verbatim. }, ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { h.logger.Warnf("cache forward %s %s: %v", r.Method, r.URL.Path, err) h.twirpError(w, r, twirpUnavailable, errors.New("cache upstream is unavailable")) }, } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(r.URL.Path, apiPath+"/") || strings.HasPrefix(r.URL.Path, cacheServiceV2Path+"/") { proxy.ServeHTTP(w, r) } else { h.forwardOrNotFound(w, r) } }), nil } func (h *Handler) ExternalURL() string { // TODO: make the external url configurable if necessary return "http://" + net.JoinHostPort(h.outboundIP, strconv.Itoa(h.port)) } func (h *Handler) baseURL(cred JobCredential) string { if base := strings.TrimRight(cred.PublicURL, "/"); base != "" { return base } return h.ExternalURL() } // RegisterJob makes token a valid bearer credential for cache requests from // the given repository and returns a function that removes it. The runner // calls this at job start and defers the returned func so that the credential // is only accepted while the job is running. // // Registrations are reference-counted: if a token is already registered, the // credential it was registered with is kept and the refcount is incremented. // The entry is removed only when every revoker returned by RegisterJob has // been called. // This keeps a stray re-registration from silently revoking a live job. func (h *Handler) RegisterJob(token string, cred JobCredential) func() { if h == nil || token == "" { return func() {} } h.credMu.Lock() if existing, ok := h.creds[token]; ok { existing.refs++ } else { h.creds[token] = &credEntry{ cred: cred, refs: 1, } } h.credMu.Unlock() return func() { h.RevokeJob(token) } } // RevokeJob explicitly revokes one registration of token, mirroring one call // of the closure returned by RegisterJob. Used by the control-plane endpoint // so a remote runner can revoke without holding the closure. func (h *Handler) RevokeJob(token string) { if h == nil || token == "" { return } h.credMu.Lock() if entry, ok := h.creds[token]; ok { entry.refs-- if entry.refs <= 0 { delete(h.creds, token) } } h.credMu.Unlock() } func (h *Handler) lookupCredential(token string) (JobCredential, bool) { h.credMu.RLock() entry, ok := h.creds[token] h.credMu.RUnlock() if !ok { return JobCredential{}, false } return entry.cred, true } // loadOrCreateSecret returns the 32-byte HMAC signing key for artifact URLs, // persisted in dir/.secret so signed URLs handed out before a restart stay // valid across the restart and so the standalone cache-server can be pointed // at by config.Cache.ExternalServer without the URL rotating. func loadOrCreateSecret(dir string) ([]byte, error) { path := filepath.Join(dir, ".secret") if data, err := os.ReadFile(path); err == nil { if secret, err := hex.DecodeString(strings.TrimSpace(string(data))); err == nil && len(secret) >= 32 { return secret, nil } } else if !os.IsNotExist(err) { return nil, fmt.Errorf("read cache secret: %w", err) } secret := make([]byte, 32) if _, err := rand.Read(secret); err != nil { return nil, fmt.Errorf("generate cache secret: %w", err) } if err := os.WriteFile(path, []byte(hex.EncodeToString(secret)), 0o600); err != nil { return nil, fmt.Errorf("write cache secret: %w", err) } return secret, nil } func (h *Handler) Close() error { if h == nil { return nil } if h.stopSync != nil { h.stopSync() h.syncing.Wait() } var retErr error if h.server != nil { retErr = h.server.Close() h.server = nil } if h.listener != nil { err := h.listener.Close() if errors.Is(err, net.ErrClosed) { err = nil } if err != nil { retErr = err } h.listener = nil } return retErr } // cacheBucket holds each entry as JSON under its decimal ID, the layout bolthold wrote, so older stores stay readable. var cacheBucket = []byte("Cache") var errCacheNotFound = errors.New("cache not found") type cacheDB struct { bolt *bbolt.DB close func() error entries *cacheEntries } // cacheEntries mirrors the bucket as of transaction tx, so queries decode it again only after another process wrote. type cacheEntries struct { tx int byID map[uint64]*Cache } func (db *cacheDB) Close() error { return db.close() } func cacheKey(id uint64) []byte { return strconv.AppendUint(nil, id, 10) } func (db *cacheDB) get(id uint64) (*Cache, error) { if err := db.bolt.View(db.load); err != nil { return nil, err } cache, ok := db.entries.byID[id] if !ok { return nil, errCacheNotFound } copied := *cache return &copied, nil } // find returns copies of the matching entries in ID order, so ties break the same way every time. func (db *cacheDB) find(match func(*Cache) bool) ([]*Cache, error) { if err := db.bolt.View(db.load); err != nil { return nil, err } var caches []*Cache for _, cache := range db.entries.byID { if match(cache) { copied := *cache caches = append(caches, &copied) } } slices.SortFunc(caches, func(left, right *Cache) int { return cmp.Compare(left.ID, right.ID) }) return caches, nil } func (db *cacheDB) load(tx *bbolt.Tx) error { if db.entries.byID != nil && db.entries.tx == tx.ID() { return nil } byID := make(map[uint64]*Cache) if bucket := tx.Bucket(cacheBucket); bucket != nil { if err := bucket.ForEach(func(key, data []byte) (err error) { cache := &Cache{} if err := json.Unmarshal(data, cache); err != nil { return err } cache.ID, err = strconv.ParseUint(string(key), 10, 64) // an interrupted legacy insert left "id":0 in the value byID[cache.ID] = cache return err }); err != nil { return err } } db.entries.byID, db.entries.tx = byID, tx.ID() return nil } // update keeps the mirror in step with its own write, and drops it when another process wrote in between or the write rolled back. func (db *cacheDB) update(fn func(*bbolt.Bucket) error) error { err := db.bolt.Update(func(tx *bbolt.Tx) error { if db.entries.tx != tx.ID()-1 { db.entries.byID = nil } db.entries.tx = tx.ID() bucket, err := tx.CreateBucketIfNotExists(cacheBucket) if err != nil { return err } return fn(bucket) }) if err != nil { db.entries.byID = nil } return err } func (db *cacheDB) put(bucket *bbolt.Bucket, cache *Cache) error { data, err := json.Marshal(cache) if err != nil { return err } if db.entries.byID != nil { copied := *cache db.entries.byID[cache.ID] = &copied } return bucket.Put(cacheKey(cache.ID), data) } func (db *cacheDB) delete(bucket *bbolt.Bucket, id uint64) error { delete(db.entries.byID, id) return bucket.Delete(cacheKey(id)) } // dropLegacyIndexes deletes the index buckets bolthold kept, all created together and never recreated, checking first so later runs do not write. func (db *cacheDB) dropLegacyIndexes() error { var found bool _ = db.bolt.View(func(tx *bbolt.Tx) error { found = tx.Bucket([]byte("_index:Cache:Key")) != nil return nil }) if !found { return nil } return db.bolt.Update(func(tx *bbolt.Tx) error { for _, field := range []string{"Repo", "Key", "Version", "Complete", "UsedAt", "CreatedAt"} { if err := tx.DeleteBucket([]byte("_index:Cache:" + field)); err != nil && !errors.Is(err, berrors.ErrBucketNotFound) { return err } } return nil }) } // save rewrites a stored entry, failing once a sweep has removed it so a late commit cannot resurrect it. func (db *cacheDB) save(cache *Cache) error { return db.update(func(bucket *bbolt.Bucket) error { if bucket.Get(cacheKey(cache.ID)) == nil { return errCacheNotFound } return db.put(bucket, cache) }) } func (h *Handler) openDB() (store *cacheDB, err error) { select { case h.dbLock <- struct{}{}: case <-time.After(dbOpenTimeout): return nil, berrors.ErrTimeout } defer func() { if store == nil { // only an opened store hands the lock to Close, even when Open panics <-h.dbLock } }() db, err := bbolt.Open(filepath.Join(h.dir, "bolt.db"), 0o644, &bbolt.Options{ Timeout: dbOpenTimeout, NoGrowSync: bbolt.DefaultOptions.NoGrowSync, FreelistType: bbolt.DefaultOptions.FreelistType, }) if err != nil { return nil, err } return &cacheDB{bolt: db, entries: &h.entries, close: sync.OnceValue(func() error { defer func() { <-h.dbLock }() return db.Close() })}, nil } // syncMetadata mirrors the bucket into the store until ctx ends, so no request waits on S3 for it. func (h *Handler) syncMetadata(ctx context.Context) { ticker := time.NewTicker(metadataSyncInterval) defer ticker.Stop() for { if err := h.importMetadata(ctx); err != nil && ctx.Err() == nil { h.logger.Warnf("sync S3 cache: %v", err) } select { case <-ctx.Done(): return case <-ticker.C: } } } // importMetadata lists after snapshotting the store, so an entry committed meanwhile, published before it is stored, stays. func (h *Handler) importMetadata(ctx context.Context) error { db, err := h.openDB() if err != nil { return err } local := h.caches(db, func(cache *Cache) bool { return cache.Complete }) db.Close() published, err := h.s3.published(ctx) if err != nil { return err } var gone []*Cache for _, cache := range local { if !published[cache.ID] { gone = append(gone, cache) } delete(published, cache.ID) } added := make([]*Cache, 0, len(published)) for id := range published { cache, err := h.s3.fetch(ctx, id) if err != nil { return err } added = append(added, cache) } db, err = h.openDB() if err != nil { return err } defer db.Close() return db.update(func(bucket *bbolt.Bucket) error { for _, cache := range added { if err := db.put(bucket, cache); err != nil { return err } } for _, cache := range gone { _ = h.s3.Storage.Remove(cache.ID) // a blob kept on disk before the runner used S3 if err := db.delete(bucket, cache.ID); err != nil { return err } } return nil }) } // GET /_apis/artifactcache/cache func (h *Handler) find(w http.ResponseWriter, r *http.Request) { cred := credFromContext(r.Context()) keys := strings.Split(r.URL.Query().Get("keys"), ",") version := r.URL.Query().Get("version") db, err := h.openDB() if err != nil { h.responseJSON(w, r, 500, err) return } defer db.Close() cache, err := h.lookupCache(db, cred.Repo, keys, version) if err != nil { h.responseJSON(w, r, 500, err) return } if cache == nil { h.responseJSON(w, r, 204) return } h.responseJSON(w, r, 200, map[string]any{ "result": "hit", "archiveLocation": h.signedArtifactURL(cred, cache.ID, time.Now().Add(artifactURLTTL)), "cacheKey": cache.Key, }) } // lookupCache returns the entry to restore for these keys, or (nil, nil) when there is none: // either nothing matched, or the match had lost its blob to a prune, in which case the dangling // entry is dropped on the way out. func (h *Handler) lookupCache(db *cacheDB, repo string, keys []string, version string) (*Cache, error) { cache, err := findCache(db, repo, keys, version) if err != nil || cache == nil { return nil, err } ok, err := h.storage.Exist(cache.ID) if err != nil { return nil, err } if !ok { h.deleteCache(db, cache) return nil, nil //nolint:nilnil // absence is not an error here } // Handing out a download URL counts as access, or eviction could drop the entry between // this call and the GET that follows it. h.touch(db, cache) return cache, nil } // POST /_apis/artifactcache/caches func (h *Handler) reserve(w http.ResponseWriter, r *http.Request) { cred := credFromContext(r.Context()) api := &Request{} if err := json.UnmarshalRead(r.Body, api); err != nil { h.responseJSON(w, r, 400, err) return } cache := &Cache{Repo: cred.Repo, Key: api.Key, Version: api.Version, Size: api.Size} if cache.Size == 0 { cache.Size = -1 } db, err := h.openDB() if err != nil { h.responseJSON(w, r, 500, err) return } defer db.Close() now := time.Now() cache.CreatedAt = now.Unix() cache.CreatedAtNanos = now.Nanosecond() cache.UsedAt = now.Unix() if err := h.insertCache(db, cache); err != nil { h.responseJSON(w, r, 500, err) return } h.responseJSON(w, r, 200, map[string]any{ "cacheId": cache.ID, }) } // PATCH /_apis/artifactcache/caches/:id func (h *Handler) upload(w http.ResponseWriter, r *http.Request) { cred := credFromContext(r.Context()) id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) if err != nil { h.responseJSON(w, r, 400, err) return } db, err := h.openDB() if err != nil { h.responseJSON(w, r, 500, err) return } defer db.Close() cache, err := db.get(uint64(id)) if err != nil { if errors.Is(err, errCacheNotFound) { h.responseJSON(w, r, 400, fmt.Errorf("cache %d: not reserved", id)) return } h.responseJSON(w, r, 500, err) return } if cache.Repo != cred.Repo { h.responseJSON(w, r, 403, fmt.Errorf("cache %d: forbidden", id)) return } if cache.Complete { h.responseJSON(w, r, 400, fmt.Errorf("cache %v %q: already complete", cache.ID, cache.Key)) return } db.Close() start, err := parseContentRange(r.Header.Get("Content-Range")) if err != nil { h.responseJSON(w, r, 400, err) return } if err := h.storage.Write(cache.ID, start, r.Body); err != nil { h.responseJSON(w, r, 500, err) return } _ = h.touchCache(uint64(id), false) h.responseJSON(w, r, 200) } // POST /_apis/artifactcache/caches/:id func (h *Handler) commit(w http.ResponseWriter, r *http.Request) { cred := credFromContext(r.Context()) id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) if err != nil { h.responseJSON(w, r, 400, err) return } var request struct { Size *int64 `json:"size"` } body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) if err == nil && len(body) > 0 { err = json.Unmarshal(body, &request) } if err != nil { h.responseJSON(w, r, http.StatusBadRequest, err) return } db, err := h.openDB() if err != nil { h.responseJSON(w, r, 500, err) return } defer db.Close() cache, err := db.get(uint64(id)) if err != nil { if errors.Is(err, errCacheNotFound) { h.responseJSON(w, r, 400, fmt.Errorf("cache %d: not reserved", id)) return } h.responseJSON(w, r, 500, err) return } if cache.Repo != cred.Repo { h.responseJSON(w, r, 403, fmt.Errorf("cache %d: forbidden", id)) return } if cache.Complete { h.responseJSON(w, r, 400, fmt.Errorf("cache %v %q: already complete", cache.ID, cache.Key)) return } if request.Size != nil { if *request.Size < 0 || cache.Size >= 0 && cache.Size != *request.Size { h.responseJSON(w, r, http.StatusBadRequest, errors.New("invalid cache size")) return } cache.Size = *request.Size } db.Close() if err := h.commitCache(cache); err != nil { h.responseJSON(w, r, 500, err) return } h.responseJSON(w, r, 200) } // commitCache assembles the uploaded parts and marks the entry complete. The caller must // have closed its store first: Commit concatenates the whole archive and would otherwise // hold bolt's exclusive file lock for the duration. func (h *Handler) commitCache(cache *Cache) error { written, err := h.storage.Commit(cache.ID, cache.Size) if err != nil { return err } // write real size back to cache, it may be different from the current value when the request doesn't specify it. cache.Size = written cache.Complete = true cache.UsedAt = time.Now().Unix() // a just-written entry counts as accessed, so it cannot be its own eviction victim if h.s3 != nil { if err := h.s3.publish(cache); err != nil { _ = h.storage.Remove(cache.ID) return err } } db, err := h.openDB() if err != nil { return err } defer db.Close() if err := db.save(cache); err != nil { return err } if h.s3 != nil { return nil // lifecycle rules on the bucket bound its size } // A commit is the only thing that grows the store, so the only thing that can push the // volume under the floor. h.evictRepo(db, cache.Repo) h.evictTotal(db) h.evictForFreeSpace(db) return nil } // GET /_apis/artifactcache/artifacts/:id // Authenticated via signed URL (see signedAuth), not bearer, because the // @actions/cache toolkit downloads archiveLocation without Authorization. // Repository scoping is already enforced at find() time; the signature binds // the URL to the specific cache ID and an expiry. func (h *Handler) get(w http.ResponseWriter, r *http.Request) { id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) if err != nil { h.responseJSON(w, r, 400, err) return } _ = h.touchCache(uint64(id), false) h.storage.Serve(w, r, uint64(id)) } // POST /_apis/artifactcache/clean func (h *Handler) clean(w http.ResponseWriter, r *http.Request) { // TODO: don't support force deleting cache entries // see: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries h.responseJSON(w, r, 200) } // bearerAuth resolves ACTIONS_RUNTIME_TOKEN against the set of currently // registered jobs. A match attaches the job's JobCredential to the request // context; a miss returns 401 before the handler body runs. func (h *Handler) bearerAuth(handler http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { h.logger.Debugf("%s %s", r.Method, r.URL.Path) token := bearerToken(r) if token == "" { h.unauthorized(w, r, errors.New("missing bearer token")) return } cred, ok := h.lookupCredential(token) if !ok { h.unauthorized(w, r, errors.New("unknown bearer token")) return } ctx := context.WithValue(r.Context(), credKey{}, cred) handler(w, r.WithContext(ctx)) go h.gcCache() } } func (h *Handler) unauthorized(w http.ResponseWriter, r *http.Request, err error) { if strings.HasPrefix(r.URL.Path, cacheServiceV2Path) { h.twirpError(w, r, twirpUnauthenticated, err) return } h.responseJSON(w, r, http.StatusUnauthorized, err) } // signedAuth authenticates a signed URL. purpose separates the flavours of URL the // handler hands out, so one cannot be replayed as another; see computeSignature. func (h *Handler) signedAuth(purpose string, handler http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { h.logger.Debugf("%s %s", r.Method, r.URL.Path) id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) if err != nil { h.responseJSON(w, r, 400, err) return } expStr := r.URL.Query().Get("exp") sig := r.URL.Query().Get("sig") if expStr == "" || sig == "" { h.responseJSON(w, r, http.StatusUnauthorized, errors.New("missing signature")) return } exp, err := strconv.ParseInt(expStr, 10, 64) if err != nil { h.responseJSON(w, r, http.StatusUnauthorized, errors.New("invalid expiry")) return } if time.Now().Unix() > exp { h.responseJSON(w, r, http.StatusUnauthorized, errors.New("signature expired")) return } expected := h.computeSignature(purpose, id, exp) if !hmac.Equal([]byte(sig), []byte(expected)) { h.responseJSON(w, r, http.StatusUnauthorized, errors.New("bad signature")) return } handler(w, r) go h.gcCache() } } // internalAuth gates the control-plane endpoints. The bearer must // constant-time-equal the configured internalSecret. If the secret is empty, // the control-plane is disabled and every request gets 404 — which matches // the upstream nektos/act behavior of "the route does not exist". func (h *Handler) internalAuth(handler http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if h.internalSecret == "" { http.NotFound(w, r) return } token := bearerToken(r) if token == "" || !hmac.Equal([]byte(token), []byte(h.internalSecret)) { h.responseJSON(w, r, http.StatusUnauthorized, errors.New("internal: bad secret")) return } handler(w, r) } } type internalRegisterBody struct { Token string `json:"token"` JobCredential } type internalRevokeBody struct { Token string `json:"token"` } // POST /_internal/register // ResultsURL is what a job registered with cred should be given as ACTIONS_RESULTS_URL, or "" when // the credential names no instance to forward the artifact half to. func (h *Handler) ResultsURL(cred JobCredential) string { if h == nil || cred.Results == "" { return "" } return h.baseURL(cred) } func (h *Handler) internalRegister(w http.ResponseWriter, r *http.Request) { var body internalRegisterBody if err := json.UnmarshalRead(r.Body, &body); err != nil { h.responseJSON(w, r, http.StatusBadRequest, err) return } if body.Token == "" { h.responseJSON(w, r, http.StatusBadRequest, errors.New("token is required")) return } h.RegisterJob(body.Token, body.JobCredential) h.responseJSON(w, r, http.StatusOK) } // POST /_internal/revoke func (h *Handler) internalRevoke(w http.ResponseWriter, r *http.Request) { var body internalRevokeBody if err := json.UnmarshalRead(r.Body, &body); err != nil { h.responseJSON(w, r, http.StatusBadRequest, err) return } if body.Token == "" { h.responseJSON(w, r, http.StatusBadRequest, errors.New("token is required")) return } h.RevokeJob(body.Token) h.responseJSON(w, r, http.StatusOK) } // hashedToken fingerprints a job's bearer, so a reservation can tell its own retry from another job. func hashedToken(token string) string { sum := sha256.Sum256([]byte(token)) return hex.EncodeToString(sum[:]) } func bearerToken(r *http.Request) string { auth := r.Header.Get("Authorization") const prefix = "Bearer " if len(auth) > len(prefix) && strings.EqualFold(auth[:len(prefix)], prefix) { return auth[len(prefix):] } return "" } func credFromContext(ctx context.Context) JobCredential { cred, _ := ctx.Value(credKey{}).(JobCredential) return cred } // computeSignature signs a URL for one cache entry and expiry. purpose is mixed into the // message so a URL handed out for writing an entry cannot be replayed to read one, and the // other way round. Downloads use the empty purpose, the message v1 has always signed. func (h *Handler) computeSignature(purpose string, cacheID, exp int64) string { mac := hmac.New(sha256.New, h.secret) fmt.Fprintf(mac, "%s%d:%d", purpose, cacheID, exp) return hex.EncodeToString(mac.Sum(nil)) } // signedURL builds a URL under path that signedAuth accepts for the same purpose. func (h *Handler) signedURL(cred JobCredential, path, purpose string, cacheID uint64, exp time.Time) string { expUnix := exp.Unix() q := url.Values{} q.Set("exp", strconv.FormatInt(expUnix, 10)) q.Set("sig", h.computeSignature(purpose, int64(cacheID), expUnix)) return fmt.Sprintf("%s%s/%d?%s", h.baseURL(cred), path, cacheID, q.Encode()) } func (h *Handler) signedArtifactURL(cred JobCredential, cacheID uint64, exp time.Time) string { return h.signedURL(cred, apiPath+"/artifacts", "", cacheID, exp) } // if not found, return (nil, nil) instead of an error. func findCache(db *cacheDB, repo string, keys []string, version string) (*Cache, error) { for _, prefix := range keys { // if a key in the list matches exactly, don't return partial matches exact, err := findExactCache(db, repo, prefix, version, true) if err != nil { return nil, err } if exact != nil { return exact, nil } caches, err := db.find(func(cache *Cache) bool { return cache.Repo == repo && strings.HasPrefix(cache.Key, prefix) && cache.Version == version && cache.Complete }) if err != nil { return nil, fmt.Errorf("find cache: %w", err) } if cache := latest(caches, (*Cache).createdAtNano); cache != nil { return cache, nil } } return nil, nil //nolint:nilnil // pre-existing issue from nektos/act } func latest(caches []*Cache, stamp func(*Cache) int64) *Cache { if len(caches) == 0 { return nil } return slices.MaxFunc(caches, func(left, right *Cache) int { return cmp.Or(cmp.Compare(stamp(left), stamp(right)), cmp.Compare(left.ID, right.ID)) }) } // findExactCache returns the entry for exactly this key and version, or (nil, nil) if there is // none. Unlike findCache it never falls back to a prefix (restore-key) match, which is what both // its callers need: a new key that is only a prefix of an existing key is not the same entry. // // A completed entry is the one to restore, sorted by when it was written. An incomplete one is a // reservation being uploaded to, sorted by when it was last written to, because the upload route // touches UsedAt on every part. func findExactCache(db *cacheDB, repo, key, version string, complete bool) (*Cache, error) { caches, err := db.find(func(cache *Cache) bool { return cache.Key == key && cache.Repo == repo && cache.Version == version && cache.Complete == complete }) if err != nil { return nil, fmt.Errorf("find cache: %w", err) } stamp := func(cache *Cache) int64 { return cache.UsedAt } if complete { stamp = (*Cache).createdAtNano } return latest(caches, stamp), nil } func (h *Handler) insertCache(db *cacheDB, cache *Cache) error { return db.update(func(bucket *bbolt.Bucket) (err error) { if h.s3 != nil { var id [8]byte _, _ = rand.Read(id[:]) cache.ID = binary.BigEndian.Uint64(id[:]) >> 11 // IDs must be unique across runners, and v1 clients parse an ID as a JS number } else if cache.ID, err = bucket.NextSequence(); err != nil { return fmt.Errorf("insert cache: %w", err) } if bucket.Get(cacheKey(cache.ID)) != nil { return fmt.Errorf("insert cache: id %d is taken", cache.ID) } return db.put(bucket, cache) }) } // touchCache stamps UsedAt so gcCache does not reap an entry mid-upload. With requireIncomplete // it also refuses an entry that is already complete, which is what the v2 blob route needs: its // upload URL outlives the finalize call, and overwriting a finished entry would leave the blob // other jobs restore no longer matching its recorded size. An entry missing from the store is // accepted, since the signature proves the id was handed out. func (h *Handler) touchCache(id uint64, requireIncomplete bool) error { db, err := h.openDB() if err != nil { return err } defer db.Close() cache, err := db.get(id) if errors.Is(err, errCacheNotFound) { return nil } if err != nil { return err } if requireIncomplete && cache.Complete { return fmt.Errorf("cache %d: already complete", id) } if !touchNeeded(cache) { return nil } cache.UsedAt = time.Now().Unix() return db.save(cache) } const ( miB = 1024 * 1024 defaultSweepInterval = time.Hour touchStale = time.Minute // how stale a completed entry's UsedAt may get before an access rewrites it, sparing hits the fsync // inUseGrace covers artifactURLTTL plus the touchStale lag so an entry outlives every signed URL // still usable for it, and no sweep cuts off a download in progress. inUseGrace = artifactURLTTL + touchStale // uploadStallTimeout is how long a reservation may sit without a chunk before it counts // as abandoned. Widening it also widens the window for findExactCache to hand a finalize // a stale reservation. uploadStallTimeout = 5 * time.Minute metadataSyncInterval = 30 * time.Second defaultMinFreeDisk = 1024 * miB ) // Policy bounds what the cache server keeps: a retention window counted from last access, // and size limits that evict least recently accessed first. A zero limit is no limit. type Policy struct { Retention time.Duration // Retention removes entries nothing has read or written within this window. Zero keeps them regardless of age. RepoSizeLimit int64 // RepoSizeLimit caps one repository's completed entries in bytes, evicting least recently accessed first. SizeLimit int64 // SizeLimit caps every repository's completed entries together, in bytes. SweepInterval time.Duration // SweepInterval is the minimum time between two eviction sweeps. MinFreeDisk int64 // MinFreeDisk is volume headroom the cache will not eat into. Tracks the runner's health-check floor rather than taking a key of its own. } func (p Policy) withDefaults() Policy { // The limits default in config.LoadDefault, so a written 0 means off. if p.MinFreeDisk <= 0 { p.MinFreeDisk = defaultMinFreeDisk } if p.SweepInterval <= 0 { p.SweepInterval = defaultSweepInterval } return p } func (h *Handler) gcCache() { if !h.gcing.CompareAndSwap(false, true) { return } defer h.gcing.Store(false) if time.Since(h.gcAt) < h.policy.SweepInterval { h.logger.Debugf("skip gc: %v", h.gcAt.String()) return } h.gcAt = time.Now() h.logger.Debugf("gc: %v", h.gcAt.String()) db, err := h.openDB() if err != nil { return } defer db.Close() if err := db.dropLegacyIndexes(); err != nil { h.logger.Warnf("drop legacy indexes: %v", err) } h.evictIncomplete(db) if h.s3 != nil { return // lifecycle rules on the bucket expire its entries } h.evictExpired(db) h.evictSuperseded(db) h.evictOversized(db) h.evictForFreeSpace(db) } // evictForFreeSpace bounds the volume itself, so it also covers bytes the cache never // accounted for. func (h *Handler) evictForFreeSpace(db *cacheDB) { free, err := h.freeDisk(h.dir) if err != nil { h.logger.Debugf("free disk check: %v", err) // unsupported platform, treat as unavailable rather than full return } if free >= uint64(h.policy.MinFreeDisk) { return } caches := h.completedByUse(db) total, shortfall := totalSize(caches), h.policy.MinFreeDisk-int64(free) if total <= shortfall { // Say so, or shedding everything and still being short reads as the backstop working. h.logger.Warnf("cache volume is %d MiB short of the free space floor with only %d MiB of cache on it; something else is filling it", shortfall/miB, total/miB) } h.evictTo(db, caches, total-shortfall, "the cache volume") } // evictIncomplete removes uploads that stopped part way, which are most likely broken. func (h *Handler) evictIncomplete(db *cacheDB) { stalled := time.Now().Add(-uploadStallTimeout).Unix() h.sweep(db, func(cache *Cache) bool { return cache.UsedAt < stalled && !cache.Complete }) } func (h *Handler) evictExpired(db *cacheDB) { if h.policy.Retention <= 0 { return } // Never below inUseGrace, or a short retention would outrun a signed URL already issued. expired := time.Now().Add(-max(h.policy.Retention+touchStale, inUseGrace)).Unix() h.sweep(db, func(cache *Cache) bool { return cache.UsedAt < expired }) } // evictSuperseded removes entries a newer one with the same key and version replaced. The // grouping includes Repo so two repos sharing a (key, version) do not evict each other. func (h *Handler) evictSuperseded(db *cacheDB) { type entry struct{ repo, key, version string } groups := make(map[entry][]*Cache) for _, cache := range h.caches(db, func(cache *Cache) bool { return cache.Complete }) { group := entry{cache.Repo, cache.Key, cache.Version} groups[group] = append(groups[group], cache) } for _, caches := range groups { slices.SortFunc(caches, func(left, right *Cache) int { return cmp.Or(cmp.Compare(left.createdAtNano(), right.createdAtNano()), cmp.Compare(left.ID, right.ID)) }) for _, cache := range caches[:len(caches)-1] { if inUse(cache) { continue } h.deleteCache(db, cache) } } } // evictOversized applies the per-repository limit, then the whole-store one. Only completed // entries count, since only those carry a size measured at commit rather than claimed. func (h *Handler) evictOversized(db *cacheDB) { if h.policy.RepoSizeLimit > 0 { byRepo := make(map[string][]*Cache) for _, cache := range h.completedByUse(db) { byRepo[cache.Repo] = append(byRepo[cache.Repo], cache) } for repo, caches := range byRepo { h.evictTo(db, caches, h.policy.RepoSizeLimit, "repository "+repo) } } h.evictTotal(db) } // evictTotal caps the store as a whole. It re-queries because the per-repo pass may have // deleted rows an earlier result still holds. func (h *Handler) evictTotal(db *cacheDB) { if h.policy.SizeLimit <= 0 { return } h.evictTo(db, h.completedByUse(db), h.policy.SizeLimit, "the cache") } // evictRepo reclaims space when a commit pushes a repo over, rather than at the next sweep. func (h *Handler) evictRepo(db *cacheDB, repo string) { if h.policy.RepoSizeLimit <= 0 { return } h.evictTo(db, h.cachesByUse(db, func(cache *Cache) bool { return cache.Repo == repo && cache.Complete }), h.policy.RepoSizeLimit, "repository "+repo) } // evictTo deletes until caches fit limit. caches must be ordered by UsedAt ascending. func (h *Handler) evictTo(db *cacheDB, caches []*Cache, limit int64, scope string) { // An entry bigger than the limit never fits, so it goes on its own account instead of // dragging every neighbour out first and then following them next sweep. fits := caches[:0] for _, cache := range caches { if cache.Size <= limit { fits = append(fits, cache) continue } if !inUse(cache) { h.logger.Warnf("cache %q is %d MiB on its own, over the limit for %s; dropping it", cache.Key, cache.Size/miB, scope) h.deleteCache(db, cache) } } caches = fits total := totalSize(caches) var freed int64 for _, cache := range caches { if total <= limit { break } if inUse(cache) || !h.deleteCache(db, cache) { continue } total -= cache.Size freed += cache.Size } if freed > 0 { h.logger.Warnf("evicted %d MiB from %s, least recently used first", freed/miB, scope) } } // inUse reports whether an entry was read or written recently enough that removing it // could break a download in progress. func inUse(cache *Cache) bool { return time.Since(time.Unix(cache.UsedAt, 0)) < inUseGrace } // touchNeeded skips a write that would not change the second, and completed entries fresher than touchStale. func touchNeeded(cache *Cache) bool { return cache.UsedAt < time.Now().Unix() && (!cache.Complete || time.Since(time.Unix(cache.UsedAt, 0)) >= touchStale) } // touch stamps UsedAt through the caller's store, a bolt write on the read path. It cannot // go through touchCache, which opens its own store and would block on the exclusive lock // for as long as the caller holds one. func (h *Handler) touch(db *cacheDB, cache *Cache) { if !touchNeeded(cache) { return } cache.UsedAt = time.Now().Unix() if err := db.save(cache); err != nil { h.logger.Warnf("touch cache: %v", err) } } func (h *Handler) sweep(db *cacheDB, match func(*Cache) bool) { for _, cache := range h.caches(db, match) { h.deleteCache(db, cache) } } func (h *Handler) caches(db *cacheDB, match func(*Cache) bool) []*Cache { caches, err := db.find(match) if err != nil { h.logger.Warnf("find caches: %v", err) } return caches } func (h *Handler) cachesByUse(db *cacheDB, match func(*Cache) bool) []*Cache { caches := h.caches(db, match) slices.SortFunc(caches, func(a, b *Cache) int { return cmp.Compare(a.UsedAt, b.UsedAt) }) return caches } // completedByUse returns every entry the size limits count, least recently accessed first. func (h *Handler) completedByUse(db *cacheDB) []*Cache { return h.cachesByUse(db, func(cache *Cache) bool { return cache.Complete }) } func totalSize(caches []*Cache) int64 { var total int64 for _, cache := range caches { total += cache.Size } return total } // deleteCache drops an entry and its bytes, reporting whether it went fully. The blob goes // first, so a failed unlink leaves the row for the next sweep instead of orphaning bytes. func (h *Handler) deleteCache(db *cacheDB, cache *Cache) bool { storage := h.storage if h.s3 != nil && !cache.Complete { storage = h.s3.Storage // an upload not yet committed has nothing in the bucket } if err := storage.Remove(cache.ID); err != nil { h.logger.Warnf("remove cache blob: %v", err) return false } if err := db.update(func(bucket *bbolt.Bucket) error { return db.delete(bucket, cache.ID) }); err != nil { h.logger.Warnf("delete cache: %v", err) return false } h.logger.Infof("deleted cache: %+v", cache) return true } func (h *Handler) responseJSON(w http.ResponseWriter, r *http.Request, code int, v ...any) { w.Header().Set("Content-Type", "application/json; charset=utf-8") var data []byte if len(v) == 0 || v[0] == nil { data, _ = json.Marshal(struct{}{}) } else if err, ok := v[0].(error); ok { h.logger.Errorf("%v %v: %v", r.Method, r.URL.Path, err) data, _ = json.Marshal(map[string]any{ "error": err.Error(), }) } else { data, _ = json.Marshal(v[0]) } w.WriteHeader(code) _, _ = w.Write(data) } func parseContentRange(s string) (int64, error) { // support the format like "bytes 11-22/*" only s, _, _ = strings.Cut(strings.TrimPrefix(s, "bytes "), "/") s1, s2, _ := strings.Cut(s, "-") start, err := strconv.ParseInt(s1, 10, 64) if err != nil { return 0, fmt.Errorf("parse %q: %w", s, err) } if _, err := strconv.ParseInt(s2, 10, 64); err != nil { return 0, fmt.Errorf("parse %q: %w", s, err) } return start, nil }