mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-18 05:43:42 +00:00
Support cache (#25)
See [Caching dependencies to speed up workflows](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows). Reviewed-on: https://gitea.com/gitea/act_runner/pulls/25 Reviewed-by: Lunny Xiao <[email protected]> Reviewed-by: Zettat123 <[email protected]> Co-authored-by: Jason Song <[email protected]> Co-committed-by: Jason Song <[email protected]>
This commit is contained in:
1 parent
552dbcdda9
commit
436b441cad
11 files changed
+1091
-6
No files matched your search
@@ -0,0 +1,7 @@
|
|||||||
|
Inspired by:
|
||||||
|
https://github.com/sp-ricard-valverde/github-act-cache-server
|
||||||
|
|
||||||
|
TODO:
|
||||||
|
- Authorization
|
||||||
|
- [Restrictions for accessing a cache](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache)
|
||||||
|
- [Force deleting cache entries](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries)
|
||||||
@@ -0,0 +1,409 @@
|
|||||||
|
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifactcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
|
"github.com/go-chi/render"
|
||||||
|
"github.com/nektos/act/pkg/common"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
_ "modernc.org/sqlite"
|
||||||
|
"xorm.io/builder"
|
||||||
|
"xorm.io/xorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
urlBase = "/_apis/artifactcache"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
logger = log.StandardLogger().WithField("module", "cache_request")
|
||||||
|
)
|
||||||
|
|
||||||
|
type Handler struct {
|
||||||
|
engine engine
|
||||||
|
storage *Storage
|
||||||
|
router *chi.Mux
|
||||||
|
listener net.Listener
|
||||||
|
|
||||||
|
gc atomic.Bool
|
||||||
|
gcAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewHandler() (*Handler, error) {
|
||||||
|
h := &Handler{}
|
||||||
|
|
||||||
|
dir := "" // TODO: make the dir configurable if necessary
|
||||||
|
if home, err := os.UserHomeDir(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
} else {
|
||||||
|
dir = filepath.Join(home, ".cache/actcache")
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
e, err := xorm.NewEngine("sqlite", filepath.Join(dir, "sqlite.db"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := e.Sync(&Cache{}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
h.engine = engine{e: e}
|
||||||
|
|
||||||
|
storage, err := NewStorage(filepath.Join(dir, "cache"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
h.storage = storage
|
||||||
|
|
||||||
|
router := chi.NewRouter()
|
||||||
|
router.Use(middleware.RequestLogger(&middleware.DefaultLogFormatter{Logger: logger}))
|
||||||
|
router.Use(func(handler http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
handler.ServeHTTP(w, r)
|
||||||
|
go h.gcCache()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
router.Use(middleware.Logger)
|
||||||
|
router.Route(urlBase, func(r chi.Router) {
|
||||||
|
r.Get("/cache", h.find)
|
||||||
|
r.Route("/caches", func(r chi.Router) {
|
||||||
|
r.Post("/", h.reserve)
|
||||||
|
r.Route("/{id}", func(r chi.Router) {
|
||||||
|
r.Patch("/", h.upload)
|
||||||
|
r.Post("/", h.commit)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
r.Get("/artifacts/{id}", h.get)
|
||||||
|
r.Post("/clean", h.clean)
|
||||||
|
})
|
||||||
|
|
||||||
|
h.router = router
|
||||||
|
|
||||||
|
h.gcCache()
|
||||||
|
|
||||||
|
// TODO: make the port configurable if necessary
|
||||||
|
listener, err := net.Listen("tcp", ":0") // random available port
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
if err := http.Serve(listener, h.router); err != nil {
|
||||||
|
logger.Error("http serve: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
h.listener = listener
|
||||||
|
|
||||||
|
return h, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) ExternalURL() string {
|
||||||
|
// TODO: make the external url configurable if necessary
|
||||||
|
return fmt.Sprintf("http://%s:%d",
|
||||||
|
common.GetOutboundIP().String(),
|
||||||
|
h.listener.Addr().(*net.TCPAddr).Port)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GET /_apis/artifactcache/cache
|
||||||
|
func (h *Handler) find(w http.ResponseWriter, r *http.Request) {
|
||||||
|
keys := strings.Split(r.URL.Query().Get("keys"), ",")
|
||||||
|
version := r.URL.Query().Get("version")
|
||||||
|
|
||||||
|
cache, err := h.findCache(r.Context(), keys, version)
|
||||||
|
if err != nil {
|
||||||
|
responseJson(w, r, 500, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cache == nil {
|
||||||
|
responseJson(w, r, 204)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if ok, err := h.storage.Exist(cache.ID); err != nil {
|
||||||
|
responseJson(w, r, 500, err)
|
||||||
|
return
|
||||||
|
} else if !ok {
|
||||||
|
_ = h.engine.Exec(func(sess *xorm.Session) error {
|
||||||
|
_, err := sess.Delete(cache)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
responseJson(w, r, 204)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
responseJson(w, r, 200, map[string]any{
|
||||||
|
"result": "hit",
|
||||||
|
"archiveLocation": fmt.Sprintf("%s%s/artifacts/%d", h.ExternalURL(), urlBase, cache.ID),
|
||||||
|
"cacheKey": cache.Key,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// POST /_apis/artifactcache/caches
|
||||||
|
func (h *Handler) reserve(w http.ResponseWriter, r *http.Request) {
|
||||||
|
cache := &Cache{}
|
||||||
|
if err := render.Bind(r, cache); err != nil {
|
||||||
|
responseJson(w, r, 400, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if ok, err := h.engine.ExecBool(func(sess *xorm.Session) (bool, error) {
|
||||||
|
return sess.Where(builder.Eq{"key": cache.Key, "version": cache.Version}).Get(&Cache{})
|
||||||
|
}); err != nil {
|
||||||
|
responseJson(w, r, 500, err)
|
||||||
|
return
|
||||||
|
} else if ok {
|
||||||
|
responseJson(w, r, 400, fmt.Errorf("already exist"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := h.engine.Exec(func(sess *xorm.Session) error {
|
||||||
|
_, err := sess.Insert(cache)
|
||||||
|
return err
|
||||||
|
}); err != nil {
|
||||||
|
responseJson(w, r, 500, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
responseJson(w, r, 200, map[string]any{
|
||||||
|
"cacheId": cache.ID,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// PATCH /_apis/artifactcache/caches/:id
|
||||||
|
func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
responseJson(w, r, 400, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cache := &Cache{
|
||||||
|
ID: id,
|
||||||
|
}
|
||||||
|
|
||||||
|
if ok, err := h.engine.ExecBool(func(sess *xorm.Session) (bool, error) {
|
||||||
|
return sess.Get(cache)
|
||||||
|
}); err != nil {
|
||||||
|
responseJson(w, r, 500, err)
|
||||||
|
return
|
||||||
|
} else if !ok {
|
||||||
|
responseJson(w, r, 400, fmt.Errorf("cache %d: not reserved", id))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if cache.Complete {
|
||||||
|
responseJson(w, r, 400, fmt.Errorf("cache %v %q: already complete", cache.ID, cache.Key))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
start, _, err := parseContentRange(r.Header.Get("Content-Range"))
|
||||||
|
if err != nil {
|
||||||
|
responseJson(w, r, 400, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := h.storage.Write(cache.ID, start, r.Body); err != nil {
|
||||||
|
responseJson(w, r, 500, err)
|
||||||
|
}
|
||||||
|
h.useCache(r.Context(), id)
|
||||||
|
responseJson(w, r, 200)
|
||||||
|
}
|
||||||
|
|
||||||
|
// POST /_apis/artifactcache/caches/:id
|
||||||
|
func (h *Handler) commit(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
responseJson(w, r, 400, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cache := &Cache{
|
||||||
|
ID: id,
|
||||||
|
}
|
||||||
|
if ok, err := h.engine.ExecBool(func(sess *xorm.Session) (bool, error) {
|
||||||
|
return sess.Get(cache)
|
||||||
|
}); err != nil {
|
||||||
|
responseJson(w, r, 500, err)
|
||||||
|
return
|
||||||
|
} else if !ok {
|
||||||
|
responseJson(w, r, 400, fmt.Errorf("cache %d: not reserved", id))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if cache.Complete {
|
||||||
|
responseJson(w, r, 400, fmt.Errorf("cache %v %q: already complete", cache.ID, cache.Key))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := h.storage.Commit(cache.ID, cache.Size); err != nil {
|
||||||
|
responseJson(w, r, 500, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cache.Complete = true
|
||||||
|
if err := h.engine.Exec(func(sess *xorm.Session) error {
|
||||||
|
_, err := sess.ID(cache.ID).Cols("complete").Update(cache)
|
||||||
|
return err
|
||||||
|
}); err != nil {
|
||||||
|
responseJson(w, r, 500, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
responseJson(w, r, 200)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GET /_apis/artifactcache/artifacts/:id
|
||||||
|
func (h *Handler) get(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
responseJson(w, r, 400, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.useCache(r.Context(), id)
|
||||||
|
h.storage.Serve(w, r, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// POST /_apis/artifactcache/clean
|
||||||
|
func (h *Handler) clean(w http.ResponseWriter, r *http.Request) {
|
||||||
|
// TODO: don't support force deleting cache entries
|
||||||
|
// see: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
|
||||||
|
|
||||||
|
responseJson(w, r, 200)
|
||||||
|
}
|
||||||
|
|
||||||
|
// if not found, return (nil, nil) instead of an error.
|
||||||
|
func (h *Handler) findCache(ctx context.Context, keys []string, version string) (*Cache, error) {
|
||||||
|
if len(keys) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
key := keys[0] // the first key is for exact match.
|
||||||
|
|
||||||
|
cache := &Cache{}
|
||||||
|
if ok, err := h.engine.ExecBool(func(sess *xorm.Session) (bool, error) {
|
||||||
|
return sess.Where(builder.Eq{"key": key, "version": version, "complete": true}).Get(cache)
|
||||||
|
}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
} else if ok {
|
||||||
|
return cache, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, prefix := range keys[1:] {
|
||||||
|
if ok, err := h.engine.ExecBool(func(sess *xorm.Session) (bool, error) {
|
||||||
|
return sess.Where(builder.And(
|
||||||
|
builder.Like{"key", prefix + "%"},
|
||||||
|
builder.Eq{"version": version, "complete": true},
|
||||||
|
)).OrderBy("id DESC").Get(cache)
|
||||||
|
}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
} else if ok {
|
||||||
|
return cache, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) useCache(ctx context.Context, id int64) {
|
||||||
|
// keep quiet
|
||||||
|
_ = h.engine.Exec(func(sess *xorm.Session) error {
|
||||||
|
_, err := sess.Context(ctx).Cols("used_at").Update(&Cache{
|
||||||
|
ID: id,
|
||||||
|
UsedAt: time.Now().Unix(),
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) gcCache() {
|
||||||
|
if h.gc.Load() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !h.gc.CompareAndSwap(false, true) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer h.gc.Store(false)
|
||||||
|
|
||||||
|
if time.Since(h.gcAt) < time.Hour {
|
||||||
|
logger.Infof("skip gc: %v", h.gcAt.String())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.gcAt = time.Now()
|
||||||
|
logger.Infof("gc: %v", h.gcAt.String())
|
||||||
|
|
||||||
|
const (
|
||||||
|
keepUsed = 30 * 24 * time.Hour
|
||||||
|
keepUnused = 7 * 24 * time.Hour
|
||||||
|
keepTemp = 5 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
var caches []*Cache
|
||||||
|
if err := h.engine.Exec(func(sess *xorm.Session) error {
|
||||||
|
return sess.Where(builder.And(builder.Lt{"used_at": time.Now().Add(-keepTemp).Unix()}, builder.Eq{"complete": false})).
|
||||||
|
Find(&caches)
|
||||||
|
}); err != nil {
|
||||||
|
logger.Warnf("find caches: %v", err)
|
||||||
|
} else {
|
||||||
|
for _, cache := range caches {
|
||||||
|
h.storage.Remove(cache.ID)
|
||||||
|
if err := h.engine.Exec(func(sess *xorm.Session) error {
|
||||||
|
_, err := sess.Delete(cache)
|
||||||
|
return err
|
||||||
|
}); err != nil {
|
||||||
|
logger.Warnf("delete cache: %v", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
logger.Infof("deleted cache: %+v", cache)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
caches = caches[:0]
|
||||||
|
if err := h.engine.Exec(func(sess *xorm.Session) error {
|
||||||
|
return sess.Where(builder.Lt{"used_at": time.Now().Add(-keepUnused).Unix()}).
|
||||||
|
Find(&caches)
|
||||||
|
}); err != nil {
|
||||||
|
logger.Warnf("find caches: %v", err)
|
||||||
|
} else {
|
||||||
|
for _, cache := range caches {
|
||||||
|
h.storage.Remove(cache.ID)
|
||||||
|
if err := h.engine.Exec(func(sess *xorm.Session) error {
|
||||||
|
_, err := sess.Delete(cache)
|
||||||
|
return err
|
||||||
|
}); err != nil {
|
||||||
|
logger.Warnf("delete cache: %v", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
logger.Infof("deleted cache: %+v", cache)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
caches = caches[:0]
|
||||||
|
if err := h.engine.Exec(func(sess *xorm.Session) error {
|
||||||
|
return sess.Where(builder.Lt{"created_at": time.Now().Add(-keepUsed).Unix()}).
|
||||||
|
Find(&caches)
|
||||||
|
}); err != nil {
|
||||||
|
logger.Warnf("find caches: %v", err)
|
||||||
|
} else {
|
||||||
|
for _, cache := range caches {
|
||||||
|
h.storage.Remove(cache.ID)
|
||||||
|
if err := h.engine.Exec(func(sess *xorm.Session) error {
|
||||||
|
_, err := sess.Delete(cache)
|
||||||
|
return err
|
||||||
|
}); err != nil {
|
||||||
|
logger.Warnf("delete cache: %v", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
logger.Infof("deleted cache: %+v", cache)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifactcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Cache struct {
|
||||||
|
ID int64 `xorm:"id pk autoincr" json:"-"`
|
||||||
|
Key string `xorm:"TEXT index unique(key_version)" json:"key"`
|
||||||
|
Version string `xorm:"TEXT unique(key_version)" json:"version"`
|
||||||
|
Size int64 `json:"cacheSize"`
|
||||||
|
Complete bool `xorm:"index(complete_used_at)" json:"-"`
|
||||||
|
UsedAt int64 `xorm:"index(complete_used_at) updated" json:"-"`
|
||||||
|
CreatedAt int64 `xorm:"index created" json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bind implements render.Binder
|
||||||
|
func (c *Cache) Bind(_ *http.Request) error {
|
||||||
|
if c.Key == "" {
|
||||||
|
return fmt.Errorf("missing key")
|
||||||
|
}
|
||||||
|
if c.Version == "" {
|
||||||
|
return fmt.Errorf("missing version")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifactcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Storage struct {
|
||||||
|
rootDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewStorage(rootDir string) (*Storage, error) {
|
||||||
|
if err := os.MkdirAll(rootDir, 0o755); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &Storage{
|
||||||
|
rootDir: rootDir,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Storage) Exist(id int64) (bool, error) {
|
||||||
|
name := s.filename(id)
|
||||||
|
if _, err := os.Stat(name); os.IsNotExist(err) {
|
||||||
|
return false, nil
|
||||||
|
} else if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Storage) Write(id int64, offset int64, reader io.Reader) error {
|
||||||
|
name := s.tempName(id, offset)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
file, err := os.Create(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
_, err = io.Copy(file, reader)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Storage) Commit(id int64, size int64) error {
|
||||||
|
defer func() {
|
||||||
|
_ = os.RemoveAll(s.tempDir(id))
|
||||||
|
}()
|
||||||
|
|
||||||
|
name := s.filename(id)
|
||||||
|
tempNames, err := s.tempNames(id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
file, err := os.Create(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
var written int64
|
||||||
|
for _, v := range tempNames {
|
||||||
|
f, err := os.Open(v)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
n, err := io.Copy(file, f)
|
||||||
|
_ = f.Close()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
written += n
|
||||||
|
}
|
||||||
|
|
||||||
|
if written != size {
|
||||||
|
_ = file.Close()
|
||||||
|
_ = os.Remove(name)
|
||||||
|
return fmt.Errorf("broken file: %v != %v", written, size)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id int64) {
|
||||||
|
name := s.filename(id)
|
||||||
|
http.ServeFile(w, r, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Storage) Remove(id int64) {
|
||||||
|
_ = os.Remove(s.filename(id))
|
||||||
|
_ = os.RemoveAll(s.tempDir(id))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Storage) filename(id int64) string {
|
||||||
|
return filepath.Join(s.rootDir, fmt.Sprintf("%02x", id%0xff), fmt.Sprint(id))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Storage) tempDir(id int64) string {
|
||||||
|
return filepath.Join(s.rootDir, "tmp", fmt.Sprint(id))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Storage) tempName(id, offset int64) string {
|
||||||
|
return filepath.Join(s.tempDir(id), fmt.Sprintf("%016x", offset))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Storage) tempNames(id int64) ([]string, error) {
|
||||||
|
dir := s.tempDir(id)
|
||||||
|
files, err := os.ReadDir(dir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
for _, v := range files {
|
||||||
|
if !v.IsDir() {
|
||||||
|
names = append(names, filepath.Join(dir, v.Name()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package artifactcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/go-chi/render"
|
||||||
|
"xorm.io/xorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
func responseJson(w http.ResponseWriter, r *http.Request, code int, v ...any) {
|
||||||
|
render.Status(r, code)
|
||||||
|
if len(v) == 0 || v[0] == nil {
|
||||||
|
render.JSON(w, r, struct{}{})
|
||||||
|
} else if err, ok := v[0].(error); ok {
|
||||||
|
logger.Errorf("%v %v: %v", r.Method, r.RequestURI, err)
|
||||||
|
render.JSON(w, r, map[string]any{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
render.JSON(w, r, v[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseContentRange(s string) (int64, int64, error) {
|
||||||
|
// support the format like "bytes 11-22/*" only
|
||||||
|
s, _, _ = strings.Cut(strings.TrimPrefix(s, "bytes "), "/")
|
||||||
|
s1, s2, _ := strings.Cut(s, "-")
|
||||||
|
|
||||||
|
start, err := strconv.ParseInt(s1, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("parse %q: %w", s, err)
|
||||||
|
}
|
||||||
|
stop, err := strconv.ParseInt(s2, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("parse %q: %w", s, err)
|
||||||
|
}
|
||||||
|
return start, stop, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// engine is a wrapper of *xorm.Engine, with a lock.
|
||||||
|
// To avoid racing of sqlite, we don't careperformance here.
|
||||||
|
type engine struct {
|
||||||
|
e *xorm.Engine
|
||||||
|
m sync.Mutex
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *engine) Exec(f func(*xorm.Session) error) error {
|
||||||
|
e.m.Lock()
|
||||||
|
defer e.m.Unlock()
|
||||||
|
|
||||||
|
sess := e.e.NewSession()
|
||||||
|
defer sess.Close()
|
||||||
|
|
||||||
|
return f(sess)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *engine) ExecBool(f func(*xorm.Session) (bool, error)) (bool, error) {
|
||||||
|
e.m.Lock()
|
||||||
|
defer e.m.Unlock()
|
||||||
|
|
||||||
|
sess := e.e.NewSession()
|
||||||
|
defer sess.Close()
|
||||||
|
|
||||||
|
return f(sess)
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
//go:build vendor
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
// for vet
|
||||||
|
_ "code.gitea.io/gitea-vet"
|
||||||
|
)
|
||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.com/gitea/act_runner/artifactcache"
|
||||||
"gitea.com/gitea/act_runner/client"
|
"gitea.com/gitea/act_runner/client"
|
||||||
"gitea.com/gitea/act_runner/config"
|
"gitea.com/gitea/act_runner/config"
|
||||||
"gitea.com/gitea/act_runner/engine"
|
"gitea.com/gitea/act_runner/engine"
|
||||||
@@ -52,6 +53,12 @@ func runDaemon(ctx context.Context, envFile string) func(cmd *cobra.Command, arg
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
handler, err := artifactcache.NewHandler()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Infof("cache handler listens on: %v", handler.ExternalURL())
|
||||||
|
|
||||||
var g errgroup.Group
|
var g errgroup.Group
|
||||||
|
|
||||||
cli := client.New(
|
cli := client.New(
|
||||||
@@ -67,6 +74,7 @@ func runDaemon(ctx context.Context, envFile string) func(cmd *cobra.Command, arg
|
|||||||
ForgeInstance: cfg.Client.Address,
|
ForgeInstance: cfg.Client.Address,
|
||||||
Environ: cfg.Runner.Environ,
|
Environ: cfg.Runner.Environ,
|
||||||
Labels: cfg.Runner.Labels,
|
Labels: cfg.Runner.Labels,
|
||||||
|
CacheHandler: handler,
|
||||||
}
|
}
|
||||||
|
|
||||||
poller := poller.New(
|
poller := poller.New(
|
||||||
|
|||||||
@@ -4,9 +4,12 @@ go 1.18
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
code.gitea.io/actions-proto-go v0.2.0
|
code.gitea.io/actions-proto-go v0.2.0
|
||||||
|
code.gitea.io/gitea-vet v0.2.3-0.20230113022436-2b1561217fa5
|
||||||
github.com/avast/retry-go/v4 v4.3.1
|
github.com/avast/retry-go/v4 v4.3.1
|
||||||
github.com/bufbuild/connect-go v1.3.1
|
github.com/bufbuild/connect-go v1.3.1
|
||||||
github.com/docker/docker v20.10.21+incompatible
|
github.com/docker/docker v20.10.21+incompatible
|
||||||
|
github.com/go-chi/chi/v5 v5.0.8
|
||||||
|
github.com/go-chi/render v1.0.2
|
||||||
github.com/joho/godotenv v1.4.0
|
github.com/joho/godotenv v1.4.0
|
||||||
github.com/kelseyhightower/envconfig v1.4.0
|
github.com/kelseyhightower/envconfig v1.4.0
|
||||||
github.com/mattn/go-isatty v0.0.16
|
github.com/mattn/go-isatty v0.0.16
|
||||||
@@ -15,15 +18,18 @@ require (
|
|||||||
github.com/spf13/cobra v1.6.1
|
github.com/spf13/cobra v1.6.1
|
||||||
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde
|
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde
|
||||||
google.golang.org/protobuf v1.28.1
|
google.golang.org/protobuf v1.28.1
|
||||||
|
modernc.org/sqlite v1.14.2
|
||||||
|
xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978
|
||||||
|
xorm.io/xorm v1.3.2
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
code.gitea.io/gitea-vet v0.2.3-0.20230113022436-2b1561217fa5 // indirect
|
|
||||||
github.com/Masterminds/semver v1.5.0 // indirect
|
github.com/Masterminds/semver v1.5.0 // indirect
|
||||||
github.com/Microsoft/go-winio v0.5.2 // indirect
|
github.com/Microsoft/go-winio v0.5.2 // indirect
|
||||||
github.com/Microsoft/hcsshim v0.9.3 // indirect
|
github.com/Microsoft/hcsshim v0.9.3 // indirect
|
||||||
github.com/ProtonMail/go-crypto v0.0.0-20220404123522-616f957b79ad // indirect
|
github.com/ProtonMail/go-crypto v0.0.0-20220404123522-616f957b79ad // indirect
|
||||||
github.com/acomagu/bufpipe v1.0.3 // indirect
|
github.com/acomagu/bufpipe v1.0.3 // indirect
|
||||||
|
github.com/ajg/form v1.5.1 // indirect
|
||||||
github.com/containerd/cgroups v1.0.3 // indirect
|
github.com/containerd/cgroups v1.0.3 // indirect
|
||||||
github.com/containerd/containerd v1.6.6 // indirect
|
github.com/containerd/containerd v1.6.6 // indirect
|
||||||
github.com/creack/pty v1.1.18 // indirect
|
github.com/creack/pty v1.1.18 // indirect
|
||||||
@@ -38,12 +44,16 @@ require (
|
|||||||
github.com/go-git/go-billy/v5 v5.3.1 // indirect
|
github.com/go-git/go-billy/v5 v5.3.1 // indirect
|
||||||
github.com/go-git/go-git/v5 v5.4.2 // indirect
|
github.com/go-git/go-git/v5 v5.4.2 // indirect
|
||||||
github.com/go-ini/ini v1.67.0 // indirect
|
github.com/go-ini/ini v1.67.0 // indirect
|
||||||
|
github.com/goccy/go-json v0.8.1 // indirect
|
||||||
github.com/gogo/protobuf v1.3.2 // indirect
|
github.com/gogo/protobuf v1.3.2 // indirect
|
||||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||||
|
github.com/golang/snappy v0.0.4 // indirect
|
||||||
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
|
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
|
||||||
|
github.com/google/uuid v1.3.0 // indirect
|
||||||
github.com/imdario/mergo v0.3.13 // indirect
|
github.com/imdario/mergo v0.3.13 // indirect
|
||||||
github.com/inconshreveable/mousetrap v1.0.1 // indirect
|
github.com/inconshreveable/mousetrap v1.0.1 // indirect
|
||||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||||
|
github.com/json-iterator/go v1.1.12 // indirect
|
||||||
github.com/julienschmidt/httprouter v1.3.0 // indirect
|
github.com/julienschmidt/httprouter v1.3.0 // indirect
|
||||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||||
@@ -54,29 +64,44 @@ require (
|
|||||||
github.com/moby/buildkit v0.10.6 // indirect
|
github.com/moby/buildkit v0.10.6 // indirect
|
||||||
github.com/moby/sys/mount v0.3.1 // indirect
|
github.com/moby/sys/mount v0.3.1 // indirect
|
||||||
github.com/moby/sys/mountinfo v0.6.0 // indirect
|
github.com/moby/sys/mountinfo v0.6.0 // indirect
|
||||||
|
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||||
|
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||||
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
|
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
|
||||||
github.com/opencontainers/runc v1.1.2 // indirect
|
github.com/opencontainers/runc v1.1.2 // indirect
|
||||||
github.com/opencontainers/selinux v1.10.2 // indirect
|
github.com/opencontainers/selinux v1.10.2 // indirect
|
||||||
github.com/pkg/errors v0.9.1 // indirect
|
github.com/pkg/errors v0.9.1 // indirect
|
||||||
|
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
|
||||||
github.com/rhysd/actionlint v1.6.22 // indirect
|
github.com/rhysd/actionlint v1.6.22 // indirect
|
||||||
github.com/rivo/uniseg v0.3.4 // indirect
|
github.com/rivo/uniseg v0.3.4 // indirect
|
||||||
github.com/robfig/cron v1.2.0 // indirect
|
github.com/robfig/cron v1.2.0 // indirect
|
||||||
github.com/sergi/go-diff v1.2.0 // indirect
|
github.com/sergi/go-diff v1.2.0 // indirect
|
||||||
github.com/spf13/pflag v1.0.5 // indirect
|
github.com/spf13/pflag v1.0.5 // indirect
|
||||||
|
github.com/syndtr/goleveldb v1.0.0 // indirect
|
||||||
github.com/xanzy/ssh-agent v0.3.1 // indirect
|
github.com/xanzy/ssh-agent v0.3.1 // indirect
|
||||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
|
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
|
||||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||||
github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f // indirect
|
github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f // indirect
|
||||||
go.opencensus.io v0.23.0 // indirect
|
go.opencensus.io v0.23.0 // indirect
|
||||||
golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect
|
golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect
|
||||||
|
golang.org/x/mod v0.4.2 // indirect
|
||||||
golang.org/x/net v0.0.0-20220906165146-f3363e06e74c // indirect
|
golang.org/x/net v0.0.0-20220906165146-f3363e06e74c // indirect
|
||||||
golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 // indirect
|
golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 // indirect
|
||||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
||||||
golang.org/x/tools v0.1.5 // indirect
|
golang.org/x/tools v0.1.5 // indirect
|
||||||
|
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
|
||||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||||
|
lukechampine.com/uint128 v1.1.1 // indirect
|
||||||
|
modernc.org/cc/v3 v3.35.18 // indirect
|
||||||
|
modernc.org/ccgo/v3 v3.12.82 // indirect
|
||||||
|
modernc.org/libc v1.11.87 // indirect
|
||||||
|
modernc.org/mathutil v1.4.1 // indirect
|
||||||
|
modernc.org/memory v1.0.5 // indirect
|
||||||
|
modernc.org/opt v0.1.1 // indirect
|
||||||
|
modernc.org/strutil v1.1.1 // indirect
|
||||||
|
modernc.org/token v1.0.0 // indirect
|
||||||
)
|
)
|
||||||
|
|
||||||
replace github.com/nektos/act => gitea.com/gitea/act v0.234.3-0.20230224062034-1252e551b867
|
replace github.com/nektos/act => gitea.com/gitea/act v0.234.3-0.20230224062034-1252e551b867
|
||||||
+9
-1
@@ -8,6 +8,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
runnerv1 "code.gitea.io/actions-proto-go/runner/v1"
|
runnerv1 "code.gitea.io/actions-proto-go/runner/v1"
|
||||||
|
|
||||||
|
"gitea.com/gitea/act_runner/artifactcache"
|
||||||
"gitea.com/gitea/act_runner/client"
|
"gitea.com/gitea/act_runner/client"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -18,11 +20,17 @@ type Runner struct {
|
|||||||
Environ map[string]string
|
Environ map[string]string
|
||||||
Client client.Client
|
Client client.Client
|
||||||
Labels []string
|
Labels []string
|
||||||
|
CacheHandler *artifactcache.Handler
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run runs the pipeline stage.
|
// Run runs the pipeline stage.
|
||||||
func (s *Runner) Run(ctx context.Context, task *runnerv1.Task) error {
|
func (s *Runner) Run(ctx context.Context, task *runnerv1.Task) error {
|
||||||
return NewTask(s.ForgeInstance, task.Id, s.Client, s.Environ, s.platformPicker).Run(ctx, task)
|
env := map[string]string{}
|
||||||
|
for k, v := range s.Environ {
|
||||||
|
env[k] = v
|
||||||
|
}
|
||||||
|
env["ACTIONS_CACHE_URL"] = s.CacheHandler.ExternalURL() + "/"
|
||||||
|
return NewTask(s.ForgeInstance, task.Id, s.Client, env, s.platformPicker).Run(ctx, task)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Runner) platformPicker(labels []string) string {
|
func (s *Runner) platformPicker(labels []string) string {
|
||||||
|
|||||||
+2
-2
@@ -14,13 +14,13 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
runnerv1 "code.gitea.io/actions-proto-go/runner/v1"
|
runnerv1 "code.gitea.io/actions-proto-go/runner/v1"
|
||||||
"gitea.com/gitea/act_runner/client"
|
|
||||||
|
|
||||||
"github.com/nektos/act/pkg/artifacts"
|
"github.com/nektos/act/pkg/artifacts"
|
||||||
"github.com/nektos/act/pkg/common"
|
"github.com/nektos/act/pkg/common"
|
||||||
"github.com/nektos/act/pkg/model"
|
"github.com/nektos/act/pkg/model"
|
||||||
"github.com/nektos/act/pkg/runner"
|
"github.com/nektos/act/pkg/runner"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
|
|
||||||
|
"gitea.com/gitea/act_runner/client"
|
||||||
)
|
)
|
||||||
|
|
||||||
var globalTaskMap sync.Map
|
var globalTaskMap sync.Map
|
||||||
|
|||||||
Reference in new issue
Block a user