mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-16 04:43:42 +00:00
refactor(container): stream copies into the job and archives out of the host (#1340)
Adds `CopyTarStream` to the job environments. It extracts directories and files into the job as they are written, kept below the destination and owned by the job user along with any directories it creates, so built-in actions can put large content in place without buffering it. The host's `GetContainerArchive` now streams instead of building the archive in memory. --------- Co-authored-by: silverwind <[email protected]> Co-authored-by: silverwind <[email protected]> Co-authored-by: Lunny Xiao <[email protected]> Reviewed-on: https://gitea.com/gitea/runner/pulls/1340 Reviewed-by: Lunny Xiao <[email protected]> Reviewed-by: silverwind <[email protected]> Co-authored-by: bircni <[email protected]>
This commit is contained in:
1 parent
b88d111cc5
commit
8c8950d9ef
9 files changed
+313
-40
No files matched your search
@@ -6,8 +6,13 @@ package container
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
@@ -15,6 +20,84 @@ import (
|
||||
"gitea.com/gitea/runner/act/filecollector"
|
||||
)
|
||||
|
||||
// TarWriter takes directories and files named relative to the destination, owned by the job user.
|
||||
type TarWriter struct {
|
||||
writer *tar.Writer
|
||||
uid, gid int
|
||||
prefix string
|
||||
dirs map[string]bool
|
||||
}
|
||||
|
||||
func (w *TarWriter) Write(data []byte) (int, error) {
|
||||
return w.writer.Write(data)
|
||||
}
|
||||
|
||||
func (w *TarWriter) WriteHeader(hdr *tar.Header) error {
|
||||
name := strings.TrimSuffix(hdr.Name, "/")
|
||||
if !fs.ValidPath(name) {
|
||||
return fmt.Errorf("tar entry %q escapes the destination", hdr.Name)
|
||||
}
|
||||
if hdr.Typeflag != tar.TypeDir && hdr.Typeflag != tar.TypeReg {
|
||||
return fmt.Errorf("tar entry %q: unsupported type %q", hdr.Name, hdr.Typeflag)
|
||||
}
|
||||
for index := range len(name) {
|
||||
if dir := name[:index]; name[index] == '/' && !w.dirs[dir] {
|
||||
if err := w.write(&tar.Header{Typeflag: tar.TypeDir, Name: path.Join(w.prefix, dir), Mode: 0o755}); err != nil {
|
||||
return err
|
||||
}
|
||||
w.dirs[dir] = true
|
||||
}
|
||||
}
|
||||
if hdr.Typeflag == tar.TypeDir {
|
||||
w.dirs[name] = true
|
||||
}
|
||||
entry := *hdr
|
||||
entry.Name = path.Join(w.prefix, name)
|
||||
return w.write(&entry)
|
||||
}
|
||||
|
||||
func (w *TarWriter) write(hdr *tar.Header) error {
|
||||
hdr.Uid, hdr.Gid, hdr.Uname, hdr.Gname = w.uid, w.gid, "", "" // tar running as root prefers names over ids
|
||||
if hdr.Typeflag == tar.TypeDir {
|
||||
hdr.Name += "/"
|
||||
}
|
||||
return w.writer.WriteHeader(hdr)
|
||||
}
|
||||
|
||||
func writeTarStream(dst io.Writer, uid, gid int, prefix string, parents []string, write func(*TarWriter) error) error {
|
||||
tw := &TarWriter{writer: tar.NewWriter(dst), uid: uid, gid: gid, prefix: prefix, dirs: map[string]bool{}}
|
||||
for _, name := range parents {
|
||||
if err := tw.write(&tar.Header{Typeflag: tar.TypeDir, Name: name, Mode: 0o755}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := write(tw); errors.Is(err, io.EOF) {
|
||||
return fmt.Errorf("tar stream: %v", err) // %v, as a wrapped io.EOF reads as the archive's end
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
return tw.writer.Close()
|
||||
}
|
||||
|
||||
func pipeTar(uid, gid int, prefix string, parents []string, write func(*TarWriter) error) *io.PipeReader {
|
||||
reader, writer := io.Pipe()
|
||||
go func() { writer.CloseWithError(writeTarStream(writer, uid, gid, prefix, parents, write)) }()
|
||||
return reader
|
||||
}
|
||||
|
||||
// startedArchive returns reader once its first byte arrives, so failing to start archiving errors here, not on read.
|
||||
func startedArchive(reader *io.PipeReader) (io.ReadCloser, error) {
|
||||
buffered := bufio.NewReader(reader)
|
||||
if _, err := buffered.Peek(1); err != nil {
|
||||
reader.Close()
|
||||
return nil, err
|
||||
}
|
||||
return struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}{buffered, reader}, nil
|
||||
}
|
||||
|
||||
func writeFilesTar(ctx context.Context, w io.Writer, uid, gid int, files ...*FileEntry) error {
|
||||
tw := tar.NewWriter(w)
|
||||
for _, file := range files {
|
||||
|
||||
Reference in new issue
Block a user