feat: auto extract archives (#1212)

pkg/base/model.go

@@ -2,13 +2,14 @@ package base
 
 import (
 	"fmt"
-	"github.com/GopeedLab/gopeed/pkg/util"
-	"github.com/mattn/go-ieproxy"
-	"golang.org/x/exp/slices"
 	"net/http"
 	"net/url"
 	"sync"
 	"time"
+
+	"github.com/GopeedLab/gopeed/pkg/util"
+	"github.com/mattn/go-ieproxy"
+	"golang.org/x/exp/slices"
 )
 
 // Request download request
@@ -183,6 +184,7 @@ type DownloaderStoreConfig struct {
 	Extra          map[string]any         `json:"extra"`
 	Proxy          *DownloaderProxyConfig `json:"proxy"`
 	Webhook        *WebhookConfig         `json:"webhook"` // Webhook is the webhook configuration
+	Archive        *ArchiveConfig         `json:"archive"` // Archive is the archive extraction configuration
 }
 
 func (cfg *DownloaderStoreConfig) Init() *DownloaderStoreConfig {
@@ -198,6 +200,12 @@ func (cfg *DownloaderStoreConfig) Init() *DownloaderStoreConfig {
 	if cfg.Webhook == nil {
 		cfg.Webhook = &WebhookConfig{}
 	}
+	if cfg.Archive == nil {
+		cfg.Archive = &ArchiveConfig{
+			AutoExtract:        false,
+			DeleteAfterExtract: false,
+		}
+	}
 	return cfg
 }
 
@@ -223,6 +231,9 @@ func (cfg *DownloaderStoreConfig) Merge(beforeCfg *DownloaderStoreConfig) *Downl
 	if cfg.Webhook == nil {
 		cfg.Webhook = beforeCfg.Webhook
 	}
+	if cfg.Archive == nil {
+		cfg.Archive = beforeCfg.Archive
+	}
 	return cfg
 }
 
@@ -232,6 +243,12 @@ type WebhookConfig struct {
 	URLs   []string `json:"urls"`   // URLs is the list of webhook URLs
 }
 
+// ArchiveConfig is the archive extraction configuration
+type ArchiveConfig struct {
+	AutoExtract        bool `json:"autoExtract"`        // AutoExtract enables automatic extraction of archives after download
+	DeleteAfterExtract bool `json:"deleteAfterExtract"` // DeleteAfterExtract deletes the archive after successful extraction
+}
+
 type DownloaderProxyConfig struct {
 	Enable bool `json:"enable"`
 	// System is the flag that use system proxy

pkg/base/model_test.go

@@ -19,6 +19,10 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
 				ProtocolConfig: map[string]any{},
 				Proxy:          &DownloaderProxyConfig{},
 				Webhook:        &WebhookConfig{},
+				Archive: &ArchiveConfig{
+					AutoExtract:        false,
+					DeleteAfterExtract: false,
+				},
 			},
 		},
 		{
@@ -31,6 +35,10 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
 				ProtocolConfig: map[string]any{},
 				Proxy:          &DownloaderProxyConfig{},
 				Webhook:        &WebhookConfig{},
+				Archive: &ArchiveConfig{
+					AutoExtract:        false,
+					DeleteAfterExtract: false,
+				},
 			},
 		},
 		{
@@ -47,6 +55,10 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
 				},
 				Proxy:   &DownloaderProxyConfig{},
 				Webhook: &WebhookConfig{},
+				Archive: &ArchiveConfig{
+					AutoExtract:        false,
+					DeleteAfterExtract: false,
+				},
 			},
 		},
 		{
@@ -63,6 +75,29 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
 					Enable: true,
 				},
 				Webhook: &WebhookConfig{},
+				Archive: &ArchiveConfig{
+					AutoExtract:        false,
+					DeleteAfterExtract: false,
+				},
+			},
+		},
+		{
+			"Init Archive",
+			&DownloaderStoreConfig{
+				Archive: &ArchiveConfig{
+					AutoExtract:        true,
+					DeleteAfterExtract: false,
+				},
+			},
+			&DownloaderStoreConfig{
+				MaxRunning:     5,
+				ProtocolConfig: map[string]any{},
+				Proxy:          &DownloaderProxyConfig{},
+				Webhook:        &WebhookConfig{},
+				Archive: &ArchiveConfig{
+					AutoExtract:        true,
+					DeleteAfterExtract: false,
+				},
 			},
 		},
 	}
@@ -76,6 +111,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
 				Extra:          tt.fields.Extra,
 				Proxy:          tt.fields.Proxy,
 				Webhook:        tt.fields.Webhook,
+				Archive:        tt.fields.Archive,
 			}
 			if got := cfg.Init(); !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("Init() = %v, want %v", got, tt.want)
@@ -250,6 +286,45 @@ func TestDownloaderStoreConfig_Merge(t *testing.T) {
 				},
 			},
 		},
+		{
+			"Merge Archive No Override",
+			&DownloaderStoreConfig{
+				Archive: &ArchiveConfig{
+					AutoExtract: true,
+				},
+			},
+			args{
+				beforeCfg: &DownloaderStoreConfig{
+					Archive: &ArchiveConfig{
+						AutoExtract:        false,
+						DeleteAfterExtract: false,
+					},
+				},
+			},
+			&DownloaderStoreConfig{
+				Archive: &ArchiveConfig{
+					AutoExtract: true,
+				},
+			},
+		},
+		{
+			"Merge Archive Override",
+			&DownloaderStoreConfig{},
+			args{
+				beforeCfg: &DownloaderStoreConfig{
+					Archive: &ArchiveConfig{
+						AutoExtract:        false,
+						DeleteAfterExtract: false,
+					},
+				},
+			},
+			&DownloaderStoreConfig{
+				Archive: &ArchiveConfig{
+					AutoExtract:        false,
+					DeleteAfterExtract: false,
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -261,6 +336,7 @@ func TestDownloaderStoreConfig_Merge(t *testing.T) {
 				Extra:          tt.fields.Extra,
 				Proxy:          tt.fields.Proxy,
 				Webhook:        tt.fields.Webhook,
+				Archive:        tt.fields.Archive,
 			}
 			if got := cfg.Merge(tt.args.beforeCfg); !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("Merge() = %v, want %v", got, tt.want)

pkg/download/downloader.go

@@ -877,6 +877,29 @@ func (d *Downloader) watch(task *Task) {
 
 			}()
 		}
+
+		// Auto-extract archive files
+		if e.AutoExtract && isArchiveFile(downloadFilePath) {
+			go func() {
+				// Extract to the same directory as the downloaded file
+				destDir := task.Meta.Opts.Path
+				extractErr := extractArchive(downloadFilePath, destDir, e.ArchivePassword)
+				if extractErr != nil {
+					d.Logger.Error().Err(extractErr).Msgf("auto extract archive failed, task id: %s", task.ID)
+				} else {
+					d.Logger.Info().Msgf("auto extract archive completed, task id: %s", task.ID)
+					// Delete archive after successful extraction if enabled
+					if e.DeleteAfterExtract {
+						deleteErr := os.Remove(downloadFilePath)
+						if deleteErr != nil {
+							d.Logger.Error().Err(deleteErr).Msgf("delete archive after extraction failed, task id: %s", task.ID)
+						} else {
+							d.Logger.Info().Msgf("archive deleted after extraction, task id: %s", task.ID)
+						}
+					}
+				}
+			}()
+		}
 	}
 }
 

pkg/download/extract.go

@@ -0,0 +1,183 @@
+package download
+
+import (
+	"context"
+	"io"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"github.com/mholt/archives"
+)
+
+// supportedArchiveExtensions contains file extensions supported by mholt/archives library
+var supportedArchiveExtensions = []string{
+	// Archive formats
+	".zip",
+	".tar",
+	".tar.gz", ".tgz",
+	".tar.bz2", ".tbz2",
+	".tar.xz", ".txz",
+	".tar.lz4", ".tlz4",
+	".tar.sz", ".tsz",
+	".tar.zst", ".tzst",
+	".rar",
+	".7z",
+	// Compression formats
+	".gz",
+	".bz2",
+	".xz",
+	".lz4",
+	".sz",
+	".zst",
+	".br",
+	".lz",
+}
+
+// isArchiveFile checks if a file is a supported archive format
+func isArchiveFile(filename string) bool {
+	lowerName := strings.ToLower(filename)
+	return slices.ContainsFunc(supportedArchiveExtensions, func(ext string) bool {
+		return strings.HasSuffix(lowerName, ext)
+	})
+}
+
+// extractArchive extracts an archive file to a destination directory
+func extractArchive(archivePath string, destDir string, password string) error {
+	// Open the archive file
+	file, err := os.Open(archivePath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	// Get file info for format identification
+	stat, err := file.Stat()
+	if err != nil {
+		return err
+	}
+
+	// Identify the archive format
+	format, input, err := archives.Identify(context.Background(), archivePath, file)
+	if err != nil {
+		return err
+	}
+
+	// Create destination directory if it doesn't exist
+	if err := os.MkdirAll(destDir, 0755); err != nil {
+		return err
+	}
+
+	// Handle password-protected archives
+	if password != "" {
+		// Try to set password for formats that support it
+		if rar, ok := format.(archives.Rar); ok {
+			rar.Password = password
+			format = rar
+		}
+		if sz, ok := format.(archives.SevenZip); ok {
+			sz.Password = password
+			format = sz
+		}
+		// Note: Zip format in archives doesn't have a Password field
+		// Password-protected zips are not fully supported
+	}
+
+	// Handle extraction based on format type
+	switch f := format.(type) {
+	case archives.Extractor:
+		// For archive formats (zip, rar, 7z, tar, etc.)
+		return f.Extract(context.Background(), input, func(ctx context.Context, fileInfo archives.FileInfo) error {
+			return extractFile(ctx, fileInfo, destDir)
+		})
+	case archives.Decompressor:
+		// For single-file compression formats (gz, bz2, xz, etc.)
+		// Decompress to a file without the compression extension
+		baseName := filepath.Base(archivePath)
+		lowerBaseName := strings.ToLower(baseName)
+		for _, ext := range supportedArchiveExtensions {
+			if strings.HasSuffix(lowerBaseName, ext) {
+				// Get the actual suffix from the original filename (preserving case)
+				actualSuffix := baseName[len(baseName)-len(ext):]
+				baseName = strings.TrimSuffix(baseName, actualSuffix)
+				break
+			}
+		}
+		destPath := filepath.Join(destDir, baseName)
+
+		reader, err := f.OpenReader(input)
+		if err != nil {
+			return err
+		}
+		defer reader.Close()
+
+		destFile, err := os.Create(destPath)
+		if err != nil {
+			return err
+		}
+		defer destFile.Close()
+
+		_, err = io.Copy(destFile, reader)
+		return err
+	case archives.Archiver:
+		// This format is an archiver, try to extract using the extractor interface
+		if ext, ok := format.(archives.Extractor); ok {
+			// Reset file position
+			if seeker, ok := input.(io.Seeker); ok {
+				seeker.Seek(0, io.SeekStart)
+			}
+			return ext.Extract(context.Background(), io.NewSectionReader(file, 0, stat.Size()), func(ctx context.Context, fileInfo archives.FileInfo) error {
+				return extractFile(ctx, fileInfo, destDir)
+			})
+		}
+	}
+
+	return nil
+}
+
+// extractFile handles extracting a single file from an archive
+func extractFile(ctx context.Context, fileInfo archives.FileInfo, destDir string) error {
+	// Skip directories, they will be created when extracting files
+	if fileInfo.IsDir() {
+		destPath := filepath.Join(destDir, fileInfo.NameInArchive)
+		return os.MkdirAll(destPath, fileInfo.Mode())
+	}
+
+	// Sanitize the path to prevent path traversal attacks
+	cleanPath := filepath.Clean(fileInfo.NameInArchive)
+	if strings.HasPrefix(cleanPath, "..") || filepath.IsAbs(cleanPath) {
+		// Skip files with suspicious paths
+		return nil
+	}
+
+	destPath := filepath.Join(destDir, cleanPath)
+
+	// Create parent directories
+	if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
+		return err
+	}
+
+	// Open the file from the archive
+	reader, err := fileInfo.Open()
+	if err != nil {
+		return err
+	}
+	defer reader.Close()
+
+	// Create the destination file
+	destFile, err := os.Create(destPath)
+	if err != nil {
+		return err
+	}
+	defer destFile.Close()
+
+	// Copy the contents
+	_, err = io.Copy(destFile, reader)
+	if err != nil {
+		return err
+	}
+
+	// Set file permissions
+	return os.Chmod(destPath, fileInfo.Mode())
+}

pkg/download/extract_test.go

@@ -0,0 +1,154 @@
+package download
+
+import (
+	"archive/zip"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestIsArchiveFile(t *testing.T) {
+	tests := []struct {
+		filename string
+		expected bool
+	}{
+		// Archive formats
+		{"file.zip", true},
+		{"file.ZIP", true},
+		{"file.tar", true},
+		{"file.tar.gz", true},
+		{"file.tgz", true},
+		{"file.tar.bz2", true},
+		{"file.tbz2", true},
+		{"file.tar.xz", true},
+		{"file.txz", true},
+		{"file.tar.zst", true},
+		{"file.tzst", true},
+		{"file.rar", true},
+		{"file.RAR", true},
+		{"file.7z", true},
+		// Compression formats
+		{"file.gz", true},
+		{"file.bz2", true},
+		{"file.xz", true},
+		{"file.lz4", true},
+		{"file.sz", true},
+		{"file.zst", true},
+		{"file.br", true},
+		{"file.lz", true},
+		// Non-archive files
+		{"file.txt", false},
+		{"file.pdf", false},
+		{"file.exe", false},
+		{"archive", false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.filename, func(t *testing.T) {
+			result := isArchiveFile(tt.filename)
+			if result != tt.expected {
+				t.Errorf("isArchiveFile(%q) = %v, expected %v", tt.filename, result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestExtractArchive_Zip(t *testing.T) {
+	// Create a temporary directory
+	tempDir, err := os.MkdirTemp("", "extract_test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	// Create a test zip file
+	zipPath := filepath.Join(tempDir, "test.zip")
+	destDir := filepath.Join(tempDir, "extracted")
+
+	if err := createTestZip(zipPath); err != nil {
+		t.Fatal(err)
+	}
+
+	// Extract the archive
+	err = extractArchive(zipPath, destDir, "")
+	if err != nil {
+		t.Fatalf("extractArchive failed: %v", err)
+	}
+
+	// Verify the extracted files
+	expectedFiles := []string{
+		filepath.Join(destDir, "test.txt"),
+		filepath.Join(destDir, "subdir", "nested.txt"),
+	}
+
+	for _, path := range expectedFiles {
+		if _, err := os.Stat(path); os.IsNotExist(err) {
+			t.Errorf("expected file %q not found after extraction", path)
+		}
+	}
+
+	// Verify content
+	content, err := os.ReadFile(filepath.Join(destDir, "test.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(content) != "Hello, World!" {
+		t.Errorf("unexpected content: %q", string(content))
+	}
+}
+
+func TestExtractArchive_NonArchive(t *testing.T) {
+	// Create a temporary directory
+	tempDir, err := os.MkdirTemp("", "extract_test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	// Create a non-archive file
+	txtPath := filepath.Join(tempDir, "test.txt")
+	if err := os.WriteFile(txtPath, []byte("not an archive"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	destDir := filepath.Join(tempDir, "extracted")
+
+	// Trying to extract a non-archive should return an error
+	err = extractArchive(txtPath, destDir, "")
+	if err == nil {
+		t.Error("expected error when extracting non-archive file")
+	}
+}
+
+// createTestZip creates a test zip file with sample content
+func createTestZip(path string) error {
+	zipFile, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer zipFile.Close()
+
+	w := zip.NewWriter(zipFile)
+
+	// Add a file to the root
+	f, err := w.Create("test.txt")
+	if err != nil {
+		return err
+	}
+	_, err = f.Write([]byte("Hello, World!"))
+	if err != nil {
+		return err
+	}
+
+	// Add a file in a subdirectory
+	f, err = w.Create("subdir/nested.txt")
+	if err != nil {
+		return err
+	}
+	_, err = f.Write([]byte("Nested content"))
+	if err != nil {
+		return err
+	}
+
+	return w.Close()
+}

pkg/protocol/http/model.go

@@ -10,6 +10,12 @@ type OptsExtra struct {
 	Connections int `json:"connections"`
 	// AutoTorrent when task download complete, and it is a .torrent file, it will be auto create a new task for the torrent file
 	AutoTorrent bool `json:"autoTorrent"`
+	// AutoExtract when task download complete, and it is an archive file, it will be auto extracted
+	AutoExtract bool `json:"autoExtract"`
+	// ArchivePassword is the password for extracting password-protected archives
+	ArchivePassword string `json:"archivePassword"`
+	// DeleteAfterExtract when true, deletes the archive file after successful extraction
+	DeleteAfterExtract bool `json:"deleteAfterExtract"`
 }
 
 // Stats for download