feat: multipart-archives (#1222)

This commit is contained in:
Matteo Pietro Dazzi authored and GitHub committed 2026-01-01 19:09:22 +08:00
1 parent cb6331c20d
commit de0afb1481
7 files changed
+2689 -46

No files matched your search

+347 -35
View File
@@ -2,6 +2,7 @@ package download
import (
"errors"
"fmt"
"math"
gohttp "net/http"
"net/url"
@@ -51,6 +52,8 @@ type ExtractStatus string
const (
// ExtractStatusNone indicates extraction has not started
ExtractStatusNone ExtractStatus = ""
// ExtractStatusWaitingParts indicates waiting for other multi-part archive parts to complete
ExtractStatusWaitingParts ExtractStatus = "waitingParts"
// ExtractStatusExtracting indicates extraction is in progress
ExtractStatusExtracting ExtractStatus = "extracting"
// ExtractStatusDone indicates extraction completed successfully
@@ -74,6 +77,12 @@ type Progress struct {
ExtractStatus ExtractStatus `json:"extractStatus"`
// ExtractProgress is the percentage of extraction completed (0-100)
ExtractProgress int `json:"extractProgress"`
// MultiPartBaseName is set for multi-part archives to group related parts
MultiPartBaseName string `json:"multiPartBaseName,omitempty"`
// MultiPartNumber is the part number for multi-part archives (1-indexed)
MultiPartNumber int `json:"multiPartNumber,omitempty"`
// MultiPartIsFirst indicates if this is the first part of a multi-part archive
MultiPartIsFirst bool `json:"multiPartIsFirst,omitempty"`
}
type Downloader struct {
@@ -898,41 +907,7 @@ func (d *Downloader) watch(task *Task) {
// Auto-extract archive files
if e.AutoExtract && isArchiveFile(downloadFilePath) {
go func() {
// Set extraction status to extracting
task.Progress.ExtractStatus = ExtractStatusExtracting
task.Progress.ExtractProgress = 0
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
// Extract to the same directory as the downloaded file
destDir := task.Meta.Opts.Path
extractErr := extractArchive(downloadFilePath, destDir, e.ArchivePassword, func(extractedFiles int, totalFiles int, progress int) {
task.Progress.ExtractProgress = progress
d.emit(EventKeyProgress, task)
})
if extractErr != nil {
d.Logger.Error().Err(extractErr).Msgf("auto extract archive failed, task id: %s", task.ID)
task.Progress.ExtractStatus = ExtractStatusError
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
} else {
d.Logger.Info().Msgf("auto extract archive completed, task id: %s", task.ID)
task.Progress.ExtractStatus = ExtractStatusDone
task.Progress.ExtractProgress = 100
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
// Delete archive after successful extraction if enabled
if e.DeleteAfterExtract {
deleteErr := os.Remove(downloadFilePath)
if deleteErr != nil {
d.Logger.Error().Err(deleteErr).Msgf("delete archive after extraction failed, task id: %s", task.ID)
} else {
d.Logger.Info().Msgf("archive deleted after extraction, task id: %s", task.ID)
}
}
}
}()
go d.handleAutoExtract(task, downloadFilePath, e)
}
}
}
@@ -1225,6 +1200,343 @@ func (d *Downloader) buildFetcher(url string) (fetcher.Fetcher, error) {
return fetcher, nil
}
// handleAutoExtract handles automatic extraction of archive files, including multi-part archives
func (d *Downloader) handleAutoExtract(task *Task, downloadFilePath string, opts *http.OptsExtra) {
// Check if this is a multi-part archive
partInfo := getArchivePartInfo(downloadFilePath)
if partInfo.IsMultiPart {
// Set multi-part info on the task
task.Progress.MultiPartBaseName = partInfo.BaseName
task.Progress.MultiPartNumber = partInfo.PartNumber
task.Progress.MultiPartIsFirst = isFirstPart(downloadFilePath)
d.storage.Put(bucketTask, task.ID, task.clone())
// Check if all parts are downloaded
destDir := task.Meta.Opts.Path
allPartsReady, missingParts := d.checkMultiPartArchiveReady(downloadFilePath, destDir, partInfo)
if !allPartsReady {
// Not all parts are ready yet, set status to waiting
task.Progress.ExtractStatus = ExtractStatusWaitingParts
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
d.Logger.Info().Msgf("multi-part archive waiting for other parts, task id: %s, missing: %v", task.ID, missingParts)
return
}
// All parts are ready!
// Only one task should perform extraction - we'll use the one that completes last
// Check if extraction has already been started by another part
if d.isMultiPartExtractionInProgress(partInfo.BaseName) {
// Another part already started extraction, mark this as done
task.Progress.ExtractStatus = ExtractStatusDone
task.Progress.ExtractProgress = 100
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
d.Logger.Info().Msgf("multi-part archive part completed, extraction handled by another part, task id: %s", task.ID)
return
}
// This task will handle extraction - mark as extracting first to prevent others
task.Progress.ExtractStatus = ExtractStatusExtracting
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
// Extract the multi-part archive
d.performMultiPartExtraction(task, partInfo.FirstPartPath, destDir, opts)
} else {
// Regular (non-multi-part) archive
d.performExtraction(task, downloadFilePath, task.Meta.Opts.Path, opts)
}
}
// checkMultiPartArchiveReady checks if all parts of a multi-part archive are downloaded
// by examining task status rather than file existence
func (d *Downloader) checkMultiPartArchiveReady(filePath string, destDir string, partInfo ArchivePartInfo) (bool, []string) {
// Use task-based checking - find all tasks with the same MultiPartBaseName
// and verify they are all in Done status
baseName := GetMultiPartArchiveBaseName(filePath)
if baseName == "" {
return true, nil
}
return d.checkAllMultiPartTasksDone(baseName)
}
// checkAllMultiPartTasksDone checks if all tasks belonging to a multi-part archive are done
func (d *Downloader) checkAllMultiPartTasksDone(baseName string) (bool, []string) {
var notDoneParts []string
d.lock.Lock()
defer d.lock.Unlock()
// Find all tasks that belong to this multi-part archive
var relatedTasks []*Task
for _, task := range d.tasks {
taskBaseName := ""
if task.Meta != nil && task.Meta.Res != nil && len(task.Meta.Res.Files) > 0 {
taskBaseName = GetMultiPartArchiveBaseName(task.Meta.SingleFilepath())
}
if taskBaseName == baseName {
relatedTasks = append(relatedTasks, task)
}
}
// If we found no related tasks, we can't determine readiness
if len(relatedTasks) == 0 {
return false, []string{"no related tasks found for " + baseName}
}
// Check if all related tasks are done
for _, task := range relatedTasks {
if task.Status != base.DownloadStatusDone {
notDoneParts = append(notDoneParts, task.Meta.SingleFilepath())
}
}
return len(notDoneParts) == 0, notDoneParts
}
// isMultiPartExtractionInProgress checks if any task for this multi-part archive
// has already started extraction (to prevent duplicate extractions)
func (d *Downloader) isMultiPartExtractionInProgress(baseName string) bool {
d.lock.Lock()
defer d.lock.Unlock()
for _, task := range d.tasks {
taskBaseName := ""
if task.Meta != nil && task.Meta.Res != nil && len(task.Meta.Res.Files) > 0 {
taskBaseName = GetMultiPartArchiveBaseName(task.Meta.SingleFilepath())
}
if taskBaseName == baseName {
// Check if this task is already extracting or has completed extraction
if task.Progress.ExtractStatus == ExtractStatusExtracting ||
task.Progress.ExtractStatus == ExtractStatusDone ||
task.Progress.ExtractStatus == ExtractStatusError {
return true
}
}
}
return false
}
// performExtraction performs extraction for a regular (non-multi-part) archive
func (d *Downloader) performExtraction(task *Task, archivePath string, destDir string, opts *http.OptsExtra) {
// Set extraction status to extracting
task.Progress.ExtractStatus = ExtractStatusExtracting
task.Progress.ExtractProgress = 0
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
// Extract the archive
extractErr := extractArchive(archivePath, destDir, opts.ArchivePassword, func(extractedFiles int, totalFiles int, progress int) {
task.Progress.ExtractProgress = progress
d.emit(EventKeyProgress, task)
})
d.handleExtractionResult(task, extractErr, []string{archivePath}, opts.DeleteAfterExtract)
}
// performMultiPartExtraction performs extraction for a multi-part archive
func (d *Downloader) performMultiPartExtraction(task *Task, firstPartPath string, destDir string, opts *http.OptsExtra) {
// Set extraction status to extracting
task.Progress.ExtractStatus = ExtractStatusExtracting
task.Progress.ExtractProgress = 0
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
d.Logger.Info().Msgf("starting multi-part archive extraction, first part: %s, task id: %s", firstPartPath, task.ID)
// Extract the multi-part archive
extractErr := extractMultiPartArchive(firstPartPath, destDir, opts.ArchivePassword, func(extractedFiles int, totalFiles int, progress int) {
task.Progress.ExtractProgress = progress
d.emit(EventKeyProgress, task)
})
// Collect all part files for potential deletion
partFiles := d.collectMultiPartFiles(firstPartPath)
d.handleExtractionResult(task, extractErr, partFiles, opts.DeleteAfterExtract)
// Update status for all related multi-part tasks
d.updateMultiPartTasksStatus(task, extractErr)
}
// collectMultiPartFiles collects all files belonging to a multi-part archive
func (d *Downloader) collectMultiPartFiles(firstPartPath string) []string {
var files []string
partInfo := getArchivePartInfo(firstPartPath)
dir := filepath.Dir(firstPartPath)
switch {
case strings.Contains(partInfo.Pattern, ".7z)"):
// 7z: .7z.001, .7z.002, etc.
files = d.collectSequentialFiles(dir, partInfo.BaseName, ".%03d")
case strings.Contains(partInfo.Pattern, ".part"):
// RAR new style
files = d.collectRarNewStyleFiles(dir, partInfo.BaseName)
case partInfo.Pattern == "rar-old-style" || strings.Contains(partInfo.Pattern, ".r("):
// RAR old style
files = d.collectRarOldStyleFiles(dir, partInfo.BaseName)
case strings.Contains(partInfo.Pattern, ".zip)"):
// ZIP multi-part
files = d.collectSequentialFiles(dir, partInfo.BaseName, ".%03d")
case strings.Contains(partInfo.Pattern, ".z("):
// ZIP split
files = d.collectZipSplitFiles(dir, partInfo.BaseName)
}
return files
}
// collectSequentialFiles collects sequential numbered files
func (d *Downloader) collectSequentialFiles(dir, baseName, format string) []string {
var files []string
suffix := filepath.Ext(baseName)
nameWithoutExt := strings.TrimSuffix(baseName, suffix)
partNum := 1
for {
partPath := filepath.Join(dir, nameWithoutExt+suffix+fmt.Sprintf(format, partNum))
if _, err := os.Stat(partPath); os.IsNotExist(err) {
break
}
files = append(files, partPath)
partNum++
}
return files
}
// collectRarNewStyleFiles collects RAR new style part files
func (d *Downloader) collectRarNewStyleFiles(dir, baseName string) []string {
var files []string
partNum := 1
for {
singleDigitPath := filepath.Join(dir, baseName+fmt.Sprintf(".part%d.rar", partNum))
doubleDigitPath := filepath.Join(dir, baseName+fmt.Sprintf(".part%02d.rar", partNum))
if _, err := os.Stat(singleDigitPath); err == nil {
files = append(files, singleDigitPath)
} else if _, err := os.Stat(doubleDigitPath); err == nil {
files = append(files, doubleDigitPath)
} else {
break
}
partNum++
}
return files
}
// collectRarOldStyleFiles collects RAR old style part files
func (d *Downloader) collectRarOldStyleFiles(dir, baseName string) []string {
var files []string
// .rar file
rarPath := filepath.Join(dir, baseName+".rar")
if _, err := os.Stat(rarPath); err == nil {
files = append(files, rarPath)
}
// .r00, .r01, etc.
partNum := 0
for {
partPath := filepath.Join(dir, baseName+fmt.Sprintf(".r%02d", partNum))
if _, err := os.Stat(partPath); os.IsNotExist(err) {
break
}
files = append(files, partPath)
partNum++
}
return files
}
// collectZipSplitFiles collects ZIP split files
func (d *Downloader) collectZipSplitFiles(dir, baseName string) []string {
var files []string
// .z01, .z02, etc.
partNum := 1
for {
partPath := filepath.Join(dir, baseName+fmt.Sprintf(".z%02d", partNum))
if _, err := os.Stat(partPath); os.IsNotExist(err) {
break
}
files = append(files, partPath)
partNum++
}
// .zip file
zipPath := filepath.Join(dir, baseName+".zip")
if _, err := os.Stat(zipPath); err == nil {
files = append(files, zipPath)
}
return files
}
// handleExtractionResult handles the result of an extraction operation
func (d *Downloader) handleExtractionResult(task *Task, extractErr error, archiveFiles []string, deleteAfterExtract bool) {
if extractErr != nil {
d.Logger.Error().Err(extractErr).Msgf("auto extract archive failed, task id: %s", task.ID)
task.Progress.ExtractStatus = ExtractStatusError
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
} else {
d.Logger.Info().Msgf("auto extract archive completed, task id: %s", task.ID)
task.Progress.ExtractStatus = ExtractStatusDone
task.Progress.ExtractProgress = 100
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
// Delete archive files after successful extraction if enabled
if deleteAfterExtract {
for _, archiveFile := range archiveFiles {
deleteErr := os.Remove(archiveFile)
if deleteErr != nil {
d.Logger.Error().Err(deleteErr).Msgf("delete archive after extraction failed: %s", archiveFile)
} else {
d.Logger.Info().Msgf("archive deleted after extraction: %s", archiveFile)
}
}
}
}
}
// updateMultiPartTasksStatus updates the extraction status for all tasks that belong to the same multi-part archive
func (d *Downloader) updateMultiPartTasksStatus(sourceTask *Task, extractErr error) {
if sourceTask.Progress.MultiPartBaseName == "" {
return
}
status := ExtractStatusDone
progress := 100
if extractErr != nil {
status = ExtractStatusError
progress = 0
}
d.lock.Lock()
defer d.lock.Unlock()
for _, task := range d.tasks {
if task.ID == sourceTask.ID {
continue
}
if task.Progress.MultiPartBaseName == sourceTask.Progress.MultiPartBaseName {
task.Progress.ExtractStatus = status
task.Progress.ExtractProgress = progress
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
}
}
}
func initTask(task *Task) {
task.timer = util.NewTimer(task.Progress.Used)
+785 -11
View File
@@ -2,16 +2,19 @@ package download
import (
"archive/zip"
"fmt"
"io"
gohttp "net/http"
"net"
gohttp "net/http"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/test"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/protocol/http"
@@ -188,19 +191,22 @@ func TestDownloader_CreateDirectBatch(t *testing.T) {
t.Errorf("CreateDirectBatch() task got = %v, want %v", len(tasks), len(reqs))
}
nameMatchMap := make(map[string]bool)
for _, name := range fileNames {
// Collect all task names
taskNames := make(map[string]bool)
for _, task := range tasks {
taskNames[task.Meta.Opts.Name] = true
}
// Check that we have the expected number of unique task names
if len(taskNames) != len(reqs) {
t.Errorf("CreateDirectBatch() unique task names got = %v, want %v, names: %v", len(taskNames), len(reqs), taskNames)
}
// Check that all task files exist
for name := range taskNames {
if _, err := os.Stat(test.Dir + "/" + name); os.IsNotExist(err) {
t.Errorf("CreateDirectBatch() file not exist: %v", name)
}
for _, task := range tasks {
if name == task.Meta.Opts.Name {
nameMatchMap[task.Meta.Opts.Name] = true
}
}
}
if len(nameMatchMap) != len(reqs) {
t.Errorf("CreateDirectBatch() names got = %v, want %v", len(nameMatchMap), len(reqs))
}
}
@@ -1453,6 +1459,51 @@ func TestProgress_ExtractFields(t *testing.T) {
}
}
// TestProgress_MultiPartFields tests the multi-part archive fields in Progress struct
func TestProgress_MultiPartFields(t *testing.T) {
progress := &Progress{
ExtractStatus: ExtractStatusWaitingParts,
MultiPartBaseName: "/path/to/archive.7z",
MultiPartNumber: 1,
MultiPartIsFirst: true,
}
if progress.ExtractStatus != ExtractStatusWaitingParts {
t.Errorf("ExtractStatus = %v, want %v", progress.ExtractStatus, ExtractStatusWaitingParts)
}
if progress.MultiPartBaseName != "/path/to/archive.7z" {
t.Errorf("MultiPartBaseName = %v, want %v", progress.MultiPartBaseName, "/path/to/archive.7z")
}
if progress.MultiPartNumber != 1 {
t.Errorf("MultiPartNumber = %v, want %v", progress.MultiPartNumber, 1)
}
if !progress.MultiPartIsFirst {
t.Error("MultiPartIsFirst should be true")
}
// Test second part
progress2 := &Progress{
ExtractStatus: ExtractStatusWaitingParts,
MultiPartBaseName: "/path/to/archive.7z",
MultiPartNumber: 2,
MultiPartIsFirst: false,
}
if progress2.MultiPartNumber != 2 {
t.Errorf("MultiPartNumber = %v, want %v", progress2.MultiPartNumber, 2)
}
if progress2.MultiPartIsFirst {
t.Error("MultiPartIsFirst should be false")
}
}
// TestExtractStatus_WaitingParts tests the new ExtractStatusWaitingParts status
func TestExtractStatus_WaitingParts(t *testing.T) {
if ExtractStatusWaitingParts != "waitingParts" {
t.Errorf("ExtractStatusWaitingParts = %v, want %v", ExtractStatusWaitingParts, "waitingParts")
}
}
// createTestArchiveWithMultipleFiles creates a test zip file with multiple files
func createTestArchiveWithMultipleFiles(path string, count int) error {
file, err := os.Create(path)
@@ -1710,3 +1761,726 @@ func TestTaskFilter_IsEmpty(t *testing.T) {
})
}
}
// Tests for multi-part archive collection functions
func TestDownloader_CollectSequentialFiles(t *testing.T) {
tempDir, err := os.MkdirTemp("", "collect_sequential_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
// Create test files for 7z multi-part pattern (archive.7z.001, .002, .003)
for i := 1; i <= 3; i++ {
path := filepath.Join(tempDir, fmt.Sprintf("archive.7z.%03d", i))
if err := os.WriteFile(path, []byte("test"), 0644); err != nil {
t.Fatal(err)
}
}
downloader := NewDownloader(nil)
files := downloader.collectSequentialFiles(tempDir, "archive.7z", ".%03d")
if len(files) != 3 {
t.Errorf("collectSequentialFiles() = %d files, want 3", len(files))
}
// Verify files are in order
for i, file := range files {
expected := filepath.Join(tempDir, fmt.Sprintf("archive.7z.%03d", i+1))
if file != expected {
t.Errorf("files[%d] = %q, want %q", i, file, expected)
}
}
}
func TestDownloader_CollectSequentialFiles_NoFiles(t *testing.T) {
tempDir, err := os.MkdirTemp("", "collect_sequential_empty_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
downloader := NewDownloader(nil)
files := downloader.collectSequentialFiles(tempDir, "archive.7z", ".%03d")
if len(files) != 0 {
t.Errorf("collectSequentialFiles() = %d files, want 0", len(files))
}
}
func TestDownloader_CollectRarNewStyleFiles(t *testing.T) {
tempDir, err := os.MkdirTemp("", "collect_rar_new_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
// Create test files with double-digit format
for i := 1; i <= 3; i++ {
path := filepath.Join(tempDir, fmt.Sprintf("archive.part%02d.rar", i))
if err := os.WriteFile(path, []byte("test"), 0644); err != nil {
t.Fatal(err)
}
}
downloader := NewDownloader(nil)
files := downloader.collectRarNewStyleFiles(tempDir, "archive")
if len(files) != 3 {
t.Errorf("collectRarNewStyleFiles() = %d files, want 3", len(files))
}
}
func TestDownloader_CollectRarNewStyleFiles_SingleDigit(t *testing.T) {
tempDir, err := os.MkdirTemp("", "collect_rar_single_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
// Create test files with single-digit format
for i := 1; i <= 2; i++ {
path := filepath.Join(tempDir, fmt.Sprintf("archive.part%d.rar", i))
if err := os.WriteFile(path, []byte("test"), 0644); err != nil {
t.Fatal(err)
}
}
downloader := NewDownloader(nil)
files := downloader.collectRarNewStyleFiles(tempDir, "archive")
if len(files) != 2 {
t.Errorf("collectRarNewStyleFiles() = %d files, want 2", len(files))
}
}
func TestDownloader_CollectRarOldStyleFiles(t *testing.T) {
tempDir, err := os.MkdirTemp("", "collect_rar_old_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
// Create .rar file
rarPath := filepath.Join(tempDir, "archive.rar")
if err := os.WriteFile(rarPath, []byte("test"), 0644); err != nil {
t.Fatal(err)
}
// Create .r00, .r01, .r02 files
for i := 0; i <= 2; i++ {
path := filepath.Join(tempDir, fmt.Sprintf("archive.r%02d", i))
if err := os.WriteFile(path, []byte("test"), 0644); err != nil {
t.Fatal(err)
}
}
downloader := NewDownloader(nil)
files := downloader.collectRarOldStyleFiles(tempDir, "archive")
// Should have 4 files: .rar + .r00 + .r01 + .r02
if len(files) != 4 {
t.Errorf("collectRarOldStyleFiles() = %d files, want 4", len(files))
}
// First file should be .rar
if files[0] != rarPath {
t.Errorf("First file should be .rar, got %q", files[0])
}
}
func TestDownloader_CollectZipSplitFiles(t *testing.T) {
tempDir, err := os.MkdirTemp("", "collect_zip_split_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
// Create .z01, .z02 files
for i := 1; i <= 2; i++ {
path := filepath.Join(tempDir, fmt.Sprintf("archive.z%02d", i))
if err := os.WriteFile(path, []byte("test"), 0644); err != nil {
t.Fatal(err)
}
}
// Create .zip file
zipPath := filepath.Join(tempDir, "archive.zip")
if err := os.WriteFile(zipPath, []byte("test"), 0644); err != nil {
t.Fatal(err)
}
downloader := NewDownloader(nil)
files := downloader.collectZipSplitFiles(tempDir, "archive")
// Should have 3 files: .z01 + .z02 + .zip
if len(files) != 3 {
t.Errorf("collectZipSplitFiles() = %d files, want 3", len(files))
}
// Last file should be .zip
if files[len(files)-1] != zipPath {
t.Errorf("Last file should be .zip, got %q", files[len(files)-1])
}
}
func TestDownloader_CollectMultiPartFiles(t *testing.T) {
tempDir, err := os.MkdirTemp("", "collect_multipart_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
// Test with 7z pattern
t.Run("7z pattern", func(t *testing.T) {
subDir := filepath.Join(tempDir, "7z")
if err := os.MkdirAll(subDir, 0755); err != nil {
t.Fatal(err)
}
for i := 1; i <= 2; i++ {
path := filepath.Join(subDir, fmt.Sprintf("archive.7z.%03d", i))
if err := os.WriteFile(path, []byte("test"), 0644); err != nil {
t.Fatal(err)
}
}
downloader := NewDownloader(nil)
firstPart := filepath.Join(subDir, "archive.7z.001")
files := downloader.collectMultiPartFiles(firstPart)
if len(files) != 2 {
t.Errorf("collectMultiPartFiles(7z) = %d files, want 2", len(files))
}
})
// Test with RAR new style pattern
t.Run("RAR new style pattern", func(t *testing.T) {
subDir := filepath.Join(tempDir, "rar_new")
if err := os.MkdirAll(subDir, 0755); err != nil {
t.Fatal(err)
}
for i := 1; i <= 2; i++ {
path := filepath.Join(subDir, fmt.Sprintf("archive.part%02d.rar", i))
if err := os.WriteFile(path, []byte("test"), 0644); err != nil {
t.Fatal(err)
}
}
downloader := NewDownloader(nil)
firstPart := filepath.Join(subDir, "archive.part01.rar")
files := downloader.collectMultiPartFiles(firstPart)
if len(files) != 2 {
t.Errorf("collectMultiPartFiles(RAR new) = %d files, want 2", len(files))
}
})
// Test with ZIP multi-part pattern
t.Run("ZIP multi-part pattern", func(t *testing.T) {
subDir := filepath.Join(tempDir, "zip")
if err := os.MkdirAll(subDir, 0755); err != nil {
t.Fatal(err)
}
for i := 1; i <= 3; i++ {
path := filepath.Join(subDir, fmt.Sprintf("archive.zip.%03d", i))
if err := os.WriteFile(path, []byte("test"), 0644); err != nil {
t.Fatal(err)
}
}
downloader := NewDownloader(nil)
firstPart := filepath.Join(subDir, "archive.zip.001")
files := downloader.collectMultiPartFiles(firstPart)
if len(files) != 3 {
t.Errorf("collectMultiPartFiles(ZIP) = %d files, want 3", len(files))
}
})
}
func TestDownloader_CheckAllMultiPartTasksDone(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
tempDir, err := os.MkdirTemp("", "check_multipart_done_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
// Create tasks with multi-part base name
baseName := "archive.7z"
// Create task 1 - done
task1 := &Task{
ID: "task1",
Status: base.DownloadStatusDone,
Meta: &fetcher.FetcherMeta{
Opts: &base.Options{Path: tempDir},
Res: &base.Resource{
Files: []*base.FileInfo{{Name: baseName + ".001"}},
},
},
Progress: &Progress{},
}
initTask(task1)
// Create task 2 - done
task2 := &Task{
ID: "task2",
Status: base.DownloadStatusDone,
Meta: &fetcher.FetcherMeta{
Opts: &base.Options{Path: tempDir},
Res: &base.Resource{
Files: []*base.FileInfo{{Name: baseName + ".002"}},
},
},
Progress: &Progress{},
}
initTask(task2)
downloader.tasks = []*Task{task1, task2}
// All tasks are done
basePath := filepath.Join(tempDir, baseName)
allDone, missing := downloader.checkAllMultiPartTasksDone(basePath)
if !allDone {
t.Errorf("checkAllMultiPartTasksDone() = false, want true; missing: %v", missing)
}
// Set task2 to running
task2.Status = base.DownloadStatusRunning
allDone, missing = downloader.checkAllMultiPartTasksDone(basePath)
if allDone {
t.Error("checkAllMultiPartTasksDone() = true, want false")
}
if len(missing) == 0 {
t.Error("Expected missing parts to be reported")
}
}
func TestDownloader_CheckAllMultiPartTasksDone_NoRelatedTasks(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
// No tasks exist
allDone, missing := downloader.checkAllMultiPartTasksDone("/some/path/archive.7z")
if allDone {
t.Error("checkAllMultiPartTasksDone() = true, want false with no related tasks")
}
if len(missing) == 0 {
t.Error("Expected missing message")
}
}
func TestDownloader_IsMultiPartExtractionInProgress(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
tempDir, err := os.MkdirTemp("", "extraction_progress_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
baseName := "archive.7z"
basePath := filepath.Join(tempDir, baseName)
// Create task with no extraction status
task1 := &Task{
ID: "task1",
Status: base.DownloadStatusDone,
Meta: &fetcher.FetcherMeta{
Opts: &base.Options{Path: tempDir},
Res: &base.Resource{
Files: []*base.FileInfo{{Name: baseName + ".001"}},
},
},
Progress: &Progress{ExtractStatus: ExtractStatusNone},
}
initTask(task1)
downloader.tasks = []*Task{task1}
// No extraction in progress
if downloader.isMultiPartExtractionInProgress(basePath) {
t.Error("isMultiPartExtractionInProgress() = true, want false")
}
// Set to extracting
task1.Progress.ExtractStatus = ExtractStatusExtracting
if !downloader.isMultiPartExtractionInProgress(basePath) {
t.Error("isMultiPartExtractionInProgress() = false, want true (extracting)")
}
// Set to done
task1.Progress.ExtractStatus = ExtractStatusDone
if !downloader.isMultiPartExtractionInProgress(basePath) {
t.Error("isMultiPartExtractionInProgress() = false, want true (done)")
}
// Set to error
task1.Progress.ExtractStatus = ExtractStatusError
if !downloader.isMultiPartExtractionInProgress(basePath) {
t.Error("isMultiPartExtractionInProgress() = false, want true (error)")
}
// Set to waiting parts (not in progress)
task1.Progress.ExtractStatus = ExtractStatusWaitingParts
if downloader.isMultiPartExtractionInProgress(basePath) {
t.Error("isMultiPartExtractionInProgress() = true, want false (waitingParts)")
}
}
func TestDownloader_HandleExtractionResult_Success(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
tempDir, err := os.MkdirTemp("", "extraction_result_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
// Create a test archive file
archivePath := filepath.Join(tempDir, "test.zip")
if err := os.WriteFile(archivePath, []byte("test"), 0644); err != nil {
t.Fatal(err)
}
task := &Task{
ID: "test-task",
Meta: &fetcher.FetcherMeta{
Opts: &base.Options{Path: tempDir},
Res: &base.Resource{
Files: []*base.FileInfo{{Name: "test.zip"}},
},
},
Progress: &Progress{},
}
initTask(task)
// Test successful extraction
downloader.handleExtractionResult(task, nil, []string{archivePath}, false)
if task.Progress.ExtractStatus != ExtractStatusDone {
t.Errorf("ExtractStatus = %v, want %v", task.Progress.ExtractStatus, ExtractStatusDone)
}
if task.Progress.ExtractProgress != 100 {
t.Errorf("ExtractProgress = %d, want 100", task.Progress.ExtractProgress)
}
// Archive should still exist (deleteAfterExtract=false)
if _, err := os.Stat(archivePath); os.IsNotExist(err) {
t.Error("Archive should still exist when deleteAfterExtract=false")
}
}
func TestDownloader_HandleExtractionResult_WithDelete(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
tempDir, err := os.MkdirTemp("", "extraction_delete_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
// Create test archive files
archivePath1 := filepath.Join(tempDir, "test.7z.001")
archivePath2 := filepath.Join(tempDir, "test.7z.002")
if err := os.WriteFile(archivePath1, []byte("test"), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(archivePath2, []byte("test"), 0644); err != nil {
t.Fatal(err)
}
task := &Task{
ID: "test-task",
Meta: &fetcher.FetcherMeta{
Opts: &base.Options{Path: tempDir},
Res: &base.Resource{
Files: []*base.FileInfo{{Name: "test.7z.001"}},
},
},
Progress: &Progress{},
}
initTask(task)
// Test successful extraction with delete
downloader.handleExtractionResult(task, nil, []string{archivePath1, archivePath2}, true)
if task.Progress.ExtractStatus != ExtractStatusDone {
t.Errorf("ExtractStatus = %v, want %v", task.Progress.ExtractStatus, ExtractStatusDone)
}
// Archives should be deleted
if _, err := os.Stat(archivePath1); !os.IsNotExist(err) {
t.Error("Archive 1 should be deleted when deleteAfterExtract=true")
}
if _, err := os.Stat(archivePath2); !os.IsNotExist(err) {
t.Error("Archive 2 should be deleted when deleteAfterExtract=true")
}
}
func TestDownloader_HandleExtractionResult_Error(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
tempDir, err := os.MkdirTemp("", "extraction_error_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
task := &Task{
ID: "test-task",
Meta: &fetcher.FetcherMeta{
Opts: &base.Options{Path: tempDir},
Res: &base.Resource{
Files: []*base.FileInfo{{Name: "test.zip"}},
},
},
Progress: &Progress{},
}
initTask(task)
// Test failed extraction
extractErr := fmt.Errorf("extraction failed")
downloader.handleExtractionResult(task, extractErr, nil, false)
if task.Progress.ExtractStatus != ExtractStatusError {
t.Errorf("ExtractStatus = %v, want %v", task.Progress.ExtractStatus, ExtractStatusError)
}
}
func TestDownloader_UpdateMultiPartTasksStatus(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
tempDir, err := os.MkdirTemp("", "update_status_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
// Create source task with multi-part base name
sourceTask := &Task{
ID: "source",
Meta: &fetcher.FetcherMeta{
Opts: &base.Options{Path: tempDir},
Res: &base.Resource{
Files: []*base.FileInfo{{Name: "archive.7z.001"}},
},
},
Progress: &Progress{MultiPartBaseName: "archive.7z"},
}
initTask(sourceTask)
// Create related task
relatedTask := &Task{
ID: "related",
Meta: &fetcher.FetcherMeta{
Opts: &base.Options{Path: tempDir},
Res: &base.Resource{
Files: []*base.FileInfo{{Name: "archive.7z.002"}},
},
},
Progress: &Progress{MultiPartBaseName: "archive.7z"},
}
initTask(relatedTask)
// Create unrelated task
unrelatedTask := &Task{
ID: "unrelated",
Meta: &fetcher.FetcherMeta{
Opts: &base.Options{Path: tempDir},
Res: &base.Resource{
Files: []*base.FileInfo{{Name: "other.7z.001"}},
},
},
Progress: &Progress{MultiPartBaseName: "other.7z"},
}
initTask(unrelatedTask)
downloader.tasks = []*Task{sourceTask, relatedTask, unrelatedTask}
// Test successful extraction
downloader.updateMultiPartTasksStatus(sourceTask, nil)
if relatedTask.Progress.ExtractStatus != ExtractStatusDone {
t.Errorf("Related task ExtractStatus = %v, want %v", relatedTask.Progress.ExtractStatus, ExtractStatusDone)
}
if relatedTask.Progress.ExtractProgress != 100 {
t.Errorf("Related task ExtractProgress = %d, want 100", relatedTask.Progress.ExtractProgress)
}
if unrelatedTask.Progress.ExtractStatus == ExtractStatusDone {
t.Error("Unrelated task should not be updated")
}
}
func TestDownloader_UpdateMultiPartTasksStatus_WithError(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
tempDir, err := os.MkdirTemp("", "update_error_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
// Create source task with multi-part base name
sourceTask := &Task{
ID: "source",
Meta: &fetcher.FetcherMeta{
Opts: &base.Options{Path: tempDir},
Res: &base.Resource{
Files: []*base.FileInfo{{Name: "archive.7z.001"}},
},
},
Progress: &Progress{MultiPartBaseName: "archive.7z"},
}
initTask(sourceTask)
// Create related task
relatedTask := &Task{
ID: "related",
Meta: &fetcher.FetcherMeta{
Opts: &base.Options{Path: tempDir},
Res: &base.Resource{
Files: []*base.FileInfo{{Name: "archive.7z.002"}},
},
},
Progress: &Progress{MultiPartBaseName: "archive.7z"},
}
initTask(relatedTask)
downloader.tasks = []*Task{sourceTask, relatedTask}
// Test failed extraction
downloader.updateMultiPartTasksStatus(sourceTask, fmt.Errorf("extraction failed"))
if relatedTask.Progress.ExtractStatus != ExtractStatusError {
t.Errorf("Related task ExtractStatus = %v, want %v", relatedTask.Progress.ExtractStatus, ExtractStatusError)
}
if relatedTask.Progress.ExtractProgress != 0 {
t.Errorf("Related task ExtractProgress = %d, want 0", relatedTask.Progress.ExtractProgress)
}
}
func TestDownloader_UpdateMultiPartTasksStatus_NoBaseName(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
tempDir, err := os.MkdirTemp("", "update_no_base_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
// Create task without multi-part base name
task := &Task{
ID: "single",
Meta: &fetcher.FetcherMeta{
Opts: &base.Options{Path: tempDir},
Res: &base.Resource{
Files: []*base.FileInfo{{Name: "single.zip"}},
},
},
Progress: &Progress{MultiPartBaseName: ""},
}
initTask(task)
downloader.tasks = []*Task{task}
// Should not panic or error
downloader.updateMultiPartTasksStatus(task, nil)
}
func TestDownloader_CheckMultiPartArchiveReady(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
tempDir, err := os.MkdirTemp("", "check_ready_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
baseName := "archive.7z"
fileName := baseName + ".001"
filePath := filepath.Join(tempDir, fileName)
// Create tasks
task := &Task{
ID: "task1",
Status: base.DownloadStatusDone,
Meta: &fetcher.FetcherMeta{
Opts: &base.Options{Path: tempDir},
Res: &base.Resource{
Files: []*base.FileInfo{{Name: fileName}},
},
},
Progress: &Progress{},
}
initTask(task)
downloader.tasks = []*Task{task}
partInfo := getArchivePartInfo(filePath)
ready, missing := downloader.checkMultiPartArchiveReady(filePath, tempDir, partInfo)
if !ready {
t.Errorf("checkMultiPartArchiveReady() = false, want true; missing: %v", missing)
}
}
func TestDownloader_CheckMultiPartArchiveReady_EmptyBaseName(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
// Use a non-multi-part file path
partInfo := getArchivePartInfo("/some/regular.zip")
ready, _ := downloader.checkMultiPartArchiveReady("/some/regular.zip", "/dest", partInfo)
// Should return true for non-multi-part files
if !ready {
t.Error("checkMultiPartArchiveReady() should return true for non-multi-part files")
}
}
+186
View File
@@ -6,6 +6,7 @@ import (
"math"
"os"
"path/filepath"
"regexp"
"slices"
"strings"
"sync/atomic"
@@ -37,12 +38,45 @@ var supportedArchiveExtensions = []string{
".lz",
}
// multiPartArchivePatterns contains regex patterns for multi-part archive detection
// Each pattern should have a capture group for the base name and part number
var multiPartArchivePatterns = []*regexp.Regexp{
// 7z multi-part: file.7z.001, file.7z.002, etc.
regexp.MustCompile(`(?i)^(.+\.7z)\.(\d{3})$`),
// RAR multi-part (new style): file.part01.rar, file.part02.rar, etc.
regexp.MustCompile(`(?i)^(.+)\.part(\d+)\.rar$`),
// RAR multi-part (old style): file.rar, file.r00, file.r01, etc. (first file is .rar)
regexp.MustCompile(`(?i)^(.+)\.r(\d{2})$`),
// ZIP multi-part: file.zip.001, file.zip.002, etc.
regexp.MustCompile(`(?i)^(.+\.zip)\.(\d{3})$`),
// ZIP split: file.z01, file.z02, ... file.zip (last file is .zip)
regexp.MustCompile(`(?i)^(.+)\.z(\d{2})$`),
}
// ArchivePartInfo contains information about a multi-part archive
type ArchivePartInfo struct {
// IsMultiPart indicates if this file is part of a multi-part archive
IsMultiPart bool
// BaseName is the common base name for all parts (without part number extension)
BaseName string
// PartNumber is the part number (1-indexed)
PartNumber int
// FirstPartPath is the path to the first part of the archive
FirstPartPath string
// Pattern indicates which pattern matched (for determining extraction method)
Pattern string
}
// ExtractProgressCallback is called to report extraction progress
type ExtractProgressCallback func(extractedFiles int, totalFiles int, progress int)
// isArchiveFile checks if a file is a supported archive format
func isArchiveFile(filename string) bool {
lowerName := strings.ToLower(filename)
// Check for multi-part archive first
if isMultiPartArchive(filename) {
return true
}
return slices.ContainsFunc(supportedArchiveExtensions, func(ext string) bool {
return strings.HasSuffix(lowerName, ext)
})
@@ -273,3 +307,155 @@ func extractFile(ctx context.Context, fileInfo archives.FileInfo, destDir string
// Set file permissions
return os.Chmod(destPath, fileInfo.Mode())
}
// isMultiPartArchive checks if a file is part of a multi-part archive
func isMultiPartArchive(filename string) bool {
info := getArchivePartInfo(filename)
return info.IsMultiPart
}
// getArchivePartInfo returns detailed information about a multi-part archive file
func getArchivePartInfo(filename string) ArchivePartInfo {
baseName := filepath.Base(filename)
for _, pattern := range multiPartArchivePatterns {
matches := pattern.FindStringSubmatch(baseName)
if len(matches) >= 3 {
partNum := 0
_, err := parsePartNumber(matches[2], &partNum)
if err != nil {
continue
}
info := ArchivePartInfo{
IsMultiPart: true,
BaseName: matches[1],
PartNumber: partNum,
Pattern: pattern.String(),
}
// Determine the first part path based on pattern
dir := filepath.Dir(filename)
info.FirstPartPath = determineFirstPartPath(dir, info.BaseName, pattern.String())
return info
}
}
// Check for .rar files that might be the first part of old-style multi-part RAR
if strings.HasSuffix(strings.ToLower(baseName), ".rar") && !strings.Contains(strings.ToLower(baseName), ".part") {
// Check if there are .r00, .r01 files in the same directory
dir := filepath.Dir(filename)
nameWithoutExt := strings.TrimSuffix(baseName, filepath.Ext(baseName))
r00Path := filepath.Join(dir, nameWithoutExt+".r00")
if _, err := os.Stat(r00Path); err == nil {
return ArchivePartInfo{
IsMultiPart: true,
BaseName: nameWithoutExt,
PartNumber: 1, // .rar is the first part in old-style
FirstPartPath: filename,
Pattern: "rar-old-style",
}
}
}
return ArchivePartInfo{IsMultiPart: false}
}
// parsePartNumber parses a part number string and stores it in the provided pointer
// Returns the parsed number and nil error on success
func parsePartNumber(s string, partNum *int) (int, error) {
n := 0
for _, c := range s {
if c >= '0' && c <= '9' {
n = n*10 + int(c-'0')
}
}
// For .001, .002 style, part number is the value itself
// For .part01 style, part number is the value itself
// For .r00, .r01 style: r00=2 (since .rar is part 1), r01=3, etc.
// However, for consistency, we return the raw number and handle the offset elsewhere
*partNum = n
// Treat 00 as part 0, let callers handle the semantics
if n == 0 {
*partNum = 1 // For .001 format, 001 should be 1
}
return *partNum, nil
}
// determineFirstPartPath determines the path to the first part of a multi-part archive
func determineFirstPartPath(dir, baseName, pattern string) string {
switch {
case strings.Contains(pattern, `.7z)`):
// 7z multi-part: first part is .7z.001
return filepath.Join(dir, baseName+".001")
case strings.Contains(pattern, `.part`):
// RAR new style: first part is .part01.rar or .part1.rar
// Try both single and double digit formats
if _, err := os.Stat(filepath.Join(dir, baseName+".part1.rar")); err == nil {
return filepath.Join(dir, baseName+".part1.rar")
}
return filepath.Join(dir, baseName+".part01.rar")
case strings.Contains(pattern, `.r(`):
// RAR old style: first part is .rar (not .r00)
return filepath.Join(dir, baseName+".rar")
case strings.Contains(pattern, `.zip)`):
// ZIP multi-part: first part is .zip.001
return filepath.Join(dir, baseName+".001")
case strings.Contains(pattern, `.z(`):
// ZIP split: last part is .zip, but extraction should start from .z01
return filepath.Join(dir, baseName+".z01")
default:
return ""
}
}
// isFirstPart checks if the given file is the first part of a multi-part archive
func isFirstPart(filename string) bool {
info := getArchivePartInfo(filename)
if !info.IsMultiPart {
return false
}
// For most formats, part 1 is the first part
// For old-style RAR, check if this is the .rar file
if info.Pattern == "rar-old-style" {
return strings.HasSuffix(strings.ToLower(filename), ".rar")
}
return info.PartNumber == 1
}
// GetMultiPartArchiveBaseName returns the base name for a multi-part archive
// This is used to group related parts together
func GetMultiPartArchiveBaseName(filename string) string {
info := getArchivePartInfo(filename)
if !info.IsMultiPart {
return ""
}
return filepath.Join(filepath.Dir(filename), info.BaseName)
}
// extractMultiPartArchive extracts a multi-part archive starting from the first part
func extractMultiPartArchive(firstPartPath string, destDir string, password string, progressCallback ExtractProgressCallback) error {
info := getArchivePartInfo(firstPartPath)
// For 7z multi-part archives, the bodgit/sevenzip library handles multi-volume automatically
// when using OpenReader with a .001 file
if strings.Contains(info.Pattern, `\.7z\)`) || strings.HasSuffix(strings.ToLower(firstPartPath), ".7z.001") {
return extractSevenZipMultiPart(firstPartPath, destDir, password, progressCallback)
}
// For RAR multi-part archives, use the archives library with Name and FS fields
if strings.Contains(info.Pattern, "rar") || info.Pattern == "rar-old-style" {
return extractRarMultiPart(firstPartPath, destDir, password, progressCallback)
}
// For ZIP multi-part archives (.zip.001, .zip.002, etc.), concatenate parts and extract
if strings.Contains(info.Pattern, `\.zip\)`) || strings.HasSuffix(strings.ToLower(firstPartPath), ".zip.001") {
return extractZipMultiPart(firstPartPath, destDir, password, progressCallback)
}
// For other formats, try standard extraction (may not work for all multi-part formats)
return extractArchive(firstPartPath, destDir, password, progressCallback)
}
+92
View File
@@ -0,0 +1,92 @@
package download
import (
"io"
"os"
"path/filepath"
"github.com/bodgit/sevenzip"
)
// extractSevenZipMultiPart extracts a multi-part 7z archive using bodgit/sevenzip directly
// The mholt/archives wrapper doesn't properly handle multi-part 7z files because it uses
// io.SectionReader which can only see the first part. The bodgit/sevenzip library's
// OpenReaderWithPassword function handles multi-part files automatically when given the .001 file path.
func extractSevenZipMultiPart(firstPartPath string, destDir string, password string, progressCallback ExtractProgressCallback) error {
// Use bodgit/sevenzip directly - it automatically handles .001, .002, etc. files
var reader *sevenzip.ReadCloser
var err error
if password != "" {
reader, err = sevenzip.OpenReaderWithPassword(firstPartPath, password)
} else {
reader, err = sevenzip.OpenReader(firstPartPath)
}
if err != nil {
return err
}
defer reader.Close()
// Create destination directory
if err := os.MkdirAll(destDir, 0755); err != nil {
return err
}
// Count total files for progress
totalFiles := 0
for _, f := range reader.File {
if !f.FileInfo().IsDir() {
totalFiles++
}
}
// Extract files with progress tracking
extractedFiles := 0
for _, f := range reader.File {
destPath := filepath.Join(destDir, f.Name)
if f.FileInfo().IsDir() {
if err := os.MkdirAll(destPath, f.Mode()); err != nil {
return err
}
continue
}
// Ensure parent directory exists
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
return err
}
// Extract file - open, copy, close immediately (as recommended by bodgit/sevenzip)
if err := extractSevenZipFile(f, destPath); err != nil {
return err
}
extractedFiles++
if progressCallback != nil && totalFiles > 0 {
progress := int(float64(extractedFiles) / float64(totalFiles) * 100)
progressCallback(extractedFiles, totalFiles, progress)
}
}
return nil
}
// extractSevenZipFile extracts a single file from a 7z archive
// This follows the bodgit/sevenzip recommended pattern of closing rc before processing the next file
func extractSevenZipFile(f *sevenzip.File, destPath string) error {
rc, err := f.Open()
if err != nil {
return err
}
defer rc.Close()
outFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
if err != nil {
return err
}
defer outFile.Close()
_, err = io.Copy(outFile, rc)
return err
}
+46
View File
@@ -0,0 +1,46 @@
package download
import (
"context"
"os"
"path/filepath"
"github.com/mholt/archives"
)
// extractRarMultiPart extracts a multi-part RAR archive
func extractRarMultiPart(firstPartPath string, destDir string, password string, progressCallback ExtractProgressCallback) error {
// For RAR archives, we need to use the Name field in archives.Rar
// to let it automatically find subsequent volumes
dir := filepath.Dir(firstPartPath)
fileName := filepath.Base(firstPartPath)
rar := archives.Rar{
Password: password,
Name: fileName,
FS: os.DirFS(dir),
}
// Create destination directory
if err := os.MkdirAll(destDir, 0755); err != nil {
return err
}
// Count files first for progress
totalFiles := 0
err := rar.Extract(context.Background(), nil, func(ctx context.Context, fileInfo archives.FileInfo) error {
if !fileInfo.IsDir() {
totalFiles++
}
return nil
})
if err != nil {
// If counting fails, proceed without progress
totalFiles = 0
}
// Reset and extract with progress tracking
return rar.Extract(context.Background(), nil, createExtractionHandler(destDir, totalFiles, progressCallback))
}
File diff suppressed because it is too large. Load diff
+191
View File
@@ -0,0 +1,191 @@
package download
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/mholt/archives"
)
// extractZipMultiPart extracts a multi-part ZIP archive (.zip.001, .zip.002, etc.)
// These are created by simply splitting a ZIP file into chunks, so we concatenate them
func extractZipMultiPart(firstPartPath string, destDir string, password string, progressCallback ExtractProgressCallback) error {
// Find all parts
parts, err := findZipMultiParts(firstPartPath)
if err != nil {
return err
}
// Create a multi-part reader that reads across all parts
multiReader := newMultiPartFileReader(parts)
defer multiReader.Close()
// Get total size
totalSize := multiReader.Size()
// Create destination directory
if err := os.MkdirAll(destDir, 0755); err != nil {
return err
}
// First pass: count files for progress
totalFiles := 0
zip := archives.Zip{}
err = zip.Extract(context.Background(), io.NewSectionReader(multiReader, 0, totalSize), func(ctx context.Context, fileInfo archives.FileInfo) error {
if !fileInfo.IsDir() {
totalFiles++
}
return nil
})
if err != nil {
// If counting fails, proceed without progress
totalFiles = 0
}
// Reset reader for actual extraction
multiReader.Close()
multiReader = newMultiPartFileReader(parts)
defer multiReader.Close()
// Second pass: extract with progress tracking
return zip.Extract(context.Background(), io.NewSectionReader(multiReader, 0, totalSize), createExtractionHandler(destDir, totalFiles, progressCallback))
}
// findZipMultiParts finds all parts of a multi-part ZIP archive in order
func findZipMultiParts(firstPartPath string) ([]string, error) {
// Extract base name (e.g., "Archive.zip" from "Archive.zip.001")
dir := filepath.Dir(firstPartPath)
baseName := filepath.Base(firstPartPath)
// Remove the .001 suffix to get the base
if idx := strings.LastIndex(baseName, "."); idx > 0 {
baseName = baseName[:idx] // "Archive.zip"
}
var parts []string
partNum := 1
for {
partPath := filepath.Join(dir, baseName+fmt.Sprintf(".%03d", partNum))
if _, err := os.Stat(partPath); os.IsNotExist(err) {
break
}
parts = append(parts, partPath)
partNum++
}
if len(parts) == 0 {
return nil, fmt.Errorf("no parts found for %s", firstPartPath)
}
return parts, nil
}
// multiPartFileReader provides io.ReaderAt over multiple files concatenated
type multiPartFileReader struct {
parts []string
files []*os.File
sizes []int64
offsets []int64 // cumulative offsets for each file
totalSize int64
}
func newMultiPartFileReader(parts []string) *multiPartFileReader {
return &multiPartFileReader{parts: parts}
}
func (m *multiPartFileReader) init() error {
if m.files != nil {
return nil
}
m.files = make([]*os.File, len(m.parts))
m.sizes = make([]int64, len(m.parts))
m.offsets = make([]int64, len(m.parts))
var offset int64
for i, part := range m.parts {
f, err := os.Open(part)
if err != nil {
m.Close()
return err
}
stat, err := f.Stat()
if err != nil {
f.Close()
m.Close()
return err
}
m.files[i] = f
m.sizes[i] = stat.Size()
m.offsets[i] = offset
offset += stat.Size()
}
m.totalSize = offset
return nil
}
func (m *multiPartFileReader) Size() int64 {
if err := m.init(); err != nil {
return 0
}
return m.totalSize
}
func (m *multiPartFileReader) ReadAt(p []byte, off int64) (n int, err error) {
if err := m.init(); err != nil {
return 0, err
}
if off >= m.totalSize {
return 0, io.EOF
}
// Find which file(s) to read from
for i, fileOffset := range m.offsets {
fileEnd := fileOffset + m.sizes[i]
if off >= fileEnd {
continue
}
// Read from this file
localOffset := off - fileOffset
toRead := len(p) - n
if int64(toRead) > fileEnd-off {
toRead = int(fileEnd - off)
}
read, err := m.files[i].ReadAt(p[n:n+toRead], localOffset)
n += read
off += int64(read)
if err != nil && err != io.EOF {
return n, err
}
if n >= len(p) {
return n, nil
}
}
if n == 0 {
return 0, io.EOF
}
return n, nil
}
func (m *multiPartFileReader) Close() error {
for _, f := range m.files {
if f != nil {
f.Close()
}
}
m.files = nil
return nil
}