mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-04 23:57:54 +00:00
feat: multipart-archives (#1222)
This commit is contained in:
1 parent
cb6331c20d
commit
de0afb1481
7 files changed
+2689
-46
No files matched your search
+347
-35
@@ -2,6 +2,7 @@ package download
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
gohttp "net/http"
|
||||
"net/url"
|
||||
@@ -51,6 +52,8 @@ type ExtractStatus string
|
||||
const (
|
||||
// ExtractStatusNone indicates extraction has not started
|
||||
ExtractStatusNone ExtractStatus = ""
|
||||
// ExtractStatusWaitingParts indicates waiting for other multi-part archive parts to complete
|
||||
ExtractStatusWaitingParts ExtractStatus = "waitingParts"
|
||||
// ExtractStatusExtracting indicates extraction is in progress
|
||||
ExtractStatusExtracting ExtractStatus = "extracting"
|
||||
// ExtractStatusDone indicates extraction completed successfully
|
||||
@@ -74,6 +77,12 @@ type Progress struct {
|
||||
ExtractStatus ExtractStatus `json:"extractStatus"`
|
||||
// ExtractProgress is the percentage of extraction completed (0-100)
|
||||
ExtractProgress int `json:"extractProgress"`
|
||||
// MultiPartBaseName is set for multi-part archives to group related parts
|
||||
MultiPartBaseName string `json:"multiPartBaseName,omitempty"`
|
||||
// MultiPartNumber is the part number for multi-part archives (1-indexed)
|
||||
MultiPartNumber int `json:"multiPartNumber,omitempty"`
|
||||
// MultiPartIsFirst indicates if this is the first part of a multi-part archive
|
||||
MultiPartIsFirst bool `json:"multiPartIsFirst,omitempty"`
|
||||
}
|
||||
|
||||
type Downloader struct {
|
||||
@@ -898,41 +907,7 @@ func (d *Downloader) watch(task *Task) {
|
||||
|
||||
// Auto-extract archive files
|
||||
if e.AutoExtract && isArchiveFile(downloadFilePath) {
|
||||
go func() {
|
||||
// Set extraction status to extracting
|
||||
task.Progress.ExtractStatus = ExtractStatusExtracting
|
||||
task.Progress.ExtractProgress = 0
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
|
||||
// Extract to the same directory as the downloaded file
|
||||
destDir := task.Meta.Opts.Path
|
||||
extractErr := extractArchive(downloadFilePath, destDir, e.ArchivePassword, func(extractedFiles int, totalFiles int, progress int) {
|
||||
task.Progress.ExtractProgress = progress
|
||||
d.emit(EventKeyProgress, task)
|
||||
})
|
||||
if extractErr != nil {
|
||||
d.Logger.Error().Err(extractErr).Msgf("auto extract archive failed, task id: %s", task.ID)
|
||||
task.Progress.ExtractStatus = ExtractStatusError
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
} else {
|
||||
d.Logger.Info().Msgf("auto extract archive completed, task id: %s", task.ID)
|
||||
task.Progress.ExtractStatus = ExtractStatusDone
|
||||
task.Progress.ExtractProgress = 100
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
// Delete archive after successful extraction if enabled
|
||||
if e.DeleteAfterExtract {
|
||||
deleteErr := os.Remove(downloadFilePath)
|
||||
if deleteErr != nil {
|
||||
d.Logger.Error().Err(deleteErr).Msgf("delete archive after extraction failed, task id: %s", task.ID)
|
||||
} else {
|
||||
d.Logger.Info().Msgf("archive deleted after extraction, task id: %s", task.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
go d.handleAutoExtract(task, downloadFilePath, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1225,6 +1200,343 @@ func (d *Downloader) buildFetcher(url string) (fetcher.Fetcher, error) {
|
||||
return fetcher, nil
|
||||
}
|
||||
|
||||
// handleAutoExtract handles automatic extraction of archive files, including multi-part archives
|
||||
func (d *Downloader) handleAutoExtract(task *Task, downloadFilePath string, opts *http.OptsExtra) {
|
||||
// Check if this is a multi-part archive
|
||||
partInfo := getArchivePartInfo(downloadFilePath)
|
||||
|
||||
if partInfo.IsMultiPart {
|
||||
// Set multi-part info on the task
|
||||
task.Progress.MultiPartBaseName = partInfo.BaseName
|
||||
task.Progress.MultiPartNumber = partInfo.PartNumber
|
||||
task.Progress.MultiPartIsFirst = isFirstPart(downloadFilePath)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
|
||||
// Check if all parts are downloaded
|
||||
destDir := task.Meta.Opts.Path
|
||||
allPartsReady, missingParts := d.checkMultiPartArchiveReady(downloadFilePath, destDir, partInfo)
|
||||
|
||||
if !allPartsReady {
|
||||
// Not all parts are ready yet, set status to waiting
|
||||
task.Progress.ExtractStatus = ExtractStatusWaitingParts
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.Logger.Info().Msgf("multi-part archive waiting for other parts, task id: %s, missing: %v", task.ID, missingParts)
|
||||
return
|
||||
}
|
||||
|
||||
// All parts are ready!
|
||||
// Only one task should perform extraction - we'll use the one that completes last
|
||||
// Check if extraction has already been started by another part
|
||||
if d.isMultiPartExtractionInProgress(partInfo.BaseName) {
|
||||
// Another part already started extraction, mark this as done
|
||||
task.Progress.ExtractStatus = ExtractStatusDone
|
||||
task.Progress.ExtractProgress = 100
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.Logger.Info().Msgf("multi-part archive part completed, extraction handled by another part, task id: %s", task.ID)
|
||||
return
|
||||
}
|
||||
|
||||
// This task will handle extraction - mark as extracting first to prevent others
|
||||
task.Progress.ExtractStatus = ExtractStatusExtracting
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
|
||||
// Extract the multi-part archive
|
||||
d.performMultiPartExtraction(task, partInfo.FirstPartPath, destDir, opts)
|
||||
} else {
|
||||
// Regular (non-multi-part) archive
|
||||
d.performExtraction(task, downloadFilePath, task.Meta.Opts.Path, opts)
|
||||
}
|
||||
}
|
||||
|
||||
// checkMultiPartArchiveReady checks if all parts of a multi-part archive are downloaded
|
||||
// by examining task status rather than file existence
|
||||
func (d *Downloader) checkMultiPartArchiveReady(filePath string, destDir string, partInfo ArchivePartInfo) (bool, []string) {
|
||||
// Use task-based checking - find all tasks with the same MultiPartBaseName
|
||||
// and verify they are all in Done status
|
||||
baseName := GetMultiPartArchiveBaseName(filePath)
|
||||
if baseName == "" {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return d.checkAllMultiPartTasksDone(baseName)
|
||||
}
|
||||
|
||||
// checkAllMultiPartTasksDone checks if all tasks belonging to a multi-part archive are done
|
||||
func (d *Downloader) checkAllMultiPartTasksDone(baseName string) (bool, []string) {
|
||||
var notDoneParts []string
|
||||
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
|
||||
// Find all tasks that belong to this multi-part archive
|
||||
var relatedTasks []*Task
|
||||
for _, task := range d.tasks {
|
||||
taskBaseName := ""
|
||||
if task.Meta != nil && task.Meta.Res != nil && len(task.Meta.Res.Files) > 0 {
|
||||
taskBaseName = GetMultiPartArchiveBaseName(task.Meta.SingleFilepath())
|
||||
}
|
||||
if taskBaseName == baseName {
|
||||
relatedTasks = append(relatedTasks, task)
|
||||
}
|
||||
}
|
||||
|
||||
// If we found no related tasks, we can't determine readiness
|
||||
if len(relatedTasks) == 0 {
|
||||
return false, []string{"no related tasks found for " + baseName}
|
||||
}
|
||||
|
||||
// Check if all related tasks are done
|
||||
for _, task := range relatedTasks {
|
||||
if task.Status != base.DownloadStatusDone {
|
||||
notDoneParts = append(notDoneParts, task.Meta.SingleFilepath())
|
||||
}
|
||||
}
|
||||
|
||||
return len(notDoneParts) == 0, notDoneParts
|
||||
}
|
||||
|
||||
// isMultiPartExtractionInProgress checks if any task for this multi-part archive
|
||||
// has already started extraction (to prevent duplicate extractions)
|
||||
func (d *Downloader) isMultiPartExtractionInProgress(baseName string) bool {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
|
||||
for _, task := range d.tasks {
|
||||
taskBaseName := ""
|
||||
if task.Meta != nil && task.Meta.Res != nil && len(task.Meta.Res.Files) > 0 {
|
||||
taskBaseName = GetMultiPartArchiveBaseName(task.Meta.SingleFilepath())
|
||||
}
|
||||
if taskBaseName == baseName {
|
||||
// Check if this task is already extracting or has completed extraction
|
||||
if task.Progress.ExtractStatus == ExtractStatusExtracting ||
|
||||
task.Progress.ExtractStatus == ExtractStatusDone ||
|
||||
task.Progress.ExtractStatus == ExtractStatusError {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// performExtraction performs extraction for a regular (non-multi-part) archive
|
||||
func (d *Downloader) performExtraction(task *Task, archivePath string, destDir string, opts *http.OptsExtra) {
|
||||
// Set extraction status to extracting
|
||||
task.Progress.ExtractStatus = ExtractStatusExtracting
|
||||
task.Progress.ExtractProgress = 0
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
|
||||
// Extract the archive
|
||||
extractErr := extractArchive(archivePath, destDir, opts.ArchivePassword, func(extractedFiles int, totalFiles int, progress int) {
|
||||
task.Progress.ExtractProgress = progress
|
||||
d.emit(EventKeyProgress, task)
|
||||
})
|
||||
|
||||
d.handleExtractionResult(task, extractErr, []string{archivePath}, opts.DeleteAfterExtract)
|
||||
}
|
||||
|
||||
// performMultiPartExtraction performs extraction for a multi-part archive
|
||||
func (d *Downloader) performMultiPartExtraction(task *Task, firstPartPath string, destDir string, opts *http.OptsExtra) {
|
||||
// Set extraction status to extracting
|
||||
task.Progress.ExtractStatus = ExtractStatusExtracting
|
||||
task.Progress.ExtractProgress = 0
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
|
||||
d.Logger.Info().Msgf("starting multi-part archive extraction, first part: %s, task id: %s", firstPartPath, task.ID)
|
||||
|
||||
// Extract the multi-part archive
|
||||
extractErr := extractMultiPartArchive(firstPartPath, destDir, opts.ArchivePassword, func(extractedFiles int, totalFiles int, progress int) {
|
||||
task.Progress.ExtractProgress = progress
|
||||
d.emit(EventKeyProgress, task)
|
||||
})
|
||||
|
||||
// Collect all part files for potential deletion
|
||||
partFiles := d.collectMultiPartFiles(firstPartPath)
|
||||
|
||||
d.handleExtractionResult(task, extractErr, partFiles, opts.DeleteAfterExtract)
|
||||
|
||||
// Update status for all related multi-part tasks
|
||||
d.updateMultiPartTasksStatus(task, extractErr)
|
||||
}
|
||||
|
||||
// collectMultiPartFiles collects all files belonging to a multi-part archive
|
||||
func (d *Downloader) collectMultiPartFiles(firstPartPath string) []string {
|
||||
var files []string
|
||||
partInfo := getArchivePartInfo(firstPartPath)
|
||||
dir := filepath.Dir(firstPartPath)
|
||||
|
||||
switch {
|
||||
case strings.Contains(partInfo.Pattern, ".7z)"):
|
||||
// 7z: .7z.001, .7z.002, etc.
|
||||
files = d.collectSequentialFiles(dir, partInfo.BaseName, ".%03d")
|
||||
case strings.Contains(partInfo.Pattern, ".part"):
|
||||
// RAR new style
|
||||
files = d.collectRarNewStyleFiles(dir, partInfo.BaseName)
|
||||
case partInfo.Pattern == "rar-old-style" || strings.Contains(partInfo.Pattern, ".r("):
|
||||
// RAR old style
|
||||
files = d.collectRarOldStyleFiles(dir, partInfo.BaseName)
|
||||
case strings.Contains(partInfo.Pattern, ".zip)"):
|
||||
// ZIP multi-part
|
||||
files = d.collectSequentialFiles(dir, partInfo.BaseName, ".%03d")
|
||||
case strings.Contains(partInfo.Pattern, ".z("):
|
||||
// ZIP split
|
||||
files = d.collectZipSplitFiles(dir, partInfo.BaseName)
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// collectSequentialFiles collects sequential numbered files
|
||||
func (d *Downloader) collectSequentialFiles(dir, baseName, format string) []string {
|
||||
var files []string
|
||||
suffix := filepath.Ext(baseName)
|
||||
nameWithoutExt := strings.TrimSuffix(baseName, suffix)
|
||||
partNum := 1
|
||||
|
||||
for {
|
||||
partPath := filepath.Join(dir, nameWithoutExt+suffix+fmt.Sprintf(format, partNum))
|
||||
if _, err := os.Stat(partPath); os.IsNotExist(err) {
|
||||
break
|
||||
}
|
||||
files = append(files, partPath)
|
||||
partNum++
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// collectRarNewStyleFiles collects RAR new style part files
|
||||
func (d *Downloader) collectRarNewStyleFiles(dir, baseName string) []string {
|
||||
var files []string
|
||||
partNum := 1
|
||||
|
||||
for {
|
||||
singleDigitPath := filepath.Join(dir, baseName+fmt.Sprintf(".part%d.rar", partNum))
|
||||
doubleDigitPath := filepath.Join(dir, baseName+fmt.Sprintf(".part%02d.rar", partNum))
|
||||
|
||||
if _, err := os.Stat(singleDigitPath); err == nil {
|
||||
files = append(files, singleDigitPath)
|
||||
} else if _, err := os.Stat(doubleDigitPath); err == nil {
|
||||
files = append(files, doubleDigitPath)
|
||||
} else {
|
||||
break
|
||||
}
|
||||
partNum++
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// collectRarOldStyleFiles collects RAR old style part files
|
||||
func (d *Downloader) collectRarOldStyleFiles(dir, baseName string) []string {
|
||||
var files []string
|
||||
|
||||
// .rar file
|
||||
rarPath := filepath.Join(dir, baseName+".rar")
|
||||
if _, err := os.Stat(rarPath); err == nil {
|
||||
files = append(files, rarPath)
|
||||
}
|
||||
|
||||
// .r00, .r01, etc.
|
||||
partNum := 0
|
||||
for {
|
||||
partPath := filepath.Join(dir, baseName+fmt.Sprintf(".r%02d", partNum))
|
||||
if _, err := os.Stat(partPath); os.IsNotExist(err) {
|
||||
break
|
||||
}
|
||||
files = append(files, partPath)
|
||||
partNum++
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// collectZipSplitFiles collects ZIP split files
|
||||
func (d *Downloader) collectZipSplitFiles(dir, baseName string) []string {
|
||||
var files []string
|
||||
|
||||
// .z01, .z02, etc.
|
||||
partNum := 1
|
||||
for {
|
||||
partPath := filepath.Join(dir, baseName+fmt.Sprintf(".z%02d", partNum))
|
||||
if _, err := os.Stat(partPath); os.IsNotExist(err) {
|
||||
break
|
||||
}
|
||||
files = append(files, partPath)
|
||||
partNum++
|
||||
}
|
||||
|
||||
// .zip file
|
||||
zipPath := filepath.Join(dir, baseName+".zip")
|
||||
if _, err := os.Stat(zipPath); err == nil {
|
||||
files = append(files, zipPath)
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// handleExtractionResult handles the result of an extraction operation
|
||||
func (d *Downloader) handleExtractionResult(task *Task, extractErr error, archiveFiles []string, deleteAfterExtract bool) {
|
||||
if extractErr != nil {
|
||||
d.Logger.Error().Err(extractErr).Msgf("auto extract archive failed, task id: %s", task.ID)
|
||||
task.Progress.ExtractStatus = ExtractStatusError
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
} else {
|
||||
d.Logger.Info().Msgf("auto extract archive completed, task id: %s", task.ID)
|
||||
task.Progress.ExtractStatus = ExtractStatusDone
|
||||
task.Progress.ExtractProgress = 100
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
|
||||
// Delete archive files after successful extraction if enabled
|
||||
if deleteAfterExtract {
|
||||
for _, archiveFile := range archiveFiles {
|
||||
deleteErr := os.Remove(archiveFile)
|
||||
if deleteErr != nil {
|
||||
d.Logger.Error().Err(deleteErr).Msgf("delete archive after extraction failed: %s", archiveFile)
|
||||
} else {
|
||||
d.Logger.Info().Msgf("archive deleted after extraction: %s", archiveFile)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// updateMultiPartTasksStatus updates the extraction status for all tasks that belong to the same multi-part archive
|
||||
func (d *Downloader) updateMultiPartTasksStatus(sourceTask *Task, extractErr error) {
|
||||
if sourceTask.Progress.MultiPartBaseName == "" {
|
||||
return
|
||||
}
|
||||
|
||||
status := ExtractStatusDone
|
||||
progress := 100
|
||||
if extractErr != nil {
|
||||
status = ExtractStatusError
|
||||
progress = 0
|
||||
}
|
||||
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
|
||||
for _, task := range d.tasks {
|
||||
if task.ID == sourceTask.ID {
|
||||
continue
|
||||
}
|
||||
if task.Progress.MultiPartBaseName == sourceTask.Progress.MultiPartBaseName {
|
||||
task.Progress.ExtractStatus = status
|
||||
task.Progress.ExtractProgress = progress
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func initTask(task *Task) {
|
||||
task.timer = util.NewTimer(task.Progress.Used)
|
||||
|
||||
|
||||
+785
-11
@@ -2,16 +2,19 @@ package download
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"fmt"
|
||||
"io"
|
||||
gohttp "net/http"
|
||||
"net"
|
||||
gohttp "net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/internal/test"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||
@@ -188,19 +191,22 @@ func TestDownloader_CreateDirectBatch(t *testing.T) {
|
||||
t.Errorf("CreateDirectBatch() task got = %v, want %v", len(tasks), len(reqs))
|
||||
}
|
||||
|
||||
nameMatchMap := make(map[string]bool)
|
||||
for _, name := range fileNames {
|
||||
// Collect all task names
|
||||
taskNames := make(map[string]bool)
|
||||
for _, task := range tasks {
|
||||
taskNames[task.Meta.Opts.Name] = true
|
||||
}
|
||||
|
||||
// Check that we have the expected number of unique task names
|
||||
if len(taskNames) != len(reqs) {
|
||||
t.Errorf("CreateDirectBatch() unique task names got = %v, want %v, names: %v", len(taskNames), len(reqs), taskNames)
|
||||
}
|
||||
|
||||
// Check that all task files exist
|
||||
for name := range taskNames {
|
||||
if _, err := os.Stat(test.Dir + "/" + name); os.IsNotExist(err) {
|
||||
t.Errorf("CreateDirectBatch() file not exist: %v", name)
|
||||
}
|
||||
for _, task := range tasks {
|
||||
if name == task.Meta.Opts.Name {
|
||||
nameMatchMap[task.Meta.Opts.Name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(nameMatchMap) != len(reqs) {
|
||||
t.Errorf("CreateDirectBatch() names got = %v, want %v", len(nameMatchMap), len(reqs))
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1453,6 +1459,51 @@ func TestProgress_ExtractFields(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestProgress_MultiPartFields tests the multi-part archive fields in Progress struct
|
||||
func TestProgress_MultiPartFields(t *testing.T) {
|
||||
progress := &Progress{
|
||||
ExtractStatus: ExtractStatusWaitingParts,
|
||||
MultiPartBaseName: "/path/to/archive.7z",
|
||||
MultiPartNumber: 1,
|
||||
MultiPartIsFirst: true,
|
||||
}
|
||||
|
||||
if progress.ExtractStatus != ExtractStatusWaitingParts {
|
||||
t.Errorf("ExtractStatus = %v, want %v", progress.ExtractStatus, ExtractStatusWaitingParts)
|
||||
}
|
||||
if progress.MultiPartBaseName != "/path/to/archive.7z" {
|
||||
t.Errorf("MultiPartBaseName = %v, want %v", progress.MultiPartBaseName, "/path/to/archive.7z")
|
||||
}
|
||||
if progress.MultiPartNumber != 1 {
|
||||
t.Errorf("MultiPartNumber = %v, want %v", progress.MultiPartNumber, 1)
|
||||
}
|
||||
if !progress.MultiPartIsFirst {
|
||||
t.Error("MultiPartIsFirst should be true")
|
||||
}
|
||||
|
||||
// Test second part
|
||||
progress2 := &Progress{
|
||||
ExtractStatus: ExtractStatusWaitingParts,
|
||||
MultiPartBaseName: "/path/to/archive.7z",
|
||||
MultiPartNumber: 2,
|
||||
MultiPartIsFirst: false,
|
||||
}
|
||||
|
||||
if progress2.MultiPartNumber != 2 {
|
||||
t.Errorf("MultiPartNumber = %v, want %v", progress2.MultiPartNumber, 2)
|
||||
}
|
||||
if progress2.MultiPartIsFirst {
|
||||
t.Error("MultiPartIsFirst should be false")
|
||||
}
|
||||
}
|
||||
|
||||
// TestExtractStatus_WaitingParts tests the new ExtractStatusWaitingParts status
|
||||
func TestExtractStatus_WaitingParts(t *testing.T) {
|
||||
if ExtractStatusWaitingParts != "waitingParts" {
|
||||
t.Errorf("ExtractStatusWaitingParts = %v, want %v", ExtractStatusWaitingParts, "waitingParts")
|
||||
}
|
||||
}
|
||||
|
||||
// createTestArchiveWithMultipleFiles creates a test zip file with multiple files
|
||||
func createTestArchiveWithMultipleFiles(path string, count int) error {
|
||||
file, err := os.Create(path)
|
||||
@@ -1710,3 +1761,726 @@ func TestTaskFilter_IsEmpty(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Tests for multi-part archive collection functions
|
||||
func TestDownloader_CollectSequentialFiles(t *testing.T) {
|
||||
tempDir, err := os.MkdirTemp("", "collect_sequential_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Create test files for 7z multi-part pattern (archive.7z.001, .002, .003)
|
||||
for i := 1; i <= 3; i++ {
|
||||
path := filepath.Join(tempDir, fmt.Sprintf("archive.7z.%03d", i))
|
||||
if err := os.WriteFile(path, []byte("test"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
downloader := NewDownloader(nil)
|
||||
files := downloader.collectSequentialFiles(tempDir, "archive.7z", ".%03d")
|
||||
|
||||
if len(files) != 3 {
|
||||
t.Errorf("collectSequentialFiles() = %d files, want 3", len(files))
|
||||
}
|
||||
|
||||
// Verify files are in order
|
||||
for i, file := range files {
|
||||
expected := filepath.Join(tempDir, fmt.Sprintf("archive.7z.%03d", i+1))
|
||||
if file != expected {
|
||||
t.Errorf("files[%d] = %q, want %q", i, file, expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_CollectSequentialFiles_NoFiles(t *testing.T) {
|
||||
tempDir, err := os.MkdirTemp("", "collect_sequential_empty_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
downloader := NewDownloader(nil)
|
||||
files := downloader.collectSequentialFiles(tempDir, "archive.7z", ".%03d")
|
||||
|
||||
if len(files) != 0 {
|
||||
t.Errorf("collectSequentialFiles() = %d files, want 0", len(files))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_CollectRarNewStyleFiles(t *testing.T) {
|
||||
tempDir, err := os.MkdirTemp("", "collect_rar_new_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Create test files with double-digit format
|
||||
for i := 1; i <= 3; i++ {
|
||||
path := filepath.Join(tempDir, fmt.Sprintf("archive.part%02d.rar", i))
|
||||
if err := os.WriteFile(path, []byte("test"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
downloader := NewDownloader(nil)
|
||||
files := downloader.collectRarNewStyleFiles(tempDir, "archive")
|
||||
|
||||
if len(files) != 3 {
|
||||
t.Errorf("collectRarNewStyleFiles() = %d files, want 3", len(files))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_CollectRarNewStyleFiles_SingleDigit(t *testing.T) {
|
||||
tempDir, err := os.MkdirTemp("", "collect_rar_single_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Create test files with single-digit format
|
||||
for i := 1; i <= 2; i++ {
|
||||
path := filepath.Join(tempDir, fmt.Sprintf("archive.part%d.rar", i))
|
||||
if err := os.WriteFile(path, []byte("test"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
downloader := NewDownloader(nil)
|
||||
files := downloader.collectRarNewStyleFiles(tempDir, "archive")
|
||||
|
||||
if len(files) != 2 {
|
||||
t.Errorf("collectRarNewStyleFiles() = %d files, want 2", len(files))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_CollectRarOldStyleFiles(t *testing.T) {
|
||||
tempDir, err := os.MkdirTemp("", "collect_rar_old_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Create .rar file
|
||||
rarPath := filepath.Join(tempDir, "archive.rar")
|
||||
if err := os.WriteFile(rarPath, []byte("test"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create .r00, .r01, .r02 files
|
||||
for i := 0; i <= 2; i++ {
|
||||
path := filepath.Join(tempDir, fmt.Sprintf("archive.r%02d", i))
|
||||
if err := os.WriteFile(path, []byte("test"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
downloader := NewDownloader(nil)
|
||||
files := downloader.collectRarOldStyleFiles(tempDir, "archive")
|
||||
|
||||
// Should have 4 files: .rar + .r00 + .r01 + .r02
|
||||
if len(files) != 4 {
|
||||
t.Errorf("collectRarOldStyleFiles() = %d files, want 4", len(files))
|
||||
}
|
||||
|
||||
// First file should be .rar
|
||||
if files[0] != rarPath {
|
||||
t.Errorf("First file should be .rar, got %q", files[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_CollectZipSplitFiles(t *testing.T) {
|
||||
tempDir, err := os.MkdirTemp("", "collect_zip_split_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Create .z01, .z02 files
|
||||
for i := 1; i <= 2; i++ {
|
||||
path := filepath.Join(tempDir, fmt.Sprintf("archive.z%02d", i))
|
||||
if err := os.WriteFile(path, []byte("test"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create .zip file
|
||||
zipPath := filepath.Join(tempDir, "archive.zip")
|
||||
if err := os.WriteFile(zipPath, []byte("test"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
downloader := NewDownloader(nil)
|
||||
files := downloader.collectZipSplitFiles(tempDir, "archive")
|
||||
|
||||
// Should have 3 files: .z01 + .z02 + .zip
|
||||
if len(files) != 3 {
|
||||
t.Errorf("collectZipSplitFiles() = %d files, want 3", len(files))
|
||||
}
|
||||
|
||||
// Last file should be .zip
|
||||
if files[len(files)-1] != zipPath {
|
||||
t.Errorf("Last file should be .zip, got %q", files[len(files)-1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_CollectMultiPartFiles(t *testing.T) {
|
||||
tempDir, err := os.MkdirTemp("", "collect_multipart_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Test with 7z pattern
|
||||
t.Run("7z pattern", func(t *testing.T) {
|
||||
subDir := filepath.Join(tempDir, "7z")
|
||||
if err := os.MkdirAll(subDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for i := 1; i <= 2; i++ {
|
||||
path := filepath.Join(subDir, fmt.Sprintf("archive.7z.%03d", i))
|
||||
if err := os.WriteFile(path, []byte("test"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
downloader := NewDownloader(nil)
|
||||
firstPart := filepath.Join(subDir, "archive.7z.001")
|
||||
files := downloader.collectMultiPartFiles(firstPart)
|
||||
|
||||
if len(files) != 2 {
|
||||
t.Errorf("collectMultiPartFiles(7z) = %d files, want 2", len(files))
|
||||
}
|
||||
})
|
||||
|
||||
// Test with RAR new style pattern
|
||||
t.Run("RAR new style pattern", func(t *testing.T) {
|
||||
subDir := filepath.Join(tempDir, "rar_new")
|
||||
if err := os.MkdirAll(subDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for i := 1; i <= 2; i++ {
|
||||
path := filepath.Join(subDir, fmt.Sprintf("archive.part%02d.rar", i))
|
||||
if err := os.WriteFile(path, []byte("test"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
downloader := NewDownloader(nil)
|
||||
firstPart := filepath.Join(subDir, "archive.part01.rar")
|
||||
files := downloader.collectMultiPartFiles(firstPart)
|
||||
|
||||
if len(files) != 2 {
|
||||
t.Errorf("collectMultiPartFiles(RAR new) = %d files, want 2", len(files))
|
||||
}
|
||||
})
|
||||
|
||||
// Test with ZIP multi-part pattern
|
||||
t.Run("ZIP multi-part pattern", func(t *testing.T) {
|
||||
subDir := filepath.Join(tempDir, "zip")
|
||||
if err := os.MkdirAll(subDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for i := 1; i <= 3; i++ {
|
||||
path := filepath.Join(subDir, fmt.Sprintf("archive.zip.%03d", i))
|
||||
if err := os.WriteFile(path, []byte("test"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
downloader := NewDownloader(nil)
|
||||
firstPart := filepath.Join(subDir, "archive.zip.001")
|
||||
files := downloader.collectMultiPartFiles(firstPart)
|
||||
|
||||
if len(files) != 3 {
|
||||
t.Errorf("collectMultiPartFiles(ZIP) = %d files, want 3", len(files))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_CheckAllMultiPartTasksDone(t *testing.T) {
|
||||
downloader := NewDownloader(nil)
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer downloader.Clear()
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "check_multipart_done_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Create tasks with multi-part base name
|
||||
baseName := "archive.7z"
|
||||
|
||||
// Create task 1 - done
|
||||
task1 := &Task{
|
||||
ID: "task1",
|
||||
Status: base.DownloadStatusDone,
|
||||
Meta: &fetcher.FetcherMeta{
|
||||
Opts: &base.Options{Path: tempDir},
|
||||
Res: &base.Resource{
|
||||
Files: []*base.FileInfo{{Name: baseName + ".001"}},
|
||||
},
|
||||
},
|
||||
Progress: &Progress{},
|
||||
}
|
||||
initTask(task1)
|
||||
|
||||
// Create task 2 - done
|
||||
task2 := &Task{
|
||||
ID: "task2",
|
||||
Status: base.DownloadStatusDone,
|
||||
Meta: &fetcher.FetcherMeta{
|
||||
Opts: &base.Options{Path: tempDir},
|
||||
Res: &base.Resource{
|
||||
Files: []*base.FileInfo{{Name: baseName + ".002"}},
|
||||
},
|
||||
},
|
||||
Progress: &Progress{},
|
||||
}
|
||||
initTask(task2)
|
||||
|
||||
downloader.tasks = []*Task{task1, task2}
|
||||
|
||||
// All tasks are done
|
||||
basePath := filepath.Join(tempDir, baseName)
|
||||
allDone, missing := downloader.checkAllMultiPartTasksDone(basePath)
|
||||
if !allDone {
|
||||
t.Errorf("checkAllMultiPartTasksDone() = false, want true; missing: %v", missing)
|
||||
}
|
||||
|
||||
// Set task2 to running
|
||||
task2.Status = base.DownloadStatusRunning
|
||||
allDone, missing = downloader.checkAllMultiPartTasksDone(basePath)
|
||||
if allDone {
|
||||
t.Error("checkAllMultiPartTasksDone() = true, want false")
|
||||
}
|
||||
if len(missing) == 0 {
|
||||
t.Error("Expected missing parts to be reported")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_CheckAllMultiPartTasksDone_NoRelatedTasks(t *testing.T) {
|
||||
downloader := NewDownloader(nil)
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer downloader.Clear()
|
||||
|
||||
// No tasks exist
|
||||
allDone, missing := downloader.checkAllMultiPartTasksDone("/some/path/archive.7z")
|
||||
if allDone {
|
||||
t.Error("checkAllMultiPartTasksDone() = true, want false with no related tasks")
|
||||
}
|
||||
if len(missing) == 0 {
|
||||
t.Error("Expected missing message")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_IsMultiPartExtractionInProgress(t *testing.T) {
|
||||
downloader := NewDownloader(nil)
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer downloader.Clear()
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "extraction_progress_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
baseName := "archive.7z"
|
||||
basePath := filepath.Join(tempDir, baseName)
|
||||
|
||||
// Create task with no extraction status
|
||||
task1 := &Task{
|
||||
ID: "task1",
|
||||
Status: base.DownloadStatusDone,
|
||||
Meta: &fetcher.FetcherMeta{
|
||||
Opts: &base.Options{Path: tempDir},
|
||||
Res: &base.Resource{
|
||||
Files: []*base.FileInfo{{Name: baseName + ".001"}},
|
||||
},
|
||||
},
|
||||
Progress: &Progress{ExtractStatus: ExtractStatusNone},
|
||||
}
|
||||
initTask(task1)
|
||||
downloader.tasks = []*Task{task1}
|
||||
|
||||
// No extraction in progress
|
||||
if downloader.isMultiPartExtractionInProgress(basePath) {
|
||||
t.Error("isMultiPartExtractionInProgress() = true, want false")
|
||||
}
|
||||
|
||||
// Set to extracting
|
||||
task1.Progress.ExtractStatus = ExtractStatusExtracting
|
||||
if !downloader.isMultiPartExtractionInProgress(basePath) {
|
||||
t.Error("isMultiPartExtractionInProgress() = false, want true (extracting)")
|
||||
}
|
||||
|
||||
// Set to done
|
||||
task1.Progress.ExtractStatus = ExtractStatusDone
|
||||
if !downloader.isMultiPartExtractionInProgress(basePath) {
|
||||
t.Error("isMultiPartExtractionInProgress() = false, want true (done)")
|
||||
}
|
||||
|
||||
// Set to error
|
||||
task1.Progress.ExtractStatus = ExtractStatusError
|
||||
if !downloader.isMultiPartExtractionInProgress(basePath) {
|
||||
t.Error("isMultiPartExtractionInProgress() = false, want true (error)")
|
||||
}
|
||||
|
||||
// Set to waiting parts (not in progress)
|
||||
task1.Progress.ExtractStatus = ExtractStatusWaitingParts
|
||||
if downloader.isMultiPartExtractionInProgress(basePath) {
|
||||
t.Error("isMultiPartExtractionInProgress() = true, want false (waitingParts)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_HandleExtractionResult_Success(t *testing.T) {
|
||||
downloader := NewDownloader(nil)
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer downloader.Clear()
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "extraction_result_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Create a test archive file
|
||||
archivePath := filepath.Join(tempDir, "test.zip")
|
||||
if err := os.WriteFile(archivePath, []byte("test"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
task := &Task{
|
||||
ID: "test-task",
|
||||
Meta: &fetcher.FetcherMeta{
|
||||
Opts: &base.Options{Path: tempDir},
|
||||
Res: &base.Resource{
|
||||
Files: []*base.FileInfo{{Name: "test.zip"}},
|
||||
},
|
||||
},
|
||||
Progress: &Progress{},
|
||||
}
|
||||
initTask(task)
|
||||
|
||||
// Test successful extraction
|
||||
downloader.handleExtractionResult(task, nil, []string{archivePath}, false)
|
||||
|
||||
if task.Progress.ExtractStatus != ExtractStatusDone {
|
||||
t.Errorf("ExtractStatus = %v, want %v", task.Progress.ExtractStatus, ExtractStatusDone)
|
||||
}
|
||||
if task.Progress.ExtractProgress != 100 {
|
||||
t.Errorf("ExtractProgress = %d, want 100", task.Progress.ExtractProgress)
|
||||
}
|
||||
|
||||
// Archive should still exist (deleteAfterExtract=false)
|
||||
if _, err := os.Stat(archivePath); os.IsNotExist(err) {
|
||||
t.Error("Archive should still exist when deleteAfterExtract=false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_HandleExtractionResult_WithDelete(t *testing.T) {
|
||||
downloader := NewDownloader(nil)
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer downloader.Clear()
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "extraction_delete_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Create test archive files
|
||||
archivePath1 := filepath.Join(tempDir, "test.7z.001")
|
||||
archivePath2 := filepath.Join(tempDir, "test.7z.002")
|
||||
if err := os.WriteFile(archivePath1, []byte("test"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(archivePath2, []byte("test"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
task := &Task{
|
||||
ID: "test-task",
|
||||
Meta: &fetcher.FetcherMeta{
|
||||
Opts: &base.Options{Path: tempDir},
|
||||
Res: &base.Resource{
|
||||
Files: []*base.FileInfo{{Name: "test.7z.001"}},
|
||||
},
|
||||
},
|
||||
Progress: &Progress{},
|
||||
}
|
||||
initTask(task)
|
||||
|
||||
// Test successful extraction with delete
|
||||
downloader.handleExtractionResult(task, nil, []string{archivePath1, archivePath2}, true)
|
||||
|
||||
if task.Progress.ExtractStatus != ExtractStatusDone {
|
||||
t.Errorf("ExtractStatus = %v, want %v", task.Progress.ExtractStatus, ExtractStatusDone)
|
||||
}
|
||||
|
||||
// Archives should be deleted
|
||||
if _, err := os.Stat(archivePath1); !os.IsNotExist(err) {
|
||||
t.Error("Archive 1 should be deleted when deleteAfterExtract=true")
|
||||
}
|
||||
if _, err := os.Stat(archivePath2); !os.IsNotExist(err) {
|
||||
t.Error("Archive 2 should be deleted when deleteAfterExtract=true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_HandleExtractionResult_Error(t *testing.T) {
|
||||
downloader := NewDownloader(nil)
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer downloader.Clear()
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "extraction_error_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
task := &Task{
|
||||
ID: "test-task",
|
||||
Meta: &fetcher.FetcherMeta{
|
||||
Opts: &base.Options{Path: tempDir},
|
||||
Res: &base.Resource{
|
||||
Files: []*base.FileInfo{{Name: "test.zip"}},
|
||||
},
|
||||
},
|
||||
Progress: &Progress{},
|
||||
}
|
||||
initTask(task)
|
||||
|
||||
// Test failed extraction
|
||||
extractErr := fmt.Errorf("extraction failed")
|
||||
downloader.handleExtractionResult(task, extractErr, nil, false)
|
||||
|
||||
if task.Progress.ExtractStatus != ExtractStatusError {
|
||||
t.Errorf("ExtractStatus = %v, want %v", task.Progress.ExtractStatus, ExtractStatusError)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_UpdateMultiPartTasksStatus(t *testing.T) {
|
||||
downloader := NewDownloader(nil)
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer downloader.Clear()
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "update_status_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Create source task with multi-part base name
|
||||
sourceTask := &Task{
|
||||
ID: "source",
|
||||
Meta: &fetcher.FetcherMeta{
|
||||
Opts: &base.Options{Path: tempDir},
|
||||
Res: &base.Resource{
|
||||
Files: []*base.FileInfo{{Name: "archive.7z.001"}},
|
||||
},
|
||||
},
|
||||
Progress: &Progress{MultiPartBaseName: "archive.7z"},
|
||||
}
|
||||
initTask(sourceTask)
|
||||
|
||||
// Create related task
|
||||
relatedTask := &Task{
|
||||
ID: "related",
|
||||
Meta: &fetcher.FetcherMeta{
|
||||
Opts: &base.Options{Path: tempDir},
|
||||
Res: &base.Resource{
|
||||
Files: []*base.FileInfo{{Name: "archive.7z.002"}},
|
||||
},
|
||||
},
|
||||
Progress: &Progress{MultiPartBaseName: "archive.7z"},
|
||||
}
|
||||
initTask(relatedTask)
|
||||
|
||||
// Create unrelated task
|
||||
unrelatedTask := &Task{
|
||||
ID: "unrelated",
|
||||
Meta: &fetcher.FetcherMeta{
|
||||
Opts: &base.Options{Path: tempDir},
|
||||
Res: &base.Resource{
|
||||
Files: []*base.FileInfo{{Name: "other.7z.001"}},
|
||||
},
|
||||
},
|
||||
Progress: &Progress{MultiPartBaseName: "other.7z"},
|
||||
}
|
||||
initTask(unrelatedTask)
|
||||
|
||||
downloader.tasks = []*Task{sourceTask, relatedTask, unrelatedTask}
|
||||
|
||||
// Test successful extraction
|
||||
downloader.updateMultiPartTasksStatus(sourceTask, nil)
|
||||
|
||||
if relatedTask.Progress.ExtractStatus != ExtractStatusDone {
|
||||
t.Errorf("Related task ExtractStatus = %v, want %v", relatedTask.Progress.ExtractStatus, ExtractStatusDone)
|
||||
}
|
||||
if relatedTask.Progress.ExtractProgress != 100 {
|
||||
t.Errorf("Related task ExtractProgress = %d, want 100", relatedTask.Progress.ExtractProgress)
|
||||
}
|
||||
if unrelatedTask.Progress.ExtractStatus == ExtractStatusDone {
|
||||
t.Error("Unrelated task should not be updated")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_UpdateMultiPartTasksStatus_WithError(t *testing.T) {
|
||||
downloader := NewDownloader(nil)
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer downloader.Clear()
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "update_error_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Create source task with multi-part base name
|
||||
sourceTask := &Task{
|
||||
ID: "source",
|
||||
Meta: &fetcher.FetcherMeta{
|
||||
Opts: &base.Options{Path: tempDir},
|
||||
Res: &base.Resource{
|
||||
Files: []*base.FileInfo{{Name: "archive.7z.001"}},
|
||||
},
|
||||
},
|
||||
Progress: &Progress{MultiPartBaseName: "archive.7z"},
|
||||
}
|
||||
initTask(sourceTask)
|
||||
|
||||
// Create related task
|
||||
relatedTask := &Task{
|
||||
ID: "related",
|
||||
Meta: &fetcher.FetcherMeta{
|
||||
Opts: &base.Options{Path: tempDir},
|
||||
Res: &base.Resource{
|
||||
Files: []*base.FileInfo{{Name: "archive.7z.002"}},
|
||||
},
|
||||
},
|
||||
Progress: &Progress{MultiPartBaseName: "archive.7z"},
|
||||
}
|
||||
initTask(relatedTask)
|
||||
|
||||
downloader.tasks = []*Task{sourceTask, relatedTask}
|
||||
|
||||
// Test failed extraction
|
||||
downloader.updateMultiPartTasksStatus(sourceTask, fmt.Errorf("extraction failed"))
|
||||
|
||||
if relatedTask.Progress.ExtractStatus != ExtractStatusError {
|
||||
t.Errorf("Related task ExtractStatus = %v, want %v", relatedTask.Progress.ExtractStatus, ExtractStatusError)
|
||||
}
|
||||
if relatedTask.Progress.ExtractProgress != 0 {
|
||||
t.Errorf("Related task ExtractProgress = %d, want 0", relatedTask.Progress.ExtractProgress)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_UpdateMultiPartTasksStatus_NoBaseName(t *testing.T) {
|
||||
downloader := NewDownloader(nil)
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer downloader.Clear()
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "update_no_base_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Create task without multi-part base name
|
||||
task := &Task{
|
||||
ID: "single",
|
||||
Meta: &fetcher.FetcherMeta{
|
||||
Opts: &base.Options{Path: tempDir},
|
||||
Res: &base.Resource{
|
||||
Files: []*base.FileInfo{{Name: "single.zip"}},
|
||||
},
|
||||
},
|
||||
Progress: &Progress{MultiPartBaseName: ""},
|
||||
}
|
||||
initTask(task)
|
||||
|
||||
downloader.tasks = []*Task{task}
|
||||
|
||||
// Should not panic or error
|
||||
downloader.updateMultiPartTasksStatus(task, nil)
|
||||
}
|
||||
|
||||
func TestDownloader_CheckMultiPartArchiveReady(t *testing.T) {
|
||||
downloader := NewDownloader(nil)
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer downloader.Clear()
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "check_ready_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
baseName := "archive.7z"
|
||||
fileName := baseName + ".001"
|
||||
filePath := filepath.Join(tempDir, fileName)
|
||||
|
||||
// Create tasks
|
||||
task := &Task{
|
||||
ID: "task1",
|
||||
Status: base.DownloadStatusDone,
|
||||
Meta: &fetcher.FetcherMeta{
|
||||
Opts: &base.Options{Path: tempDir},
|
||||
Res: &base.Resource{
|
||||
Files: []*base.FileInfo{{Name: fileName}},
|
||||
},
|
||||
},
|
||||
Progress: &Progress{},
|
||||
}
|
||||
initTask(task)
|
||||
downloader.tasks = []*Task{task}
|
||||
|
||||
partInfo := getArchivePartInfo(filePath)
|
||||
ready, missing := downloader.checkMultiPartArchiveReady(filePath, tempDir, partInfo)
|
||||
|
||||
if !ready {
|
||||
t.Errorf("checkMultiPartArchiveReady() = false, want true; missing: %v", missing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_CheckMultiPartArchiveReady_EmptyBaseName(t *testing.T) {
|
||||
downloader := NewDownloader(nil)
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer downloader.Clear()
|
||||
|
||||
// Use a non-multi-part file path
|
||||
partInfo := getArchivePartInfo("/some/regular.zip")
|
||||
ready, _ := downloader.checkMultiPartArchiveReady("/some/regular.zip", "/dest", partInfo)
|
||||
|
||||
// Should return true for non-multi-part files
|
||||
if !ready {
|
||||
t.Error("checkMultiPartArchiveReady() should return true for non-multi-part files")
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
@@ -37,12 +38,45 @@ var supportedArchiveExtensions = []string{
|
||||
".lz",
|
||||
}
|
||||
|
||||
// multiPartArchivePatterns contains regex patterns for multi-part archive detection
|
||||
// Each pattern should have a capture group for the base name and part number
|
||||
var multiPartArchivePatterns = []*regexp.Regexp{
|
||||
// 7z multi-part: file.7z.001, file.7z.002, etc.
|
||||
regexp.MustCompile(`(?i)^(.+\.7z)\.(\d{3})$`),
|
||||
// RAR multi-part (new style): file.part01.rar, file.part02.rar, etc.
|
||||
regexp.MustCompile(`(?i)^(.+)\.part(\d+)\.rar$`),
|
||||
// RAR multi-part (old style): file.rar, file.r00, file.r01, etc. (first file is .rar)
|
||||
regexp.MustCompile(`(?i)^(.+)\.r(\d{2})$`),
|
||||
// ZIP multi-part: file.zip.001, file.zip.002, etc.
|
||||
regexp.MustCompile(`(?i)^(.+\.zip)\.(\d{3})$`),
|
||||
// ZIP split: file.z01, file.z02, ... file.zip (last file is .zip)
|
||||
regexp.MustCompile(`(?i)^(.+)\.z(\d{2})$`),
|
||||
}
|
||||
|
||||
// ArchivePartInfo contains information about a multi-part archive
|
||||
type ArchivePartInfo struct {
|
||||
// IsMultiPart indicates if this file is part of a multi-part archive
|
||||
IsMultiPart bool
|
||||
// BaseName is the common base name for all parts (without part number extension)
|
||||
BaseName string
|
||||
// PartNumber is the part number (1-indexed)
|
||||
PartNumber int
|
||||
// FirstPartPath is the path to the first part of the archive
|
||||
FirstPartPath string
|
||||
// Pattern indicates which pattern matched (for determining extraction method)
|
||||
Pattern string
|
||||
}
|
||||
|
||||
// ExtractProgressCallback is called to report extraction progress
|
||||
type ExtractProgressCallback func(extractedFiles int, totalFiles int, progress int)
|
||||
|
||||
// isArchiveFile checks if a file is a supported archive format
|
||||
func isArchiveFile(filename string) bool {
|
||||
lowerName := strings.ToLower(filename)
|
||||
// Check for multi-part archive first
|
||||
if isMultiPartArchive(filename) {
|
||||
return true
|
||||
}
|
||||
return slices.ContainsFunc(supportedArchiveExtensions, func(ext string) bool {
|
||||
return strings.HasSuffix(lowerName, ext)
|
||||
})
|
||||
@@ -273,3 +307,155 @@ func extractFile(ctx context.Context, fileInfo archives.FileInfo, destDir string
|
||||
// Set file permissions
|
||||
return os.Chmod(destPath, fileInfo.Mode())
|
||||
}
|
||||
|
||||
// isMultiPartArchive checks if a file is part of a multi-part archive
|
||||
func isMultiPartArchive(filename string) bool {
|
||||
info := getArchivePartInfo(filename)
|
||||
return info.IsMultiPart
|
||||
}
|
||||
|
||||
// getArchivePartInfo returns detailed information about a multi-part archive file
|
||||
func getArchivePartInfo(filename string) ArchivePartInfo {
|
||||
baseName := filepath.Base(filename)
|
||||
|
||||
for _, pattern := range multiPartArchivePatterns {
|
||||
matches := pattern.FindStringSubmatch(baseName)
|
||||
if len(matches) >= 3 {
|
||||
partNum := 0
|
||||
_, err := parsePartNumber(matches[2], &partNum)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
info := ArchivePartInfo{
|
||||
IsMultiPart: true,
|
||||
BaseName: matches[1],
|
||||
PartNumber: partNum,
|
||||
Pattern: pattern.String(),
|
||||
}
|
||||
|
||||
// Determine the first part path based on pattern
|
||||
dir := filepath.Dir(filename)
|
||||
info.FirstPartPath = determineFirstPartPath(dir, info.BaseName, pattern.String())
|
||||
|
||||
return info
|
||||
}
|
||||
}
|
||||
|
||||
// Check for .rar files that might be the first part of old-style multi-part RAR
|
||||
if strings.HasSuffix(strings.ToLower(baseName), ".rar") && !strings.Contains(strings.ToLower(baseName), ".part") {
|
||||
// Check if there are .r00, .r01 files in the same directory
|
||||
dir := filepath.Dir(filename)
|
||||
nameWithoutExt := strings.TrimSuffix(baseName, filepath.Ext(baseName))
|
||||
r00Path := filepath.Join(dir, nameWithoutExt+".r00")
|
||||
if _, err := os.Stat(r00Path); err == nil {
|
||||
return ArchivePartInfo{
|
||||
IsMultiPart: true,
|
||||
BaseName: nameWithoutExt,
|
||||
PartNumber: 1, // .rar is the first part in old-style
|
||||
FirstPartPath: filename,
|
||||
Pattern: "rar-old-style",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ArchivePartInfo{IsMultiPart: false}
|
||||
}
|
||||
|
||||
// parsePartNumber parses a part number string and stores it in the provided pointer
|
||||
// Returns the parsed number and nil error on success
|
||||
func parsePartNumber(s string, partNum *int) (int, error) {
|
||||
n := 0
|
||||
for _, c := range s {
|
||||
if c >= '0' && c <= '9' {
|
||||
n = n*10 + int(c-'0')
|
||||
}
|
||||
}
|
||||
// For .001, .002 style, part number is the value itself
|
||||
// For .part01 style, part number is the value itself
|
||||
// For .r00, .r01 style: r00=2 (since .rar is part 1), r01=3, etc.
|
||||
// However, for consistency, we return the raw number and handle the offset elsewhere
|
||||
*partNum = n
|
||||
// Treat 00 as part 0, let callers handle the semantics
|
||||
if n == 0 {
|
||||
*partNum = 1 // For .001 format, 001 should be 1
|
||||
}
|
||||
return *partNum, nil
|
||||
}
|
||||
|
||||
// determineFirstPartPath determines the path to the first part of a multi-part archive
|
||||
func determineFirstPartPath(dir, baseName, pattern string) string {
|
||||
switch {
|
||||
case strings.Contains(pattern, `.7z)`):
|
||||
// 7z multi-part: first part is .7z.001
|
||||
return filepath.Join(dir, baseName+".001")
|
||||
case strings.Contains(pattern, `.part`):
|
||||
// RAR new style: first part is .part01.rar or .part1.rar
|
||||
// Try both single and double digit formats
|
||||
if _, err := os.Stat(filepath.Join(dir, baseName+".part1.rar")); err == nil {
|
||||
return filepath.Join(dir, baseName+".part1.rar")
|
||||
}
|
||||
return filepath.Join(dir, baseName+".part01.rar")
|
||||
case strings.Contains(pattern, `.r(`):
|
||||
// RAR old style: first part is .rar (not .r00)
|
||||
return filepath.Join(dir, baseName+".rar")
|
||||
case strings.Contains(pattern, `.zip)`):
|
||||
// ZIP multi-part: first part is .zip.001
|
||||
return filepath.Join(dir, baseName+".001")
|
||||
case strings.Contains(pattern, `.z(`):
|
||||
// ZIP split: last part is .zip, but extraction should start from .z01
|
||||
return filepath.Join(dir, baseName+".z01")
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// isFirstPart checks if the given file is the first part of a multi-part archive
|
||||
func isFirstPart(filename string) bool {
|
||||
info := getArchivePartInfo(filename)
|
||||
if !info.IsMultiPart {
|
||||
return false
|
||||
}
|
||||
|
||||
// For most formats, part 1 is the first part
|
||||
// For old-style RAR, check if this is the .rar file
|
||||
if info.Pattern == "rar-old-style" {
|
||||
return strings.HasSuffix(strings.ToLower(filename), ".rar")
|
||||
}
|
||||
|
||||
return info.PartNumber == 1
|
||||
}
|
||||
|
||||
// GetMultiPartArchiveBaseName returns the base name for a multi-part archive
|
||||
// This is used to group related parts together
|
||||
func GetMultiPartArchiveBaseName(filename string) string {
|
||||
info := getArchivePartInfo(filename)
|
||||
if !info.IsMultiPart {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(filepath.Dir(filename), info.BaseName)
|
||||
}
|
||||
|
||||
// extractMultiPartArchive extracts a multi-part archive starting from the first part
|
||||
func extractMultiPartArchive(firstPartPath string, destDir string, password string, progressCallback ExtractProgressCallback) error {
|
||||
info := getArchivePartInfo(firstPartPath)
|
||||
|
||||
// For 7z multi-part archives, the bodgit/sevenzip library handles multi-volume automatically
|
||||
// when using OpenReader with a .001 file
|
||||
if strings.Contains(info.Pattern, `\.7z\)`) || strings.HasSuffix(strings.ToLower(firstPartPath), ".7z.001") {
|
||||
return extractSevenZipMultiPart(firstPartPath, destDir, password, progressCallback)
|
||||
}
|
||||
|
||||
// For RAR multi-part archives, use the archives library with Name and FS fields
|
||||
if strings.Contains(info.Pattern, "rar") || info.Pattern == "rar-old-style" {
|
||||
return extractRarMultiPart(firstPartPath, destDir, password, progressCallback)
|
||||
}
|
||||
|
||||
// For ZIP multi-part archives (.zip.001, .zip.002, etc.), concatenate parts and extract
|
||||
if strings.Contains(info.Pattern, `\.zip\)`) || strings.HasSuffix(strings.ToLower(firstPartPath), ".zip.001") {
|
||||
return extractZipMultiPart(firstPartPath, destDir, password, progressCallback)
|
||||
}
|
||||
|
||||
// For other formats, try standard extraction (may not work for all multi-part formats)
|
||||
return extractArchive(firstPartPath, destDir, password, progressCallback)
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/bodgit/sevenzip"
|
||||
)
|
||||
|
||||
// extractSevenZipMultiPart extracts a multi-part 7z archive using bodgit/sevenzip directly
|
||||
// The mholt/archives wrapper doesn't properly handle multi-part 7z files because it uses
|
||||
// io.SectionReader which can only see the first part. The bodgit/sevenzip library's
|
||||
// OpenReaderWithPassword function handles multi-part files automatically when given the .001 file path.
|
||||
func extractSevenZipMultiPart(firstPartPath string, destDir string, password string, progressCallback ExtractProgressCallback) error {
|
||||
// Use bodgit/sevenzip directly - it automatically handles .001, .002, etc. files
|
||||
var reader *sevenzip.ReadCloser
|
||||
var err error
|
||||
|
||||
if password != "" {
|
||||
reader, err = sevenzip.OpenReaderWithPassword(firstPartPath, password)
|
||||
} else {
|
||||
reader, err = sevenzip.OpenReader(firstPartPath)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
// Create destination directory
|
||||
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Count total files for progress
|
||||
totalFiles := 0
|
||||
for _, f := range reader.File {
|
||||
if !f.FileInfo().IsDir() {
|
||||
totalFiles++
|
||||
}
|
||||
}
|
||||
|
||||
// Extract files with progress tracking
|
||||
extractedFiles := 0
|
||||
for _, f := range reader.File {
|
||||
destPath := filepath.Join(destDir, f.Name)
|
||||
|
||||
if f.FileInfo().IsDir() {
|
||||
if err := os.MkdirAll(destPath, f.Mode()); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Ensure parent directory exists
|
||||
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Extract file - open, copy, close immediately (as recommended by bodgit/sevenzip)
|
||||
if err := extractSevenZipFile(f, destPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
extractedFiles++
|
||||
if progressCallback != nil && totalFiles > 0 {
|
||||
progress := int(float64(extractedFiles) / float64(totalFiles) * 100)
|
||||
progressCallback(extractedFiles, totalFiles, progress)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// extractSevenZipFile extracts a single file from a 7z archive
|
||||
// This follows the bodgit/sevenzip recommended pattern of closing rc before processing the next file
|
||||
func extractSevenZipFile(f *sevenzip.File, destPath string) error {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
outFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer outFile.Close()
|
||||
|
||||
_, err = io.Copy(outFile, rc)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
)
|
||||
|
||||
// extractRarMultiPart extracts a multi-part RAR archive
|
||||
func extractRarMultiPart(firstPartPath string, destDir string, password string, progressCallback ExtractProgressCallback) error {
|
||||
// For RAR archives, we need to use the Name field in archives.Rar
|
||||
// to let it automatically find subsequent volumes
|
||||
|
||||
dir := filepath.Dir(firstPartPath)
|
||||
fileName := filepath.Base(firstPartPath)
|
||||
|
||||
rar := archives.Rar{
|
||||
Password: password,
|
||||
Name: fileName,
|
||||
FS: os.DirFS(dir),
|
||||
}
|
||||
|
||||
// Create destination directory
|
||||
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Count files first for progress
|
||||
totalFiles := 0
|
||||
err := rar.Extract(context.Background(), nil, func(ctx context.Context, fileInfo archives.FileInfo) error {
|
||||
if !fileInfo.IsDir() {
|
||||
totalFiles++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
// If counting fails, proceed without progress
|
||||
totalFiles = 0
|
||||
}
|
||||
|
||||
// Reset and extract with progress tracking
|
||||
return rar.Extract(context.Background(), nil, createExtractionHandler(destDir, totalFiles, progressCallback))
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,191 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
)
|
||||
|
||||
// extractZipMultiPart extracts a multi-part ZIP archive (.zip.001, .zip.002, etc.)
|
||||
// These are created by simply splitting a ZIP file into chunks, so we concatenate them
|
||||
func extractZipMultiPart(firstPartPath string, destDir string, password string, progressCallback ExtractProgressCallback) error {
|
||||
// Find all parts
|
||||
parts, err := findZipMultiParts(firstPartPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create a multi-part reader that reads across all parts
|
||||
multiReader := newMultiPartFileReader(parts)
|
||||
defer multiReader.Close()
|
||||
|
||||
// Get total size
|
||||
totalSize := multiReader.Size()
|
||||
|
||||
// Create destination directory
|
||||
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// First pass: count files for progress
|
||||
totalFiles := 0
|
||||
zip := archives.Zip{}
|
||||
err = zip.Extract(context.Background(), io.NewSectionReader(multiReader, 0, totalSize), func(ctx context.Context, fileInfo archives.FileInfo) error {
|
||||
if !fileInfo.IsDir() {
|
||||
totalFiles++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
// If counting fails, proceed without progress
|
||||
totalFiles = 0
|
||||
}
|
||||
|
||||
// Reset reader for actual extraction
|
||||
multiReader.Close()
|
||||
multiReader = newMultiPartFileReader(parts)
|
||||
defer multiReader.Close()
|
||||
|
||||
// Second pass: extract with progress tracking
|
||||
return zip.Extract(context.Background(), io.NewSectionReader(multiReader, 0, totalSize), createExtractionHandler(destDir, totalFiles, progressCallback))
|
||||
}
|
||||
|
||||
// findZipMultiParts finds all parts of a multi-part ZIP archive in order
|
||||
func findZipMultiParts(firstPartPath string) ([]string, error) {
|
||||
// Extract base name (e.g., "Archive.zip" from "Archive.zip.001")
|
||||
dir := filepath.Dir(firstPartPath)
|
||||
baseName := filepath.Base(firstPartPath)
|
||||
|
||||
// Remove the .001 suffix to get the base
|
||||
if idx := strings.LastIndex(baseName, "."); idx > 0 {
|
||||
baseName = baseName[:idx] // "Archive.zip"
|
||||
}
|
||||
|
||||
var parts []string
|
||||
partNum := 1
|
||||
|
||||
for {
|
||||
partPath := filepath.Join(dir, baseName+fmt.Sprintf(".%03d", partNum))
|
||||
if _, err := os.Stat(partPath); os.IsNotExist(err) {
|
||||
break
|
||||
}
|
||||
parts = append(parts, partPath)
|
||||
partNum++
|
||||
}
|
||||
|
||||
if len(parts) == 0 {
|
||||
return nil, fmt.Errorf("no parts found for %s", firstPartPath)
|
||||
}
|
||||
|
||||
return parts, nil
|
||||
}
|
||||
|
||||
// multiPartFileReader provides io.ReaderAt over multiple files concatenated
|
||||
type multiPartFileReader struct {
|
||||
parts []string
|
||||
files []*os.File
|
||||
sizes []int64
|
||||
offsets []int64 // cumulative offsets for each file
|
||||
totalSize int64
|
||||
}
|
||||
|
||||
func newMultiPartFileReader(parts []string) *multiPartFileReader {
|
||||
return &multiPartFileReader{parts: parts}
|
||||
}
|
||||
|
||||
func (m *multiPartFileReader) init() error {
|
||||
if m.files != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
m.files = make([]*os.File, len(m.parts))
|
||||
m.sizes = make([]int64, len(m.parts))
|
||||
m.offsets = make([]int64, len(m.parts))
|
||||
|
||||
var offset int64
|
||||
for i, part := range m.parts {
|
||||
f, err := os.Open(part)
|
||||
if err != nil {
|
||||
m.Close()
|
||||
return err
|
||||
}
|
||||
stat, err := f.Stat()
|
||||
if err != nil {
|
||||
f.Close()
|
||||
m.Close()
|
||||
return err
|
||||
}
|
||||
m.files[i] = f
|
||||
m.sizes[i] = stat.Size()
|
||||
m.offsets[i] = offset
|
||||
offset += stat.Size()
|
||||
}
|
||||
m.totalSize = offset
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *multiPartFileReader) Size() int64 {
|
||||
if err := m.init(); err != nil {
|
||||
return 0
|
||||
}
|
||||
return m.totalSize
|
||||
}
|
||||
|
||||
func (m *multiPartFileReader) ReadAt(p []byte, off int64) (n int, err error) {
|
||||
if err := m.init(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if off >= m.totalSize {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
// Find which file(s) to read from
|
||||
for i, fileOffset := range m.offsets {
|
||||
fileEnd := fileOffset + m.sizes[i]
|
||||
|
||||
if off >= fileEnd {
|
||||
continue
|
||||
}
|
||||
|
||||
// Read from this file
|
||||
localOffset := off - fileOffset
|
||||
toRead := len(p) - n
|
||||
if int64(toRead) > fileEnd-off {
|
||||
toRead = int(fileEnd - off)
|
||||
}
|
||||
|
||||
read, err := m.files[i].ReadAt(p[n:n+toRead], localOffset)
|
||||
n += read
|
||||
off += int64(read)
|
||||
|
||||
if err != nil && err != io.EOF {
|
||||
return n, err
|
||||
}
|
||||
|
||||
if n >= len(p) {
|
||||
return n, nil
|
||||
}
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (m *multiPartFileReader) Close() error {
|
||||
for _, f := range m.files {
|
||||
if f != nil {
|
||||
f.Close()
|
||||
}
|
||||
}
|
||||
m.files = nil
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user