mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-15 04:43:41 +00:00
feat: multipart-archives (#1222)
This commit is contained in:
1 parent
cb6331c20d
commit
de0afb1481
7 files changed
+2689
-46
No files matched your search
+347
-35
@@ -2,6 +2,7 @@ package download
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
gohttp "net/http"
|
||||
"net/url"
|
||||
@@ -51,6 +52,8 @@ type ExtractStatus string
|
||||
const (
|
||||
// ExtractStatusNone indicates extraction has not started
|
||||
ExtractStatusNone ExtractStatus = ""
|
||||
// ExtractStatusWaitingParts indicates waiting for other multi-part archive parts to complete
|
||||
ExtractStatusWaitingParts ExtractStatus = "waitingParts"
|
||||
// ExtractStatusExtracting indicates extraction is in progress
|
||||
ExtractStatusExtracting ExtractStatus = "extracting"
|
||||
// ExtractStatusDone indicates extraction completed successfully
|
||||
@@ -74,6 +77,12 @@ type Progress struct {
|
||||
ExtractStatus ExtractStatus `json:"extractStatus"`
|
||||
// ExtractProgress is the percentage of extraction completed (0-100)
|
||||
ExtractProgress int `json:"extractProgress"`
|
||||
// MultiPartBaseName is set for multi-part archives to group related parts
|
||||
MultiPartBaseName string `json:"multiPartBaseName,omitempty"`
|
||||
// MultiPartNumber is the part number for multi-part archives (1-indexed)
|
||||
MultiPartNumber int `json:"multiPartNumber,omitempty"`
|
||||
// MultiPartIsFirst indicates if this is the first part of a multi-part archive
|
||||
MultiPartIsFirst bool `json:"multiPartIsFirst,omitempty"`
|
||||
}
|
||||
|
||||
type Downloader struct {
|
||||
@@ -898,41 +907,7 @@ func (d *Downloader) watch(task *Task) {
|
||||
|
||||
// Auto-extract archive files
|
||||
if e.AutoExtract && isArchiveFile(downloadFilePath) {
|
||||
go func() {
|
||||
// Set extraction status to extracting
|
||||
task.Progress.ExtractStatus = ExtractStatusExtracting
|
||||
task.Progress.ExtractProgress = 0
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
|
||||
// Extract to the same directory as the downloaded file
|
||||
destDir := task.Meta.Opts.Path
|
||||
extractErr := extractArchive(downloadFilePath, destDir, e.ArchivePassword, func(extractedFiles int, totalFiles int, progress int) {
|
||||
task.Progress.ExtractProgress = progress
|
||||
d.emit(EventKeyProgress, task)
|
||||
})
|
||||
if extractErr != nil {
|
||||
d.Logger.Error().Err(extractErr).Msgf("auto extract archive failed, task id: %s", task.ID)
|
||||
task.Progress.ExtractStatus = ExtractStatusError
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
} else {
|
||||
d.Logger.Info().Msgf("auto extract archive completed, task id: %s", task.ID)
|
||||
task.Progress.ExtractStatus = ExtractStatusDone
|
||||
task.Progress.ExtractProgress = 100
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
// Delete archive after successful extraction if enabled
|
||||
if e.DeleteAfterExtract {
|
||||
deleteErr := os.Remove(downloadFilePath)
|
||||
if deleteErr != nil {
|
||||
d.Logger.Error().Err(deleteErr).Msgf("delete archive after extraction failed, task id: %s", task.ID)
|
||||
} else {
|
||||
d.Logger.Info().Msgf("archive deleted after extraction, task id: %s", task.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
go d.handleAutoExtract(task, downloadFilePath, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1225,6 +1200,343 @@ func (d *Downloader) buildFetcher(url string) (fetcher.Fetcher, error) {
|
||||
return fetcher, nil
|
||||
}
|
||||
|
||||
// handleAutoExtract handles automatic extraction of archive files, including multi-part archives
|
||||
func (d *Downloader) handleAutoExtract(task *Task, downloadFilePath string, opts *http.OptsExtra) {
|
||||
// Check if this is a multi-part archive
|
||||
partInfo := getArchivePartInfo(downloadFilePath)
|
||||
|
||||
if partInfo.IsMultiPart {
|
||||
// Set multi-part info on the task
|
||||
task.Progress.MultiPartBaseName = partInfo.BaseName
|
||||
task.Progress.MultiPartNumber = partInfo.PartNumber
|
||||
task.Progress.MultiPartIsFirst = isFirstPart(downloadFilePath)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
|
||||
// Check if all parts are downloaded
|
||||
destDir := task.Meta.Opts.Path
|
||||
allPartsReady, missingParts := d.checkMultiPartArchiveReady(downloadFilePath, destDir, partInfo)
|
||||
|
||||
if !allPartsReady {
|
||||
// Not all parts are ready yet, set status to waiting
|
||||
task.Progress.ExtractStatus = ExtractStatusWaitingParts
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.Logger.Info().Msgf("multi-part archive waiting for other parts, task id: %s, missing: %v", task.ID, missingParts)
|
||||
return
|
||||
}
|
||||
|
||||
// All parts are ready!
|
||||
// Only one task should perform extraction - we'll use the one that completes last
|
||||
// Check if extraction has already been started by another part
|
||||
if d.isMultiPartExtractionInProgress(partInfo.BaseName) {
|
||||
// Another part already started extraction, mark this as done
|
||||
task.Progress.ExtractStatus = ExtractStatusDone
|
||||
task.Progress.ExtractProgress = 100
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.Logger.Info().Msgf("multi-part archive part completed, extraction handled by another part, task id: %s", task.ID)
|
||||
return
|
||||
}
|
||||
|
||||
// This task will handle extraction - mark as extracting first to prevent others
|
||||
task.Progress.ExtractStatus = ExtractStatusExtracting
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
|
||||
// Extract the multi-part archive
|
||||
d.performMultiPartExtraction(task, partInfo.FirstPartPath, destDir, opts)
|
||||
} else {
|
||||
// Regular (non-multi-part) archive
|
||||
d.performExtraction(task, downloadFilePath, task.Meta.Opts.Path, opts)
|
||||
}
|
||||
}
|
||||
|
||||
// checkMultiPartArchiveReady checks if all parts of a multi-part archive are downloaded
|
||||
// by examining task status rather than file existence
|
||||
func (d *Downloader) checkMultiPartArchiveReady(filePath string, destDir string, partInfo ArchivePartInfo) (bool, []string) {
|
||||
// Use task-based checking - find all tasks with the same MultiPartBaseName
|
||||
// and verify they are all in Done status
|
||||
baseName := GetMultiPartArchiveBaseName(filePath)
|
||||
if baseName == "" {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return d.checkAllMultiPartTasksDone(baseName)
|
||||
}
|
||||
|
||||
// checkAllMultiPartTasksDone checks if all tasks belonging to a multi-part archive are done
|
||||
func (d *Downloader) checkAllMultiPartTasksDone(baseName string) (bool, []string) {
|
||||
var notDoneParts []string
|
||||
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
|
||||
// Find all tasks that belong to this multi-part archive
|
||||
var relatedTasks []*Task
|
||||
for _, task := range d.tasks {
|
||||
taskBaseName := ""
|
||||
if task.Meta != nil && task.Meta.Res != nil && len(task.Meta.Res.Files) > 0 {
|
||||
taskBaseName = GetMultiPartArchiveBaseName(task.Meta.SingleFilepath())
|
||||
}
|
||||
if taskBaseName == baseName {
|
||||
relatedTasks = append(relatedTasks, task)
|
||||
}
|
||||
}
|
||||
|
||||
// If we found no related tasks, we can't determine readiness
|
||||
if len(relatedTasks) == 0 {
|
||||
return false, []string{"no related tasks found for " + baseName}
|
||||
}
|
||||
|
||||
// Check if all related tasks are done
|
||||
for _, task := range relatedTasks {
|
||||
if task.Status != base.DownloadStatusDone {
|
||||
notDoneParts = append(notDoneParts, task.Meta.SingleFilepath())
|
||||
}
|
||||
}
|
||||
|
||||
return len(notDoneParts) == 0, notDoneParts
|
||||
}
|
||||
|
||||
// isMultiPartExtractionInProgress checks if any task for this multi-part archive
|
||||
// has already started extraction (to prevent duplicate extractions)
|
||||
func (d *Downloader) isMultiPartExtractionInProgress(baseName string) bool {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
|
||||
for _, task := range d.tasks {
|
||||
taskBaseName := ""
|
||||
if task.Meta != nil && task.Meta.Res != nil && len(task.Meta.Res.Files) > 0 {
|
||||
taskBaseName = GetMultiPartArchiveBaseName(task.Meta.SingleFilepath())
|
||||
}
|
||||
if taskBaseName == baseName {
|
||||
// Check if this task is already extracting or has completed extraction
|
||||
if task.Progress.ExtractStatus == ExtractStatusExtracting ||
|
||||
task.Progress.ExtractStatus == ExtractStatusDone ||
|
||||
task.Progress.ExtractStatus == ExtractStatusError {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// performExtraction performs extraction for a regular (non-multi-part) archive
|
||||
func (d *Downloader) performExtraction(task *Task, archivePath string, destDir string, opts *http.OptsExtra) {
|
||||
// Set extraction status to extracting
|
||||
task.Progress.ExtractStatus = ExtractStatusExtracting
|
||||
task.Progress.ExtractProgress = 0
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
|
||||
// Extract the archive
|
||||
extractErr := extractArchive(archivePath, destDir, opts.ArchivePassword, func(extractedFiles int, totalFiles int, progress int) {
|
||||
task.Progress.ExtractProgress = progress
|
||||
d.emit(EventKeyProgress, task)
|
||||
})
|
||||
|
||||
d.handleExtractionResult(task, extractErr, []string{archivePath}, opts.DeleteAfterExtract)
|
||||
}
|
||||
|
||||
// performMultiPartExtraction performs extraction for a multi-part archive
|
||||
func (d *Downloader) performMultiPartExtraction(task *Task, firstPartPath string, destDir string, opts *http.OptsExtra) {
|
||||
// Set extraction status to extracting
|
||||
task.Progress.ExtractStatus = ExtractStatusExtracting
|
||||
task.Progress.ExtractProgress = 0
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
|
||||
d.Logger.Info().Msgf("starting multi-part archive extraction, first part: %s, task id: %s", firstPartPath, task.ID)
|
||||
|
||||
// Extract the multi-part archive
|
||||
extractErr := extractMultiPartArchive(firstPartPath, destDir, opts.ArchivePassword, func(extractedFiles int, totalFiles int, progress int) {
|
||||
task.Progress.ExtractProgress = progress
|
||||
d.emit(EventKeyProgress, task)
|
||||
})
|
||||
|
||||
// Collect all part files for potential deletion
|
||||
partFiles := d.collectMultiPartFiles(firstPartPath)
|
||||
|
||||
d.handleExtractionResult(task, extractErr, partFiles, opts.DeleteAfterExtract)
|
||||
|
||||
// Update status for all related multi-part tasks
|
||||
d.updateMultiPartTasksStatus(task, extractErr)
|
||||
}
|
||||
|
||||
// collectMultiPartFiles collects all files belonging to a multi-part archive
|
||||
func (d *Downloader) collectMultiPartFiles(firstPartPath string) []string {
|
||||
var files []string
|
||||
partInfo := getArchivePartInfo(firstPartPath)
|
||||
dir := filepath.Dir(firstPartPath)
|
||||
|
||||
switch {
|
||||
case strings.Contains(partInfo.Pattern, ".7z)"):
|
||||
// 7z: .7z.001, .7z.002, etc.
|
||||
files = d.collectSequentialFiles(dir, partInfo.BaseName, ".%03d")
|
||||
case strings.Contains(partInfo.Pattern, ".part"):
|
||||
// RAR new style
|
||||
files = d.collectRarNewStyleFiles(dir, partInfo.BaseName)
|
||||
case partInfo.Pattern == "rar-old-style" || strings.Contains(partInfo.Pattern, ".r("):
|
||||
// RAR old style
|
||||
files = d.collectRarOldStyleFiles(dir, partInfo.BaseName)
|
||||
case strings.Contains(partInfo.Pattern, ".zip)"):
|
||||
// ZIP multi-part
|
||||
files = d.collectSequentialFiles(dir, partInfo.BaseName, ".%03d")
|
||||
case strings.Contains(partInfo.Pattern, ".z("):
|
||||
// ZIP split
|
||||
files = d.collectZipSplitFiles(dir, partInfo.BaseName)
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// collectSequentialFiles collects sequential numbered files
|
||||
func (d *Downloader) collectSequentialFiles(dir, baseName, format string) []string {
|
||||
var files []string
|
||||
suffix := filepath.Ext(baseName)
|
||||
nameWithoutExt := strings.TrimSuffix(baseName, suffix)
|
||||
partNum := 1
|
||||
|
||||
for {
|
||||
partPath := filepath.Join(dir, nameWithoutExt+suffix+fmt.Sprintf(format, partNum))
|
||||
if _, err := os.Stat(partPath); os.IsNotExist(err) {
|
||||
break
|
||||
}
|
||||
files = append(files, partPath)
|
||||
partNum++
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// collectRarNewStyleFiles collects RAR new style part files
|
||||
func (d *Downloader) collectRarNewStyleFiles(dir, baseName string) []string {
|
||||
var files []string
|
||||
partNum := 1
|
||||
|
||||
for {
|
||||
singleDigitPath := filepath.Join(dir, baseName+fmt.Sprintf(".part%d.rar", partNum))
|
||||
doubleDigitPath := filepath.Join(dir, baseName+fmt.Sprintf(".part%02d.rar", partNum))
|
||||
|
||||
if _, err := os.Stat(singleDigitPath); err == nil {
|
||||
files = append(files, singleDigitPath)
|
||||
} else if _, err := os.Stat(doubleDigitPath); err == nil {
|
||||
files = append(files, doubleDigitPath)
|
||||
} else {
|
||||
break
|
||||
}
|
||||
partNum++
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// collectRarOldStyleFiles collects RAR old style part files
|
||||
func (d *Downloader) collectRarOldStyleFiles(dir, baseName string) []string {
|
||||
var files []string
|
||||
|
||||
// .rar file
|
||||
rarPath := filepath.Join(dir, baseName+".rar")
|
||||
if _, err := os.Stat(rarPath); err == nil {
|
||||
files = append(files, rarPath)
|
||||
}
|
||||
|
||||
// .r00, .r01, etc.
|
||||
partNum := 0
|
||||
for {
|
||||
partPath := filepath.Join(dir, baseName+fmt.Sprintf(".r%02d", partNum))
|
||||
if _, err := os.Stat(partPath); os.IsNotExist(err) {
|
||||
break
|
||||
}
|
||||
files = append(files, partPath)
|
||||
partNum++
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// collectZipSplitFiles collects ZIP split files
|
||||
func (d *Downloader) collectZipSplitFiles(dir, baseName string) []string {
|
||||
var files []string
|
||||
|
||||
// .z01, .z02, etc.
|
||||
partNum := 1
|
||||
for {
|
||||
partPath := filepath.Join(dir, baseName+fmt.Sprintf(".z%02d", partNum))
|
||||
if _, err := os.Stat(partPath); os.IsNotExist(err) {
|
||||
break
|
||||
}
|
||||
files = append(files, partPath)
|
||||
partNum++
|
||||
}
|
||||
|
||||
// .zip file
|
||||
zipPath := filepath.Join(dir, baseName+".zip")
|
||||
if _, err := os.Stat(zipPath); err == nil {
|
||||
files = append(files, zipPath)
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// handleExtractionResult handles the result of an extraction operation
|
||||
func (d *Downloader) handleExtractionResult(task *Task, extractErr error, archiveFiles []string, deleteAfterExtract bool) {
|
||||
if extractErr != nil {
|
||||
d.Logger.Error().Err(extractErr).Msgf("auto extract archive failed, task id: %s", task.ID)
|
||||
task.Progress.ExtractStatus = ExtractStatusError
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
} else {
|
||||
d.Logger.Info().Msgf("auto extract archive completed, task id: %s", task.ID)
|
||||
task.Progress.ExtractStatus = ExtractStatusDone
|
||||
task.Progress.ExtractProgress = 100
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
|
||||
// Delete archive files after successful extraction if enabled
|
||||
if deleteAfterExtract {
|
||||
for _, archiveFile := range archiveFiles {
|
||||
deleteErr := os.Remove(archiveFile)
|
||||
if deleteErr != nil {
|
||||
d.Logger.Error().Err(deleteErr).Msgf("delete archive after extraction failed: %s", archiveFile)
|
||||
} else {
|
||||
d.Logger.Info().Msgf("archive deleted after extraction: %s", archiveFile)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// updateMultiPartTasksStatus updates the extraction status for all tasks that belong to the same multi-part archive
|
||||
func (d *Downloader) updateMultiPartTasksStatus(sourceTask *Task, extractErr error) {
|
||||
if sourceTask.Progress.MultiPartBaseName == "" {
|
||||
return
|
||||
}
|
||||
|
||||
status := ExtractStatusDone
|
||||
progress := 100
|
||||
if extractErr != nil {
|
||||
status = ExtractStatusError
|
||||
progress = 0
|
||||
}
|
||||
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
|
||||
for _, task := range d.tasks {
|
||||
if task.ID == sourceTask.ID {
|
||||
continue
|
||||
}
|
||||
if task.Progress.MultiPartBaseName == sourceTask.Progress.MultiPartBaseName {
|
||||
task.Progress.ExtractStatus = status
|
||||
task.Progress.ExtractProgress = progress
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func initTask(task *Task) {
|
||||
task.timer = util.NewTimer(task.Progress.Used)
|
||||
|
||||
|
||||
+785
-11
File diff suppressed because it is too large.
Load diff
@@ -6,6 +6,7 @@ import (
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
@@ -37,12 +38,45 @@ var supportedArchiveExtensions = []string{
|
||||
".lz",
|
||||
}
|
||||
|
||||
// multiPartArchivePatterns contains regex patterns for multi-part archive detection
|
||||
// Each pattern should have a capture group for the base name and part number
|
||||
var multiPartArchivePatterns = []*regexp.Regexp{
|
||||
// 7z multi-part: file.7z.001, file.7z.002, etc.
|
||||
regexp.MustCompile(`(?i)^(.+\.7z)\.(\d{3})$`),
|
||||
// RAR multi-part (new style): file.part01.rar, file.part02.rar, etc.
|
||||
regexp.MustCompile(`(?i)^(.+)\.part(\d+)\.rar$`),
|
||||
// RAR multi-part (old style): file.rar, file.r00, file.r01, etc. (first file is .rar)
|
||||
regexp.MustCompile(`(?i)^(.+)\.r(\d{2})$`),
|
||||
// ZIP multi-part: file.zip.001, file.zip.002, etc.
|
||||
regexp.MustCompile(`(?i)^(.+\.zip)\.(\d{3})$`),
|
||||
// ZIP split: file.z01, file.z02, ... file.zip (last file is .zip)
|
||||
regexp.MustCompile(`(?i)^(.+)\.z(\d{2})$`),
|
||||
}
|
||||
|
||||
// ArchivePartInfo contains information about a multi-part archive
|
||||
type ArchivePartInfo struct {
|
||||
// IsMultiPart indicates if this file is part of a multi-part archive
|
||||
IsMultiPart bool
|
||||
// BaseName is the common base name for all parts (without part number extension)
|
||||
BaseName string
|
||||
// PartNumber is the part number (1-indexed)
|
||||
PartNumber int
|
||||
// FirstPartPath is the path to the first part of the archive
|
||||
FirstPartPath string
|
||||
// Pattern indicates which pattern matched (for determining extraction method)
|
||||
Pattern string
|
||||
}
|
||||
|
||||
// ExtractProgressCallback is called to report extraction progress
|
||||
type ExtractProgressCallback func(extractedFiles int, totalFiles int, progress int)
|
||||
|
||||
// isArchiveFile checks if a file is a supported archive format
|
||||
func isArchiveFile(filename string) bool {
|
||||
lowerName := strings.ToLower(filename)
|
||||
// Check for multi-part archive first
|
||||
if isMultiPartArchive(filename) {
|
||||
return true
|
||||
}
|
||||
return slices.ContainsFunc(supportedArchiveExtensions, func(ext string) bool {
|
||||
return strings.HasSuffix(lowerName, ext)
|
||||
})
|
||||
@@ -273,3 +307,155 @@ func extractFile(ctx context.Context, fileInfo archives.FileInfo, destDir string
|
||||
// Set file permissions
|
||||
return os.Chmod(destPath, fileInfo.Mode())
|
||||
}
|
||||
|
||||
// isMultiPartArchive checks if a file is part of a multi-part archive
|
||||
func isMultiPartArchive(filename string) bool {
|
||||
info := getArchivePartInfo(filename)
|
||||
return info.IsMultiPart
|
||||
}
|
||||
|
||||
// getArchivePartInfo returns detailed information about a multi-part archive file
|
||||
func getArchivePartInfo(filename string) ArchivePartInfo {
|
||||
baseName := filepath.Base(filename)
|
||||
|
||||
for _, pattern := range multiPartArchivePatterns {
|
||||
matches := pattern.FindStringSubmatch(baseName)
|
||||
if len(matches) >= 3 {
|
||||
partNum := 0
|
||||
_, err := parsePartNumber(matches[2], &partNum)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
info := ArchivePartInfo{
|
||||
IsMultiPart: true,
|
||||
BaseName: matches[1],
|
||||
PartNumber: partNum,
|
||||
Pattern: pattern.String(),
|
||||
}
|
||||
|
||||
// Determine the first part path based on pattern
|
||||
dir := filepath.Dir(filename)
|
||||
info.FirstPartPath = determineFirstPartPath(dir, info.BaseName, pattern.String())
|
||||
|
||||
return info
|
||||
}
|
||||
}
|
||||
|
||||
// Check for .rar files that might be the first part of old-style multi-part RAR
|
||||
if strings.HasSuffix(strings.ToLower(baseName), ".rar") && !strings.Contains(strings.ToLower(baseName), ".part") {
|
||||
// Check if there are .r00, .r01 files in the same directory
|
||||
dir := filepath.Dir(filename)
|
||||
nameWithoutExt := strings.TrimSuffix(baseName, filepath.Ext(baseName))
|
||||
r00Path := filepath.Join(dir, nameWithoutExt+".r00")
|
||||
if _, err := os.Stat(r00Path); err == nil {
|
||||
return ArchivePartInfo{
|
||||
IsMultiPart: true,
|
||||
BaseName: nameWithoutExt,
|
||||
PartNumber: 1, // .rar is the first part in old-style
|
||||
FirstPartPath: filename,
|
||||
Pattern: "rar-old-style",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ArchivePartInfo{IsMultiPart: false}
|
||||
}
|
||||
|
||||
// parsePartNumber parses a part number string and stores it in the provided pointer
|
||||
// Returns the parsed number and nil error on success
|
||||
func parsePartNumber(s string, partNum *int) (int, error) {
|
||||
n := 0
|
||||
for _, c := range s {
|
||||
if c >= '0' && c <= '9' {
|
||||
n = n*10 + int(c-'0')
|
||||
}
|
||||
}
|
||||
// For .001, .002 style, part number is the value itself
|
||||
// For .part01 style, part number is the value itself
|
||||
// For .r00, .r01 style: r00=2 (since .rar is part 1), r01=3, etc.
|
||||
// However, for consistency, we return the raw number and handle the offset elsewhere
|
||||
*partNum = n
|
||||
// Treat 00 as part 0, let callers handle the semantics
|
||||
if n == 0 {
|
||||
*partNum = 1 // For .001 format, 001 should be 1
|
||||
}
|
||||
return *partNum, nil
|
||||
}
|
||||
|
||||
// determineFirstPartPath determines the path to the first part of a multi-part archive
|
||||
func determineFirstPartPath(dir, baseName, pattern string) string {
|
||||
switch {
|
||||
case strings.Contains(pattern, `.7z)`):
|
||||
// 7z multi-part: first part is .7z.001
|
||||
return filepath.Join(dir, baseName+".001")
|
||||
case strings.Contains(pattern, `.part`):
|
||||
// RAR new style: first part is .part01.rar or .part1.rar
|
||||
// Try both single and double digit formats
|
||||
if _, err := os.Stat(filepath.Join(dir, baseName+".part1.rar")); err == nil {
|
||||
return filepath.Join(dir, baseName+".part1.rar")
|
||||
}
|
||||
return filepath.Join(dir, baseName+".part01.rar")
|
||||
case strings.Contains(pattern, `.r(`):
|
||||
// RAR old style: first part is .rar (not .r00)
|
||||
return filepath.Join(dir, baseName+".rar")
|
||||
case strings.Contains(pattern, `.zip)`):
|
||||
// ZIP multi-part: first part is .zip.001
|
||||
return filepath.Join(dir, baseName+".001")
|
||||
case strings.Contains(pattern, `.z(`):
|
||||
// ZIP split: last part is .zip, but extraction should start from .z01
|
||||
return filepath.Join(dir, baseName+".z01")
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// isFirstPart checks if the given file is the first part of a multi-part archive
|
||||
func isFirstPart(filename string) bool {
|
||||
info := getArchivePartInfo(filename)
|
||||
if !info.IsMultiPart {
|
||||
return false
|
||||
}
|
||||
|
||||
// For most formats, part 1 is the first part
|
||||
// For old-style RAR, check if this is the .rar file
|
||||
if info.Pattern == "rar-old-style" {
|
||||
return strings.HasSuffix(strings.ToLower(filename), ".rar")
|
||||
}
|
||||
|
||||
return info.PartNumber == 1
|
||||
}
|
||||
|
||||
// GetMultiPartArchiveBaseName returns the base name for a multi-part archive
|
||||
// This is used to group related parts together
|
||||
func GetMultiPartArchiveBaseName(filename string) string {
|
||||
info := getArchivePartInfo(filename)
|
||||
if !info.IsMultiPart {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(filepath.Dir(filename), info.BaseName)
|
||||
}
|
||||
|
||||
// extractMultiPartArchive extracts a multi-part archive starting from the first part
|
||||
func extractMultiPartArchive(firstPartPath string, destDir string, password string, progressCallback ExtractProgressCallback) error {
|
||||
info := getArchivePartInfo(firstPartPath)
|
||||
|
||||
// For 7z multi-part archives, the bodgit/sevenzip library handles multi-volume automatically
|
||||
// when using OpenReader with a .001 file
|
||||
if strings.Contains(info.Pattern, `\.7z\)`) || strings.HasSuffix(strings.ToLower(firstPartPath), ".7z.001") {
|
||||
return extractSevenZipMultiPart(firstPartPath, destDir, password, progressCallback)
|
||||
}
|
||||
|
||||
// For RAR multi-part archives, use the archives library with Name and FS fields
|
||||
if strings.Contains(info.Pattern, "rar") || info.Pattern == "rar-old-style" {
|
||||
return extractRarMultiPart(firstPartPath, destDir, password, progressCallback)
|
||||
}
|
||||
|
||||
// For ZIP multi-part archives (.zip.001, .zip.002, etc.), concatenate parts and extract
|
||||
if strings.Contains(info.Pattern, `\.zip\)`) || strings.HasSuffix(strings.ToLower(firstPartPath), ".zip.001") {
|
||||
return extractZipMultiPart(firstPartPath, destDir, password, progressCallback)
|
||||
}
|
||||
|
||||
// For other formats, try standard extraction (may not work for all multi-part formats)
|
||||
return extractArchive(firstPartPath, destDir, password, progressCallback)
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/bodgit/sevenzip"
|
||||
)
|
||||
|
||||
// extractSevenZipMultiPart extracts a multi-part 7z archive using bodgit/sevenzip directly
|
||||
// The mholt/archives wrapper doesn't properly handle multi-part 7z files because it uses
|
||||
// io.SectionReader which can only see the first part. The bodgit/sevenzip library's
|
||||
// OpenReaderWithPassword function handles multi-part files automatically when given the .001 file path.
|
||||
func extractSevenZipMultiPart(firstPartPath string, destDir string, password string, progressCallback ExtractProgressCallback) error {
|
||||
// Use bodgit/sevenzip directly - it automatically handles .001, .002, etc. files
|
||||
var reader *sevenzip.ReadCloser
|
||||
var err error
|
||||
|
||||
if password != "" {
|
||||
reader, err = sevenzip.OpenReaderWithPassword(firstPartPath, password)
|
||||
} else {
|
||||
reader, err = sevenzip.OpenReader(firstPartPath)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
// Create destination directory
|
||||
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Count total files for progress
|
||||
totalFiles := 0
|
||||
for _, f := range reader.File {
|
||||
if !f.FileInfo().IsDir() {
|
||||
totalFiles++
|
||||
}
|
||||
}
|
||||
|
||||
// Extract files with progress tracking
|
||||
extractedFiles := 0
|
||||
for _, f := range reader.File {
|
||||
destPath := filepath.Join(destDir, f.Name)
|
||||
|
||||
if f.FileInfo().IsDir() {
|
||||
if err := os.MkdirAll(destPath, f.Mode()); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Ensure parent directory exists
|
||||
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Extract file - open, copy, close immediately (as recommended by bodgit/sevenzip)
|
||||
if err := extractSevenZipFile(f, destPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
extractedFiles++
|
||||
if progressCallback != nil && totalFiles > 0 {
|
||||
progress := int(float64(extractedFiles) / float64(totalFiles) * 100)
|
||||
progressCallback(extractedFiles, totalFiles, progress)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// extractSevenZipFile extracts a single file from a 7z archive
|
||||
// This follows the bodgit/sevenzip recommended pattern of closing rc before processing the next file
|
||||
func extractSevenZipFile(f *sevenzip.File, destPath string) error {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
outFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer outFile.Close()
|
||||
|
||||
_, err = io.Copy(outFile, rc)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
)
|
||||
|
||||
// extractRarMultiPart extracts a multi-part RAR archive
|
||||
func extractRarMultiPart(firstPartPath string, destDir string, password string, progressCallback ExtractProgressCallback) error {
|
||||
// For RAR archives, we need to use the Name field in archives.Rar
|
||||
// to let it automatically find subsequent volumes
|
||||
|
||||
dir := filepath.Dir(firstPartPath)
|
||||
fileName := filepath.Base(firstPartPath)
|
||||
|
||||
rar := archives.Rar{
|
||||
Password: password,
|
||||
Name: fileName,
|
||||
FS: os.DirFS(dir),
|
||||
}
|
||||
|
||||
// Create destination directory
|
||||
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Count files first for progress
|
||||
totalFiles := 0
|
||||
err := rar.Extract(context.Background(), nil, func(ctx context.Context, fileInfo archives.FileInfo) error {
|
||||
if !fileInfo.IsDir() {
|
||||
totalFiles++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
// If counting fails, proceed without progress
|
||||
totalFiles = 0
|
||||
}
|
||||
|
||||
// Reset and extract with progress tracking
|
||||
return rar.Extract(context.Background(), nil, createExtractionHandler(destDir, totalFiles, progressCallback))
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,191 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
)
|
||||
|
||||
// extractZipMultiPart extracts a multi-part ZIP archive (.zip.001, .zip.002, etc.)
|
||||
// These are created by simply splitting a ZIP file into chunks, so we concatenate them
|
||||
func extractZipMultiPart(firstPartPath string, destDir string, password string, progressCallback ExtractProgressCallback) error {
|
||||
// Find all parts
|
||||
parts, err := findZipMultiParts(firstPartPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create a multi-part reader that reads across all parts
|
||||
multiReader := newMultiPartFileReader(parts)
|
||||
defer multiReader.Close()
|
||||
|
||||
// Get total size
|
||||
totalSize := multiReader.Size()
|
||||
|
||||
// Create destination directory
|
||||
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// First pass: count files for progress
|
||||
totalFiles := 0
|
||||
zip := archives.Zip{}
|
||||
err = zip.Extract(context.Background(), io.NewSectionReader(multiReader, 0, totalSize), func(ctx context.Context, fileInfo archives.FileInfo) error {
|
||||
if !fileInfo.IsDir() {
|
||||
totalFiles++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
// If counting fails, proceed without progress
|
||||
totalFiles = 0
|
||||
}
|
||||
|
||||
// Reset reader for actual extraction
|
||||
multiReader.Close()
|
||||
multiReader = newMultiPartFileReader(parts)
|
||||
defer multiReader.Close()
|
||||
|
||||
// Second pass: extract with progress tracking
|
||||
return zip.Extract(context.Background(), io.NewSectionReader(multiReader, 0, totalSize), createExtractionHandler(destDir, totalFiles, progressCallback))
|
||||
}
|
||||
|
||||
// findZipMultiParts finds all parts of a multi-part ZIP archive in order
|
||||
func findZipMultiParts(firstPartPath string) ([]string, error) {
|
||||
// Extract base name (e.g., "Archive.zip" from "Archive.zip.001")
|
||||
dir := filepath.Dir(firstPartPath)
|
||||
baseName := filepath.Base(firstPartPath)
|
||||
|
||||
// Remove the .001 suffix to get the base
|
||||
if idx := strings.LastIndex(baseName, "."); idx > 0 {
|
||||
baseName = baseName[:idx] // "Archive.zip"
|
||||
}
|
||||
|
||||
var parts []string
|
||||
partNum := 1
|
||||
|
||||
for {
|
||||
partPath := filepath.Join(dir, baseName+fmt.Sprintf(".%03d", partNum))
|
||||
if _, err := os.Stat(partPath); os.IsNotExist(err) {
|
||||
break
|
||||
}
|
||||
parts = append(parts, partPath)
|
||||
partNum++
|
||||
}
|
||||
|
||||
if len(parts) == 0 {
|
||||
return nil, fmt.Errorf("no parts found for %s", firstPartPath)
|
||||
}
|
||||
|
||||
return parts, nil
|
||||
}
|
||||
|
||||
// multiPartFileReader provides io.ReaderAt over multiple files concatenated
|
||||
type multiPartFileReader struct {
|
||||
parts []string
|
||||
files []*os.File
|
||||
sizes []int64
|
||||
offsets []int64 // cumulative offsets for each file
|
||||
totalSize int64
|
||||
}
|
||||
|
||||
func newMultiPartFileReader(parts []string) *multiPartFileReader {
|
||||
return &multiPartFileReader{parts: parts}
|
||||
}
|
||||
|
||||
func (m *multiPartFileReader) init() error {
|
||||
if m.files != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
m.files = make([]*os.File, len(m.parts))
|
||||
m.sizes = make([]int64, len(m.parts))
|
||||
m.offsets = make([]int64, len(m.parts))
|
||||
|
||||
var offset int64
|
||||
for i, part := range m.parts {
|
||||
f, err := os.Open(part)
|
||||
if err != nil {
|
||||
m.Close()
|
||||
return err
|
||||
}
|
||||
stat, err := f.Stat()
|
||||
if err != nil {
|
||||
f.Close()
|
||||
m.Close()
|
||||
return err
|
||||
}
|
||||
m.files[i] = f
|
||||
m.sizes[i] = stat.Size()
|
||||
m.offsets[i] = offset
|
||||
offset += stat.Size()
|
||||
}
|
||||
m.totalSize = offset
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *multiPartFileReader) Size() int64 {
|
||||
if err := m.init(); err != nil {
|
||||
return 0
|
||||
}
|
||||
return m.totalSize
|
||||
}
|
||||
|
||||
func (m *multiPartFileReader) ReadAt(p []byte, off int64) (n int, err error) {
|
||||
if err := m.init(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if off >= m.totalSize {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
// Find which file(s) to read from
|
||||
for i, fileOffset := range m.offsets {
|
||||
fileEnd := fileOffset + m.sizes[i]
|
||||
|
||||
if off >= fileEnd {
|
||||
continue
|
||||
}
|
||||
|
||||
// Read from this file
|
||||
localOffset := off - fileOffset
|
||||
toRead := len(p) - n
|
||||
if int64(toRead) > fileEnd-off {
|
||||
toRead = int(fileEnd - off)
|
||||
}
|
||||
|
||||
read, err := m.files[i].ReadAt(p[n:n+toRead], localOffset)
|
||||
n += read
|
||||
off += int64(read)
|
||||
|
||||
if err != nil && err != io.EOF {
|
||||
return n, err
|
||||
}
|
||||
|
||||
if n >= len(p) {
|
||||
return n, nil
|
||||
}
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (m *multiPartFileReader) Close() error {
|
||||
for _, f := range m.files {
|
||||
if f != nil {
|
||||
f.Close()
|
||||
}
|
||||
}
|
||||
m.files = nil
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user