feat: queue extractions (#1227)

This commit is contained in:
Matteo Pietro Dazzi authored and GitHub committed 2026-01-05 15:53:19 +08:00
1 parent de0afb1481
commit 24db9bb58b
35 files changed
+1047 -159

No files matched your search

+118 -61
View File
@@ -52,6 +52,8 @@ type ExtractStatus string
const (
// ExtractStatusNone indicates extraction has not started
ExtractStatusNone ExtractStatus = ""
// ExtractStatusQueued indicates extraction is waiting in the queue
ExtractStatusQueued ExtractStatus = "queued"
// ExtractStatusWaitingParts indicates waiting for other multi-part archive parts to complete
ExtractStatusWaitingParts ExtractStatus = "waitingParts"
// ExtractStatusExtracting indicates extraction is in progress
@@ -102,6 +104,10 @@ type Downloader struct {
checkDuplicateLock *sync.Mutex
closed atomic.Bool
// claimedExtractions tracks which multi-part archives have been claimed for extraction
// Key: fullBaseName (e.g., "/path/archive.7z"), Value: taskID that claimed it
claimedExtractions sync.Map
extensions []*Extension
}
@@ -736,6 +742,9 @@ func (d *Downloader) emit(eventKey EventKey, task *Task, errs ...error) {
}
func (d *Downloader) GetTask(id string) *Task {
d.lock.Lock()
defer d.lock.Unlock()
for _, task := range d.tasks {
if task.ID == id {
return task
@@ -745,12 +754,18 @@ func (d *Downloader) GetTask(id string) *Task {
}
func (d *Downloader) GetTasks() []*Task {
d.lock.Lock()
defer d.lock.Unlock()
return d.tasks
}
// GetTasksByFilter get tasks by filter, if filter is nil, return all tasks
// return tasks and if match all tasks
func (d *Downloader) GetTasksByFilter(filter *TaskFilter) []*Task {
d.lock.Lock()
defer d.lock.Unlock()
if filter == nil || filter.IsEmpty() {
return d.tasks
}
@@ -905,9 +920,10 @@ func (d *Downloader) watch(task *Task) {
}()
}
// Auto-extract archive files
// Auto-extract archive files using the extraction queue
// This ensures only one extraction runs at a time to prevent resource exhaustion
if e.AutoExtract && isArchiveFile(downloadFilePath) {
go d.handleAutoExtract(task, downloadFilePath, e)
d.enqueueExtraction(task, downloadFilePath, e)
}
}
}
@@ -1200,55 +1216,93 @@ func (d *Downloader) buildFetcher(url string) (fetcher.Fetcher, error) {
return fetcher, nil
}
// handleAutoExtract handles automatic extraction of archive files, including multi-part archives
func (d *Downloader) handleAutoExtract(task *Task, downloadFilePath string, opts *http.OptsExtra) {
// Check if this is a multi-part archive
// enqueueExtraction adds an extraction job to the global extraction queue
// This ensures only one extraction (or one multi-part archive extraction) runs at a time
// to prevent resource exhaustion
func (d *Downloader) enqueueExtraction(task *Task, downloadFilePath string, opts *http.OptsExtra) {
partInfo := getArchivePartInfo(downloadFilePath)
if partInfo.IsMultiPart {
// Set multi-part info on the task
task.Progress.MultiPartBaseName = partInfo.BaseName
task.Progress.MultiPartNumber = partInfo.PartNumber
task.Progress.MultiPartIsFirst = isFirstPart(downloadFilePath)
d.storage.Put(bucketTask, task.ID, task.clone())
// For multi-part archives, handle specially
d.enqueueMultiPartExtraction(task, downloadFilePath, partInfo, opts)
} else {
// For single archives, queue immediately
d.enqueueSingleExtraction(task, downloadFilePath, opts)
}
}
// Check if all parts are downloaded
destDir := task.Meta.Opts.Path
allPartsReady, missingParts := d.checkMultiPartArchiveReady(downloadFilePath, destDir, partInfo)
// enqueueSingleExtraction queues extraction for a single (non-multi-part) archive
func (d *Downloader) enqueueSingleExtraction(task *Task, downloadFilePath string, opts *http.OptsExtra) {
jobID := "single:" + task.ID
if !allPartsReady {
// Not all parts are ready yet, set status to waiting
task.Progress.ExtractStatus = ExtractStatusWaitingParts
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
d.Logger.Info().Msgf("multi-part archive waiting for other parts, task id: %s, missing: %v", task.ID, missingParts)
return
}
// Set extraction status to queued
task.Progress.ExtractStatus = ExtractStatusQueued
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
d.Logger.Info().Msgf("extraction queued, task id: %s, job id: %s", task.ID, jobID)
// All parts are ready!
// Only one task should perform extraction - we'll use the one that completes last
// Check if extraction has already been started by another part
if d.isMultiPartExtractionInProgress(partInfo.BaseName) {
// Another part already started extraction, mark this as done
task.Progress.ExtractStatus = ExtractStatusDone
task.Progress.ExtractProgress = 100
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
d.Logger.Info().Msgf("multi-part archive part completed, extraction handled by another part, task id: %s", task.ID)
return
}
// Create and enqueue the extraction job
job := NewExtractionJob(jobID, func() {
d.performExtraction(task, downloadFilePath, task.Meta.Opts.Path, opts)
})
// This task will handle extraction - mark as extracting first to prevent others
task.Progress.ExtractStatus = ExtractStatusExtracting
go func() {
GetExtractionQueue().Enqueue(job)
}()
}
// enqueueMultiPartExtraction handles queueing for multi-part archives
// It ensures only ONE extraction job is queued when ALL parts are ready
func (d *Downloader) enqueueMultiPartExtraction(task *Task, downloadFilePath string, partInfo ArchivePartInfo, opts *http.OptsExtra) {
// Set multi-part info on the task
task.Progress.MultiPartBaseName = partInfo.BaseName
task.Progress.MultiPartNumber = partInfo.PartNumber
task.Progress.MultiPartIsFirst = isFirstPart(downloadFilePath)
// Check if all parts are downloaded
destDir := task.Meta.Opts.Path
allPartsReady, missingParts := d.checkMultiPartArchiveReady(downloadFilePath, destDir, partInfo)
if !allPartsReady {
// Not all parts are ready yet - just set status to waiting, don't queue anything
task.Progress.ExtractStatus = ExtractStatusWaitingParts
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
// Extract the multi-part archive
d.performMultiPartExtraction(task, partInfo.FirstPartPath, destDir, opts)
} else {
// Regular (non-multi-part) archive
d.performExtraction(task, downloadFilePath, task.Meta.Opts.Path, opts)
d.Logger.Info().Msgf("multi-part archive waiting for other parts, task id: %s, missing: %v", task.ID, missingParts)
return
}
// All parts are ready! Atomically check if extraction has already been started/queued
// and if not, mark this task as the one that will handle it
// Use GetMultiPartArchiveBaseName to get the full path for comparison
fullBaseName := GetMultiPartArchiveBaseName(downloadFilePath)
shouldQueue := d.tryClaimMultiPartExtraction(task, fullBaseName)
if !shouldQueue {
// Another part already started/queued extraction, mark this task as done
task.Progress.ExtractStatus = ExtractStatusDone
task.Progress.ExtractProgress = 100
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
d.Logger.Info().Msgf("multi-part archive extraction already handled by another part, task id: %s", task.ID)
return
}
// This task claimed the extraction - status already set to queued in tryClaimMultiPartExtraction
d.emit(EventKeyProgress, task)
d.storage.Put(bucketTask, task.ID, task.clone())
jobID := "multipart:" + fullBaseName
d.Logger.Info().Msgf("multi-part extraction queued, task id: %s, job id: %s", task.ID, jobID)
// Create and enqueue the extraction job
job := NewExtractionJob(jobID, func() {
d.performMultiPartExtraction(task, partInfo.FirstPartPath, destDir, opts)
})
go func() {
GetExtractionQueue().Enqueue(job)
}()
}
// checkMultiPartArchiveReady checks if all parts of a multi-part archive are downloaded
@@ -1298,28 +1352,25 @@ func (d *Downloader) checkAllMultiPartTasksDone(baseName string) (bool, []string
return len(notDoneParts) == 0, notDoneParts
}
// isMultiPartExtractionInProgress checks if any task for this multi-part archive
// has already started extraction (to prevent duplicate extractions)
func (d *Downloader) isMultiPartExtractionInProgress(baseName string) bool {
d.lock.Lock()
defer d.lock.Unlock()
for _, task := range d.tasks {
taskBaseName := ""
if task.Meta != nil && task.Meta.Res != nil && len(task.Meta.Res.Files) > 0 {
taskBaseName = GetMultiPartArchiveBaseName(task.Meta.SingleFilepath())
}
if taskBaseName == baseName {
// Check if this task is already extracting or has completed extraction
if task.Progress.ExtractStatus == ExtractStatusExtracting ||
task.Progress.ExtractStatus == ExtractStatusDone ||
task.Progress.ExtractStatus == ExtractStatusError {
return true
}
}
// tryClaimMultiPartExtraction atomically checks if extraction can be claimed for a multi-part archive
// and if so, marks the task as queued. Returns true if this task should proceed with queueing.
// This uses sync.Map.LoadOrStore for atomic claim to prevent race conditions.
func (d *Downloader) tryClaimMultiPartExtraction(task *Task, baseName string) bool {
// Use LoadOrStore for atomic claim - if another goroutine already stored a value, we get that value back
_, alreadyClaimed := d.claimedExtractions.LoadOrStore(baseName, task.ID)
if alreadyClaimed {
return false // Another task already claimed it
}
return false
// This task successfully claimed it
task.Progress.ExtractStatus = ExtractStatusQueued
return true
}
// releaseMultiPartExtractionClaim releases the extraction claim for a multi-part archive
// This is primarily used for testing purposes
func (d *Downloader) releaseMultiPartExtractionClaim(baseName string) {
d.claimedExtractions.Delete(baseName)
}
// performExtraction performs extraction for a regular (non-multi-part) archive
@@ -1341,6 +1392,9 @@ func (d *Downloader) performExtraction(task *Task, archivePath string, destDir s
// performMultiPartExtraction performs extraction for a multi-part archive
func (d *Downloader) performMultiPartExtraction(task *Task, firstPartPath string, destDir string, opts *http.OptsExtra) {
// Get the baseName for releasing the claim later
fullBaseName := GetMultiPartArchiveBaseName(firstPartPath)
// Set extraction status to extracting
task.Progress.ExtractStatus = ExtractStatusExtracting
task.Progress.ExtractProgress = 0
@@ -1362,6 +1416,9 @@ func (d *Downloader) performMultiPartExtraction(task *Task, firstPartPath string
// Update status for all related multi-part tasks
d.updateMultiPartTasksStatus(task, extractErr)
// Release the claim so future downloads of the same archive can be extracted
d.releaseMultiPartExtractionClaim(fullBaseName)
}
// collectMultiPartFiles collects all files belonging to a multi-part archive