mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-19 06:43:42 +00:00
feat: queue extractions (#1227)
This commit is contained in:
1 parent
de0afb1481
commit
24db9bb58b
35 files changed
+1047
-159
No files matched your search
+118
-61
@@ -52,6 +52,8 @@ type ExtractStatus string
|
||||
const (
|
||||
// ExtractStatusNone indicates extraction has not started
|
||||
ExtractStatusNone ExtractStatus = ""
|
||||
// ExtractStatusQueued indicates extraction is waiting in the queue
|
||||
ExtractStatusQueued ExtractStatus = "queued"
|
||||
// ExtractStatusWaitingParts indicates waiting for other multi-part archive parts to complete
|
||||
ExtractStatusWaitingParts ExtractStatus = "waitingParts"
|
||||
// ExtractStatusExtracting indicates extraction is in progress
|
||||
@@ -102,6 +104,10 @@ type Downloader struct {
|
||||
checkDuplicateLock *sync.Mutex
|
||||
closed atomic.Bool
|
||||
|
||||
// claimedExtractions tracks which multi-part archives have been claimed for extraction
|
||||
// Key: fullBaseName (e.g., "/path/archive.7z"), Value: taskID that claimed it
|
||||
claimedExtractions sync.Map
|
||||
|
||||
extensions []*Extension
|
||||
}
|
||||
|
||||
@@ -736,6 +742,9 @@ func (d *Downloader) emit(eventKey EventKey, task *Task, errs ...error) {
|
||||
}
|
||||
|
||||
func (d *Downloader) GetTask(id string) *Task {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
|
||||
for _, task := range d.tasks {
|
||||
if task.ID == id {
|
||||
return task
|
||||
@@ -745,12 +754,18 @@ func (d *Downloader) GetTask(id string) *Task {
|
||||
}
|
||||
|
||||
func (d *Downloader) GetTasks() []*Task {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
|
||||
return d.tasks
|
||||
}
|
||||
|
||||
// GetTasksByFilter get tasks by filter, if filter is nil, return all tasks
|
||||
// return tasks and if match all tasks
|
||||
func (d *Downloader) GetTasksByFilter(filter *TaskFilter) []*Task {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
|
||||
if filter == nil || filter.IsEmpty() {
|
||||
return d.tasks
|
||||
}
|
||||
@@ -905,9 +920,10 @@ func (d *Downloader) watch(task *Task) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Auto-extract archive files
|
||||
// Auto-extract archive files using the extraction queue
|
||||
// This ensures only one extraction runs at a time to prevent resource exhaustion
|
||||
if e.AutoExtract && isArchiveFile(downloadFilePath) {
|
||||
go d.handleAutoExtract(task, downloadFilePath, e)
|
||||
d.enqueueExtraction(task, downloadFilePath, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1200,55 +1216,93 @@ func (d *Downloader) buildFetcher(url string) (fetcher.Fetcher, error) {
|
||||
return fetcher, nil
|
||||
}
|
||||
|
||||
// handleAutoExtract handles automatic extraction of archive files, including multi-part archives
|
||||
func (d *Downloader) handleAutoExtract(task *Task, downloadFilePath string, opts *http.OptsExtra) {
|
||||
// Check if this is a multi-part archive
|
||||
// enqueueExtraction adds an extraction job to the global extraction queue
|
||||
// This ensures only one extraction (or one multi-part archive extraction) runs at a time
|
||||
// to prevent resource exhaustion
|
||||
func (d *Downloader) enqueueExtraction(task *Task, downloadFilePath string, opts *http.OptsExtra) {
|
||||
partInfo := getArchivePartInfo(downloadFilePath)
|
||||
|
||||
if partInfo.IsMultiPart {
|
||||
// Set multi-part info on the task
|
||||
task.Progress.MultiPartBaseName = partInfo.BaseName
|
||||
task.Progress.MultiPartNumber = partInfo.PartNumber
|
||||
task.Progress.MultiPartIsFirst = isFirstPart(downloadFilePath)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
// For multi-part archives, handle specially
|
||||
d.enqueueMultiPartExtraction(task, downloadFilePath, partInfo, opts)
|
||||
} else {
|
||||
// For single archives, queue immediately
|
||||
d.enqueueSingleExtraction(task, downloadFilePath, opts)
|
||||
}
|
||||
}
|
||||
|
||||
// Check if all parts are downloaded
|
||||
destDir := task.Meta.Opts.Path
|
||||
allPartsReady, missingParts := d.checkMultiPartArchiveReady(downloadFilePath, destDir, partInfo)
|
||||
// enqueueSingleExtraction queues extraction for a single (non-multi-part) archive
|
||||
func (d *Downloader) enqueueSingleExtraction(task *Task, downloadFilePath string, opts *http.OptsExtra) {
|
||||
jobID := "single:" + task.ID
|
||||
|
||||
if !allPartsReady {
|
||||
// Not all parts are ready yet, set status to waiting
|
||||
task.Progress.ExtractStatus = ExtractStatusWaitingParts
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.Logger.Info().Msgf("multi-part archive waiting for other parts, task id: %s, missing: %v", task.ID, missingParts)
|
||||
return
|
||||
}
|
||||
// Set extraction status to queued
|
||||
task.Progress.ExtractStatus = ExtractStatusQueued
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.Logger.Info().Msgf("extraction queued, task id: %s, job id: %s", task.ID, jobID)
|
||||
|
||||
// All parts are ready!
|
||||
// Only one task should perform extraction - we'll use the one that completes last
|
||||
// Check if extraction has already been started by another part
|
||||
if d.isMultiPartExtractionInProgress(partInfo.BaseName) {
|
||||
// Another part already started extraction, mark this as done
|
||||
task.Progress.ExtractStatus = ExtractStatusDone
|
||||
task.Progress.ExtractProgress = 100
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.Logger.Info().Msgf("multi-part archive part completed, extraction handled by another part, task id: %s", task.ID)
|
||||
return
|
||||
}
|
||||
// Create and enqueue the extraction job
|
||||
job := NewExtractionJob(jobID, func() {
|
||||
d.performExtraction(task, downloadFilePath, task.Meta.Opts.Path, opts)
|
||||
})
|
||||
|
||||
// This task will handle extraction - mark as extracting first to prevent others
|
||||
task.Progress.ExtractStatus = ExtractStatusExtracting
|
||||
go func() {
|
||||
GetExtractionQueue().Enqueue(job)
|
||||
}()
|
||||
}
|
||||
|
||||
// enqueueMultiPartExtraction handles queueing for multi-part archives
|
||||
// It ensures only ONE extraction job is queued when ALL parts are ready
|
||||
func (d *Downloader) enqueueMultiPartExtraction(task *Task, downloadFilePath string, partInfo ArchivePartInfo, opts *http.OptsExtra) {
|
||||
// Set multi-part info on the task
|
||||
task.Progress.MultiPartBaseName = partInfo.BaseName
|
||||
task.Progress.MultiPartNumber = partInfo.PartNumber
|
||||
task.Progress.MultiPartIsFirst = isFirstPart(downloadFilePath)
|
||||
|
||||
// Check if all parts are downloaded
|
||||
destDir := task.Meta.Opts.Path
|
||||
allPartsReady, missingParts := d.checkMultiPartArchiveReady(downloadFilePath, destDir, partInfo)
|
||||
|
||||
if !allPartsReady {
|
||||
// Not all parts are ready yet - just set status to waiting, don't queue anything
|
||||
task.Progress.ExtractStatus = ExtractStatusWaitingParts
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
|
||||
// Extract the multi-part archive
|
||||
d.performMultiPartExtraction(task, partInfo.FirstPartPath, destDir, opts)
|
||||
} else {
|
||||
// Regular (non-multi-part) archive
|
||||
d.performExtraction(task, downloadFilePath, task.Meta.Opts.Path, opts)
|
||||
d.Logger.Info().Msgf("multi-part archive waiting for other parts, task id: %s, missing: %v", task.ID, missingParts)
|
||||
return
|
||||
}
|
||||
|
||||
// All parts are ready! Atomically check if extraction has already been started/queued
|
||||
// and if not, mark this task as the one that will handle it
|
||||
// Use GetMultiPartArchiveBaseName to get the full path for comparison
|
||||
fullBaseName := GetMultiPartArchiveBaseName(downloadFilePath)
|
||||
shouldQueue := d.tryClaimMultiPartExtraction(task, fullBaseName)
|
||||
|
||||
if !shouldQueue {
|
||||
// Another part already started/queued extraction, mark this task as done
|
||||
task.Progress.ExtractStatus = ExtractStatusDone
|
||||
task.Progress.ExtractProgress = 100
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.Logger.Info().Msgf("multi-part archive extraction already handled by another part, task id: %s", task.ID)
|
||||
return
|
||||
}
|
||||
|
||||
// This task claimed the extraction - status already set to queued in tryClaimMultiPartExtraction
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
|
||||
jobID := "multipart:" + fullBaseName
|
||||
d.Logger.Info().Msgf("multi-part extraction queued, task id: %s, job id: %s", task.ID, jobID)
|
||||
|
||||
// Create and enqueue the extraction job
|
||||
job := NewExtractionJob(jobID, func() {
|
||||
d.performMultiPartExtraction(task, partInfo.FirstPartPath, destDir, opts)
|
||||
})
|
||||
|
||||
go func() {
|
||||
GetExtractionQueue().Enqueue(job)
|
||||
}()
|
||||
}
|
||||
|
||||
// checkMultiPartArchiveReady checks if all parts of a multi-part archive are downloaded
|
||||
@@ -1298,28 +1352,25 @@ func (d *Downloader) checkAllMultiPartTasksDone(baseName string) (bool, []string
|
||||
return len(notDoneParts) == 0, notDoneParts
|
||||
}
|
||||
|
||||
// isMultiPartExtractionInProgress checks if any task for this multi-part archive
|
||||
// has already started extraction (to prevent duplicate extractions)
|
||||
func (d *Downloader) isMultiPartExtractionInProgress(baseName string) bool {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
|
||||
for _, task := range d.tasks {
|
||||
taskBaseName := ""
|
||||
if task.Meta != nil && task.Meta.Res != nil && len(task.Meta.Res.Files) > 0 {
|
||||
taskBaseName = GetMultiPartArchiveBaseName(task.Meta.SingleFilepath())
|
||||
}
|
||||
if taskBaseName == baseName {
|
||||
// Check if this task is already extracting or has completed extraction
|
||||
if task.Progress.ExtractStatus == ExtractStatusExtracting ||
|
||||
task.Progress.ExtractStatus == ExtractStatusDone ||
|
||||
task.Progress.ExtractStatus == ExtractStatusError {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// tryClaimMultiPartExtraction atomically checks if extraction can be claimed for a multi-part archive
|
||||
// and if so, marks the task as queued. Returns true if this task should proceed with queueing.
|
||||
// This uses sync.Map.LoadOrStore for atomic claim to prevent race conditions.
|
||||
func (d *Downloader) tryClaimMultiPartExtraction(task *Task, baseName string) bool {
|
||||
// Use LoadOrStore for atomic claim - if another goroutine already stored a value, we get that value back
|
||||
_, alreadyClaimed := d.claimedExtractions.LoadOrStore(baseName, task.ID)
|
||||
if alreadyClaimed {
|
||||
return false // Another task already claimed it
|
||||
}
|
||||
|
||||
return false
|
||||
// This task successfully claimed it
|
||||
task.Progress.ExtractStatus = ExtractStatusQueued
|
||||
return true
|
||||
}
|
||||
|
||||
// releaseMultiPartExtractionClaim releases the extraction claim for a multi-part archive
|
||||
// This is primarily used for testing purposes
|
||||
func (d *Downloader) releaseMultiPartExtractionClaim(baseName string) {
|
||||
d.claimedExtractions.Delete(baseName)
|
||||
}
|
||||
|
||||
// performExtraction performs extraction for a regular (non-multi-part) archive
|
||||
@@ -1341,6 +1392,9 @@ func (d *Downloader) performExtraction(task *Task, archivePath string, destDir s
|
||||
|
||||
// performMultiPartExtraction performs extraction for a multi-part archive
|
||||
func (d *Downloader) performMultiPartExtraction(task *Task, firstPartPath string, destDir string, opts *http.OptsExtra) {
|
||||
// Get the baseName for releasing the claim later
|
||||
fullBaseName := GetMultiPartArchiveBaseName(firstPartPath)
|
||||
|
||||
// Set extraction status to extracting
|
||||
task.Progress.ExtractStatus = ExtractStatusExtracting
|
||||
task.Progress.ExtractProgress = 0
|
||||
@@ -1362,6 +1416,9 @@ func (d *Downloader) performMultiPartExtraction(task *Task, firstPartPath string
|
||||
|
||||
// Update status for all related multi-part tasks
|
||||
d.updateMultiPartTasksStatus(task, extractErr)
|
||||
|
||||
// Release the claim so future downloads of the same archive can be extracted
|
||||
d.releaseMultiPartExtractionClaim(fullBaseName)
|
||||
}
|
||||
|
||||
// collectMultiPartFiles collects all files belonging to a multi-part archive
|
||||
|
||||
Reference in new issue
Block a user