mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-09 10:00:27 +00:00
feat(protocol/http, mcpserver, download): add checksum verification, MCP speed tool, pause webhook, and CLI flags (#1542)
This commit is contained in:
1 parent
fa622a4063
commit
6f20790b17
41 files changed
+1852
-25
No files matched your search
@@ -2,8 +2,13 @@ package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -280,6 +285,12 @@ type Fetcher struct {
|
||||
primaryReadyOnce sync.Once
|
||||
primaryReadyCh chan struct{}
|
||||
|
||||
// checksumFailed marks that the last completed download failed
|
||||
// checksum verification, so the next retry must force a full
|
||||
// re-download instead of skipping connections that already
|
||||
// reported Completed.
|
||||
checksumFailed atomic.Bool
|
||||
|
||||
// Start pending mechanism
|
||||
startPending atomic.Bool
|
||||
resolvedCh chan struct{} // Signal when resolve completes
|
||||
@@ -664,6 +675,22 @@ func (f *Fetcher) cleanupPrefetchFile() {
|
||||
f.resolveFallback.Store(false)
|
||||
}
|
||||
|
||||
// prepareFullRedownload resets all fetcher connection and prefetch state
|
||||
// after a checksum mismatch so the subsequent Start takes the fresh-start path.
|
||||
func (f *Fetcher) prepareFullRedownload() {
|
||||
f.connMu.Lock()
|
||||
if f.meta.Res.Range {
|
||||
f.connections = nil
|
||||
f.resolveConn = nil
|
||||
f.rangeReprobeEligible = false
|
||||
f.rangeValidatorPinned = false
|
||||
}
|
||||
f.connMu.Unlock()
|
||||
|
||||
f.cleanupPrefetchFile()
|
||||
f.resolveDataPos.Store(0)
|
||||
}
|
||||
|
||||
// Options may be replaced between Resolve and Start when creating a resolved task.
|
||||
func (f *Fetcher) initOptions() error {
|
||||
opts := f.meta.Opts
|
||||
@@ -681,6 +708,9 @@ func (f *Fetcher) initOptions() error {
|
||||
extra.Connections = 1
|
||||
}
|
||||
}
|
||||
if err := extra.Checksum.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -735,11 +765,26 @@ func (f *Fetcher) doStart() error {
|
||||
|
||||
// If retrying after error, reset connection states for retry
|
||||
if state == stateError {
|
||||
fullRedownload := f.checksumFailed.Swap(false)
|
||||
if fullRedownload {
|
||||
// Checksum mismatch: the file content itself is wrong even
|
||||
// though every connection reported Completed. Truncate the
|
||||
// corrupt file so stale bytes cannot linger if the retry is
|
||||
// interrupted before every connection re-writes its chunk.
|
||||
if err := os.Truncate(f.meta.SingleFilepath(), 0); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("failed to reset file after checksum mismatch: %w", err)
|
||||
}
|
||||
f.prepareFullRedownload()
|
||||
}
|
||||
f.connMu.Lock()
|
||||
for _, conn := range f.connections {
|
||||
// Reset connections that can be retried
|
||||
if !conn.Completed && conn.State != connCompleted {
|
||||
if !f.hasSequentialPrefixLocked(conn) {
|
||||
// On a normal error, only reset connections that didn't
|
||||
// finish. On a checksum mismatch, reset EVERY connection —
|
||||
// "Completed" only means the download loop finished, not
|
||||
// that the bytes were verified correct.
|
||||
shouldReset := fullRedownload || (!conn.Completed && conn.State != connCompleted)
|
||||
if shouldReset {
|
||||
if fullRedownload || !f.hasSequentialPrefixLocked(conn) {
|
||||
f.resetConnectionForRestart(conn)
|
||||
}
|
||||
conn.State = connNotStarted
|
||||
@@ -970,8 +1015,14 @@ func (f *Fetcher) expandConnections() {
|
||||
}
|
||||
f.fileMu.Unlock()
|
||||
|
||||
f.setState(stateDone)
|
||||
f.doneCh <- nil
|
||||
var err error
|
||||
if chkErr := f.verifyChecksum(); chkErr != nil {
|
||||
err = chkErr
|
||||
f.setState(stateError)
|
||||
} else {
|
||||
f.setState(stateDone)
|
||||
}
|
||||
f.doneCh <- err
|
||||
return
|
||||
}
|
||||
|
||||
@@ -2067,7 +2118,7 @@ func (f *Fetcher) resetConnectionForRestart(conn *connection) {
|
||||
// so pause/retry must restart instead of continuing from the old offset.
|
||||
if conn.Chunk == nil {
|
||||
conn.Chunk = newChunk(0, 0)
|
||||
} else {
|
||||
} else if !f.meta.Res.Range {
|
||||
conn.Chunk.Begin = 0
|
||||
conn.Chunk.setEnd(0)
|
||||
conn.Chunk.Downloaded = 0
|
||||
@@ -2238,10 +2289,15 @@ func (f *Fetcher) onDownloadComplete() {
|
||||
}
|
||||
f.fileMu.Unlock()
|
||||
|
||||
if finalErr != nil {
|
||||
f.setState(stateError)
|
||||
if finalErr == nil {
|
||||
if chkErr := f.verifyChecksum(); chkErr != nil {
|
||||
finalErr = chkErr
|
||||
f.setState(stateError)
|
||||
} else {
|
||||
f.setState(stateDone)
|
||||
}
|
||||
} else {
|
||||
f.setState(stateDone)
|
||||
f.setState(stateError)
|
||||
}
|
||||
|
||||
select {
|
||||
@@ -2250,6 +2306,51 @@ func (f *Fetcher) onDownloadComplete() {
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fetcher) verifyChecksum() error {
|
||||
if f.meta == nil || f.meta.Opts == nil || f.meta.Opts.Extra == nil {
|
||||
return nil
|
||||
}
|
||||
extra, ok := f.meta.Opts.Extra.(*fhttp.OptsExtra)
|
||||
if !ok || extra.Checksum == nil {
|
||||
return nil
|
||||
}
|
||||
chk := extra.Checksum
|
||||
if chk.Algorithm == "" && chk.Expected == "" {
|
||||
return nil
|
||||
}
|
||||
var h hash.Hash
|
||||
switch strings.ToLower(strings.TrimSpace(chk.Algorithm)) {
|
||||
case "md5":
|
||||
h = md5.New()
|
||||
case "sha1", "sha-1":
|
||||
h = sha1.New()
|
||||
case "sha256", "sha-256":
|
||||
h = sha256.New()
|
||||
default:
|
||||
return fmt.Errorf("unsupported checksum algorithm: %s", chk.Algorithm)
|
||||
}
|
||||
|
||||
filePath := f.meta.SingleFilepath()
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open file for checksum verification failed: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
buf := make([]byte, 64*1024)
|
||||
if _, err := io.CopyBuffer(h, file, buf); err != nil {
|
||||
return fmt.Errorf("calculate checksum failed: %w", err)
|
||||
}
|
||||
|
||||
actual := hex.EncodeToString(h.Sum(nil))
|
||||
expected := strings.ToLower(strings.TrimSpace(chk.Expected))
|
||||
if !strings.EqualFold(actual, expected) {
|
||||
f.checksumFailed.Store(true)
|
||||
return fmt.Errorf("checksum mismatch: expected %s, got %s", expected, actual)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) checkCompletion() bool {
|
||||
// Check if all data has been downloaded
|
||||
f.connMu.Lock()
|
||||
|
||||
@@ -0,0 +1,388 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
gohttp "net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||
)
|
||||
|
||||
// serveRangeContent serves byte range requests or full responses from data.
|
||||
func serveRangeContent(w gohttp.ResponseWriter, r *gohttp.Request, data []byte) {
|
||||
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
|
||||
rangeHdr := r.Header.Get(base.HttpHeaderRange)
|
||||
total := int64(len(data))
|
||||
if rangeHdr == "" {
|
||||
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", total))
|
||||
w.WriteHeader(gohttp.StatusOK)
|
||||
_, _ = w.Write(data)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse bytes=start-end or bytes=start-
|
||||
if !strings.HasPrefix(rangeHdr, "bytes=") {
|
||||
w.WriteHeader(gohttp.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
spec := strings.TrimPrefix(rangeHdr, "bytes=")
|
||||
parts := strings.Split(spec, "-")
|
||||
start, err := strconv.ParseInt(parts[0], 10, 64)
|
||||
if err != nil || start < 0 || start >= total {
|
||||
w.WriteHeader(gohttp.StatusRequestedRangeNotSatisfiable)
|
||||
return
|
||||
}
|
||||
end := total - 1
|
||||
if len(parts) > 1 && parts[1] != "" {
|
||||
if parsedEnd, err := strconv.ParseInt(parts[1], 10, 64); err == nil && parsedEnd >= start && parsedEnd < total {
|
||||
end = parsedEnd
|
||||
}
|
||||
}
|
||||
|
||||
contentLength := end - start + 1
|
||||
w.Header().Set(base.HttpHeaderContentRange, fmt.Sprintf("bytes %d-%d/%d", start, end, total))
|
||||
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", contentLength))
|
||||
w.WriteHeader(gohttp.StatusPartialContent)
|
||||
_, _ = w.Write(data[start : end+1])
|
||||
}
|
||||
|
||||
func TestFetcher_ChecksumMismatch_Retry(t *testing.T) {
|
||||
corruptContent := []byte("Corrupt Initial Download Data 33B")
|
||||
correctContent := []byte("Correct Retried Download Data 33B")
|
||||
|
||||
correctSha256Hasher := sha256.New()
|
||||
correctSha256Hasher.Write(correctContent)
|
||||
correctSha256 := hex.EncodeToString(correctSha256Hasher.Sum(nil))
|
||||
|
||||
var phase atomic.Int32 // 0: serve corruptContent, 1: serve correctContent
|
||||
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
var served []byte
|
||||
if phase.Load() == 0 {
|
||||
served = corruptContent
|
||||
} else {
|
||||
served = correctContent
|
||||
}
|
||||
serveRangeContent(w, r, served)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
downloadDir := t.TempDir()
|
||||
fileName := "test_checksum_retry.txt"
|
||||
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
|
||||
opts := &base.Options{
|
||||
Path: downloadDir,
|
||||
Name: fileName,
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
Checksum: &http.ChecksumOption{
|
||||
Algorithm: "sha256",
|
||||
Expected: correctSha256,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// 1. Initial attempt -> will download corruptContent, failing checksum verification
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/retry_test.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
err := f.Wait()
|
||||
if err == nil {
|
||||
t.Fatal("expected checksum mismatch error on initial attempt, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "checksum mismatch") {
|
||||
t.Fatalf("expected 'checksum mismatch' in error, got %v", err)
|
||||
}
|
||||
if f.getState() != stateError {
|
||||
t.Fatalf("expected stateError after checksum mismatch, got %v", f.getState())
|
||||
}
|
||||
|
||||
// Flip phase to 1 after observing checksum mismatch
|
||||
phase.Store(1)
|
||||
|
||||
// 2. Retry download -> should trigger forced full reset, truncate file, and re-download correctContent
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error on retry: %v", err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("Wait error on retry (expected success): %v", err)
|
||||
}
|
||||
if f.getState() != stateDone {
|
||||
t.Fatalf("expected stateDone after successful retry, got %v", f.getState())
|
||||
}
|
||||
|
||||
// 3. Verify on-disk file content matches the re-downloaded bytes
|
||||
filePath := f.meta.SingleFilepath()
|
||||
fileBytes, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read completed file: %v", err)
|
||||
}
|
||||
if string(fileBytes) != string(correctContent) {
|
||||
t.Fatalf("on-disk file content mismatch:\ngot: %q\nwant: %q", string(fileBytes), string(correctContent))
|
||||
}
|
||||
|
||||
// 4. Compute hash independently and verify
|
||||
actualHasher := sha256.New()
|
||||
actualHasher.Write(fileBytes)
|
||||
actualHash := hex.EncodeToString(actualHasher.Sum(nil))
|
||||
if actualHash != correctSha256 {
|
||||
t.Fatalf("re-computed on-disk hash mismatch: got %s, want %s", actualHash, correctSha256)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_ChecksumMismatch_Retry_PartialPrefetch(t *testing.T) {
|
||||
corruptContent := []byte("Corrupt Initial Download Data 33B")
|
||||
correctContent := []byte("Correct Retried Download Data 33B")
|
||||
|
||||
correctSha256Hasher := sha256.New()
|
||||
correctSha256Hasher.Write(correctContent)
|
||||
correctSha256 := hex.EncodeToString(correctSha256Hasher.Sum(nil))
|
||||
|
||||
var phase atomic.Int32 // 0: corrupt, 1: correct
|
||||
var isFirstRequest atomic.Bool
|
||||
isFirstRequest.Store(true)
|
||||
|
||||
releasePrefetch := make(chan struct{})
|
||||
var closeReleaseOnce sync.Once
|
||||
|
||||
var mu sync.Mutex
|
||||
var phase1Ranges []string
|
||||
|
||||
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
curPhase := phase.Load()
|
||||
if curPhase == 1 {
|
||||
mu.Lock()
|
||||
phase1Ranges = append(phase1Ranges, r.Header.Get(base.HttpHeaderRange))
|
||||
mu.Unlock()
|
||||
serveRangeContent(w, r, correctContent)
|
||||
return
|
||||
}
|
||||
|
||||
// Phase 0: First request is Resolve GET without Range header.
|
||||
// Write exactly 7 bytes and flush, then block until release channel closes or context is done.
|
||||
if isFirstRequest.CompareAndSwap(true, false) {
|
||||
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
|
||||
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(corruptContent)))
|
||||
w.WriteHeader(gohttp.StatusOK)
|
||||
_, _ = w.Write(corruptContent[:7])
|
||||
if flusher, ok := w.(gohttp.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
select {
|
||||
case <-releasePrefetch:
|
||||
case <-r.Context().Done():
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Subsequent requests in Phase 0 (such as Range request from Start)
|
||||
serveRangeContent(w, r, corruptContent)
|
||||
}))
|
||||
defer server.Close()
|
||||
defer closeReleaseOnce.Do(func() { close(releasePrefetch) })
|
||||
|
||||
downloadDir := t.TempDir()
|
||||
fileName := "test_checksum_prefetch_retry.txt"
|
||||
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
|
||||
opts := &base.Options{
|
||||
Path: downloadDir,
|
||||
Name: fileName,
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
Checksum: &http.ChecksumOption{
|
||||
Algorithm: "sha256",
|
||||
Expected: correctSha256,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// 1. Resolve phase
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/prefetch_retry.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
|
||||
// Wait until prefetch has received exactly 7 bytes
|
||||
for i := 0; i < 200; i++ {
|
||||
if f.prefetchSize.Load() >= 7 {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if f.prefetchSize.Load() < 7 {
|
||||
t.Fatalf("expected at least 7 prefetched bytes, got %d", f.prefetchSize.Load())
|
||||
}
|
||||
|
||||
// Release prefetch handler right before Start
|
||||
closeReleaseOnce.Do(func() { close(releasePrefetch) })
|
||||
|
||||
// 2. Start initial download attempt (will complete with corruptContent and fail checksum)
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
err := f.Wait()
|
||||
if err == nil {
|
||||
t.Fatal("expected checksum mismatch error on initial attempt, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "checksum mismatch") {
|
||||
t.Fatalf("expected 'checksum mismatch' in error, got %v", err)
|
||||
}
|
||||
if f.getState() != stateError {
|
||||
t.Fatalf("expected stateError after checksum mismatch, got %v", f.getState())
|
||||
}
|
||||
|
||||
// 3. Flip to phase 1 (correctContent) and retry
|
||||
phase.Store(1)
|
||||
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error on retry: %v", err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("Wait error on retry (expected success): %v", err)
|
||||
}
|
||||
if f.getState() != stateDone {
|
||||
t.Fatalf("expected stateDone after successful retry, got %v", f.getState())
|
||||
}
|
||||
|
||||
// Verify that in Phase 1 at least one Range request started at byte 0
|
||||
mu.Lock()
|
||||
rangesSeen := append([]string(nil), phase1Ranges...)
|
||||
mu.Unlock()
|
||||
|
||||
hasByteZero := false
|
||||
for _, r := range rangesSeen {
|
||||
if strings.HasPrefix(r, "bytes=0-") {
|
||||
hasByteZero = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasByteZero {
|
||||
t.Fatalf("expected at least one Phase 1 Range request starting at byte 0, saw: %v", rangesSeen)
|
||||
}
|
||||
|
||||
// Verify on-disk content matches correctContent exactly
|
||||
filePath := f.meta.SingleFilepath()
|
||||
fileBytes, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read completed file: %v", err)
|
||||
}
|
||||
if string(fileBytes) != string(correctContent) {
|
||||
t.Fatalf("on-disk file content mismatch:\ngot: %q\nwant: %q", string(fileBytes), string(correctContent))
|
||||
}
|
||||
|
||||
// Compute hash independently and verify
|
||||
actualHasher := sha256.New()
|
||||
actualHasher.Write(fileBytes)
|
||||
actualHash := hex.EncodeToString(actualHasher.Sum(nil))
|
||||
if actualHash != correctSha256 {
|
||||
t.Fatalf("re-computed on-disk hash mismatch: got %s, want %s", actualHash, correctSha256)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_ChecksumMismatch_Retry_NonRange(t *testing.T) {
|
||||
corruptContent := []byte("Corrupt Non-Range Initial Data 33B")
|
||||
correctContent := []byte("Correct Non-Range Retried Data 33B")
|
||||
|
||||
correctSha256Hasher := sha256.New()
|
||||
correctSha256Hasher.Write(correctContent)
|
||||
correctSha256 := hex.EncodeToString(correctSha256Hasher.Sum(nil))
|
||||
|
||||
var phase atomic.Int32 // 0: corrupt, 1: correct
|
||||
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
// Server does not support Range (always returns full content 200 OK)
|
||||
var served []byte
|
||||
if phase.Load() == 0 {
|
||||
served = corruptContent
|
||||
} else {
|
||||
served = correctContent
|
||||
}
|
||||
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(served)))
|
||||
w.WriteHeader(gohttp.StatusOK)
|
||||
_, _ = w.Write(served)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
downloadDir := t.TempDir()
|
||||
fileName := "test_checksum_nonrange_retry.txt"
|
||||
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
|
||||
opts := &base.Options{
|
||||
Path: downloadDir,
|
||||
Name: fileName,
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
Checksum: &http.ChecksumOption{
|
||||
Algorithm: "sha256",
|
||||
Expected: correctSha256,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// 1. Resolve and start initial attempt
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/nonrange_retry.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
err := f.Wait()
|
||||
if err == nil {
|
||||
t.Fatal("expected checksum mismatch error on initial non-range attempt, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "checksum mismatch") {
|
||||
t.Fatalf("expected 'checksum mismatch' in error, got %v", err)
|
||||
}
|
||||
if f.getState() != stateError {
|
||||
t.Fatalf("expected stateError after checksum mismatch, got %v", f.getState())
|
||||
}
|
||||
|
||||
// 2. Switch phase and retry
|
||||
phase.Store(1)
|
||||
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error on non-range retry: %v", err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("Wait error on non-range retry (expected success): %v", err)
|
||||
}
|
||||
if f.getState() != stateDone {
|
||||
t.Fatalf("expected stateDone after successful non-range retry, got %v", f.getState())
|
||||
}
|
||||
|
||||
// 3. Verify on-disk file content
|
||||
filePath := f.meta.SingleFilepath()
|
||||
fileBytes, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read completed file: %v", err)
|
||||
}
|
||||
if string(fileBytes) != string(correctContent) {
|
||||
t.Fatalf("on-disk file content mismatch:\ngot: %q\nwant: %q", string(fileBytes), string(correctContent))
|
||||
}
|
||||
|
||||
// 4. Compute hash independently and verify
|
||||
actualHasher := sha256.New()
|
||||
actualHasher.Write(fileBytes)
|
||||
actualHash := hex.EncodeToString(actualHasher.Sum(nil))
|
||||
if actualHash != correctSha256 {
|
||||
t.Fatalf("re-computed on-disk hash mismatch: got %s, want %s", actualHash, correctSha256)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
gohttp "net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/test"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||
)
|
||||
|
||||
func TestFetcher_ChecksumVerification(t *testing.T) {
|
||||
content := []byte("Hello, Gopeed Checksum Verification Test Data!")
|
||||
md5Hasher := md5.New()
|
||||
md5Hasher.Write(content)
|
||||
correctMd5 := hex.EncodeToString(md5Hasher.Sum(nil))
|
||||
|
||||
sha256Hasher := sha256.New()
|
||||
sha256Hasher.Write(content)
|
||||
correctSha256 := hex.EncodeToString(sha256Hasher.Sum(nil))
|
||||
|
||||
wrongHash := "0123456789abcdef0123456789abcdef"
|
||||
|
||||
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(content)))
|
||||
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
|
||||
w.WriteHeader(gohttp.StatusOK)
|
||||
_, _ = w.Write(content)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
t.Run("Valid MD5", func(t *testing.T) {
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
|
||||
opts := &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "test_valid_md5.txt",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
Checksum: &http.ChecksumOption{
|
||||
Algorithm: "md5",
|
||||
Expected: correctMd5,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("Wait error (expected success): %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Valid SHA256", func(t *testing.T) {
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
|
||||
opts := &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "test_valid_sha256.txt",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
Checksum: &http.ChecksumOption{
|
||||
Algorithm: "sha256",
|
||||
Expected: correctSha256,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("Wait error (expected success): %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Wrong MD5 mismatch", func(t *testing.T) {
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
|
||||
opts := &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "test_wrong_md5.txt",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
Checksum: &http.ChecksumOption{
|
||||
Algorithm: "md5",
|
||||
Expected: wrongHash,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
err := f.Wait()
|
||||
if err == nil {
|
||||
t.Fatal("Wait expected checksum mismatch error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "checksum mismatch") {
|
||||
t.Fatalf("expected 'checksum mismatch' in error, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Unsupported Algorithm", func(t *testing.T) {
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
|
||||
opts := &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "test_unsupported.txt",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
Checksum: &http.ChecksumOption{
|
||||
Algorithm: "invalid-algo",
|
||||
Expected: wrongHash,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts)
|
||||
if err == nil {
|
||||
t.Fatal("Resolve expected error for unsupported algorithm, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "unsupported checksum algorithm") {
|
||||
t.Fatalf("expected 'unsupported checksum algorithm' in error, got %v", err)
|
||||
}
|
||||
if f.checksumFailed.Load() {
|
||||
t.Fatal("checksumFailed flag should NOT be set on unsupported algorithm error")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Case-Insensitive Hash Comparison", func(t *testing.T) {
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
|
||||
opts := &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "test_case_insensitive.txt",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
Checksum: &http.ChecksumOption{
|
||||
Algorithm: "sha256",
|
||||
Expected: strings.ToUpper(correctSha256),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("Wait error (expected success with upper case hash): %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Empty Expected Checksum (Skipped)", func(t *testing.T) {
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
|
||||
opts := &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "test_empty_expected.txt",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
Checksum: &http.ChecksumOption{
|
||||
Algorithm: "",
|
||||
Expected: "",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("Wait error (expected skip and success): %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Large File Multi-Chunk Streaming (>64KB)", func(t *testing.T) {
|
||||
largeData := make([]byte, 128*1024)
|
||||
for i := range largeData {
|
||||
largeData[i] = byte(i % 256)
|
||||
}
|
||||
largeSha256Hasher := sha256.New()
|
||||
largeSha256Hasher.Write(largeData)
|
||||
largeSha256 := hex.EncodeToString(largeSha256Hasher.Sum(nil))
|
||||
|
||||
largeServer := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(largeData)))
|
||||
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
|
||||
w.WriteHeader(gohttp.StatusOK)
|
||||
_, _ = w.Write(largeData)
|
||||
}))
|
||||
defer largeServer.Close()
|
||||
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
|
||||
opts := &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "test_large_file.bin",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
Checksum: &http.ChecksumOption{
|
||||
Algorithm: "sha256",
|
||||
Expected: largeSha256,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if err := f.Resolve(&base.Request{URL: largeServer.URL + "/large.bin"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("Wait error (expected large file checksum success): %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("No Checksum (Opt-in backward compatibility)", func(t *testing.T) {
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
|
||||
opts := &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: test.DownloadName,
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
},
|
||||
}
|
||||
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("Wait error: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Invalid Checksum In Resolve Makes Zero Requests", func(t *testing.T) {
|
||||
var reqReceived atomic.Int32
|
||||
customServer := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
reqReceived.Add(1)
|
||||
w.WriteHeader(gohttp.StatusOK)
|
||||
_, _ = w.Write([]byte("some data"))
|
||||
}))
|
||||
defer customServer.Close()
|
||||
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
|
||||
opts := &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "test_invalid_checksum.txt",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
Checksum: &http.ChecksumOption{
|
||||
Algorithm: "sha256",
|
||||
Expected: "invalid-hex-chars-not-valid-hex-00000000000000000000000000000000",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
err := f.Resolve(&base.Request{URL: customServer.URL + "/test.txt"}, opts)
|
||||
if err == nil {
|
||||
t.Fatal("Resolve expected error for invalid checksum hash, got nil")
|
||||
}
|
||||
if reqReceived.Load() != 0 {
|
||||
t.Fatalf("expected 0 requests to server on invalid checksum in Resolve, got %d", reqReceived.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user