Merge PR #1542 into 2.0.0-beta.2

internal/protocol/http/fetcher.go

@@ -2,8 +2,13 @@ package http
 
 import (
 	"context"
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"fmt"
+	"hash"
 	"io"
 	"net/http"
 	"net/url"
@@ -280,6 +285,12 @@ type Fetcher struct {
 	primaryReadyOnce sync.Once
 	primaryReadyCh   chan struct{}
 
+	// checksumFailed marks that the last completed download failed
+	// checksum verification, so the next retry must force a full
+	// re-download instead of skipping connections that already
+	// reported Completed.
+	checksumFailed atomic.Bool
+
 	// Start pending mechanism
 	startPending   atomic.Bool
 	resolvedCh     chan struct{} // Signal when resolve completes
@@ -735,12 +746,26 @@ func (f *Fetcher) doStart() error {
 
 	// If retrying after error, reset connection states for retry
 	if state == stateError {
+		forceFullReset := f.checksumFailed.Swap(false)
+		if forceFullReset {
+			// Checksum mismatch: the file content itself is wrong even
+			// though every connection reported Completed. Truncate the
+			// corrupt file so stale bytes cannot linger if the retry is
+			// interrupted before every connection re-writes its chunk.
+			if err := os.Truncate(f.meta.SingleFilepath(), 0); err != nil && !os.IsNotExist(err) {
+				return fmt.Errorf("failed to reset file after checksum mismatch: %w", err)
+			}
+		}
 		f.connMu.Lock()
 		for _, conn := range f.connections {
-			// Reset connections that can be retried
-			if !conn.Completed && conn.State != connCompleted {
+			// On a normal error, only reset connections that didn't
+			// finish. On a checksum mismatch, reset EVERY connection —
+			// "Completed" only means the download loop finished, not
+			// that the bytes were verified correct.
+			shouldReset := forceFullReset || (!conn.Completed && conn.State != connCompleted)
+			if shouldReset {
 				if !f.hasSequentialPrefixLocked(conn) {
-					f.resetConnectionForRestart(conn)
+					f.resetConnectionForRestart(conn, forceFullReset)
 				}
 				conn.State = connNotStarted
 				conn.failed = false
@@ -970,8 +995,14 @@ func (f *Fetcher) expandConnections() {
 			}
 			f.fileMu.Unlock()
 
-			f.setState(stateDone)
-			f.doneCh <- nil
+			var err error
+			if chkErr := f.verifyChecksum(); chkErr != nil {
+				err = chkErr
+				f.setState(stateError)
+			} else {
+				f.setState(stateDone)
+			}
+			f.doneCh <- err
 			return
 		}
 
@@ -1107,7 +1138,7 @@ func (f *Fetcher) runConnection(conn *connection) {
 			// it only when no safe validator exists or If-Range returns a full 200.
 			f.connMu.Lock()
 			if !f.hasSequentialPrefixLocked(conn) {
-				f.resetConnectionForRestart(conn)
+				f.resetConnectionForRestart(conn, false)
 			}
 			f.connMu.Unlock()
 		}
@@ -1214,7 +1245,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
 	resumeProbe := f.canProbeSequentialResumeLocked(conn)
 	intentionalRestart := sequentialSizeUnknown
 	if sequentialSizeUnknown || (hasSequentialPrefix && !resumeProbe) {
-		f.resetConnectionForRestart(conn)
+		f.resetConnectionForRestart(conn, false)
 		f.resolveDataPos.Store(0)
 		f.rangeValidatorPinned = false
 		intentionalRestart = true
@@ -1659,7 +1690,7 @@ func (f *Fetcher) fallbackToSequentialDownload(conn *connection, ifRange string)
 	f.rangeReprobeEligible = true
 	f.rangeValidatorPinned = false
 	f.ifRange = ifRange
-	f.resetConnectionForRestart(conn)
+	f.resetConnectionForRestart(conn, false)
 	f.resolveDataPos.Store(0)
 	return nil
 }
@@ -1675,7 +1706,7 @@ func (f *Fetcher) restartSequentialDownload(conn *connection, ifRange string) {
 	f.rangeReprobeEligible = true
 	f.rangeValidatorPinned = false
 	f.ifRange = ifRange
-	f.resetConnectionForRestart(conn)
+	f.resetConnectionForRestart(conn, false)
 	f.resolveDataPos.Store(0)
 }
 
@@ -2058,16 +2089,25 @@ func (f *Fetcher) helpOtherConnection(helper *connection) bool {
 	return true
 }
 
-func (f *Fetcher) resetConnectionForRestart(conn *connection) {
-	if f.meta.Res.Range {
+func (f *Fetcher) resetConnectionForRestart(conn *connection, force bool) {
+	if f.meta.Res.Range && !force {
 		return
 	}
+	if f.meta.Res.Range && force {
+		// Checksum mismatch: the on-disk content for this chunk is
+		// confirmed wrong, so its bytes must be re-fetched from the
+		// origin server even though range support would normally let
+		// us just resume from the existing Downloaded position.
+		if conn.Chunk != nil {
+			conn.Chunk.Downloaded = 0
+		}
+	}
 
 	// Without range support a new request always starts from byte 0,
 	// so pause/retry must restart instead of continuing from the old offset.
 	if conn.Chunk == nil {
 		conn.Chunk = newChunk(0, 0)
-	} else {
+	} else if !f.meta.Res.Range {
 		conn.Chunk.Begin = 0
 		conn.Chunk.setEnd(0)
 		conn.Chunk.Downloaded = 0
@@ -2131,7 +2171,7 @@ func (f *Fetcher) resumeConnections() {
 			}
 		}
 		if !f.hasSequentialPrefixLocked(conn) {
-			f.resetConnectionForRestart(conn)
+			f.resetConnectionForRestart(conn, false)
 		}
 		// Reset the connection state for resume
 		conn.ctx, conn.cancel = context.WithCancel(f.ctx)
@@ -2238,10 +2278,15 @@ func (f *Fetcher) onDownloadComplete() {
 	}
 	f.fileMu.Unlock()
 
-	if finalErr != nil {
-		f.setState(stateError)
+	if finalErr == nil {
+		if chkErr := f.verifyChecksum(); chkErr != nil {
+			finalErr = chkErr
+			f.setState(stateError)
+		} else {
+			f.setState(stateDone)
+		}
 	} else {
-		f.setState(stateDone)
+		f.setState(stateError)
 	}
 
 	select {
@@ -2250,6 +2295,51 @@ func (f *Fetcher) onDownloadComplete() {
 	}
 }
 
+func (f *Fetcher) verifyChecksum() error {
+	if f.meta == nil || f.meta.Opts == nil || f.meta.Opts.Extra == nil {
+		return nil
+	}
+	extra, ok := f.meta.Opts.Extra.(*fhttp.OptsExtra)
+	if !ok || extra.Checksum == nil {
+		return nil
+	}
+	chk := extra.Checksum
+	if chk.Algorithm == "" && chk.Expected == "" {
+		return nil
+	}
+	var h hash.Hash
+	switch strings.ToLower(strings.TrimSpace(chk.Algorithm)) {
+	case "md5":
+		h = md5.New()
+	case "sha1", "sha-1":
+		h = sha1.New()
+	case "sha256", "sha-256":
+		h = sha256.New()
+	default:
+		return fmt.Errorf("unsupported checksum algorithm: %s", chk.Algorithm)
+	}
+
+	filePath := f.meta.SingleFilepath()
+	file, err := os.Open(filePath)
+	if err != nil {
+		return fmt.Errorf("open file for checksum verification failed: %w", err)
+	}
+	defer file.Close()
+
+	buf := make([]byte, 64*1024)
+	if _, err := io.CopyBuffer(h, file, buf); err != nil {
+		return fmt.Errorf("calculate checksum failed: %w", err)
+	}
+
+	actual := hex.EncodeToString(h.Sum(nil))
+	expected := strings.ToLower(strings.TrimSpace(chk.Expected))
+	if !strings.EqualFold(actual, expected) {
+		f.checksumFailed.Store(true)
+		return fmt.Errorf("checksum mismatch: expected %s, got %s", expected, actual)
+	}
+	return nil
+}
+
 func (f *Fetcher) checkCompletion() bool {
 	// Check if all data has been downloaded
 	f.connMu.Lock()

internal/protocol/http/fetcher_checksum_retry_test.go

@@ -0,0 +1,108 @@
+package http
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	gohttp "net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"github.com/GopeedLab/gopeed/pkg/base"
+	"github.com/GopeedLab/gopeed/pkg/protocol/http"
+)
+
+func TestFetcher_ChecksumMismatch_Retry(t *testing.T) {
+	corruptContent := []byte("Corrupt Initial Download Data!")
+	correctContent := []byte("Correct Retried Download Data!")
+
+	correctSha256Hasher := sha256.New()
+	correctSha256Hasher.Write(correctContent)
+	correctSha256 := hex.EncodeToString(correctSha256Hasher.Sum(nil))
+
+	var reqCount atomic.Int32
+	server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
+		count := reqCount.Add(1)
+		var served []byte
+		if count == 1 {
+			// First request: serve corrupted content
+			served = corruptContent
+		} else {
+			// Second / subsequent request: serve correct content
+			served = correctContent
+		}
+		w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(served)))
+		w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
+		w.WriteHeader(gohttp.StatusOK)
+		_, _ = w.Write(served)
+	}))
+	defer server.Close()
+
+	downloadDir := t.TempDir()
+	fileName := "test_checksum_retry.txt"
+
+	f := buildFetcher()
+	defer f.Close()
+
+	opts := &base.Options{
+		Path: downloadDir,
+		Name: fileName,
+		Extra: &http.OptsExtra{
+			Connections: 1,
+			Checksum: &http.ChecksumOption{
+				Algorithm: "sha256",
+				Expected:  correctSha256,
+			},
+		},
+	}
+
+	// 1. Initial attempt -> will download corruptContent, failing checksum verification
+	if err := f.Resolve(&base.Request{URL: server.URL + "/retry_test.txt"}, opts); err != nil {
+		t.Fatalf("Resolve error: %v", err)
+	}
+	if err := f.Start(); err != nil {
+		t.Fatalf("Start error: %v", err)
+	}
+	err := f.Wait()
+	if err == nil {
+		t.Fatal("expected checksum mismatch error on initial attempt, got nil")
+	}
+	if !strings.Contains(err.Error(), "checksum mismatch") {
+		t.Fatalf("expected 'checksum mismatch' in error, got %v", err)
+	}
+	if f.getState() != stateError {
+		t.Fatalf("expected stateError after checksum mismatch, got %v", f.getState())
+	}
+
+	// 2. Retry download -> should trigger forced full reset, truncate file, and re-download correctContent
+	if err := f.Start(); err != nil {
+		t.Fatalf("Start error on retry: %v", err)
+	}
+	if err := f.Wait(); err != nil {
+		t.Fatalf("Wait error on retry (expected success): %v", err)
+	}
+	if f.getState() != stateDone {
+		t.Fatalf("expected stateDone after successful retry, got %v", f.getState())
+	}
+
+	// 3. Verify on-disk file content matches the re-downloaded bytes
+	filePath := f.meta.SingleFilepath()
+	fileBytes, err := os.ReadFile(filePath)
+	if err != nil {
+		t.Fatalf("failed to read completed file: %v", err)
+	}
+	if string(fileBytes) != string(correctContent) {
+		t.Fatalf("on-disk file content mismatch:\ngot:  %q\nwant: %q", string(fileBytes), string(correctContent))
+	}
+
+	// 4. Compute hash independently and verify
+	actualHasher := sha256.New()
+	actualHasher.Write(fileBytes)
+	actualHash := hex.EncodeToString(actualHasher.Sum(nil))
+	if actualHash != correctSha256 {
+		t.Fatalf("re-computed on-disk hash mismatch: got %s, want %s", actualHash, correctSha256)
+	}
+}

internal/protocol/http/fetcher_checksum_test.go

@@ -0,0 +1,307 @@
+package http
+
+import (
+	"crypto/md5"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	gohttp "net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/GopeedLab/gopeed/internal/fetcher"
+	"github.com/GopeedLab/gopeed/internal/test"
+	"github.com/GopeedLab/gopeed/pkg/base"
+	"github.com/GopeedLab/gopeed/pkg/protocol/http"
+)
+
+func TestFetcher_ChecksumVerification(t *testing.T) {
+	content := []byte("Hello, Gopeed Checksum Verification Test Data!")
+	md5Hasher := md5.New()
+	md5Hasher.Write(content)
+	correctMd5 := hex.EncodeToString(md5Hasher.Sum(nil))
+
+	sha256Hasher := sha256.New()
+	sha256Hasher.Write(content)
+	correctSha256 := hex.EncodeToString(sha256Hasher.Sum(nil))
+
+	wrongHash := "0123456789abcdef0123456789abcdef"
+
+	server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
+		w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(content)))
+		w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
+		w.WriteHeader(gohttp.StatusOK)
+		_, _ = w.Write(content)
+	}))
+	defer server.Close()
+
+	t.Run("Valid MD5", func(t *testing.T) {
+		f := buildFetcher()
+		defer f.Close()
+
+		opts := &base.Options{
+			Path: t.TempDir(),
+			Name: "test_valid_md5.txt",
+			Extra: &http.OptsExtra{
+				Connections: 1,
+				Checksum: &http.ChecksumOption{
+					Algorithm: "md5",
+					Expected:  correctMd5,
+				},
+			},
+		}
+
+		if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
+			t.Fatalf("Resolve error: %v", err)
+		}
+		if err := f.Start(); err != nil {
+			t.Fatalf("Start error: %v", err)
+		}
+		if err := f.Wait(); err != nil {
+			t.Fatalf("Wait error (expected success): %v", err)
+		}
+	})
+
+	t.Run("Valid SHA256", func(t *testing.T) {
+		f := buildFetcher()
+		defer f.Close()
+
+		opts := &base.Options{
+			Path: t.TempDir(),
+			Name: "test_valid_sha256.txt",
+			Extra: &http.OptsExtra{
+				Connections: 1,
+				Checksum: &http.ChecksumOption{
+					Algorithm: "sha256",
+					Expected:  correctSha256,
+				},
+			},
+		}
+
+		if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
+			t.Fatalf("Resolve error: %v", err)
+		}
+		if err := f.Start(); err != nil {
+			t.Fatalf("Start error: %v", err)
+		}
+		if err := f.Wait(); err != nil {
+			t.Fatalf("Wait error (expected success): %v", err)
+		}
+	})
+
+	t.Run("Wrong MD5 mismatch", func(t *testing.T) {
+		f := buildFetcher()
+		defer f.Close()
+
+		opts := &base.Options{
+			Path: t.TempDir(),
+			Name: "test_wrong_md5.txt",
+			Extra: &http.OptsExtra{
+				Connections: 1,
+				Checksum: &http.ChecksumOption{
+					Algorithm: "md5",
+					Expected:  wrongHash,
+				},
+			},
+		}
+
+		if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
+			t.Fatalf("Resolve error: %v", err)
+		}
+		if err := f.Start(); err != nil {
+			t.Fatalf("Start error: %v", err)
+		}
+		err := f.Wait()
+		if err == nil {
+			t.Fatal("Wait expected checksum mismatch error, got nil")
+		}
+		if !strings.Contains(err.Error(), "checksum mismatch") {
+			t.Fatalf("expected 'checksum mismatch' in error, got %v", err)
+		}
+	})
+
+	t.Run("Unsupported Algorithm", func(t *testing.T) {
+		f := buildFetcher()
+		defer f.Close()
+
+		opts := &base.Options{
+			Path: t.TempDir(),
+			Name: "test_unsupported.txt",
+			Extra: &http.OptsExtra{
+				Connections: 1,
+				Checksum: &http.ChecksumOption{
+					Algorithm: "invalid-algo",
+					Expected:  wrongHash,
+				},
+			},
+		}
+
+		if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
+			t.Fatalf("Resolve error: %v", err)
+		}
+		if err := f.Start(); err != nil {
+			t.Fatalf("Start error: %v", err)
+		}
+		err := f.Wait()
+		if err == nil {
+			t.Fatal("Wait expected error for unsupported algorithm, got nil")
+		}
+		if !strings.Contains(err.Error(), "unsupported checksum algorithm") {
+			t.Fatalf("expected 'unsupported checksum algorithm' in error, got %v", err)
+		}
+		if f.checksumFailed.Load() {
+			t.Fatal("checksumFailed flag should NOT be set on unsupported algorithm error")
+		}
+	})
+
+	t.Run("Case-Insensitive Hash Comparison", func(t *testing.T) {
+		f := buildFetcher()
+		defer f.Close()
+
+		opts := &base.Options{
+			Path: t.TempDir(),
+			Name: "test_case_insensitive.txt",
+			Extra: &http.OptsExtra{
+				Connections: 1,
+				Checksum: &http.ChecksumOption{
+					Algorithm: "sha256",
+					Expected:  strings.ToUpper(correctSha256),
+				},
+			},
+		}
+
+		if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
+			t.Fatalf("Resolve error: %v", err)
+		}
+		if err := f.Start(); err != nil {
+			t.Fatalf("Start error: %v", err)
+		}
+		if err := f.Wait(); err != nil {
+			t.Fatalf("Wait error (expected success with upper case hash): %v", err)
+		}
+	})
+
+	t.Run("Empty Expected Checksum (Skipped)", func(t *testing.T) {
+		f := buildFetcher()
+		defer f.Close()
+
+		opts := &base.Options{
+			Path: t.TempDir(),
+			Name: "test_empty_expected.txt",
+			Extra: &http.OptsExtra{
+				Connections: 1,
+				Checksum: &http.ChecksumOption{
+					Algorithm: "",
+					Expected:  "",
+				},
+			},
+		}
+
+		if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
+			t.Fatalf("Resolve error: %v", err)
+		}
+		if err := f.Start(); err != nil {
+			t.Fatalf("Start error: %v", err)
+		}
+		if err := f.Wait(); err != nil {
+			t.Fatalf("Wait error (expected skip and success): %v", err)
+		}
+	})
+
+	t.Run("Large File Multi-Chunk Streaming (>64KB)", func(t *testing.T) {
+		largeData := make([]byte, 128*1024)
+		for i := range largeData {
+			largeData[i] = byte(i % 256)
+		}
+		largeSha256Hasher := sha256.New()
+		largeSha256Hasher.Write(largeData)
+		largeSha256 := hex.EncodeToString(largeSha256Hasher.Sum(nil))
+
+		largeServer := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
+			w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(largeData)))
+			w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
+			w.WriteHeader(gohttp.StatusOK)
+			_, _ = w.Write(largeData)
+		}))
+		defer largeServer.Close()
+
+		f := buildFetcher()
+		defer f.Close()
+
+		opts := &base.Options{
+			Path: t.TempDir(),
+			Name: "test_large_file.bin",
+			Extra: &http.OptsExtra{
+				Connections: 1,
+				Checksum: &http.ChecksumOption{
+					Algorithm: "sha256",
+					Expected:  largeSha256,
+				},
+			},
+		}
+
+		if err := f.Resolve(&base.Request{URL: largeServer.URL + "/large.bin"}, opts); err != nil {
+			t.Fatalf("Resolve error: %v", err)
+		}
+		if err := f.Start(); err != nil {
+			t.Fatalf("Start error: %v", err)
+		}
+		if err := f.Wait(); err != nil {
+			t.Fatalf("Wait error (expected large file checksum success): %v", err)
+		}
+	})
+
+	t.Run("No Checksum (Opt-in backward compatibility)", func(t *testing.T) {
+		f := buildFetcher()
+		defer f.Close()
+
+		opts := &base.Options{
+			Path: t.TempDir(),
+			Name: test.DownloadName,
+			Extra: &http.OptsExtra{
+				Connections: 1,
+			},
+		}
+
+		if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
+			t.Fatalf("Resolve error: %v", err)
+		}
+		if err := f.Start(); err != nil {
+			t.Fatalf("Start error: %v", err)
+		}
+		if err := f.Wait(); err != nil {
+			t.Fatalf("Wait error: %v", err)
+		}
+	})
+}
+
+func TestResetConnectionForRestart_ForceOnRangeDownload(t *testing.T) {
+	f := buildFetcher()
+	f.meta = &fetcher.FetcherMeta{
+		Res: &base.Resource{
+			Range: true,
+		},
+	}
+	conn := &connection{
+		Chunk:      newChunk(0, 1024),
+		Downloaded: 512,
+		Completed:  true,
+	}
+	conn.Chunk.Downloaded = 512
+
+	// Non-forced reset on range-supported download does not reset Chunk.Downloaded
+	f.resetConnectionForRestart(conn, false)
+	if conn.Chunk.Downloaded != 512 {
+		t.Fatalf("expected Chunk.Downloaded to remain 512 when force is false, got %d", conn.Chunk.Downloaded)
+	}
+
+	// Forced reset on range-supported download resets Chunk.Downloaded to 0
+	f.resetConnectionForRestart(conn, true)
+	if conn.Chunk.Downloaded != 0 {
+		t.Fatalf("expected Chunk.Downloaded to be reset to 0 when force is true, got %d", conn.Chunk.Downloaded)
+	}
+	if conn.Completed {
+		t.Fatal("expected Completed to be false after reset")
+	}
+}