From 2f5fbfb17c284a9db3d66df8a84626ded8f82d4d Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sat, 29 Nov 2025 11:34:50 +0800 Subject: [PATCH] fix: HTTP filename parsing for mixed encodings and GBK-encoded (#1189) --- go.mod | 2 +- internal/protocol/http/fetcher.go | 177 +++++++++++++++++++++---- internal/protocol/http/fetcher_test.go | 42 +++++- internal/test/httptest.go | 62 ++++++++- 4 files changed, 253 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index a98d899e..62dbc0be 100644 --- a/go.mod +++ b/go.mod @@ -132,7 +132,7 @@ require ( golang.org/x/mod v0.24.0 // indirect golang.org/x/net v0.39.0 // indirect golang.org/x/sys v0.32.0 // indirect - golang.org/x/text v0.24.0 // indirect + golang.org/x/text v0.24.0 golang.org/x/time v0.8.0 // indirect golang.org/x/tools v0.32.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/internal/protocol/http/fetcher.go b/internal/protocol/http/fetcher.go index b3e9035f..3a408a93 100644 --- a/internal/protocol/http/fetcher.go +++ b/internal/protocol/http/fetcher.go @@ -18,6 +18,7 @@ import ( "strings" "sync" "time" + "unicode/utf8" "github.com/GopeedLab/gopeed/internal/controller" "github.com/GopeedLab/gopeed/internal/fetcher" @@ -26,6 +27,7 @@ import ( "github.com/GopeedLab/gopeed/pkg/util" "github.com/xiaoqidun/setft" "golang.org/x/sync/errgroup" + "golang.org/x/text/encoding/simplifiedchinese" ) const ( @@ -164,24 +166,7 @@ func (f *Fetcher) Resolve(req *base.Request) error { } contentDisposition := httpResp.Header.Get(base.HttpHeaderContentDisposition) if contentDisposition != "" { - _, params, _ := mime.ParseMediaType(contentDisposition) - filename := params["filename"] - if filename != "" { - // Check if the filename is MIME encoded-word - if strings.HasPrefix(filename, "=?") { - decoder := new(mime.WordDecoder) - filename = strings.Replace(filename, "UTF8", "UTF-8", 1) - file.Name, _ = decoder.Decode(filename) - } else { - file.Name = util.TryUrlQueryUnescape(filename) - } - } else { - substr := "attachment; filename=" - index := strings.Index(contentDisposition, substr) - if index != -1 { - file.Name = util.TryUrlQueryUnescape(contentDisposition[index+len(substr):]) - } - } + file.Name = parseFilename(contentDisposition) } // get file filePath by URL if file.Name == "" { @@ -618,12 +603,158 @@ func (f *Fetcher) buildClient() *http.Client { } } -func decodeMangledString(mangled string) string { - rawBytes := make([]byte, 0, len(mangled)*3) - for _, r := range mangled { - rawBytes = append(rawBytes, byte(r)) +// parseFilename extracts filename from Content-Disposition header +// It handles multiple encoding scenarios: +// 1. RFC 5987/RFC 2231 format: filename*=UTF-8''%E6%B5%8B%E8%AF%95.zip (preferred, checked first) +// 2. MIME encoded-word: filename="=?UTF-8?B?5rWL6K+VLnppcA==?=" +// 3. URL-encoded: filename="%E6%B5%8B%E8%AF%95.zip" +// 4. Plain ASCII filename +// +// The key fix is checking filename*= first before mime.ParseMediaType, because +// some servers send Content-Disposition headers with invalid characters that cause +// mime.ParseMediaType to fail, but the filename*= parameter is still valid. +func parseFilename(contentDisposition string) string { + // First, try to find filename*= (RFC 5987 format, most reliable for non-ASCII) + if filename := parseFilenameExtended(contentDisposition); filename != "" { + return filename } - return string(rawBytes) + + // Try standard MIME parsing for regular filename= parameter + _, params, err := mime.ParseMediaType(contentDisposition) + if err == nil { + if filename := params["filename"]; filename != "" { + return decodeFilenameParam(filename) + } + } + + // Fallback: manual parsing if mime.ParseMediaType fails + return parseFilenameFallback(contentDisposition) +} + +// parseFilenameExtended parses RFC 5987/RFC 2231 extended parameter format +// Format: filename*=charset'language'value (e.g., UTF-8''%E6%B5%8B%E8%AF%95.zip) +func parseFilenameExtended(cd string) string { + // Look for filename*= (case-insensitive) + lower := strings.ToLower(cd) + idx := strings.Index(lower, "filename*=") + if idx == -1 { + return "" + } + + // Extract the value after filename*= + value := cd[idx+len("filename*="):] + + // Find the end of the value (next ; or end of string) + if endIdx := strings.Index(value, ";"); endIdx != -1 { + value = value[:endIdx] + } + value = strings.TrimSpace(value) + + // Parse charset'language'encoded-value format + // Common format: UTF-8''%E6%B5%8B%E8%AF%95.zip + parts := strings.SplitN(value, "''", 2) + if len(parts) == 2 { + // parts[0] is charset (e.g., "UTF-8") + // parts[1] is percent-encoded value + decoded, err := url.QueryUnescape(parts[1]) + if err == nil { + return decoded + } + } + + // Try with single quote delimiter as well (some servers use this) + parts = strings.SplitN(value, "'", 3) + if len(parts) >= 3 { + decoded, err := url.QueryUnescape(parts[2]) + if err == nil { + return decoded + } + } + + return "" +} + +// decodeFilenameParam decodes a filename parameter value +// Handles MIME encoded-word, URL encoding, and GBK encoding fallback +func decodeFilenameParam(filename string) string { + // Check if the filename is MIME encoded-word (e.g., =?UTF-8?B?...?=) + if strings.HasPrefix(filename, "=?") { + decoder := new(mime.WordDecoder) + // Some servers use "UTF8" instead of "UTF-8", create a normalized copy + normalizedFilename := strings.Replace(filename, "UTF8", "UTF-8", 1) + if decoded, err := decoder.Decode(normalizedFilename); err == nil { + return decoded + } + } + + // Try URL decoding + decoded := util.TryUrlQueryUnescape(filename) + + // Check if the result is valid UTF-8. If not, try GBK decoding. + // This handles the case where Chinese Windows servers send GBK-encoded filenames + // which appear as garbled characters (e.g., "下载地址.zip" -> "���ص�ַ.zip") + if !utf8.ValidString(decoded) { + if gbkDecoded := tryDecodeGBK(decoded); gbkDecoded != "" { + return gbkDecoded + } + } + + return decoded +} + +// gbkDecoder is a reusable GBK decoder for better performance +var gbkDecoder = simplifiedchinese.GBK.NewDecoder() + +// tryDecodeGBK attempts to decode a string as GBK/GB2312/GB18030 encoding +// Returns empty string if decoding fails or result is not valid UTF-8 +func tryDecodeGBK(s string) string { + // GBK uses 1-2 bytes per character. Single-byte chars are 0x00-0x7F (ASCII compatible). + // Double-byte chars have first byte 0x81-0xFE and second byte 0x40-0xFE. + // Skip if string is empty or all ASCII (valid UTF-8) + if len(s) == 0 { + return "" + } + + // Create a fresh decoder since the transform state may be corrupted + decoder := simplifiedchinese.GBK.NewDecoder() + decoded, err := decoder.Bytes([]byte(s)) + if err != nil { + return "" + } + result := string(decoded) + if utf8.ValidString(result) { + return result + } + return "" +} + +// parseFilenameFallback manually parses filename= when mime.ParseMediaType fails +func parseFilenameFallback(cd string) string { + // Look for filename= (case-insensitive) + lower := strings.ToLower(cd) + idx := strings.Index(lower, "filename=") + if idx == -1 { + return "" + } + + // Skip "filename=" prefix + value := cd[idx+len("filename="):] + + // Find the end of the value + if endIdx := strings.Index(value, ";"); endIdx != -1 { + value = value[:endIdx] + } + value = strings.TrimSpace(value) + + // Remove quotes if present + if len(value) >= 2 { + if (value[0] == '"' && value[len(value)-1] == '"') || + (value[0] == '\'' && value[len(value)-1] == '\'') { + value = value[1 : len(value)-1] + } + } + + return decodeFilenameParam(value) } type fetcherData struct { diff --git a/internal/protocol/http/fetcher_test.go b/internal/protocol/http/fetcher_test.go index 7ccc9abc..c709c46e 100644 --- a/internal/protocol/http/fetcher_test.go +++ b/internal/protocol/http/fetcher_test.go @@ -42,7 +42,7 @@ func TestFetcher_Resolve(t *testing.T) { Range: false, Files: []*base.FileInfo{ { - Name: "测试.zip", + Name: test.TestChineseFileName, Size: test.BuildSize, }, }, @@ -52,7 +52,7 @@ func TestFetcher_Resolve(t *testing.T) { Range: false, Files: []*base.FileInfo{ { - Name: "测试.zip", + Name: test.TestChineseFileName, Size: test.BuildSize, }, }, @@ -62,7 +62,7 @@ func TestFetcher_Resolve(t *testing.T) { Range: false, Files: []*base.FileInfo{ { - Name: "测试.zip", + Name: test.TestChineseFileName, Size: 0, }, }, @@ -77,6 +77,42 @@ func TestFetcher_Resolve(t *testing.T) { }, }, }, t) + // Test mixed encoding Content-Disposition where mime.ParseMediaType fails + // due to invalid characters, but filename*= contains the correct UTF-8 encoded name + testResolve(test.StartTestCustomServer, "mixed-encoding", &base.Resource{ + Size: test.BuildSize, + Range: false, + Files: []*base.FileInfo{ + { + Name: test.TestChineseFileName, + Size: test.BuildSize, + }, + }, + }, t) + // Test filename*= only (RFC 5987 format) + testResolve(test.StartTestCustomServer, "filename-star", &base.Resource{ + Size: test.BuildSize, + Range: false, + Files: []*base.FileInfo{ + { + Name: test.TestChineseFileName, + Size: test.BuildSize, + }, + }, + }, t) + // Test GBK-encoded filename (common on Chinese Windows servers) + // Before fix: "测试.zip" sent as GBK bytes -> parsed as "²âÊÔ.zip" (garbled) + // After fix: correctly decoded back to "测试.zip" + testResolve(test.StartTestCustomServer, "gbk-encoded", &base.Resource{ + Size: test.BuildSize, + Range: false, + Files: []*base.FileInfo{ + { + Name: test.TestChineseFileName, + Size: test.BuildSize, + }, + }, + }, t) fetcher := buildFetcher() err := fetcher.Resolve(&base.Request{ diff --git a/internal/test/httptest.go b/internal/test/httptest.go index 9910b5bc..36467851 100644 --- a/internal/test/httptest.go +++ b/internal/test/httptest.go @@ -6,8 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/GopeedLab/gopeed/pkg/base" - "github.com/armon/go-socks5" "io" "net" "net/http" @@ -17,6 +15,10 @@ import ( "strings" "sync/atomic" "time" + + "github.com/GopeedLab/gopeed/pkg/base" + "github.com/armon/go-socks5" + "golang.org/x/text/encoding/simplifiedchinese" ) const ( @@ -34,6 +36,10 @@ const ( DownloadRename = "download (1).data" DownloadFile = Dir + DownloadName DownloadRenameFile = Dir + DownloadRename + + // TestChineseFileName is a common test filename with Chinese characters + // Used to test Content-Disposition parsing with various encodings + TestChineseFileName = "测试.zip" ) func StartTestFileServer() net.Listener { @@ -103,7 +109,57 @@ func StartTestCustomServer() net.Listener { io.Copy(writer, file) }) mux.HandleFunc("/no-encode", func(writer http.ResponseWriter, request *http.Request) { - writer.Header().Set("Content-Disposition", "attachment; filename=测试.zip") + writer.Header().Set("Content-Disposition", "attachment; filename="+TestChineseFileName) + writer.Header().Set("Content-Type", "application/octet-stream") + writer.Header().Set("Content-Length", fmt.Sprintf("%d", BuildSize)) + file, err := os.Open(BuildFile) + if err != nil { + panic(err) + } + defer file.Close() + io.Copy(writer, file) + }) + // Test endpoint for mixed encoding: filename= with garbled characters (including special chars) + // and filename*= with proper UTF-8. This tests the case where mime.ParseMediaType fails + // due to invalid characters like tags in the filename. + mux.HandleFunc("/mixed-encoding", func(writer http.ResponseWriter, request *http.Request) { + // This simulates a server that sends a garbled filename= with special chars that cause + // mime.ParseMediaType to fail, plus a proper filename*=UTF-8''... + // The filename*= should be preferred and correctly parsed. + writer.Header().Set("Content-Disposition", `attachment;filename="garbledchars.zip";filename*=UTF-8''%E6%B5%8B%E8%AF%95.zip`) + writer.Header().Set("Content-Type", "application/octet-stream") + writer.Header().Set("Content-Length", fmt.Sprintf("%d", BuildSize)) + file, err := os.Open(BuildFile) + if err != nil { + panic(err) + } + defer file.Close() + io.Copy(writer, file) + }) + // Test endpoint for filename*= only (RFC 5987 format) + mux.HandleFunc("/filename-star", func(writer http.ResponseWriter, request *http.Request) { + // URL-encoded TestChineseFileName: 测试.zip -> %E6%B5%8B%E8%AF%95.zip + writer.Header().Set("Content-Disposition", `attachment; filename*=UTF-8''%E6%B5%8B%E8%AF%95.zip`) + writer.Header().Set("Content-Type", "application/octet-stream") + writer.Header().Set("Content-Length", fmt.Sprintf("%d", BuildSize)) + file, err := os.Open(BuildFile) + if err != nil { + panic(err) + } + defer file.Close() + io.Copy(writer, file) + }) + // Test endpoint for GBK-encoded filename (common on Chinese Windows servers) + // This simulates the case where Chinese characters are sent as GBK bytes + // which appear as garbled characters when interpreted as UTF-8. + // For example, "测试" in GBK is [B2 E2 CA D4] which is invalid UTF-8. + // Our fix detects invalid UTF-8 and attempts GBK decoding. + mux.HandleFunc("/gbk-encoded", func(writer http.ResponseWriter, request *http.Request) { + // Encode TestChineseFileName as GBK + gbkEncoder := simplifiedchinese.GBK.NewEncoder() + gbkBytes, _ := gbkEncoder.Bytes([]byte(TestChineseFileName)) + // Send GBK bytes directly in filename (simulating broken server behavior) + writer.Header().Set("Content-Disposition", `attachment; filename="`+string(gbkBytes)+`"`) writer.Header().Set("Content-Type", "application/octet-stream") writer.Header().Set("Content-Length", fmt.Sprintf("%d", BuildSize)) file, err := os.Open(BuildFile)