From 68d42eb39778be4cc2e3b43a47944e3649202985 Mon Sep 17 00:00:00 2001 From: Levi Date: Fri, 14 Mar 2025 18:31:05 +0800 Subject: [PATCH] feat: web http file server caching and gzip support (#961) --- .github/workflows/build.yml | 1 - pkg/rest/gizp_middleware.go | 34 ++++++++++++++++ pkg/rest/server.go | 66 +++++++++++++++++++++++++++++- pkg/rest/server_test.go | 81 ++++++++++++++++++++++++++++++++++++- 4 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 pkg/rest/gizp_middleware.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index deba082c..cac2707c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -529,7 +529,6 @@ jobs: env: VERSION: ${{ needs.get-release.outputs.version }} run: | - VERSION=${VERSION:-$(date +%y%m%d%H%M)} # Use current time as version if not specified cd ui/flutter flutter build web --no-web-resources-cdn cd ../../ diff --git a/pkg/rest/gizp_middleware.go b/pkg/rest/gizp_middleware.go new file mode 100644 index 00000000..8170db7b --- /dev/null +++ b/pkg/rest/gizp_middleware.go @@ -0,0 +1,34 @@ +package rest + +import ( + "compress/gzip" + "io" + "net/http" + "strings" +) + +type gzipResponseWriter struct { + io.Writer + http.ResponseWriter +} + +func (g gzipResponseWriter) Write(b []byte) (int, error) { + return g.Writer.Write(b) +} + +func gzipMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + next.ServeHTTP(w, r) + return + } + + w.Header().Set("Content-Encoding", "gzip") + w.Header().Add("Vary", "Accept-Encoding") + + gz := gzip.NewWriter(w) + defer gz.Close() + + next.ServeHTTP(gzipResponseWriter{Writer: gz, ResponseWriter: w}, r) + }) +} diff --git a/pkg/rest/server.go b/pkg/rest/server.go index 9c0836d4..1e4a10ec 100644 --- a/pkg/rest/server.go +++ b/pkg/rest/server.go @@ -10,12 +10,14 @@ import ( "github.com/gorilla/handlers" "github.com/gorilla/mux" "github.com/pkg/errors" + "io/fs" "net" "net/http" "os" "path/filepath" "regexp" "strings" + "time" ) var ( @@ -126,7 +128,7 @@ func BuildServer(startCfg *model.StartConfig) (*http.Server, net.Listener, error if startCfg.WebEnable { r.PathPrefix("/fs/tasks").Handler(http.FileServer(new(taskFileSystem))) r.PathPrefix("/fs/extensions").Handler(http.FileServer(new(extensionFileSystem))) - r.PathPrefix("/").Handler(http.FileServer(http.FS(startCfg.WebFS))) + r.PathPrefix("/").Handler(gzipMiddleware(http.FileServer(newEmbedCacheFileSystem(http.FS(startCfg.WebFS))))) } if startCfg.ApiToken != "" || (startCfg.WebEnable && startCfg.WebBasicAuth != nil) { @@ -222,6 +224,68 @@ func (e *extensionFileSystem) Open(name string) (http.File, error) { return os.Open(filepath.Join(extensionPath, path)) } +type embedCacheFileSystem struct { + fs http.FileSystem + lastModTime time.Time +} + +func newEmbedCacheFileSystem(fs http.FileSystem) *embedCacheFileSystem { + efs := &embedCacheFileSystem{ + fs: fs, + lastModTime: time.Now(), + } + + exe, err := os.Executable() + if err != nil { + return efs + } + + fi, err := os.Stat(exe) + if err != nil { + return efs + } + + efs.lastModTime = fi.ModTime() + return efs +} + +func (e *embedCacheFileSystem) Open(name string) (http.File, error) { + file, err := e.fs.Open(name) + if err != nil { + return nil, err + } + + return &embedFile{ + File: file, + lastModTime: e.lastModTime, + }, nil +} + +type embedFile struct { + http.File + lastModTime time.Time +} + +type embedFileInfo struct { + fs.FileInfo + lastModTime time.Time +} + +func (e *embedFileInfo) ModTime() time.Time { + return e.lastModTime +} + +func (e *embedFile) Stat() (fs.FileInfo, error) { + fi, err := e.File.Stat() + if err != nil { + return nil, err + } + return &embedFileInfo{ + FileInfo: fi, + lastModTime: e.lastModTime, + }, nil +} + func ReadJson(r *http.Request, w http.ResponseWriter, v any) bool { if err := json.NewDecoder(r.Body).Decode(v); err != nil { WriteJson(w, model.NewErrorResult(err.Error())) diff --git a/pkg/rest/server_test.go b/pkg/rest/server_test.go index 930d9817..9dee79f3 100644 --- a/pkg/rest/server_test.go +++ b/pkg/rest/server_test.go @@ -14,6 +14,7 @@ import ( "net" "net/http" "os" + "path/filepath" "reflect" "sync" "testing" @@ -478,6 +479,66 @@ func TestFsExtensionFail(t *testing.T) { }) } +func TestWebFsEnhance(t *testing.T) { + indexHtml := ` + + + + + index + + +

index

+ + +` + webDistPath := "dist" + os.MkdirAll("dist", os.ModePerm) + if err := os.WriteFile(filepath.Join(webDistPath, "index.html"), []byte(indexHtml), os.ModePerm); err != nil { + panic(err) + } + defer os.RemoveAll(webDistPath) + + doTest0(func(cfg *model.StartConfig) { + cfg.WebFS = os.DirFS(webDistPath) + }, func() { + // First request no cache + code, header, _ := doHttpRequest1(http.MethodGet, "/index.html", map[string]string{ + "Accept-Encoding": "gzip", + }, nil) + if code != http.StatusOK { + t.Errorf("TestWebFsEnhance() got = %v, want %v", code, http.StatusOK) + } + // Check header last-modified + if _, ok := header["Last-Modified"]; !ok { + t.Errorf("TestWebFsEnhance() missing key = %v", "Last-Modified") + } + // Check gzip compress + if _, ok := header["Content-Encoding"]; !ok || header["Content-Encoding"] != "gzip" { + t.Errorf("TestWebFsEnhance() no gzip compress") + } + + // Request with If-Modified-Since + ifModifiedSince := header["Last-Modified"] + code, _, _ = doHttpRequest1(http.MethodGet, "/index.html", map[string]string{ + "If-Modified-Since": ifModifiedSince, + }, nil) + if code != http.StatusNotModified { + t.Errorf("TestWebFsEnhance() got = %v, want %v", code, http.StatusNotModified) + } + + // Request with un gzip + code, header, _ = doHttpRequest1(http.MethodGet, "/index.html?t=123", nil, nil) + if code != http.StatusOK { + t.Errorf("TestWebFsEnhance() got = %v, want %v", code, http.StatusOK) + } + // Check no gzip compress + if _, ok := header["Content-Encoding"]; ok && header["Content-Encoding"] == "gzip" { + t.Errorf("TestWebFsEnhance() has gzip compress") + } + }) +} + func TestDoProxy(t *testing.T) { doTest(func() { code, respBody := doHttpRequest0(http.MethodGet, "/api/v1/proxy", map[string]string{ @@ -567,12 +628,19 @@ func TestAuthorization(t *testing.T) { } func doTest(handler func()) { + doTest0(nil, handler) +} + +func doTest0(onStart func(cfg *model.StartConfig), handler func()) { testFunc := func(storage model.Storage) { var cfg = &model.StartConfig{} cfg.Init() cfg.Storage = storage cfg.StorageDir = ".test_storage" cfg.WebEnable = true + if onStart != nil { + onStart(cfg) + } fileListener := doStart(cfg) defer func() { if err := fileListener.Close(); err != nil { @@ -603,6 +671,11 @@ func doStart(cfg *model.StartConfig) net.Listener { } func doHttpRequest0(method string, path string, headers map[string]string, body any) (int, []byte) { + r1, _, r3 := doHttpRequest1(method, path, headers, body) + return r1, r3 +} + +func doHttpRequest1(method string, path string, headers map[string]string, body any) (int, map[string]string, []byte) { var reader io.Reader if body != nil { buf, _ := json.Marshal(body) @@ -623,12 +696,18 @@ func doHttpRequest0(method string, path string, headers map[string]string, body panic(err) } defer response.Body.Close() + + respHeader := make(map[string]string) + for k, vv := range response.Header { + respHeader[k] = vv[0] + } + respBody, err := io.ReadAll(response.Body) if err != nil { panic(err) } - return response.StatusCode, respBody + return response.StatusCode, respHeader, respBody } func doHttpRequest[T any](method string, path string, headers map[string]string, body any) (int, *model.Result[T]) {