feat: web http file server caching and gzip support (#961)

.github/workflows/build.yml

@@ -529,7 +529,6 @@ jobs:
         env:
           VERSION: ${{ needs.get-release.outputs.version }}
         run: |
-          VERSION=${VERSION:-$(date +%y%m%d%H%M)} # Use current time as version if not specified
           cd ui/flutter
           flutter build web --no-web-resources-cdn
           cd ../../

pkg/rest/gizp_middleware.go

@@ -0,0 +1,34 @@
+package rest
+
+import (
+	"compress/gzip"
+	"io"
+	"net/http"
+	"strings"
+)
+
+type gzipResponseWriter struct {
+	io.Writer
+	http.ResponseWriter
+}
+
+func (g gzipResponseWriter) Write(b []byte) (int, error) {
+	return g.Writer.Write(b)
+}
+
+func gzipMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+			next.ServeHTTP(w, r)
+			return
+		}
+
+		w.Header().Set("Content-Encoding", "gzip")
+		w.Header().Add("Vary", "Accept-Encoding")
+
+		gz := gzip.NewWriter(w)
+		defer gz.Close()
+
+		next.ServeHTTP(gzipResponseWriter{Writer: gz, ResponseWriter: w}, r)
+	})
+}

pkg/rest/server.go

@@ -10,12 +10,14 @@ import (
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
+	"io/fs"
 	"net"
 	"net/http"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
+	"time"
 )
 
 var (
@@ -126,7 +128,7 @@ func BuildServer(startCfg *model.StartConfig) (*http.Server, net.Listener, error
 	if startCfg.WebEnable {
 		r.PathPrefix("/fs/tasks").Handler(http.FileServer(new(taskFileSystem)))
 		r.PathPrefix("/fs/extensions").Handler(http.FileServer(new(extensionFileSystem)))
-		r.PathPrefix("/").Handler(http.FileServer(http.FS(startCfg.WebFS)))
+		r.PathPrefix("/").Handler(gzipMiddleware(http.FileServer(newEmbedCacheFileSystem(http.FS(startCfg.WebFS)))))
 	}
 
 	if startCfg.ApiToken != "" || (startCfg.WebEnable && startCfg.WebBasicAuth != nil) {
@@ -222,6 +224,68 @@ func (e *extensionFileSystem) Open(name string) (http.File, error) {
 	return os.Open(filepath.Join(extensionPath, path))
 }
 
+type embedCacheFileSystem struct {
+	fs          http.FileSystem
+	lastModTime time.Time
+}
+
+func newEmbedCacheFileSystem(fs http.FileSystem) *embedCacheFileSystem {
+	efs := &embedCacheFileSystem{
+		fs:          fs,
+		lastModTime: time.Now(),
+	}
+
+	exe, err := os.Executable()
+	if err != nil {
+		return efs
+	}
+
+	fi, err := os.Stat(exe)
+	if err != nil {
+		return efs
+	}
+
+	efs.lastModTime = fi.ModTime()
+	return efs
+}
+
+func (e *embedCacheFileSystem) Open(name string) (http.File, error) {
+	file, err := e.fs.Open(name)
+	if err != nil {
+		return nil, err
+	}
+
+	return &embedFile{
+		File:        file,
+		lastModTime: e.lastModTime,
+	}, nil
+}
+
+type embedFile struct {
+	http.File
+	lastModTime time.Time
+}
+
+type embedFileInfo struct {
+	fs.FileInfo
+	lastModTime time.Time
+}
+
+func (e *embedFileInfo) ModTime() time.Time {
+	return e.lastModTime
+}
+
+func (e *embedFile) Stat() (fs.FileInfo, error) {
+	fi, err := e.File.Stat()
+	if err != nil {
+		return nil, err
+	}
+	return &embedFileInfo{
+		FileInfo:    fi,
+		lastModTime: e.lastModTime,
+	}, nil
+}
+
 func ReadJson(r *http.Request, w http.ResponseWriter, v any) bool {
 	if err := json.NewDecoder(r.Body).Decode(v); err != nil {
 		WriteJson(w, model.NewErrorResult(err.Error()))

pkg/rest/server_test.go

@@ -14,6 +14,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"path/filepath"
 	"reflect"
 	"sync"
 	"testing"
@@ -478,6 +479,66 @@ func TestFsExtensionFail(t *testing.T) {
 	})
 }
 
+func TestWebFsEnhance(t *testing.T) {
+	indexHtml := `
+<!DOCTYPE html>
+<html lang="en">
+<head>
+	<meta charset="UTF-8">
+	<title>index</title>
+</head>
+<body>
+	<h1>index</h1>
+</body>
+</html>
+`
+	webDistPath := "dist"
+	os.MkdirAll("dist", os.ModePerm)
+	if err := os.WriteFile(filepath.Join(webDistPath, "index.html"), []byte(indexHtml), os.ModePerm); err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(webDistPath)
+
+	doTest0(func(cfg *model.StartConfig) {
+		cfg.WebFS = os.DirFS(webDistPath)
+	}, func() {
+		// First request no cache
+		code, header, _ := doHttpRequest1(http.MethodGet, "/index.html", map[string]string{
+			"Accept-Encoding": "gzip",
+		}, nil)
+		if code != http.StatusOK {
+			t.Errorf("TestWebFsEnhance() got = %v, want %v", code, http.StatusOK)
+		}
+		// Check header last-modified
+		if _, ok := header["Last-Modified"]; !ok {
+			t.Errorf("TestWebFsEnhance() missing key = %v", "Last-Modified")
+		}
+		// Check gzip compress
+		if _, ok := header["Content-Encoding"]; !ok || header["Content-Encoding"] != "gzip" {
+			t.Errorf("TestWebFsEnhance() no gzip compress")
+		}
+
+		// Request with If-Modified-Since
+		ifModifiedSince := header["Last-Modified"]
+		code, _, _ = doHttpRequest1(http.MethodGet, "/index.html", map[string]string{
+			"If-Modified-Since": ifModifiedSince,
+		}, nil)
+		if code != http.StatusNotModified {
+			t.Errorf("TestWebFsEnhance() got = %v, want %v", code, http.StatusNotModified)
+		}
+
+		// Request with un gzip
+		code, header, _ = doHttpRequest1(http.MethodGet, "/index.html?t=123", nil, nil)
+		if code != http.StatusOK {
+			t.Errorf("TestWebFsEnhance() got = %v, want %v", code, http.StatusOK)
+		}
+		// Check no gzip compress
+		if _, ok := header["Content-Encoding"]; ok && header["Content-Encoding"] == "gzip" {
+			t.Errorf("TestWebFsEnhance() has gzip compress")
+		}
+	})
+}
+
 func TestDoProxy(t *testing.T) {
 	doTest(func() {
 		code, respBody := doHttpRequest0(http.MethodGet, "/api/v1/proxy", map[string]string{
@@ -567,12 +628,19 @@ func TestAuthorization(t *testing.T) {
 }
 
 func doTest(handler func()) {
+	doTest0(nil, handler)
+}
+
+func doTest0(onStart func(cfg *model.StartConfig), handler func()) {
 	testFunc := func(storage model.Storage) {
 		var cfg = &model.StartConfig{}
 		cfg.Init()
 		cfg.Storage = storage
 		cfg.StorageDir = ".test_storage"
 		cfg.WebEnable = true
+		if onStart != nil {
+			onStart(cfg)
+		}
 		fileListener := doStart(cfg)
 		defer func() {
 			if err := fileListener.Close(); err != nil {
@@ -603,6 +671,11 @@ func doStart(cfg *model.StartConfig) net.Listener {
 }
 
 func doHttpRequest0(method string, path string, headers map[string]string, body any) (int, []byte) {
+	r1, _, r3 := doHttpRequest1(method, path, headers, body)
+	return r1, r3
+}
+
+func doHttpRequest1(method string, path string, headers map[string]string, body any) (int, map[string]string, []byte) {
 	var reader io.Reader
 	if body != nil {
 		buf, _ := json.Marshal(body)
@@ -623,12 +696,18 @@ func doHttpRequest0(method string, path string, headers map[string]string, body
 		panic(err)
 	}
 	defer response.Body.Close()
+
+	respHeader := make(map[string]string)
+	for k, vv := range response.Header {
+		respHeader[k] = vv[0]
+	}
+
 	respBody, err := io.ReadAll(response.Body)
 	if err != nil {
 		panic(err)
 	}
 
-	return response.StatusCode, respBody
+	return response.StatusCode, respHeader, respBody
 }
 
 func doHttpRequest[T any](method string, path string, headers map[string]string, body any) (int, *model.Result[T]) {