diff --git a/pkg/rest/server.go b/pkg/rest/server.go
index 1e4a10ec..0d7c4dc9 100644
--- a/pkg/rest/server.go
+++ b/pkg/rest/server.go
@@ -131,13 +131,25 @@ func BuildServer(startCfg *model.StartConfig) (*http.Server, net.Listener, error
 		r.PathPrefix("/").Handler(gzipMiddleware(http.FileServer(newEmbedCacheFileSystem(http.FS(startCfg.WebFS)))))
 	}
 
-	if startCfg.ApiToken != "" || (startCfg.WebEnable && startCfg.WebBasicAuth != nil) {
+	enableApiToken := startCfg.ApiToken != ""
+	enableBasicAuth := startCfg.WebEnable && startCfg.WebBasicAuth != nil
+	if enableApiToken || enableBasicAuth {
 		r.Use(func(h http.Handler) http.Handler {
 			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				if startCfg.ApiToken != "" && r.Header.Get("X-Api-Token") == startCfg.ApiToken {
-					h.ServeHTTP(w, r)
-					return
+				if enableApiToken {
+					apiTokenHeader := r.Header["X-Api-Token"]
+					// If api token header is set, only check api token ignore basic auth
+					if len(apiTokenHeader) > 0 {
+						if apiTokenHeader[0] == startCfg.ApiToken {
+							h.ServeHTTP(w, r)
+							return
+						}
+
+						WriteStatusJson(w, http.StatusUnauthorized, model.NewErrorResult("unauthorized", model.CodeUnauthorized))
+						return
+					}
 				}
+
 				if startCfg.WebEnable && startCfg.WebBasicAuth != nil {
 					if r.Header.Get("Authorization") == startCfg.WebBasicAuth.Authorization() {
 						h.ServeHTTP(w, r)
diff --git a/pkg/rest/server_test.go b/pkg/rest/server_test.go
index 9dee79f3..6e2e9228 100644
--- a/pkg/rest/server_test.go
+++ b/pkg/rest/server_test.go
@@ -625,6 +625,22 @@ func TestAuthorization(t *testing.T) {
 	if status != http.StatusOK {
 		t.Errorf("TestAuthorization() got = %v, want %v", status, http.StatusOK)
 	}
+
+	status, _ = doHttpRequest0(http.MethodGet, "/api/v1/config", map[string]string{
+		"Authorization": cfg.WebBasicAuth.Authorization(),
+		"X-Api-Token":   cfg.ApiToken,
+	}, nil)
+	if status != http.StatusOK {
+		t.Errorf("TestAuthorization() got = %v, want %v", status, http.StatusOK)
+	}
+
+	status, _ = doHttpRequest0(http.MethodGet, "/api/v1/config", map[string]string{
+		"Authorization": cfg.WebBasicAuth.Authorization(),
+		"X-Api-Token":   "",
+	}, nil)
+	if status != http.StatusUnauthorized {
+		t.Errorf("TestAuthorization() got = %v, want %v", status, http.StatusUnauthorized)
+	}
 }
 
 func doTest(handler func()) {