优化部分代码，补充未实现功能

README.md

@@ -7,10 +7,10 @@
 ## Feature
 
 - [x] 内容缓存
+- [x] CNAME 自定义域名
 
 ## TODO
 
-- [ ] CNAME 自定义域名
 - [ ] http01 自动签发证书 (CNAME)
 - [ ] Web 钩子触发更新
 - [ ] OAuth2 授权访问私有页面

config.go

@@ -1,17 +1,118 @@
 package main
 
 import (
+	_ "embed"
+	"html/template"
+	"net/http"
+	"os"
+	"path/filepath"
 	"time"
+
+	"github.com/alecthomas/units"
+
+	"code.d7z.net/d7z-project/gitea-pages/pkg"
+	"code.d7z.net/d7z-project/gitea-pages/pkg/utils"
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	"gopkg.in/yaml.v3"
 )
 
-type Config struct {
+//go:embed errors.html.tmpl
-	Bind string `yaml:"bind"` // HTTP 绑定
+var defaultErrPage string
 
+type Config struct {
+	Bind   string `yaml:"bind"`   // HTTP 绑定
 	Domain string `yaml:"domain"` // 基础域名
 
-	Auth ConfigAuth `yaml:"auth"` // 后端认证配置
+	Auth  ConfigAuth  `yaml:"auth"`  // 后端认证配置
-
 	Cache ConfigCache `yaml:"cache"` // 缓存配置
+	Page  ConfigPage  `yaml:"page"`  // 页面配置
+
+	pageErrNotFound, pageErrUnknown *template.Template
+}
+
+func (c *Config) NewPageServerOptions() (*pkg.ServerOptions, error) {
+	if c.Domain == "" {
+		return nil, errors.New("domain is required")
+	}
+	var err error
+	var cacheSize, cacheMaxSize units.Base2Bytes
+	cacheSize, err = units.ParseBase2Bytes(c.Cache.FileSize)
+	if err != nil {
+		return nil, errors.Wrap(err, "parse cache size")
+	}
+
+	cacheMaxSize, err = units.ParseBase2Bytes(c.Cache.MaxSize)
+	if err != nil {
+		return nil, errors.Wrap(err, "parse cache max size")
+	}
+	if cacheMaxSize <= cacheSize {
+		return nil, errors.New("cache max size must be greater than or equal to file max size")
+	}
+	if c.Page.DefaultBranch == "" {
+		c.Page.DefaultBranch = "gh-pages"
+	}
+	defaultErr := template.Must(template.New("err").Parse(defaultErrPage))
+	if c.Page.ErrUnknownPage != "" {
+		data, err := os.ReadFile(c.Page.ErrUnknownPage)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to read file %s", string(data))
+		}
+		c.pageErrUnknown = template.Must(template.New("err").Parse(c.Page.ErrUnknownPage))
+	} else {
+		c.pageErrUnknown = defaultErr
+	}
+	if c.Page.ErrNotFoundPage != "" {
+		data, err := os.ReadFile(c.Page.ErrNotFoundPage)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to read file %s", c.Page.ErrNotFoundPage)
+		}
+		c.pageErrNotFound = template.Must(template.New("err").Parse(string(data)))
+	} else {
+		c.pageErrNotFound = defaultErr
+	}
+
+	rel := pkg.ServerOptions{
+		Domain:              c.Domain,
+		DefaultBranch:       c.Page.DefaultBranch,
+		MaxCacheSize:        int(cacheSize),
+		HttpClient:          http.DefaultClient,
+		DefaultErrorHandler: c.ErrorHandler,
+		Cache:               utils.NewCacheMemory(int(cacheMaxSize), int(cacheMaxSize)),
+	}
+	if c.Cache.Storage != "" {
+		if err := os.MkdirAll(filepath.Dir(c.Cache.Storage), 0o755); err != nil && !os.IsExist(err) {
+			return nil, err
+		}
+	}
+	memory, err := utils.NewConfigMemory(c.Cache.Storage)
+	if err != nil {
+		return nil, err
+	}
+	rel.Config = memory
+	return &rel, nil
+}
+
+func (c *Config) ErrorHandler(w http.ResponseWriter, r *http.Request, err error) {
+	if errors.Is(err, os.ErrNotExist) {
+		w.WriteHeader(http.StatusNotFound)
+		if err = c.pageErrNotFound.Execute(w, map[string]any{
+			"err":  err,
+			"req":  r,
+			"code": 404,
+		}); err != nil {
+			zap.L().Error("failed to render error page", zap.Error(err))
+		}
+	} else {
+		w.WriteHeader(http.StatusInternalServerError)
+		if err = c.pageErrUnknown.Execute(w, map[string]any{
+			"err":  err,
+			"req":  r,
+			"code": 500,
+		}); err != nil {
+			zap.L().Error("failed to render error page", zap.Error(err))
+		}
+	}
 }
 
 type ConfigAuth struct {
@@ -21,8 +122,31 @@ type ConfigAuth struct {
 	Token string `yaml:"token"`
 }
 
-type ConfigCache struct {
+type ConfigPage struct {
-	ttl        time.Duration `yaml:"ttl"`         // 缓存时间
+	DefaultBranch   string `yaml:"default_branch"`
-	singleSize int           `yaml:"single_size"` // 单个文件最大大小
+	ErrNotFoundPage string `yaml:"404"`
-	maxSize    int           `yaml:"max_size"`    // 最大文件大小
+	ErrUnknownPage  string `yaml:"500"`
+}
+
+type ConfigCache struct {
+	Storage string `yaml:"storage"` // 缓存归档位置
+
+	Ttl      time.Duration `yaml:"ttl"`  // 缓存时间
+	FileSize string        `yaml:"size"` // 单个文件最大大小
+	MaxSize  string        `yaml:"max"`  // 最大文件大小
+}
+
+func LoadConfig(path string) (*Config, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	var config Config
+	decoder := yaml.NewDecoder(f)
+	err = decoder.Decode(&config)
+	if err != nil {
+		return nil, err
+	}
+	return &config, nil
 }

errors.html.tmpl

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport"
+          content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
+    <meta http-equiv="X-UA-Compatible" content="ie=edge">
+{{ if eq .code 404 }}<title>404 Not Found</title>{{ else }}<title>500 Unknown Error</title>{{ end }}
+</head>
+<Body>
+<div style="text-align: center;">
+    {{ if eq .code 404 }}<h1>404 Not Found</h1>{{ else }}<h1>500 Unknown Error</h1>{{ end }}
+</div>
+<hr>
+<div style="text-align: center;">Gitea Pages</div>
+</Body>
+</html>

go.mod

@@ -4,10 +4,11 @@ go 1.23.2
 
 require (
 	code.gitea.io/sdk/gitea v0.19.0
-	github.com/stretchr/testify v1.8.1
+	github.com/stretchr/testify v1.9.0
 )
 
 require (
+	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/davidmz/go-pageant v1.0.2 // indirect
 	github.com/go-fed/httpsig v1.1.0 // indirect

go.sum

@@ -1,5 +1,7 @@
 code.gitea.io/sdk/gitea v0.19.0 h1:8I6s1s4RHgzxiPHhOQdgim1RWIRcr0LVMbHBjBFXq4Y=
 code.gitea.io/sdk/gitea v0.19.0/go.mod h1:IG9xZJoltDNeDSW0qiF2Vqx5orMWa7OhVWrjvrd5NpI=
+github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0=
+github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -18,10 +20,13 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=

main.go

@@ -13,14 +13,11 @@ import (
 
 	"code.d7z.net/d7z-project/gitea-pages/pkg"
 	"code.d7z.net/d7z-project/gitea-pages/pkg/providers"
-
-	"gopkg.in/yaml.v3"
 )
 
 var (
 	configPath = "config-local.yaml"
 	debug      = false
-	config     = &Config{}
 )
 
 func init() {
@@ -32,12 +29,19 @@ func main() {
 	flag.Parse()
 	call := logInject()
 	defer call()
-	loadConf()
+	config, err := LoadConfig(configPath)
+	if err != nil {
+		log.Fatalf("fail to load config file: %v", err)
+	}
+	options, err := config.NewPageServerOptions()
+	if err != nil {
+		log.Fatalf("fail to create page server: %v", err)
+	}
 	gitea, err := providers.NewGitea(config.Auth.Server, config.Auth.Token)
 	if err != nil {
 		log.Fatalln(err)
 	}
-	giteaServer := pkg.NewPageServer(gitea, pkg.DefaultOptions(config.Domain))
+	giteaServer := pkg.NewPageServer(gitea, *options)
 	defer giteaServer.Close()
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
 	defer stop()
@@ -67,14 +71,3 @@ func logInject() func() error {
 	zap.L().Debug("debug enabled")
 	return logger.Sync
 }
-
-func loadConf() {
-	file, err := os.ReadFile(configPath)
-	if err != nil {
-		log.Fatalf("read config file failed: %v", err)
-	}
-	err = yaml.Unmarshal(file, &config)
-	if err != nil {
-		log.Fatalf("parse config file failed: %v", err)
-	}
-}

pkg/core/backend.go

@@ -147,22 +147,24 @@ func (c *CacheBackendBlobReader) Open(owner, repo, commit, path string) (io.Read
 		return open.Body, nil
 	}
 	// 没法计算大小，跳过
-	lengthStr := open.Header.Get("Content-Length")
+	length, err := strconv.Atoi(open.Header.Get("Content-Length"))
-	if lengthStr == "" {
+	if err != nil {
-		return open.Body, nil
-	}
-	length, err := strconv.Atoi(lengthStr)
-	if err != nil || length > c.maxSize {
-		// 超过最大大小，跳过
 		return open.Body, err
 	}
+	if length > c.maxSize {
+		// 超过最大大小，跳过
+		return &utils.SizeReadCloser{
+			ReadCloser: open.Body,
+			Size:       length,
+		}, err
+	}
 	defer open.Body.Close()
 	allBytes, err := io.ReadAll(open.Body)
 	if err != nil {
 		return nil, err
 	}
 	if err = c.cache.Put(key, bytes.NewBuffer(allBytes)); err != nil {
-		zap.S().Warn("缓存归档失败", zap.Error(err))
+		zap.L().Warn("缓存归档失败", zap.Error(err), zap.Int("Size", len(allBytes)), zap.Int("MaxSize", c.maxSize))
 	}
 	return &utils.CacheContent{
 		ReadSeekCloser: utils.NopCloser{

pkg/server.go

@@ -6,12 +6,15 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strconv"
 	"time"
 
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+
 	"code.d7z.net/d7z-project/gitea-pages/pkg/core"
 	"code.d7z.net/d7z-project/gitea-pages/pkg/utils"
 	"github.com/pbnjay/memory"
-	"github.com/pkg/errors"
 )
 
 type ServerOptions struct {
@@ -24,6 +27,8 @@ type ServerOptions struct {
 	MaxCacheSize int
 
 	HttpClient *http.Client
+
+	DefaultErrorHandler func(w http.ResponseWriter, r *http.Request, err error)
 }
 
 func DefaultOptions(domain string) ServerOptions {
@@ -35,12 +40,19 @@ func DefaultOptions(domain string) ServerOptions {
 		Cache:         utils.NewCacheMemory(1024*1024*10, int(memory.FreeMemory()/3*2)),
 		MaxCacheSize:  1024 * 1024 * 10,
 		HttpClient:    http.DefaultClient,
+		DefaultErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
+			if errors.Is(err, os.ErrNotExist) {
+				http.Error(w, "page not found.", http.StatusNotFound)
+			} else if err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+			}
+		},
 	}
 }
 
 type Server struct {
-	meta    *core.PageDomain
 	options *ServerOptions
+	meta    *core.PageDomain
 	reader  *core.CacheBackendBlobReader
 }
 
@@ -50,18 +62,24 @@ func NewPageServer(backend core.Backend, options ServerOptions) *Server {
 	pageMeta := core.NewPageDomain(svcMeta, options.Config, options.Domain, options.DefaultBranch)
 	reader := core.NewCacheBackendBlobReader(options.HttpClient, backend, options.Cache, options.MaxCacheSize)
 	return &Server{
-		meta:    pageMeta,
 		options: &options,
+		meta:    pageMeta,
 		reader:  reader,
 	}
 }
 
 func (s *Server) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
+	defer func() {
+		if e := recover(); e != nil {
+			zap.L().Error("panic!", zap.Any("error", e))
+			if err, ok := e.(error); ok {
+				s.options.DefaultErrorHandler(writer, request, err)
+			}
+		}
+	}()
 	err := s.Serve(writer, request)
-	if errors.Is(err, os.ErrNotExist) {
+	if err != nil {
-		http.Error(writer, "page not found.", http.StatusNotFound)
+		s.options.DefaultErrorHandler(writer, request, err)
-	} else {
-		http.Error(writer, err.Error(), http.StatusInternalServerError)
 	}
 }
 
@@ -70,17 +88,41 @@ func (s *Server) Serve(writer http.ResponseWriter, request *http.Request) error
 	if err != nil {
 		return err
 	}
+	zap.L().Debug("获取请求", zap.Any("meta", meta))
+	// todo(feat) : 支持 http range
 	result, err := s.reader.Open(meta.Owner, meta.Repo, meta.CommitID, meta.Path)
 	if err != nil {
-		return err
+		if meta.HistoryRouteMode && errors.Is(err, os.ErrNotExist) {
+			result, err = s.reader.Open(meta.Owner, meta.Repo, meta.CommitID, "index.html")
+		} else {
+			return err
+		}
+	}
+	if err != nil && meta.CustomNotFound && errors.Is(err, os.ErrNotExist) {
+		// 存在 404 页面的情况
+		result, err = s.reader.Open(meta.Owner, meta.Repo, meta.CommitID, "404.html")
+		if err != nil {
+			return err
+		}
+		writer.Header().Set("Content-Type", mime.TypeByExtension(".html"))
+		writer.WriteHeader(http.StatusNotFound)
+		_, _ = io.Copy(writer, result)
+		_ = result.Close()
+		return nil
 	}
 	fileName := filepath.Base(meta.Path)
 	if reader, ok := result.(*utils.CacheContent); ok {
 		writer.Header().Add("X-Cache", "HIT")
+		writer.Header().Add("Cache-Control", "public, max-age=86400")
 		http.ServeContent(writer, request, fileName, reader.LastModified, reader)
 		_ = reader.Close()
 	} else {
+		if reader, ok := result.(*utils.SizeReadCloser); ok {
+			writer.Header().Add("Content-Length", strconv.Itoa(reader.Size))
+		}
+		// todo(bug) : 直连模式下告知数据长度
 		writer.Header().Add("X-Cache", "MISS")
+		writer.Header().Add("Cache-Control", "public, max-age=86400")
 		writer.Header().Set("Content-Type", mime.TypeByExtension(meta.Path))
 		writer.WriteHeader(http.StatusOK)
 		_, _ = io.Copy(writer, result)

pkg/utils/reader.go

@@ -0,0 +1,8 @@
+package utils
+
+import "io"
+
+type SizeReadCloser struct {
+	io.ReadCloser
+	Size int
+}