From e663684a0ade3427dac60bf0ede99e58f97ccd2e Mon Sep 17 00:00:00 2001 From: dragon Date: Fri, 10 Jan 2025 16:23:02 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=83=A8=E5=88=86=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=EF=BC=8C=E8=A1=A5=E5=85=85=E6=9C=AA=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- config.go | 140 +++++++++++++++++++++++++++++++++++++++++--- errors.html.tmpl | 17 ++++++ go.mod | 3 +- go.sum | 5 ++ main.go | 25 +++----- pkg/core/backend.go | 18 +++--- pkg/server.go | 58 +++++++++++++++--- pkg/utils/reader.go | 8 +++ 9 files changed, 234 insertions(+), 42 deletions(-) create mode 100644 errors.html.tmpl create mode 100644 pkg/utils/reader.go diff --git a/README.md b/README.md index 7c57363..6619237 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,10 @@ ## Feature - [x] 内容缓存 +- [x] CNAME 自定义域名 ## TODO -- [ ] CNAME 自定义域名 - [ ] http01 自动签发证书 (CNAME) - [ ] Web 钩子触发更新 - [ ] OAuth2 授权访问私有页面 diff --git a/config.go b/config.go index 60fc0cc..aa08193 100644 --- a/config.go +++ b/config.go @@ -1,17 +1,118 @@ package main import ( + _ "embed" + "html/template" + "net/http" + "os" + "path/filepath" "time" + + "github.com/alecthomas/units" + + "code.d7z.net/d7z-project/gitea-pages/pkg" + "code.d7z.net/d7z-project/gitea-pages/pkg/utils" + "github.com/pkg/errors" + "go.uber.org/zap" + "gopkg.in/yaml.v3" ) -type Config struct { - Bind string `yaml:"bind"` // HTTP 绑定 +//go:embed errors.html.tmpl +var defaultErrPage string +type Config struct { + Bind string `yaml:"bind"` // HTTP 绑定 Domain string `yaml:"domain"` // 基础域名 - Auth ConfigAuth `yaml:"auth"` // 后端认证配置 - + Auth ConfigAuth `yaml:"auth"` // 后端认证配置 Cache ConfigCache `yaml:"cache"` // 缓存配置 + Page ConfigPage `yaml:"page"` // 页面配置 + + pageErrNotFound, pageErrUnknown *template.Template +} + +func (c *Config) NewPageServerOptions() (*pkg.ServerOptions, error) { + if c.Domain == "" { + return nil, errors.New("domain is required") + } + var err error + var cacheSize, cacheMaxSize units.Base2Bytes + cacheSize, err = units.ParseBase2Bytes(c.Cache.FileSize) + if err != nil { + return nil, errors.Wrap(err, "parse cache size") + } + + cacheMaxSize, err = units.ParseBase2Bytes(c.Cache.MaxSize) + if err != nil { + return nil, errors.Wrap(err, "parse cache max size") + } + if cacheMaxSize <= cacheSize { + return nil, errors.New("cache max size must be greater than or equal to file max size") + } + if c.Page.DefaultBranch == "" { + c.Page.DefaultBranch = "gh-pages" + } + defaultErr := template.Must(template.New("err").Parse(defaultErrPage)) + if c.Page.ErrUnknownPage != "" { + data, err := os.ReadFile(c.Page.ErrUnknownPage) + if err != nil { + return nil, errors.Wrapf(err, "failed to read file %s", string(data)) + } + c.pageErrUnknown = template.Must(template.New("err").Parse(c.Page.ErrUnknownPage)) + } else { + c.pageErrUnknown = defaultErr + } + if c.Page.ErrNotFoundPage != "" { + data, err := os.ReadFile(c.Page.ErrNotFoundPage) + if err != nil { + return nil, errors.Wrapf(err, "failed to read file %s", c.Page.ErrNotFoundPage) + } + c.pageErrNotFound = template.Must(template.New("err").Parse(string(data))) + } else { + c.pageErrNotFound = defaultErr + } + + rel := pkg.ServerOptions{ + Domain: c.Domain, + DefaultBranch: c.Page.DefaultBranch, + MaxCacheSize: int(cacheSize), + HttpClient: http.DefaultClient, + DefaultErrorHandler: c.ErrorHandler, + Cache: utils.NewCacheMemory(int(cacheMaxSize), int(cacheMaxSize)), + } + if c.Cache.Storage != "" { + if err := os.MkdirAll(filepath.Dir(c.Cache.Storage), 0o755); err != nil && !os.IsExist(err) { + return nil, err + } + } + memory, err := utils.NewConfigMemory(c.Cache.Storage) + if err != nil { + return nil, err + } + rel.Config = memory + return &rel, nil +} + +func (c *Config) ErrorHandler(w http.ResponseWriter, r *http.Request, err error) { + if errors.Is(err, os.ErrNotExist) { + w.WriteHeader(http.StatusNotFound) + if err = c.pageErrNotFound.Execute(w, map[string]any{ + "err": err, + "req": r, + "code": 404, + }); err != nil { + zap.L().Error("failed to render error page", zap.Error(err)) + } + } else { + w.WriteHeader(http.StatusInternalServerError) + if err = c.pageErrUnknown.Execute(w, map[string]any{ + "err": err, + "req": r, + "code": 500, + }); err != nil { + zap.L().Error("failed to render error page", zap.Error(err)) + } + } } type ConfigAuth struct { @@ -21,8 +122,31 @@ type ConfigAuth struct { Token string `yaml:"token"` } -type ConfigCache struct { - ttl time.Duration `yaml:"ttl"` // 缓存时间 - singleSize int `yaml:"single_size"` // 单个文件最大大小 - maxSize int `yaml:"max_size"` // 最大文件大小 +type ConfigPage struct { + DefaultBranch string `yaml:"default_branch"` + ErrNotFoundPage string `yaml:"404"` + ErrUnknownPage string `yaml:"500"` +} + +type ConfigCache struct { + Storage string `yaml:"storage"` // 缓存归档位置 + + Ttl time.Duration `yaml:"ttl"` // 缓存时间 + FileSize string `yaml:"size"` // 单个文件最大大小 + MaxSize string `yaml:"max"` // 最大文件大小 +} + +func LoadConfig(path string) (*Config, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + var config Config + decoder := yaml.NewDecoder(f) + err = decoder.Decode(&config) + if err != nil { + return nil, err + } + return &config, nil } diff --git a/errors.html.tmpl b/errors.html.tmpl new file mode 100644 index 0000000..710a58c --- /dev/null +++ b/errors.html.tmpl @@ -0,0 +1,17 @@ + + + + + + +{{ if eq .code 404 }}404 Not Found{{ else }}500 Unknown Error{{ end }} + + +
+ {{ if eq .code 404 }}

404 Not Found

{{ else }}

500 Unknown Error

{{ end }} +
+
+
Gitea Pages
+ + \ No newline at end of file diff --git a/go.mod b/go.mod index 9ec68aa..7458737 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,11 @@ go 1.23.2 require ( code.gitea.io/sdk/gitea v0.19.0 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.9.0 ) require ( + github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidmz/go-pageant v1.0.2 // indirect github.com/go-fed/httpsig v1.1.0 // indirect diff --git a/go.sum b/go.sum index 139cccb..781338d 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ code.gitea.io/sdk/gitea v0.19.0 h1:8I6s1s4RHgzxiPHhOQdgim1RWIRcr0LVMbHBjBFXq4Y= code.gitea.io/sdk/gitea v0.19.0/go.mod h1:IG9xZJoltDNeDSW0qiF2Vqx5orMWa7OhVWrjvrd5NpI= +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -18,10 +20,13 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= diff --git a/main.go b/main.go index c5a130f..65fb7b3 100644 --- a/main.go +++ b/main.go @@ -13,14 +13,11 @@ import ( "code.d7z.net/d7z-project/gitea-pages/pkg" "code.d7z.net/d7z-project/gitea-pages/pkg/providers" - - "gopkg.in/yaml.v3" ) var ( configPath = "config-local.yaml" debug = false - config = &Config{} ) func init() { @@ -32,12 +29,19 @@ func main() { flag.Parse() call := logInject() defer call() - loadConf() + config, err := LoadConfig(configPath) + if err != nil { + log.Fatalf("fail to load config file: %v", err) + } + options, err := config.NewPageServerOptions() + if err != nil { + log.Fatalf("fail to create page server: %v", err) + } gitea, err := providers.NewGitea(config.Auth.Server, config.Auth.Token) if err != nil { log.Fatalln(err) } - giteaServer := pkg.NewPageServer(gitea, pkg.DefaultOptions(config.Domain)) + giteaServer := pkg.NewPageServer(gitea, *options) defer giteaServer.Close() ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT) defer stop() @@ -67,14 +71,3 @@ func logInject() func() error { zap.L().Debug("debug enabled") return logger.Sync } - -func loadConf() { - file, err := os.ReadFile(configPath) - if err != nil { - log.Fatalf("read config file failed: %v", err) - } - err = yaml.Unmarshal(file, &config) - if err != nil { - log.Fatalf("parse config file failed: %v", err) - } -} diff --git a/pkg/core/backend.go b/pkg/core/backend.go index 3accb20..d9fcd89 100644 --- a/pkg/core/backend.go +++ b/pkg/core/backend.go @@ -147,22 +147,24 @@ func (c *CacheBackendBlobReader) Open(owner, repo, commit, path string) (io.Read return open.Body, nil } // 没法计算大小，跳过 - lengthStr := open.Header.Get("Content-Length") - if lengthStr == "" { - return open.Body, nil - } - length, err := strconv.Atoi(lengthStr) - if err != nil || length > c.maxSize { - // 超过最大大小，跳过 + length, err := strconv.Atoi(open.Header.Get("Content-Length")) + if err != nil { return open.Body, err } + if length > c.maxSize { + // 超过最大大小，跳过 + return &utils.SizeReadCloser{ + ReadCloser: open.Body, + Size: length, + }, err + } defer open.Body.Close() allBytes, err := io.ReadAll(open.Body) if err != nil { return nil, err } if err = c.cache.Put(key, bytes.NewBuffer(allBytes)); err != nil { - zap.S().Warn("缓存归档失败", zap.Error(err)) + zap.L().Warn("缓存归档失败", zap.Error(err), zap.Int("Size", len(allBytes)), zap.Int("MaxSize", c.maxSize)) } return &utils.CacheContent{ ReadSeekCloser: utils.NopCloser{ diff --git a/pkg/server.go b/pkg/server.go index 878f1ce..06ce679 100644 --- a/pkg/server.go +++ b/pkg/server.go @@ -6,12 +6,15 @@ import ( "net/http" "os" "path/filepath" + "strconv" "time" + "github.com/pkg/errors" + "go.uber.org/zap" + "code.d7z.net/d7z-project/gitea-pages/pkg/core" "code.d7z.net/d7z-project/gitea-pages/pkg/utils" "github.com/pbnjay/memory" - "github.com/pkg/errors" ) type ServerOptions struct { @@ -24,6 +27,8 @@ type ServerOptions struct { MaxCacheSize int HttpClient *http.Client + + DefaultErrorHandler func(w http.ResponseWriter, r *http.Request, err error) } func DefaultOptions(domain string) ServerOptions { @@ -35,12 +40,19 @@ func DefaultOptions(domain string) ServerOptions { Cache: utils.NewCacheMemory(1024*1024*10, int(memory.FreeMemory()/3*2)), MaxCacheSize: 1024 * 1024 * 10, HttpClient: http.DefaultClient, + DefaultErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { + if errors.Is(err, os.ErrNotExist) { + http.Error(w, "page not found.", http.StatusNotFound) + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + }, } } type Server struct { - meta *core.PageDomain options *ServerOptions + meta *core.PageDomain reader *core.CacheBackendBlobReader } @@ -50,18 +62,24 @@ func NewPageServer(backend core.Backend, options ServerOptions) *Server { pageMeta := core.NewPageDomain(svcMeta, options.Config, options.Domain, options.DefaultBranch) reader := core.NewCacheBackendBlobReader(options.HttpClient, backend, options.Cache, options.MaxCacheSize) return &Server{ - meta: pageMeta, options: &options, + meta: pageMeta, reader: reader, } } func (s *Server) ServeHTTP(writer http.ResponseWriter, request *http.Request) { + defer func() { + if e := recover(); e != nil { + zap.L().Error("panic!", zap.Any("error", e)) + if err, ok := e.(error); ok { + s.options.DefaultErrorHandler(writer, request, err) + } + } + }() err := s.Serve(writer, request) - if errors.Is(err, os.ErrNotExist) { - http.Error(writer, "page not found.", http.StatusNotFound) - } else { - http.Error(writer, err.Error(), http.StatusInternalServerError) + if err != nil { + s.options.DefaultErrorHandler(writer, request, err) } } @@ -70,17 +88,41 @@ func (s *Server) Serve(writer http.ResponseWriter, request *http.Request) error if err != nil { return err } + zap.L().Debug("获取请求", zap.Any("meta", meta)) + // todo(feat) : 支持 http range result, err := s.reader.Open(meta.Owner, meta.Repo, meta.CommitID, meta.Path) if err != nil { - return err + if meta.HistoryRouteMode && errors.Is(err, os.ErrNotExist) { + result, err = s.reader.Open(meta.Owner, meta.Repo, meta.CommitID, "index.html") + } else { + return err + } + } + if err != nil && meta.CustomNotFound && errors.Is(err, os.ErrNotExist) { + // 存在 404 页面的情况 + result, err = s.reader.Open(meta.Owner, meta.Repo, meta.CommitID, "404.html") + if err != nil { + return err + } + writer.Header().Set("Content-Type", mime.TypeByExtension(".html")) + writer.WriteHeader(http.StatusNotFound) + _, _ = io.Copy(writer, result) + _ = result.Close() + return nil } fileName := filepath.Base(meta.Path) if reader, ok := result.(*utils.CacheContent); ok { writer.Header().Add("X-Cache", "HIT") + writer.Header().Add("Cache-Control", "public, max-age=86400") http.ServeContent(writer, request, fileName, reader.LastModified, reader) _ = reader.Close() } else { + if reader, ok := result.(*utils.SizeReadCloser); ok { + writer.Header().Add("Content-Length", strconv.Itoa(reader.Size)) + } + // todo(bug) : 直连模式下告知数据长度 writer.Header().Add("X-Cache", "MISS") + writer.Header().Add("Cache-Control", "public, max-age=86400") writer.Header().Set("Content-Type", mime.TypeByExtension(meta.Path)) writer.WriteHeader(http.StatusOK) _, _ = io.Copy(writer, result) diff --git a/pkg/utils/reader.go b/pkg/utils/reader.go new file mode 100644 index 0000000..54934af --- /dev/null +++ b/pkg/utils/reader.go @@ -0,0 +1,8 @@ +package utils + +import "io" + +type SizeReadCloser struct { + io.ReadCloser + Size int +}