From 0172dceaa86afddf9a67ba0ac0a75562713e6a4a Mon Sep 17 00:00:00 2001 From: dragon Date: Tue, 7 Jan 2025 17:04:39 +0800 Subject: [PATCH] =?UTF-8?q?=E8=A1=A5=E5=85=85=E5=AE=9E=E7=8E=B0=E7=BB=86?= =?UTF-8?q?=E8=8A=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 5 ++- pkg/core/backend.go | 85 ++++++++++++++++++++++++++++++++++++++--- pkg/core/meta.go | 37 +++++++++--------- pkg/core/page.go | 77 +++++++++++++++++++++++++++++++++++-- pkg/providers/gitea.go | 19 ++++++--- pkg/server.go | 66 ++++++++++++++++++++++++++++---- pkg/utils/cache.go | 36 +++++++++++++---- pkg/utils/cache_test.go | 1 + pkg/utils/locker.go | 1 + 9 files changed, 278 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 642049a..bca7862 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,14 @@ > 新一代 Gitea Pages，替换之前的 caddy-gitea-proxy -注意，默认实现未考虑高并发高负载环境， +## Feature + +- [x] 内容缓存 ## TODO - [ ] CNAME 自定义域名 - [ ] http01 自动签发证书 (CNAME) - [ ] Web 钩子触发更新 -- [ ] 内容缓存 - [ ] OAuth2 授权访问私有页面 diff --git a/pkg/core/backend.go b/pkg/core/backend.go index 5415fc7..a3d6718 100644 --- a/pkg/core/backend.go +++ b/pkg/core/backend.go @@ -1,23 +1,32 @@ package core import ( + "bytes" "encoding/json" "errors" "fmt" + "io" + "log/slog" "net/http" "os" + "strconv" "time" "code.d7z.net/d7z-project/gitea-pages/pkg/utils" ) +type BranchInfo struct { + ID string `json:"id"` + LastModified time.Time `json:"last_modified"` +} + type Backend interface { // Repos return repo name + default branch Repos(owner string) (map[string]string, error) // Branches return branch + commit id - Branches(owner, repo string) (map[string]string, error) + Branches(owner, repo string) (map[string]*BranchInfo, error) // Open return file or error - Open(client *http.Client, owner, repo, commit, path string, headers map[string]string) (*http.Response, error) + Open(client *http.Client, owner, repo, commit, path string, headers http.Header) (*http.Response, error) } type CacheBackend struct { @@ -60,8 +69,8 @@ func (c *CacheBackend) Repos(owner string) (map[string]string, error) { return ret, nil } -func (c *CacheBackend) Branches(owner, repo string) (map[string]string, error) { - ret := make(map[string]string) +func (c *CacheBackend) Branches(owner, repo string) (map[string]*BranchInfo, error) { + ret := make(map[string]*BranchInfo) key := fmt.Sprintf("branches/%s/%s", owner, repo) data, err := c.config.Get(key) if err != nil { @@ -90,6 +99,72 @@ func (c *CacheBackend) Branches(owner, repo string) (map[string]string, error) { return ret, nil } -func (c *CacheBackend) Open(client *http.Client, owner, repo, commit, path string, headers map[string]string) (*http.Response, error) { +func (c *CacheBackend) Open(client *http.Client, owner, repo, commit, path string, headers http.Header) (*http.Response, error) { return c.backend.Open(client, owner, repo, commit, path, headers) } + +type CacheBackendBlobReader struct { + client *http.Client + cache utils.Cache + base Backend + maxSize int +} + +func NewCacheBackendBlobReader(client *http.Client, base Backend, cache utils.Cache, maxCacheSize int) *CacheBackendBlobReader { + return &CacheBackendBlobReader{client: client, base: base, cache: cache, maxSize: maxCacheSize} +} + +func (c *CacheBackendBlobReader) Open(owner, repo, commit, path string) (io.ReadCloser, error) { + key := fmt.Sprintf("%s/%s/%s%s", owner, repo, commit, path) + lastCache, err := c.cache.Get(key) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } else if lastCache == nil && err == nil { + // 边界缓存 + return nil, os.ErrNotExist + } else if lastCache != nil { + return lastCache, nil + } + open, err := c.base.Open(c.client, owner, repo, commit, path, http.Header{}) + if err != nil { + if open != nil { + if open.StatusCode == http.StatusNotFound { + // 缓存 404 路由 + _ = c.cache.Put(key, nil) + } + _ = open.Body.Close() + } + return nil, errors.Join(err, os.ErrNotExist) + } + + lastMod, err := time.Parse(http.TimeFormat, open.Header.Get("Last-Modified")) + if err != nil { + // 无时间，跳过 + return open.Body, nil + } + // 没法计算大小，跳过 + lengthStr := open.Header.Get("Content-Length") + if lengthStr == "" { + return open.Body, nil + } + length, err := strconv.Atoi(lengthStr) + if err != nil || length > c.maxSize { + // 超过最大大小，跳过 + return open.Body, err + } + defer open.Body.Close() + allBytes, err := io.ReadAll(open.Body) + if err != nil { + return nil, err + } + if err = c.cache.Put(key, bytes.NewBuffer(allBytes)); err != nil { + slog.Warn("缓存归档失败", "error", err) + } + return &utils.CacheContent{ + ReadSeekCloser: utils.NopCloser{ + ReadSeeker: bytes.NewReader(allBytes), + }, + LastModified: lastMod, + Length: length, + }, nil +} diff --git a/pkg/core/meta.go b/pkg/core/meta.go index 223b3c4..13ae92f 100644 --- a/pkg/core/meta.go +++ b/pkg/core/meta.go @@ -14,38 +14,39 @@ import ( ) type ServerMeta struct { - client *http.Client Backend - cache utils.Config - ttl time.Duration + + client *http.Client + cache utils.Config + ttl time.Duration locker *utils.Locker } -type PageMeta struct { - CommitID string `json:"id"` // 提交 COMMIT ID - IsPage bool `json:"pg"` // 是否为 Page - Domain string `json:"dm"` // 匹配的域名 - HistoryRouteMode bool `json:"rt"` // 路由模式 - CustomNotFound bool `json:"404"` // 注册了自定义 404 页面 - +type PageMetaContent struct { + CommitID string `json:"id"` // 提交 COMMIT ID + IsPage bool `json:"pg"` // 是否为 Page + Domain string `json:"dm"` // 匹配的域名 + HistoryRouteMode bool `json:"rt"` // 路由模式 + CustomNotFound bool `json:"404"` // 注册了自定义 404 页面 + LastModified time.Time `json:"up"` // 上次更新时间 } -func (m *PageMeta) From(data string) error { +func (m *PageMetaContent) From(data string) error { return json.Unmarshal([]byte(data), m) } -func (m *PageMeta) String() string { +func (m *PageMetaContent) String() string { marshal, _ := json.Marshal(m) return string(marshal) } func NewServerMeta(client *http.Client, backend Backend, config utils.Config, ttl time.Duration) *ServerMeta { - return &ServerMeta{client, backend, config, ttl, utils.NewLocker()} + return &ServerMeta{backend, client, config, ttl, utils.NewLocker()} } -func (s *ServerMeta) GetMeta(owner, repo, branch string) (*PageMeta, error) { - rel := &PageMeta{ +func (s *ServerMeta) GetMeta(owner, repo, branch string) (*PageMetaContent, error) { + rel := &PageMetaContent{ IsPage: false, } if repos, err := s.Repos(owner); err != nil { @@ -62,10 +63,12 @@ func (s *ServerMeta) GetMeta(owner, repo, branch string) (*PageMeta, error) { if branches, err := s.Branches(owner, repo); err != nil { return nil, err } else { - rel.CommitID = branches[branch] - if rel.CommitID == "" { + info := branches[branch] + if info == nil { return nil, os.ErrNotExist } + rel.CommitID = info.ID + rel.LastModified = info.LastModified } key := fmt.Sprintf("meta/%s/%s/%s", owner, repo, branch) diff --git a/pkg/core/page.go b/pkg/core/page.go index f692a5e..4c71889 100644 --- a/pkg/core/page.go +++ b/pkg/core/page.go @@ -1,9 +1,78 @@ package core -type PageContent struct { - meta PageMeta +import ( + "errors" + "fmt" + "os" + "strings" +) + +type PageDomain struct { + *ServerMeta + + baseDomain string + defaultBranch string } -func (p *PageContent) GetMeta(domain, path string) (*PageMeta, error) { - +func NewPageDomain(meta *ServerMeta, baseDomain, defaultBranch string) *PageDomain { + return &PageDomain{ + baseDomain: baseDomain, + defaultBranch: defaultBranch, + ServerMeta: meta, + } +} + +type PageDomainContent struct { + *PageMetaContent + + Owner string + Repo string + Path string +} + +func (m *PageDomainContent) CacheKey() string { + return fmt.Sprintf("%s/%s/%s%s", m.Owner, m.Repo, m.CommitID, m.Path) +} + +func (p *PageDomain) ParseDomainMeta(domain, path, branch string) (*PageDomainContent, error) { + if branch == "" { + branch = p.defaultBranch + } + + rel := &PageDomainContent{} + if !strings.HasSuffix(domain, "."+p.baseDomain) { + return nil, os.ErrNotExist + } + + rel.Owner = strings.TrimSuffix(domain, "."+p.baseDomain) + pathS := strings.Split(strings.TrimPrefix(path, "/"), "/") + repo := pathS[0] + defaultRepo := rel.Owner + "." + p.baseDomain + if repo == "" { + // 回退到默认仓库 + rel.Repo = defaultRepo + } + + meta, err := p.GetMeta(rel.Owner, rel.Repo, branch) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + if err == nil { + rel.Path = "/" + strings.Join(pathS[1:], "/") + rel.PageMetaContent = meta + return rel, nil + } + if defaultRepo == rel.Repo { + return nil, os.ErrNotExist + } + if meta, err := p.GetMeta(rel.Owner, defaultRepo, branch); err == nil { + rel.PageMetaContent = meta + rel.Repo = defaultRepo + rel.Path = "/" + strings.Join(pathS, "/") + return rel, nil + } + if strings.HasSuffix(path, "/") { + rel.Path = rel.Path + "/index.html" + } + return nil, os.ErrNotExist } diff --git a/pkg/providers/gitea.go b/pkg/providers/gitea.go index 9d0d391..6891eff 100644 --- a/pkg/providers/gitea.go +++ b/pkg/providers/gitea.go @@ -5,6 +5,8 @@ import ( "net/url" "os" + "code.d7z.net/d7z-project/gitea-pages/pkg/core" + "code.gitea.io/sdk/gitea" ) @@ -73,8 +75,8 @@ func (g *ProviderGitea) Repos(owner string) (map[string]string, error) { return result, nil } -func (g *ProviderGitea) Branches(owner, repo string) (map[string]string, error) { - result := make(map[string]string) +func (g *ProviderGitea) Branches(owner, repo string) (map[string]*core.BranchInfo, error) { + result := make(map[string]*core.BranchInfo) if branches, resp, err := g.gitea.ListRepoBranches(owner, repo, gitea.ListRepoBranchesOptions{ ListOptions: gitea.ListOptions{ PageSize: GiteaMaxCount, @@ -89,7 +91,10 @@ func (g *ProviderGitea) Branches(owner, repo string) (map[string]string, error) _ = resp.Body.Close() } for _, branch := range branches { - result[branch.Name] = branch.Commit.ID + result[branch.Name] = &core.BranchInfo{ + ID: branch.Commit.ID, + LastModified: branch.Commit.Timestamp, + } } } if len(result) == 0 { @@ -98,7 +103,7 @@ func (g *ProviderGitea) Branches(owner, repo string) (map[string]string, error) return result, nil } -func (g *ProviderGitea) Open(client *http.Client, owner, repo, commit, path string, headers map[string]string) (*http.Response, error) { +func (g *ProviderGitea) Open(client *http.Client, owner, repo, commit, path string, headers http.Header) (*http.Response, error) { giteaURL, err := url.JoinPath(g.BaseUrl, "api/v1/repos", owner, repo, "media", path) if err != nil { return nil, err @@ -109,8 +114,10 @@ func (g *ProviderGitea) Open(client *http.Client, owner, repo, commit, path stri return nil, err } if headers != nil { - for key, value := range headers { - req.Header.Add(key, value) + for key, values := range headers { + for _, value := range values { + req.Header.Add(key, value) + } } } req.Header.Add("Authorization", "token "+g.Token) diff --git a/pkg/server.go b/pkg/server.go index 733a929..a11ce0c 100644 --- a/pkg/server.go +++ b/pkg/server.go @@ -1,7 +1,12 @@ package pkg import ( + "errors" + "io" + "mime" "net/http" + "os" + "path/filepath" "time" "code.d7z.net/d7z-project/gitea-pages/pkg/core" @@ -10,7 +15,9 @@ import ( ) type ServerOptions struct { - Domain string + Domain string + DefaultBranch string + Config utils.Config Cache utils.Cache @@ -22,29 +29,72 @@ type ServerOptions struct { func DefaultOptions(domain string) ServerOptions { configMemory, _ := utils.NewConfigMemory("") return ServerOptions{ - Domain: domain, - Config: configMemory, - Cache: utils.NewCacheMemory(1024*1024*10, int(memory.FreeMemory()/3*2)), - MaxCacheSize: 1024 * 1024 * 10, - HttpClient: http.DefaultClient, + Domain: domain, + DefaultBranch: "gh-pages", + Config: configMemory, + Cache: utils.NewCacheMemory(1024*1024*10, int(memory.FreeMemory()/3*2)), + MaxCacheSize: 1024 * 1024 * 10, + HttpClient: http.DefaultClient, } } type Server struct { - meta *core.ServerMeta + meta *core.PageDomain options *ServerOptions + reader *core.CacheBackendBlobReader } func NewPageServer(backend core.Backend, options ServerOptions) *Server { backend = core.NewCacheBackend(backend, options.Config, time.Minute) + svcMeta := core.NewServerMeta(options.HttpClient, backend, options.Config, time.Minute) + pageMeta := core.NewPageDomain(svcMeta, options.Domain, options.DefaultBranch) + reader := core.NewCacheBackendBlobReader(options.HttpClient, backend, options.Cache, options.MaxCacheSize) return &Server{ - meta: core.NewServerMeta(options.HttpClient, backend, options.Config, time.Minute), + meta: pageMeta, options: &options, + reader: reader, } } func (s *Server) ServeHTTP(writer http.ResponseWriter, request *http.Request) { + meta, err := s.meta.ParseDomainMeta(request.Method, request.RequestURI, request.URL.Query().Get("branch")) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + s.writeError(writer, err) + } else { + s.writeNotfoundError(writer, request.RequestURI) + } + return + } + result, err := s.reader.Open(meta.Owner, meta.Repo, meta.CommitID, meta.Path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + s.writeError(writer, err) + } else { + s.writeNotfoundError(writer, request.RequestURI) + } + return + } + fileName := filepath.Base(meta.Path) + if reader, ok := result.(*utils.CacheContent); ok { + http.ServeContent(writer, request, fileName, reader.LastModified, reader) + _ = reader.Close() + return + } else { + writer.Header().Set("Content-Type", mime.TypeByExtension(meta.Path)) + writer.WriteHeader(http.StatusOK) + _, _ = io.Copy(writer, reader) + _ = reader.Close() + return + } +} +func (s *Server) writeError(writer http.ResponseWriter, err error) { + http.Error(writer, err.Error(), http.StatusInternalServerError) +} + +func (s *Server) writeNotfoundError(writer http.ResponseWriter, path string) { + http.Error(writer, "page not found.", http.StatusNotFound) } func (s *Server) Close() error { diff --git a/pkg/utils/cache.go b/pkg/utils/cache.go index 14c7670..f82642d 100644 --- a/pkg/utils/cache.go +++ b/pkg/utils/cache.go @@ -7,21 +7,31 @@ import ( "os" "strings" "sync" + "time" ) +type CacheContent struct { + io.ReadSeekCloser + Length int + LastModified time.Time +} + type Cache interface { Put(key string, reader io.Reader) error - // Get return io.ReadSeekCloser or nil when put nil io.reader - Get(key string) (io.ReadSeekCloser, error) + // Get return CacheContent or nil when put nil io.reader + Get(key string) (*CacheContent, error) Delete(pattern string) error io.Closer } var ErrCacheOutOfMemory = errors.New("内容无法被缓存，超过最大限定值") +// TODO: 优化锁结构 + type CacheMemory struct { l sync.RWMutex data map[string]*[]byte + lastModify map[string]time.Time sizeGlobal int sizeItem int @@ -33,6 +43,7 @@ type CacheMemory struct { func NewCacheMemory(maxUsage, maxGlobalUsage int) *CacheMemory { return &CacheMemory{ data: make(map[string]*[]byte), + lastModify: make(map[string]time.Time), l: sync.RWMutex{}, sizeGlobal: maxGlobalUsage, sizeItem: maxUsage, @@ -79,6 +90,7 @@ func (c *CacheMemory) Put(key string, reader io.Reader) error { } for _, s := range c.ordered[:count] { delete(c.data, s) + delete(c.lastModify, s) } c.ordered = c.ordered[count:] } @@ -87,11 +99,14 @@ func (c *CacheMemory) Put(key string, reader io.Reader) error { dest := make([]byte, size) copy(dest, c.cache[:size]) c.data[key] = &dest + c.lastModify[key] = time.Now() c.current -= currentItemSize c.current += len(dest) } else { c.data[key] = nil + c.lastModify[key] = time.Now() + c.current -= currentItemSize } @@ -105,15 +120,20 @@ func (c *CacheMemory) Put(key string, reader io.Reader) error { return nil } -func (c *CacheMemory) Get(key string) (io.ReadSeekCloser, error) { +func (c *CacheMemory) Get(key string) (*CacheContent, error) { c.l.RLock() defer c.l.RUnlock() if i, ok := c.data[key]; ok { if i == nil { return nil, nil } - return nopCloser{ - bytes.NewReader(*i), + + return &CacheContent{ + ReadSeekCloser: NopCloser{ + bytes.NewReader(*i), + }, + Length: len(*i), + LastModified: c.lastModify[key], }, nil } return nil, os.ErrNotExist @@ -127,6 +147,7 @@ func (c *CacheMemory) Delete(pattern string) error { if strings.HasPrefix(key, pattern) { c.current -= len(*c.data[key]) delete(c.data, key) + delete(c.lastModify, key) } else { nextOrder = append(nextOrder, key) } @@ -141,12 +162,13 @@ func (c *CacheMemory) Close() error { defer c.l.Unlock() clear(c.ordered) clear(c.data) + clear(c.lastModify) c.current = 0 return nil } -type nopCloser struct { +type NopCloser struct { io.ReadSeeker } -func (nopCloser) Close() error { return nil } +func (NopCloser) Close() error { return nil } diff --git a/pkg/utils/cache_test.go b/pkg/utils/cache_test.go index 45de382..ba0c558 100644 --- a/pkg/utils/cache_test.go +++ b/pkg/utils/cache_test.go @@ -71,4 +71,5 @@ func TestCacheLimit(t *testing.T) { require.Equal(t, 5, len(memory.data)) require.Equal(t, 5, len(memory.ordered)) + require.Equal(t, 5, len(memory.lastModify)) } diff --git a/pkg/utils/locker.go b/pkg/utils/locker.go index 728953b..0e4616a 100644 --- a/pkg/utils/locker.go +++ b/pkg/utils/locker.go @@ -11,6 +11,7 @@ func NewLocker() *Locker { sy: sync.Map{}, } } + func (l *Locker) Open(key string) *sync.Mutex { actual, _ := l.sy.LoadOrStore(key, &sync.Mutex{}) return actual.(*sync.Mutex)