From 246bf13a6c570a0d33215e4ebb7aaf76616bc378 Mon Sep 17 00:00:00 2001 From: dragon Date: Fri, 3 Jan 2025 17:25:41 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E5=96=84=20cache=20=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 3 + config.go | 2 - go.mod | 1 + go.sum | 2 + main.go | 1 - pkg/core/backend.go | 81 ++++++++++++++++++ pkg/core/meta.go | 118 ++++++++++++++++++++++++++ pkg/models/page.go | 10 --- pkg/providers/gitea.go | 6 +- pkg/server.go | 45 ++++++++-- pkg/services/backend.go | 14 --- pkg/services/config.go | 47 ---------- pkg/{services => utils}/cache.go | 47 ++++++---- pkg/{services => utils}/cache_test.go | 9 +- pkg/utils/config.go | 93 ++++++++++++++++++++ 15 files changed, 377 insertions(+), 102 deletions(-) create mode 100644 Makefile create mode 100644 pkg/core/backend.go create mode 100644 pkg/core/meta.go delete mode 100644 pkg/models/page.go delete mode 100644 pkg/services/backend.go delete mode 100644 pkg/services/config.go rename pkg/{services => utils}/cache.go (74%) rename pkg/{services => utils}/cache_test.go (93%) create mode 100644 pkg/utils/config.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..70eb817 --- /dev/null +++ b/Makefile @@ -0,0 +1,3 @@ +fmt: + @(test -f "$(GOPATH)/bin/gofumpt" || go install mvdan.cc/gofumpt@latest) && \ + "$(GOPATH)/bin/gofumpt" -l -w . \ No newline at end of file diff --git a/config.go b/config.go index 62608ca..dfc48e6 100644 --- a/config.go +++ b/config.go @@ -17,8 +17,6 @@ type Config struct { } type ConfigAuth struct { - // 后端类型 - Type string `yaml:"type"` // 服务器地址 Server string `yaml:"server"` // 会话 Id diff --git a/go.mod b/go.mod index 46f4c76..8d01e42 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/davidmz/go-pageant v1.0.2 // indirect github.com/go-fed/httpsig v1.1.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/crypto v0.31.0 // indirect golang.org/x/sys v0.28.0 // indirect diff --git a/go.sum b/go.sum index d41f108..09adaf7 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/main.go b/main.go index 7905807..da29a2c 100644 --- a/main.go +++ b/main.go @@ -1,5 +1,4 @@ package main func main() { - } diff --git a/pkg/core/backend.go b/pkg/core/backend.go new file mode 100644 index 0000000..9af6ca5 --- /dev/null +++ b/pkg/core/backend.go @@ -0,0 +1,81 @@ +package core + +import ( + "encoding/json" + "fmt" + "net/http" + "time" + + "code.d7z.net/d7z-project/gitea-pages/pkg/utils" +) + +type Backend interface { + // Repos return repo name + default branch + Repos(owner string) (map[string]string, error) + // Branches return branch + commit id + Branches(owner, repo string) (map[string]string, error) + // Open return file or error + Open(client *http.Client, owner, repo, commit, path string, headers map[string]string) (*http.Response, error) +} + +type CacheBackend struct { + backend Backend + config utils.Config + ttl time.Duration +} + +func NewCacheBackend(backend Backend, config utils.Config, ttl time.Duration) *CacheBackend { + return &CacheBackend{backend: backend, config: config, ttl: ttl} +} + +func (c *CacheBackend) Repos(owner string) (map[string]string, error) { + ret := make(map[string]string) + key := fmt.Sprintf("repos/%s", owner) + data, err := c.config.Get(key) + if err != nil { + ret, err = c.backend.Repos(owner) + if err != nil { + return nil, err + } + data, err := json.Marshal(data) + if err != nil { + return nil, err + } + if err = c.config.Put(key, string(data), c.ttl); err != nil { + return nil, err + } + } else { + if err := json.Unmarshal([]byte(data), &ret); err != nil { + return nil, err + } + } + return ret, nil +} + +func (c *CacheBackend) Branches(owner, repo string) (map[string]string, error) { + ret := make(map[string]string) + key := fmt.Sprintf("branches/%s/%s", owner, repo) + data, err := c.config.Get(key) + if err != nil { + ret, err = c.backend.Branches(owner, repo) + if err != nil { + return nil, err + } + data, err := json.Marshal(data) + if err != nil { + return nil, err + } + if err = c.config.Put(key, string(data), c.ttl); err != nil { + return nil, err + } + } else { + if err := json.Unmarshal([]byte(data), &ret); err != nil { + return nil, err + } + } + return ret, nil +} + +func (c *CacheBackend) Open(client *http.Client, owner, repo, commit, path string, headers map[string]string) (*http.Response, error) { + return c.backend.Open(client, owner, repo, commit, path, headers) +} diff --git a/pkg/core/meta.go b/pkg/core/meta.go new file mode 100644 index 0000000..a784b1e --- /dev/null +++ b/pkg/core/meta.go @@ -0,0 +1,118 @@ +package core + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" + + "code.d7z.net/d7z-project/gitea-pages/pkg/utils" +) + +type ServerMeta struct { + client *http.Client + backend Backend + cache utils.Config + ttl time.Duration +} + +type PageMeta struct { + CommitID string `json:"commit_id"` // 提交 COMMIT ID + IsPage bool `json:"is_page"` // 是否为 Page + Domain string `json:"domain"` // 匹配的域名和路径 + HistoryRouteMode bool `json:"route_history"` // 路由模式 +} + +func NewServerMeta(client *http.Client, backend Backend, config utils.Config, ttl time.Duration) *ServerMeta { + return &ServerMeta{client, backend, config, ttl} +} + +func (s *ServerMeta) Meta(owner, repo, branch string) (*PageMeta, error) { + rel := &PageMeta{ + IsPage: false, + } + key := fmt.Sprintf("meta/%s/%s/%s", owner, repo, branch) + pushMeta := func() error { + data, err := json.Marshal(rel) + if err != nil { + return err + } + return s.cache.Put(key, string(data), s.ttl) + } + + cache, err := s.cache.Get(key) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + } else { + if err = json.Unmarshal([]byte(cache), rel); err == nil { + return rel, nil + } + } + repos, err := s.backend.Repos(owner) + if err != nil { + return nil, err + } + rel.CommitID = repos[repo] + if rel.CommitID == "" { + _ = pushMeta() + return nil, os.ErrNotExist + } + if branch != "" { + branches, err := s.backend.Branches(owner, repo) + if err != nil { + return nil, err + } + rel.CommitID = branches[branch] + } + if rel.CommitID == "" { + _ = pushMeta() + return nil, os.ErrNotExist + } + if cname, err := s.ReadString(owner, repo, rel.CommitID, "CNAME"); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } else { + rel.Domain = strings.TrimSpace(cname) + } + if find, _ := s.FileExists(owner, repo, rel.CommitID, "index.html"); find { + rel.IsPage = true + } + if find, _ := s.FileExists(owner, repo, rel.CommitID, ".history-mode"); find { + rel.HistoryRouteMode = true + } + _ = pushMeta() + return rel, nil +} + +func (s *ServerMeta) ReadString(owner, repo, branch, path string) (string, error) { + resp, err := s.backend.Open(s.client, owner, repo, branch, path, nil) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", os.ErrNotExist + } + all, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(all), nil +} + +func (s *ServerMeta) FileExists(owner, repo, branch, path string) (bool, error) { + resp, err := s.backend.Open(s.client, owner, repo, branch, path, nil) + if err != nil { + return false, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return true, nil + } + return false, nil +} diff --git a/pkg/models/page.go b/pkg/models/page.go deleted file mode 100644 index 65ce77a..0000000 --- a/pkg/models/page.go +++ /dev/null @@ -1,10 +0,0 @@ -package models - -type PageConfig struct { - // pages 分支 - Branch string `json:"branch" yaml:"branch"` - // 匹配的域名和路径 - Domain string `json:"domain" yaml:"domain"` - // 路由模式 (default / history) - RouteMode string `json:"route" yaml:"route"` -} diff --git a/pkg/providers/gitea.go b/pkg/providers/gitea.go index 44bcdb5..9d0d391 100644 --- a/pkg/providers/gitea.go +++ b/pkg/providers/gitea.go @@ -1,10 +1,11 @@ package providers import ( - "code.gitea.io/sdk/gitea" "net/http" "net/url" "os" + + "code.gitea.io/sdk/gitea" ) const GiteaMaxCount = 9999 @@ -70,7 +71,6 @@ func (g *ProviderGitea) Repos(owner string) (map[string]string, error) { return nil, os.ErrNotExist } return result, nil - } func (g *ProviderGitea) Branches(owner, repo string) (map[string]string, error) { @@ -98,7 +98,7 @@ func (g *ProviderGitea) Branches(owner, repo string) (map[string]string, error) return result, nil } -func (g *ProviderGitea) Open(client http.Client, owner, repo, commit, path string, headers map[string]string) (*http.Response, error) { +func (g *ProviderGitea) Open(client *http.Client, owner, repo, commit, path string, headers map[string]string) (*http.Response, error) { giteaURL, err := url.JoinPath(g.BaseUrl, "api/v1/repos", owner, repo, "media", path) if err != nil { return nil, err diff --git a/pkg/server.go b/pkg/server.go index 992391e..18f220e 100644 --- a/pkg/server.go +++ b/pkg/server.go @@ -1,24 +1,53 @@ package pkg import ( - "code.d7z.net/d7z-project/gitea-pages/pkg/services" + "net/http" + "time" + + "code.d7z.net/d7z-project/gitea-pages/pkg/core" + "code.d7z.net/d7z-project/gitea-pages/pkg/utils" + "github.com/pbnjay/memory" ) type ServerOptions struct { Domain string - Cache services.Config + Config utils.Config + Cache utils.Cache + + MaxCacheSize int } -func DefaultOptions(domain string) *ServerOptions { - return &ServerOptions{ - Domain: domain, - Cache: services.NewConfigMemory(), +func DefaultOptions(domain string) ServerOptions { + configMemory, _ := utils.NewConfigMemory("") + return ServerOptions{ + Domain: domain, + Config: configMemory, + Cache: utils.NewCacheMemory(1024*1024*10, int(memory.FreeMemory()/3*2)), + MaxCacheSize: 1024 * 1024 * 10, } } type Server struct { + backend core.Backend + options *ServerOptions } -func NewServer(backend services.Backend, options *ServerOptions) *Server { - +func NewPageServer(backend core.Backend, options ServerOptions) *Server { + return &Server{ + backend: core.NewCacheBackend(backend, options.Config, time.Minute), + options: &options, + } +} + +func (s *Server) ServeHTTP(writer http.ResponseWriter, request *http.Request) { +} + +func (s *Server) Close() error { + if err := s.options.Config.Close(); err != nil { + return err + } + if err := s.options.Cache.Close(); err != nil { + return err + } + return nil } diff --git a/pkg/services/backend.go b/pkg/services/backend.go deleted file mode 100644 index 153f2eb..0000000 --- a/pkg/services/backend.go +++ /dev/null @@ -1,14 +0,0 @@ -package services - -import ( - "net/http" -) - -type Backend interface { - // Repos return repo name + default branch - Repos(owner string) (map[string]string, error) - // Branches return branch + commit id - Branches(owner, repo string) (map[string]string, error) - // Open return file or error - Open(client http.Client, owner, repo, commit, path string, headers map[string]string) (*http.Response, error) -} diff --git a/pkg/services/config.go b/pkg/services/config.go deleted file mode 100644 index 407322d..0000000 --- a/pkg/services/config.go +++ /dev/null @@ -1,47 +0,0 @@ -package services - -import ( - "io" - "os" - "sync" -) - -type Config interface { - Put(key string, value string) error - Get(key string) (string, error) - Delete(key string) error - io.Closer -} - -func NewConfigMemory() Config { - return &ConfigMemory{ - data: sync.Map{}, - } -} - -// ConfigMemory 一个简单的内存配置归档，仅用于测试 -type ConfigMemory struct { - data sync.Map -} - -func (m *ConfigMemory) Put(key string, value string) error { - m.data.Store(key, value) - return nil -} - -func (m *ConfigMemory) Get(key string) (string, error) { - if value, ok := m.data.Load(key); ok { - return value.(string), nil - } - return "", os.ErrNotExist -} - -func (m *ConfigMemory) Delete(key string) error { - m.data.Delete(key) - return nil -} - -func (m *ConfigMemory) Close() error { - m.data.Clear() - return nil -} diff --git a/pkg/services/cache.go b/pkg/utils/cache.go similarity index 74% rename from pkg/services/cache.go rename to pkg/utils/cache.go index 1c5311f..14c7670 100644 --- a/pkg/services/cache.go +++ b/pkg/utils/cache.go @@ -1,4 +1,4 @@ -package services +package utils import ( "bytes" @@ -11,6 +11,7 @@ import ( type Cache interface { Put(key string, reader io.Reader) error + // Get return io.ReadSeekCloser or nil when put nil io.reader Get(key string) (io.ReadSeekCloser, error) Delete(pattern string) error io.Closer @@ -20,7 +21,7 @@ var ErrCacheOutOfMemory = errors.New("内容无法被缓存，超过最大限定 type CacheMemory struct { l sync.RWMutex - data map[string][]byte + data map[string]*[]byte sizeGlobal int sizeItem int @@ -31,7 +32,7 @@ type CacheMemory struct { func NewCacheMemory(maxUsage, maxGlobalUsage int) *CacheMemory { return &CacheMemory{ - data: make(map[string][]byte), + data: make(map[string]*[]byte), l: sync.RWMutex{}, sizeGlobal: maxGlobalUsage, sizeItem: maxUsage, @@ -44,20 +45,28 @@ func NewCacheMemory(maxUsage, maxGlobalUsage int) *CacheMemory { func (c *CacheMemory) Put(key string, reader io.Reader) error { c.l.Lock() defer c.l.Unlock() - size, err := io.ReadAtLeast(reader, c.cache, 1) - if err != nil { - return err + size := 0 + // 可以指定空的 reader 作为 404 缓存 + if reader != nil { + var err error + size, err = io.ReadAtLeast(reader, c.cache, 1) + if err != nil { + return err + } } if size == len(c.cache) { return ErrCacheOutOfMemory } - currentItemSize := len(c.data[key]) + currentItemSize := 0 + if data, ok := c.data[key]; ok { + currentItemSize = len(*data) + } available := c.sizeGlobal + currentItemSize - (c.current + size) if available < 0 { // 清理旧的内容 count := 0 for i, k := range c.ordered { - available += len(c.data[k]) + available += len(*c.data[k]) if available > 0 { break } @@ -74,12 +83,17 @@ func (c *CacheMemory) Put(key string, reader io.Reader) error { c.ordered = c.ordered[count:] } - dest := make([]byte, size) - copy(dest, c.cache[:size]) - c.data[key] = dest + if reader != nil { + dest := make([]byte, size) + copy(dest, c.cache[:size]) + c.data[key] = &dest - c.current -= currentItemSize - c.current += len(dest) + c.current -= currentItemSize + c.current += len(dest) + } else { + c.data[key] = nil + c.current -= currentItemSize + } nextOrdered := make([]string, 0, len(c.ordered)) for _, s := range c.ordered { @@ -95,8 +109,11 @@ func (c *CacheMemory) Get(key string) (io.ReadSeekCloser, error) { c.l.RLock() defer c.l.RUnlock() if i, ok := c.data[key]; ok { + if i == nil { + return nil, nil + } return nopCloser{ - bytes.NewReader(i), + bytes.NewReader(*i), }, nil } return nil, os.ErrNotExist @@ -108,7 +125,7 @@ func (c *CacheMemory) Delete(pattern string) error { nextOrder := make([]string, 0, len(c.ordered)) for _, key := range c.ordered { if strings.HasPrefix(key, pattern) { - c.current -= len(c.data[key]) + c.current -= len(*c.data[key]) delete(c.data, key) } else { nextOrder = append(nextOrder, key) diff --git a/pkg/services/cache_test.go b/pkg/utils/cache_test.go similarity index 93% rename from pkg/services/cache_test.go rename to pkg/utils/cache_test.go index c45797b..45de382 100644 --- a/pkg/services/cache_test.go +++ b/pkg/utils/cache_test.go @@ -1,12 +1,13 @@ -package services +package utils import ( "fmt" - "github.com/stretchr/testify/require" "io" "os" "strings" "testing" + + "github.com/stretchr/testify/require" ) func TestCacheGetPutDelete(t *testing.T) { @@ -42,6 +43,10 @@ func TestCacheGetPutDelete(t *testing.T) { require.Equal(t, 1, len(memory.data)) require.Equal(t, 1, len(memory.ordered)) + require.NoError(t, memory.Put("hello", nil)) + value, err = memory.Get("hello") + require.NoError(t, err) + require.Nil(t, value) } func TestCacheLimit(t *testing.T) { diff --git a/pkg/utils/config.go b/pkg/utils/config.go new file mode 100644 index 0000000..8cc420e --- /dev/null +++ b/pkg/utils/config.go @@ -0,0 +1,93 @@ +package utils + +import ( + "encoding/json" + "io" + "os" + "sync" + "time" +) + +type Config interface { + Put(key string, value string, ttl time.Duration) error + Get(key string) (string, error) + Delete(key string) error + io.Closer +} + +// ConfigMemory 一个简单的内存配置归档，仅用于测试 +type ConfigMemory struct { + data sync.Map + store string +} + +func NewConfigMemory(store string) (Config, error) { + ret := &ConfigMemory{ + store: store, + data: sync.Map{}, + } + if store != "" { + item := make(map[string]configContent) + data, err := os.ReadFile(store) + if err == nil && os.IsNotExist(err) { + err := json.Unmarshal(data, &item) + if err != nil { + return nil, err + } + } + for key, content := range item { + if time.Now().Before(content.ttl) { + ret.data.Store(key, content) + } + } + clear(item) + } + return ret, nil +} + +type configContent struct { + data string + ttl time.Time +} + +func (m *ConfigMemory) Put(key string, value string, ttl time.Duration) error { + m.data.Store(key, configContent{ + data: value, + ttl: time.Now().Add(ttl), + }) + return nil +} + +func (m *ConfigMemory) Get(key string) (string, error) { + if value, ok := m.data.Load(key); ok { + content := value.(configContent) + if time.Now().After(content.ttl) { + return "", os.ErrNotExist + } + return content.data, nil + } + return "", os.ErrNotExist +} + +func (m *ConfigMemory) Delete(key string) error { + m.data.Delete(key) + return nil +} + +func (m *ConfigMemory) Close() error { + defer m.data.Clear() + if m.store != "" { + item := make(map[string]configContent) + m.data.Range( + func(key, value interface{}) bool { + item[key.(string)] = value.(configContent) + return true + }) + saved, err := json.Marshal(item) + if err != nil { + return err + } + return os.WriteFile(m.store, saved, 0o600) + } + return nil +}