diff --git a/config.go b/config.go index 471bdb5..17b29d0 100644 --- a/config.go +++ b/config.go @@ -8,6 +8,8 @@ import ( "time" "github.com/alecthomas/units" + "gopkg.d7z.net/gitea-pages/pkg/middleware/cache" + "gopkg.d7z.net/gitea-pages/pkg/middleware/config" "github.com/pkg/errors" "go.uber.org/zap" @@ -95,9 +97,9 @@ func (c *Config) NewPageServerOptions() (*pkg.ServerOptions, error) { EnableProxy: c.Proxy.Enable, DefaultErrorHandler: c.ErrorHandler, StaticDir: c.StaticDir, - Cache: utils.NewCacheMemory(int(cacheMaxSize), int(cacheMaxSize)), + Cache: cache.NewCacheMemory(int(cacheMaxSize), int(cacheMaxSize)), } - cfg, err := utils.NewAutoConfig(c.Cache.Storage) + cfg, err := config.NewAutoConfig(c.Cache.Storage) if err != nil { return nil, errors.Wrapf(err, "failed to init config memory") } diff --git a/pkg/core/alias.go b/pkg/core/alias.go index e00fff3..601601e 100644 --- a/pkg/core/alias.go +++ b/pkg/core/alias.go @@ -5,7 +5,7 @@ import ( "encoding/json" "fmt" - "gopkg.d7z.net/gitea-pages/pkg/utils" + "gopkg.d7z.net/gitea-pages/pkg/middleware/config" ) type Alias struct { @@ -15,10 +15,10 @@ type Alias struct { } type DomainAlias struct { - config utils.KVConfig + config config.KVConfig } -func NewDomainAlias(config utils.KVConfig) *DomainAlias { +func NewDomainAlias(config config.KVConfig) *DomainAlias { return &DomainAlias{config: config} } @@ -55,9 +55,9 @@ func (a *DomainAlias) Bind(ctx context.Context, domains []string, owner, repo, b } aliasMetaRaw, _ := json.Marshal(aliasMeta) domainsRaw, _ := json.Marshal(domains) - _ = a.config.Put(ctx, rKey, string(domainsRaw), utils.TtlKeep) + _ = a.config.Put(ctx, rKey, string(domainsRaw), config.TtlKeep) for _, domain := range domains { - if err := a.config.Put(ctx, "domain/alias/"+domain, string(aliasMetaRaw), utils.TtlKeep); err != nil { + if err := a.config.Put(ctx, "domain/alias/"+domain, string(aliasMetaRaw), config.TtlKeep); err != nil { return err } } diff --git a/pkg/core/backend.go b/pkg/core/backend.go index 196cde4..82acb08 100644 --- a/pkg/core/backend.go +++ b/pkg/core/backend.go @@ -14,6 +14,8 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" + "gopkg.d7z.net/gitea-pages/pkg/middleware/cache" + "gopkg.d7z.net/gitea-pages/pkg/middleware/config" "gopkg.d7z.net/gitea-pages/pkg/utils" ) @@ -35,7 +37,7 @@ type Backend interface { type CacheBackend struct { backend Backend - config utils.KVConfig + config config.KVConfig ttl time.Duration } @@ -43,7 +45,7 @@ func (c *CacheBackend) Close() error { return c.backend.Close() } -func NewCacheBackend(backend Backend, config utils.KVConfig, ttl time.Duration) *CacheBackend { +func NewCacheBackend(backend Backend, config config.KVConfig, ttl time.Duration) *CacheBackend { return &CacheBackend{backend: backend, config: config, ttl: ttl} } @@ -113,12 +115,12 @@ func (c *CacheBackend) Open(ctx context.Context, client *http.Client, owner, rep type CacheBackendBlobReader struct { client *http.Client - cache utils.Cache + cache cache.Cache base Backend maxSize int } -func NewCacheBackendBlobReader(client *http.Client, base Backend, cache utils.Cache, maxCacheSize int) *CacheBackendBlobReader { +func NewCacheBackendBlobReader(client *http.Client, base Backend, cache cache.Cache, maxCacheSize int) *CacheBackendBlobReader { return &CacheBackendBlobReader{client: client, base: base, cache: cache, maxSize: maxCacheSize} } @@ -173,7 +175,7 @@ func (c *CacheBackendBlobReader) Open(ctx context.Context, owner, repo, commit, if err = c.cache.Put(key, bytes.NewBuffer(allBytes)); err != nil { zap.L().Warn("缓存归档失败", zap.Error(err), zap.Int("Size", len(allBytes)), zap.Int("MaxSize", c.maxSize)) } - return &utils.CacheContent{ + return &cache.CacheContent{ ReadSeekCloser: utils.NopCloser{ ReadSeeker: bytes.NewReader(allBytes), }, diff --git a/pkg/core/domain.go b/pkg/core/domain.go index f750441..30cd998 100644 --- a/pkg/core/domain.go +++ b/pkg/core/domain.go @@ -6,8 +6,7 @@ import ( "strings" "github.com/pkg/errors" - - "gopkg.d7z.net/gitea-pages/pkg/utils" + "gopkg.d7z.net/gitea-pages/pkg/middleware/config" "go.uber.org/zap" ) @@ -20,7 +19,7 @@ type PageDomain struct { defaultBranch string } -func NewPageDomain(meta *ServerMeta, config utils.KVConfig, baseDomain, defaultBranch string) *PageDomain { +func NewPageDomain(meta *ServerMeta, config config.KVConfig, baseDomain, defaultBranch string) *PageDomain { return &PageDomain{ baseDomain: baseDomain, defaultBranch: defaultBranch, diff --git a/pkg/core/meta.go b/pkg/core/meta.go index 104c766..66cc36b 100644 --- a/pkg/core/meta.go +++ b/pkg/core/meta.go @@ -13,6 +13,7 @@ import ( "time" "go.uber.org/zap" + "gopkg.d7z.net/gitea-pages/pkg/middleware/config" "gopkg.in/yaml.v3" "github.com/gobwas/glob" @@ -30,13 +31,13 @@ type ServerMeta struct { Domain string client *http.Client - cache utils.KVConfig + cache config.KVConfig ttl time.Duration locker *utils.Locker } -func NewServerMeta(client *http.Client, backend Backend, kv utils.KVConfig, domain string, ttl time.Duration) *ServerMeta { +func NewServerMeta(client *http.Client, backend Backend, kv config.KVConfig, domain string, ttl time.Duration) *ServerMeta { return &ServerMeta{backend, domain, client, kv, ttl, utils.NewLocker()} } diff --git a/pkg/middleware/cache/cache.go b/pkg/middleware/cache/cache.go new file mode 100644 index 0000000..ed63c6f --- /dev/null +++ b/pkg/middleware/cache/cache.go @@ -0,0 +1,48 @@ +package cache + +import ( + "io" + "sync" + "time" + + "github.com/pkg/errors" +) + +type CacheContent struct { + io.ReadSeekCloser + Length int + LastModified time.Time +} + +func (c *CacheContent) ReadToString() (string, error) { + all, err := io.ReadAll(c) + if err != nil { + return "", err + } + return string(all), nil +} + +type Cache interface { + Put(key string, reader io.Reader) error + // Get return CacheContent or nil when put nil io.reader + Get(key string) (*CacheContent, error) + Delete(pattern string) error + io.Closer +} + +var ErrCacheOutOfMemory = errors.New("内容无法被缓存,超过最大限定值") + +// TODO: 优化锁结构 +// 复杂场景请使用其他缓存服务 + +type CacheMemory struct { + l sync.RWMutex + data map[string]*[]byte + lastModify map[string]time.Time + sizeGlobal int + sizeItem int + + current int + cache []byte + ordered []string +} diff --git a/pkg/utils/cache.go b/pkg/middleware/cache/cache_memory.go similarity index 73% rename from pkg/utils/cache.go rename to pkg/middleware/cache/cache_memory.go index f74e88d..adf7bca 100644 --- a/pkg/utils/cache.go +++ b/pkg/middleware/cache/cache_memory.go @@ -1,4 +1,4 @@ -package utils +package cache import ( "bytes" @@ -8,48 +8,9 @@ import ( "sync" "time" - "github.com/pkg/errors" + "gopkg.d7z.net/gitea-pages/pkg/utils" ) -type CacheContent struct { - io.ReadSeekCloser - Length int - LastModified time.Time -} - -func (c *CacheContent) ReadToString() (string, error) { - all, err := io.ReadAll(c) - if err != nil { - return "", err - } - return string(all), nil -} - -type Cache interface { - Put(key string, reader io.Reader) error - // Get return CacheContent or nil when put nil io.reader - Get(key string) (*CacheContent, error) - Delete(pattern string) error - io.Closer -} - -var ErrCacheOutOfMemory = errors.New("内容无法被缓存,超过最大限定值") - -// TODO: 优化锁结构 -// 复杂场景请使用其他缓存服务 - -type CacheMemory struct { - l sync.RWMutex - data map[string]*[]byte - lastModify map[string]time.Time - sizeGlobal int - sizeItem int - - current int - cache []byte - ordered []string -} - func NewCacheMemory(maxUsage, maxGlobalUsage int) *CacheMemory { return &CacheMemory{ data: make(map[string]*[]byte), @@ -139,7 +100,7 @@ func (c *CacheMemory) Get(key string) (*CacheContent, error) { } return &CacheContent{ - ReadSeekCloser: NopCloser{ + ReadSeekCloser: utils.NopCloser{ bytes.NewReader(*i), }, Length: len(*i), @@ -176,9 +137,3 @@ func (c *CacheMemory) Close() error { c.current = 0 return nil } - -type NopCloser struct { - io.ReadSeeker -} - -func (NopCloser) Close() error { return nil } diff --git a/pkg/utils/cache_test.go b/pkg/middleware/cache/cache_test.go similarity index 99% rename from pkg/utils/cache_test.go rename to pkg/middleware/cache/cache_test.go index ba0c558..a3e5be2 100644 --- a/pkg/utils/cache_test.go +++ b/pkg/middleware/cache/cache_test.go @@ -1,4 +1,4 @@ -package utils +package cache import ( "fmt" diff --git a/pkg/utils/config.go b/pkg/middleware/config/config.go similarity index 98% rename from pkg/utils/config.go rename to pkg/middleware/config/config.go index 782219a..9b8c852 100644 --- a/pkg/utils/config.go +++ b/pkg/middleware/config/config.go @@ -1,4 +1,4 @@ -package utils +package config import ( "context" diff --git a/pkg/utils/config_memory.go b/pkg/middleware/config/config_memory.go similarity index 99% rename from pkg/utils/config_memory.go rename to pkg/middleware/config/config_memory.go index 7a6fae3..db4e749 100644 --- a/pkg/utils/config_memory.go +++ b/pkg/middleware/config/config_memory.go @@ -1,4 +1,4 @@ -package utils +package config import ( "context" diff --git a/pkg/utils/config_redis.go b/pkg/middleware/config/config_redis.go similarity index 98% rename from pkg/utils/config_redis.go rename to pkg/middleware/config/config_redis.go index 453c653..09d8497 100644 --- a/pkg/utils/config_redis.go +++ b/pkg/middleware/config/config_redis.go @@ -1,4 +1,4 @@ -package utils +package config import ( "context" diff --git a/pkg/server.go b/pkg/server.go index 5e8aad5..84c4aa4 100644 --- a/pkg/server.go +++ b/pkg/server.go @@ -17,6 +17,8 @@ import ( "time" "github.com/google/uuid" + "gopkg.d7z.net/gitea-pages/pkg/middleware/cache" + "gopkg.d7z.net/gitea-pages/pkg/middleware/config" "github.com/pkg/errors" "go.uber.org/zap" @@ -34,8 +36,8 @@ type ServerOptions struct { Domain string DefaultBranch string - KVConfig utils.KVConfig - Cache utils.Cache + KVConfig config.KVConfig + Cache cache.Cache MaxCacheSize int @@ -52,12 +54,12 @@ type ServerOptions struct { } func DefaultOptions(domain string) ServerOptions { - configMemory, _ := utils.NewAutoConfig("") + configMemory, _ := config.NewAutoConfig("") return ServerOptions{ Domain: domain, DefaultBranch: "gh-pages", KVConfig: configMemory, - Cache: utils.NewCacheMemory(1024*1024*10, int(memory.FreeMemory()/3*2)), + Cache: cache.NewCacheMemory(1024*1024*10, int(memory.FreeMemory()/3*2)), MaxCacheSize: 1024 * 1024 * 10, HttpClient: http.DefaultClient, MetaTTL: time.Minute, @@ -235,7 +237,7 @@ func (s *Server) Serve(writer http.ResponseWriter, request *http.Request) error render = nil } defer result.Close() - if reader, ok := result.(*utils.CacheContent); ok { + if reader, ok := result.(*cache.CacheContent); ok { writer.Header().Add("X-Cache", "HIT") writer.Header().Set("Content-Type", mime.TypeByExtension(filepath.Ext(fileName))) writer.Header().Add("Cache-Control", "public, max-age=86400") diff --git a/pkg/utils/closer.go b/pkg/utils/closer.go new file mode 100644 index 0000000..31c33fe --- /dev/null +++ b/pkg/utils/closer.go @@ -0,0 +1,9 @@ +package utils + +import "io" + +type NopCloser struct { + io.ReadSeeker +} + +func (NopCloser) Close() error { return nil }