diff --git a/config.go b/config.go index 17b29d0..1137367 100644 --- a/config.go +++ b/config.go @@ -8,13 +8,12 @@ import ( "time" "github.com/alecthomas/units" - "gopkg.d7z.net/gitea-pages/pkg/middleware/cache" - "gopkg.d7z.net/gitea-pages/pkg/middleware/config" - "github.com/pkg/errors" "go.uber.org/zap" "gopkg.d7z.net/gitea-pages/pkg" "gopkg.d7z.net/gitea-pages/pkg/utils" + "gopkg.d7z.net/middleware/cache" + "gopkg.d7z.net/middleware/kv" "gopkg.in/yaml.v3" ) @@ -25,9 +24,13 @@ type Config struct { Bind string `yaml:"bind"` // HTTP 绑定 Domain string `yaml:"domain"` // 基础域名 - Auth ConfigAuth `yaml:"auth"` // 后端认证配置 + Config string `yaml:"config"` // 配置 + + Auth ConfigAuth `yaml:"auth"` // 后端认证配置 + Cache ConfigCache `yaml:"cache"` // 缓存配置 - Page ConfigPage `yaml:"page"` // 页面配置 + + Page ConfigPage `yaml:"page"` // 页面配置 Render ConfigRender `yaml:"render"` // 渲染配置 Proxy ConfigProxy `yaml:"proxy"` // 反向代理配置 @@ -42,12 +45,10 @@ func (c *Config) NewPageServerOptions() (*pkg.ServerOptions, error) { return nil, errors.New("domain is required") } var err error - var cacheSize, cacheMaxSize units.Base2Bytes - cacheSize, err = units.ParseBase2Bytes(c.Cache.FileSize) - if err != nil { - return nil, errors.Wrap(err, "parse cache size") - } + if c.Config == "" { + return nil, errors.New("config is required") + } if c.StaticDir != "" { stat, err := os.Stat(c.StaticDir) if err != nil { @@ -57,13 +58,6 @@ func (c *Config) NewPageServerOptions() (*pkg.ServerOptions, error) { return nil, errors.New("static dir is not a directory") } } - cacheMaxSize, err = units.ParseBase2Bytes(c.Cache.MaxSize) - if err != nil { - return nil, errors.Wrap(err, "parse cache max size") - } - if cacheMaxSize <= cacheSize { - return nil, errors.New("cache max size must be greater than or equal to file max size") - } if c.Page.DefaultBranch == "" { c.Page.DefaultBranch = "gh-pages" } @@ -87,23 +81,36 @@ func (c *Config) NewPageServerOptions() (*pkg.ServerOptions, error) { c.pageErrNotFound = defaultErr } + memoryCache, err := cache.NewMemoryCache(cache.MemoryCacheConfig{ + MaxCapacity: 8102, + CleanupInt: time.Hour, + }) + if err != nil { + return nil, errors.Wrap(err, "create cache") + } + alias, err := kv.NewKVFromURL(c.Config) + if err != nil { + return nil, errors.Wrapf(err, "failed to init alias config") + } + cacheMeta, err := kv.NewKVFromURL(c.Cache.Meta) + if err != nil { + return nil, errors.Wrapf(err, "failed to init cache meta") + } rel := pkg.ServerOptions{ Domain: c.Domain, DefaultBranch: c.Page.DefaultBranch, - MaxCacheSize: int(cacheSize), + Alias: alias, + CacheMeta: cacheMeta, + CacheMetaTTL: c.Cache.MetaTTL, + CacheBlob: memoryCache, + CacheBlobTTL: c.Cache.BlobTTL, + CacheBlobLimit: uint64(c.Cache.BlobLimit), HttpClient: http.DefaultClient, - MetaTTL: time.Minute, EnableRender: c.Render.Enable, EnableProxy: c.Proxy.Enable, - DefaultErrorHandler: c.ErrorHandler, StaticDir: c.StaticDir, - Cache: cache.NewCacheMemory(int(cacheMaxSize), int(cacheMaxSize)), + DefaultErrorHandler: c.ErrorHandler, } - cfg, err := config.NewAutoConfig(c.Cache.Storage) - if err != nil { - return nil, errors.Wrapf(err, "failed to init config memory") - } - rel.KVConfig = cfg return &rel, nil } @@ -153,11 +160,12 @@ type ConfigRender struct { } type ConfigCache struct { - Storage string `yaml:"storage"` // 缓存归档位置 + Meta string `yaml:"meta"` // 元数据缓存 + MetaTTL time.Duration `yaml:"meta_ttl"` // 缓存时间 - Ttl time.Duration `yaml:"ttl"` // 缓存时间 - FileSize string `yaml:"size"` // 单个文件最大大小 - MaxSize string `yaml:"max"` // 最大文件大小 + Blob string `yaml:"blob"` // 缓存归档位置 + BlobTTL time.Duration `yaml:"blob_ttl"` // 缓存归档位置 + BlobLimit units.Base2Bytes `yaml:"blob_limit"` // 单个文件最大大小 } func LoadConfig(path string) (*Config, error) { diff --git a/go.mod b/go.mod index 351221a..99ce570 100644 --- a/go.mod +++ b/go.mod @@ -1,43 +1,58 @@ module gopkg.d7z.net/gitea-pages -go 1.24.2 +go 1.25.3 require ( - code.gitea.io/sdk/gitea v0.22.0 + code.gitea.io/sdk/gitea v0.22.1 github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b github.com/go-task/slim-sprig/v3 v3.0.0 github.com/gobwas/glob v0.2.3 github.com/google/uuid v1.6.0 - github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.11.1 - github.com/valkey-io/valkey-go v1.0.66 - go.etcd.io/etcd/client/v3 v3.6.5 go.uber.org/zap v1.27.0 + gopkg.d7z.net/middleware v0.0.0-20251110085441-55e78e556d53 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/42wim/httpsig v1.2.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.6.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidmz/go-pageant v1.0.2 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-fed/httpsig v1.1.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-redis/redis/v8 v8.11.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/hashicorp/go-version v1.7.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/klauspost/compress v1.18.1 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.97 // indirect + github.com/onsi/gomega v1.36.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.5.0 // indirect go.etcd.io/etcd/api/v3 v3.6.5 // indirect go.etcd.io/etcd/client/pkg/v3 v3.6.5 // indirect + go.etcd.io/etcd/client/v3 v3.6.5 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.43.0 // indirect golang.org/x/net v0.46.0 // indirect - golang.org/x/sys v0.37.0 // indirect + golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.30.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20251007200510-49b9836ed3ff // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251007200510-49b9836ed3ff // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251103181224-f26f9409b101 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 // indirect google.golang.org/grpc v1.76.0 // indirect google.golang.org/protobuf v1.36.10 // indirect ) diff --git a/go.sum b/go.sum index feb07c8..e5245c2 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,13 @@ -code.gitea.io/sdk/gitea v0.22.0 h1:HCKq7bX/HQ85Nw7c/HAhWgRye+vBp5nQOE8Md1+9Ef0= -code.gitea.io/sdk/gitea v0.22.0/go.mod h1:yyF5+GhljqvA30sRDreoyHILruNiy4ASufugzYg0VHM= +code.gitea.io/sdk/gitea v0.22.1 h1:7K05KjRORyTcTYULQ/AwvlVS6pawLcWyXZcTr7gHFyA= +code.gitea.io/sdk/gitea v0.22.1/go.mod h1:yyF5+GhljqvA30sRDreoyHILruNiy4ASufugzYg0VHM= github.com/42wim/httpsig v1.2.3 h1:xb0YyWhkYj57SPtfSttIobJUPJZB9as1nsfo7KWVcEs= github.com/42wim/httpsig v1.2.3/go.mod h1:nZq9OlYKDrUBhptd77IHx4/sZZD+IxTBADvAPI9G/EM= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -15,17 +15,26 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= +github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -34,28 +43,47 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= +github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ= +github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= -github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= -github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -66,10 +94,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/valkey-io/valkey-go v1.0.65 h1:oOyMrlmea/RidpcYLojrfimm72C4iPfRFWHkj839REU= -github.com/valkey-io/valkey-go v1.0.65/go.mod h1:bHmwjIEOrGq/ubOJfh5uMRs7Xj6mV3mQ/ZXUbmqpjqY= -github.com/valkey-io/valkey-go v1.0.66 h1:DIEF1XpwbO78xK2sMTghYE3Bz6pePWJTNxKtgoAuA3A= -github.com/valkey-io/valkey-go v1.0.66/go.mod h1:bHmwjIEOrGq/ubOJfh5uMRs7Xj6mV3mQ/ZXUbmqpjqY= +github.com/tinylib/msgp v1.5.0 h1:GWnqAE54wmnlFazjq2+vgr736Akg58iiHImh+kPY2pc= +github.com/tinylib/msgp v1.5.0/go.mod h1:cvjFkb4RiC8qSBOPMGPSzSAx47nAsfhLVTCZZNuHv5o= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/etcd/api/v3 v3.6.5 h1:pMMc42276sgR1j1raO/Qv3QI9Af/AuyQUW6CBAWuntA= @@ -80,16 +106,16 @@ go.etcd.io/etcd/client/v3 v3.6.5 h1:yRwZNFBx/35VKHTcLDeO7XVLbCBFbPi+XV4OC3QJf2U= go.etcd.io/etcd/client/v3 v3.6.5/go.mod h1:ZqwG/7TAFZ0BJ0jXRPoJjKQJtbFo/9NIY8uoFFKcCyo= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= -go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= -go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= -go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= -go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= -go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -100,8 +126,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= -golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -111,8 +135,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -122,18 +144,13 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= -golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -144,25 +161,29 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= -google.golang.org/genproto/googleapis/api v0.0.0-20251007200510-49b9836ed3ff h1:8Zg5TdmcbU8A7CXGjGXF1Slqu/nIFCRaR3S5gT2plIA= -google.golang.org/genproto/googleapis/api v0.0.0-20251007200510-49b9836ed3ff/go.mod h1:dbWfpVPvW/RqafStmRWBUpMN14puDezDMHxNYiRfQu0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251007200510-49b9836ed3ff h1:A90eA31Wq6HOMIQlLfzFwzqGKBTuaVztYu/g8sn+8Zc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251007200510-49b9836ed3ff/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= -google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20251103181224-f26f9409b101 h1:vk5TfqZHNn0obhPIYeS+cxIFKFQgser/M2jnI+9c6MM= +google.golang.org/genproto/googleapis/api v0.0.0-20251103181224-f26f9409b101/go.mod h1:E17fc4PDhkr22dE3RgnH2hEubUaky6ZwW4VhANxyspg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 h1:tRPGkdGHuewF4UisLzzHHr1spKw92qLM98nIzxbC0wY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.d7z.net/middleware v0.0.0-20251110014340-4da77101e34d h1:BKLMKcR1iOlAwZbJ2Gb6V2km42E0xrQN9omlMh813OE= +gopkg.d7z.net/middleware v0.0.0-20251110014340-4da77101e34d/go.mod h1:BJ8ySXqmlBpM9B2zFJfmvYQ61XPA+G0O1VDmYomxyrM= +gopkg.d7z.net/middleware v0.0.0-20251110035142-024d2e09b4ea h1:8FBEO3UWeUxslckvlXSizikNfgmHauj6CarkJHlW3hs= +gopkg.d7z.net/middleware v0.0.0-20251110035142-024d2e09b4ea/go.mod h1:BJ8ySXqmlBpM9B2zFJfmvYQ61XPA+G0O1VDmYomxyrM= +gopkg.d7z.net/middleware v0.0.0-20251110035951-40e0de46e3c4 h1:wIHzqRwujNatx9ueCNMJtPDU09c+FvAqOT2kJk12+VE= +gopkg.d7z.net/middleware v0.0.0-20251110035951-40e0de46e3c4/go.mod h1:BJ8ySXqmlBpM9B2zFJfmvYQ61XPA+G0O1VDmYomxyrM= +gopkg.d7z.net/middleware v0.0.0-20251110085441-55e78e556d53 h1:Mw9UU8AAv0tk86rco7CkdhDe2HNBbBF/VFPHNhBnJpk= +gopkg.d7z.net/middleware v0.0.0-20251110085441-55e78e556d53/go.mod h1:BJ8ySXqmlBpM9B2zFJfmvYQ61XPA+G0O1VDmYomxyrM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/core/alias.go b/pkg/core/alias.go index 601601e..b26cfe2 100644 --- a/pkg/core/alias.go +++ b/pkg/core/alias.go @@ -5,7 +5,7 @@ import ( "encoding/json" "fmt" - "gopkg.d7z.net/gitea-pages/pkg/middleware/config" + "gopkg.d7z.net/middleware/kv" ) type Alias struct { @@ -15,10 +15,10 @@ type Alias struct { } type DomainAlias struct { - config config.KVConfig + config kv.KV } -func NewDomainAlias(config config.KVConfig) *DomainAlias { +func NewDomainAlias(config kv.KV) *DomainAlias { return &DomainAlias{config: config} } @@ -55,9 +55,9 @@ func (a *DomainAlias) Bind(ctx context.Context, domains []string, owner, repo, b } aliasMetaRaw, _ := json.Marshal(aliasMeta) domainsRaw, _ := json.Marshal(domains) - _ = a.config.Put(ctx, rKey, string(domainsRaw), config.TtlKeep) + _ = a.config.Put(ctx, rKey, string(domainsRaw), kv.TTLKeep) for _, domain := range domains { - if err := a.config.Put(ctx, "domain/alias/"+domain, string(aliasMetaRaw), config.TtlKeep); err != nil { + if err := a.config.Put(ctx, "domain/alias/"+domain, string(aliasMetaRaw), kv.TTLKeep); err != nil { return err } } @@ -65,5 +65,6 @@ func (a *DomainAlias) Bind(ctx context.Context, domains []string, owner, repo, b } func (a *DomainAlias) Unbind(ctx context.Context, domain string) error { - return a.config.Delete(ctx, "domain/alias/"+domain) + _, err := a.config.Delete(ctx, "domain/alias/"+domain) + return err } diff --git a/pkg/core/backend.go b/pkg/core/backend.go index 82acb08..ea432ff 100644 --- a/pkg/core/backend.go +++ b/pkg/core/backend.go @@ -1,23 +1,9 @@ package core import ( - "bytes" "context" - "encoding/json" - stdErr "errors" - "fmt" - "io" "net/http" - "os" - "strconv" "time" - - "github.com/pkg/errors" - "go.uber.org/zap" - "gopkg.d7z.net/gitea-pages/pkg/middleware/cache" - "gopkg.d7z.net/gitea-pages/pkg/middleware/config" - - "gopkg.d7z.net/gitea-pages/pkg/utils" ) type BranchInfo struct { @@ -34,152 +20,3 @@ type Backend interface { // Open return file or error (error) Open(ctx context.Context, client *http.Client, owner, repo, commit, path string, headers http.Header) (*http.Response, error) } - -type CacheBackend struct { - backend Backend - config config.KVConfig - ttl time.Duration -} - -func (c *CacheBackend) Close() error { - return c.backend.Close() -} - -func NewCacheBackend(backend Backend, config config.KVConfig, ttl time.Duration) *CacheBackend { - return &CacheBackend{backend: backend, config: config, ttl: ttl} -} - -func (c *CacheBackend) Repos(ctx context.Context, owner string) (map[string]string, error) { - ret := make(map[string]string) - key := fmt.Sprintf("repos/%s", owner) - store, err := c.config.Get(ctx, key) - if err != nil { - ret, err = c.backend.Repos(ctx, owner) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - _ = c.config.Put(ctx, key, "{}", c.ttl) - } - return nil, err - } - storeBin, err := json.Marshal(ret) - if err != nil { - return nil, err - } - if err = c.config.Put(ctx, key, string(storeBin), c.ttl); err != nil { - return nil, err - } - } else { - if err := json.Unmarshal([]byte(store), &ret); err != nil { - return nil, err - } - } - if len(ret) == 0 { - return ret, os.ErrNotExist - } - return ret, nil -} - -func (c *CacheBackend) Branches(ctx context.Context, owner, repo string) (map[string]*BranchInfo, error) { - ret := make(map[string]*BranchInfo) - key := fmt.Sprintf("branches/%s/%s", owner, repo) - data, err := c.config.Get(ctx, key) - if err != nil { - ret, err = c.backend.Branches(ctx, owner, repo) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - _ = c.config.Put(ctx, key, "{}", c.ttl) - } - return nil, err - } - data, err := json.Marshal(ret) - if err != nil { - return nil, err - } - if err = c.config.Put(ctx, key, string(data), c.ttl); err != nil { - return nil, err - } - } else { - if err := json.Unmarshal([]byte(data), &ret); err != nil { - return nil, err - } - } - if len(ret) == 0 { - return ret, os.ErrNotExist - } - return ret, nil -} - -func (c *CacheBackend) Open(ctx context.Context, client *http.Client, owner, repo, commit, path string, headers http.Header) (*http.Response, error) { - return c.backend.Open(ctx, client, owner, repo, commit, path, headers) -} - -type CacheBackendBlobReader struct { - client *http.Client - cache cache.Cache - base Backend - maxSize int -} - -func NewCacheBackendBlobReader(client *http.Client, base Backend, cache cache.Cache, maxCacheSize int) *CacheBackendBlobReader { - return &CacheBackendBlobReader{client: client, base: base, cache: cache, maxSize: maxCacheSize} -} - -func (c *CacheBackendBlobReader) Open(ctx context.Context, owner, repo, commit, path string) (io.ReadCloser, error) { - key := fmt.Sprintf("%s/%s/%s/%s", owner, repo, commit, path) - lastCache, err := c.cache.Get(key) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return nil, err - } else if lastCache == nil && err == nil { - // 边界缓存 - return nil, os.ErrNotExist - } else if lastCache != nil { - return lastCache, nil - } - open, err := c.base.Open(ctx, c.client, owner, repo, commit, path, http.Header{}) - if err != nil || open == nil { - if open != nil { - _ = open.Body.Close() - } - return nil, stdErr.Join(err, os.ErrNotExist) - } - if open.StatusCode == http.StatusNotFound { - // 缓存 404 路由 - _ = c.cache.Put(key, nil) - _ = open.Body.Close() - return nil, os.ErrNotExist - } - - lastMod, err := time.Parse(http.TimeFormat, open.Header.Get("Last-Modified")) - if err != nil { - // 无时间,跳过 - return open.Body, nil - } - length, err := strconv.Atoi(open.Header.Get("Content-Length")) - // 无法计算大小,跳过 - if err != nil { - return open.Body, nil - } - if length > c.maxSize { - // 超过最大大小,跳过 - return &utils.SizeReadCloser{ - ReadCloser: open.Body, - Size: length, - }, nil - } - - defer open.Body.Close() - allBytes, err := io.ReadAll(open.Body) - if err != nil { - return nil, err - } - if err = c.cache.Put(key, bytes.NewBuffer(allBytes)); err != nil { - zap.L().Warn("缓存归档失败", zap.Error(err), zap.Int("Size", len(allBytes)), zap.Int("MaxSize", c.maxSize)) - } - return &cache.CacheContent{ - ReadSeekCloser: utils.NopCloser{ - ReadSeeker: bytes.NewReader(allBytes), - }, - LastModified: lastMod, - Length: length, - }, nil -} diff --git a/pkg/core/backend_cache.go b/pkg/core/backend_cache.go new file mode 100644 index 0000000..8aa29db --- /dev/null +++ b/pkg/core/backend_cache.go @@ -0,0 +1,157 @@ +package core + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "os" + "strconv" + "time" + + "github.com/pkg/errors" + "go.uber.org/zap" + "gopkg.d7z.net/gitea-pages/pkg/utils" + "gopkg.d7z.net/middleware/cache" + "gopkg.d7z.net/middleware/kv" + "gopkg.d7z.net/middleware/tools" +) + +type CacheBackend struct { + backend Backend + cacheRepo *tools.Cache[map[string]string] + cacheBranch *tools.Cache[map[string]*BranchInfo] +} + +func (c *CacheBackend) Close() error { + return c.backend.Close() +} + +func NewCacheBackend(backend Backend, cache kv.KV, ttl time.Duration) *CacheBackend { + repoCache := tools.NewCache[map[string]string](cache, "repos", ttl) + branchCache := tools.NewCache[map[string]*BranchInfo](cache, "branches", ttl) + return &CacheBackend{ + backend: backend, + cacheRepo: repoCache, + cacheBranch: branchCache, + } +} + +func (c *CacheBackend) Repos(ctx context.Context, owner string) (map[string]string, error) { + if load, b := c.cacheRepo.Load(ctx, owner); b { + return load, nil + } + ret, err := c.backend.Repos(ctx, owner) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + _ = c.cacheRepo.Store(ctx, owner, map[string]string{}) + } + return nil, err + } + err = c.cacheRepo.Store(ctx, owner, ret) + if len(ret) == 0 { + return nil, os.ErrNotExist + } + return ret, err +} + +func (c *CacheBackend) Branches(ctx context.Context, owner, repo string) (map[string]*BranchInfo, error) { + ret := make(map[string]*BranchInfo) + key := fmt.Sprintf("%s/%s", owner, repo) + if load, b := c.cacheBranch.Load(ctx, key); b { + return load, nil + } + ret, err := c.backend.Branches(ctx, owner, repo) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + _ = c.cacheBranch.Store(ctx, key, map[string]*BranchInfo{}) + } + return nil, err + } + err = c.cacheBranch.Store(ctx, key, ret) + if len(ret) == 0 { + return nil, os.ErrNotExist + } + return ret, err +} + +func (c *CacheBackend) Open(ctx context.Context, client *http.Client, owner, repo, commit, path string, headers http.Header) (*http.Response, error) { + return c.backend.Open(ctx, client, owner, repo, commit, path, headers) +} + +type CacheBackendBlobReader struct { + client *http.Client + cache cache.Cache + base Backend + limit uint64 +} + +func NewCacheBackendBlobReader( + client *http.Client, + base Backend, + cache cache.Cache, + limit uint64, +) *CacheBackendBlobReader { + return &CacheBackendBlobReader{client: client, base: base, cache: cache, limit: limit} +} + +func (c *CacheBackendBlobReader) Open(ctx context.Context, owner, repo, commit, path string) (io.ReadCloser, error) { + key := fmt.Sprintf("%s/%s/%s/%s", owner, repo, commit, path) + lastCache, err := c.cache.Get(ctx, key) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } else if lastCache == nil && err == nil { + // 边界缓存 + return nil, os.ErrNotExist + } else if lastCache != nil { + return lastCache, nil + } + open, err := c.base.Open(ctx, c.client, owner, repo, commit, path, http.Header{}) + if err != nil || open == nil { + if open != nil { + _ = open.Body.Close() + } + return nil, err + } + if open.StatusCode == http.StatusNotFound { + // 缓存 404 路由 + _ = c.cache.Put(ctx, key, nil, time.Hour) + _ = open.Body.Close() + return nil, os.ErrNotExist + } + + lastMod, err := time.Parse(http.TimeFormat, open.Header.Get("Last-Modified")) + if err != nil { + // 无时间,跳过 + return open.Body, nil + } + length, err := strconv.ParseUint(open.Header.Get("Content-Length"), 10, 64) + // 无法计算大小,跳过 + if err != nil { + return open.Body, nil + } + if length > c.limit { + // 超过最大大小,跳过 + return &utils.SizeReadCloser{ + ReadCloser: open.Body, + Size: length, + }, nil + } + + defer open.Body.Close() + allBytes, err := io.ReadAll(open.Body) + if err != nil { + return nil, err + } + if err = c.cache.Put(ctx, key, bytes.NewBuffer(allBytes), time.Hour); err != nil { + zap.L().Warn("缓存归档失败", zap.Error(err), zap.Int("Size", len(allBytes)), zap.Uint64("MaxSize", c.limit)) + } + return &cache.Content{ + ReadSeekCloser: utils.NopCloser{ + ReadSeeker: bytes.NewReader(allBytes), + }, + LastModified: lastMod, + Length: length, + }, nil +} diff --git a/pkg/core/domain.go b/pkg/core/domain.go index 30cd998..ab4d8da 100644 --- a/pkg/core/domain.go +++ b/pkg/core/domain.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/pkg/errors" - "gopkg.d7z.net/gitea-pages/pkg/middleware/config" + "gopkg.d7z.net/middleware/kv" "go.uber.org/zap" ) @@ -19,12 +19,12 @@ type PageDomain struct { defaultBranch string } -func NewPageDomain(meta *ServerMeta, config config.KVConfig, baseDomain, defaultBranch string) *PageDomain { +func NewPageDomain(meta *ServerMeta, alias kv.KV, baseDomain, defaultBranch string) *PageDomain { return &PageDomain{ baseDomain: baseDomain, defaultBranch: defaultBranch, ServerMeta: meta, - alias: NewDomainAlias(config), + alias: NewDomainAlias(alias), } } diff --git a/pkg/core/meta.go b/pkg/core/meta.go index 66cc36b..01f7b73 100644 --- a/pkg/core/meta.go +++ b/pkg/core/meta.go @@ -13,7 +13,7 @@ import ( "time" "go.uber.org/zap" - "gopkg.d7z.net/gitea-pages/pkg/middleware/config" + "gopkg.d7z.net/middleware/kv" "gopkg.in/yaml.v3" "github.com/gobwas/glob" @@ -31,13 +31,14 @@ type ServerMeta struct { Domain string client *http.Client - cache config.KVConfig - ttl time.Duration + + cache kv.KV + ttl time.Duration locker *utils.Locker } -func NewServerMeta(client *http.Client, backend Backend, kv config.KVConfig, domain string, ttl time.Duration) *ServerMeta { +func NewServerMeta(client *http.Client, backend Backend, kv kv.KV, domain string, ttl time.Duration) *ServerMeta { return &ServerMeta{backend, domain, client, kv, ttl, utils.NewLocker()} } diff --git a/pkg/middleware/cache/cache.go b/pkg/middleware/cache/cache.go deleted file mode 100644 index ed63c6f..0000000 --- a/pkg/middleware/cache/cache.go +++ /dev/null @@ -1,48 +0,0 @@ -package cache - -import ( - "io" - "sync" - "time" - - "github.com/pkg/errors" -) - -type CacheContent struct { - io.ReadSeekCloser - Length int - LastModified time.Time -} - -func (c *CacheContent) ReadToString() (string, error) { - all, err := io.ReadAll(c) - if err != nil { - return "", err - } - return string(all), nil -} - -type Cache interface { - Put(key string, reader io.Reader) error - // Get return CacheContent or nil when put nil io.reader - Get(key string) (*CacheContent, error) - Delete(pattern string) error - io.Closer -} - -var ErrCacheOutOfMemory = errors.New("内容无法被缓存,超过最大限定值") - -// TODO: 优化锁结构 -// 复杂场景请使用其他缓存服务 - -type CacheMemory struct { - l sync.RWMutex - data map[string]*[]byte - lastModify map[string]time.Time - sizeGlobal int - sizeItem int - - current int - cache []byte - ordered []string -} diff --git a/pkg/middleware/cache/cache_memory.go b/pkg/middleware/cache/cache_memory.go deleted file mode 100644 index adf7bca..0000000 --- a/pkg/middleware/cache/cache_memory.go +++ /dev/null @@ -1,139 +0,0 @@ -package cache - -import ( - "bytes" - "io" - "os" - "strings" - "sync" - "time" - - "gopkg.d7z.net/gitea-pages/pkg/utils" -) - -func NewCacheMemory(maxUsage, maxGlobalUsage int) *CacheMemory { - return &CacheMemory{ - data: make(map[string]*[]byte), - lastModify: make(map[string]time.Time), - l: sync.RWMutex{}, - sizeGlobal: maxGlobalUsage, - sizeItem: maxUsage, - - cache: make([]byte, maxUsage+1), - ordered: make([]string, 0), - } -} - -func (c *CacheMemory) Put(key string, reader io.Reader) error { - c.l.Lock() - defer c.l.Unlock() - size := 0 - // 可以指定空的 reader 作为 404 缓存 - if reader != nil { - var err error - size, err = io.ReadAtLeast(reader, c.cache, 1) - if err != nil { - return err - } - } - if size == len(c.cache) { - return ErrCacheOutOfMemory - } - currentItemSize := 0 - if data, ok := c.data[key]; ok { - currentItemSize = len(*data) - } - available := c.sizeGlobal + currentItemSize - (c.current + size) - if available < 0 { - // 清理旧的内容 - count := 0 - for i, k := range c.ordered { - available += len(*c.data[k]) - if available > 0 { - break - } - count = i + 1 - } - - if available < 0 { - // 清理全部内容也无法留出空间 - return ErrCacheOutOfMemory - } - for _, s := range c.ordered[:count] { - delete(c.data, s) - delete(c.lastModify, s) - } - c.ordered = c.ordered[count:] - } - - if reader != nil { - dest := make([]byte, size) - copy(dest, c.cache[:size]) - c.data[key] = &dest - c.lastModify[key] = time.Now() - - c.current -= currentItemSize - c.current += len(dest) - } else { - c.data[key] = nil - c.lastModify[key] = time.Now() - - c.current -= currentItemSize - } - - nextOrdered := make([]string, 0, len(c.ordered)) - for _, s := range c.ordered { - if s != key { - nextOrdered = append(nextOrdered, s) - } - } - c.ordered = append(nextOrdered, key) - return nil -} - -func (c *CacheMemory) Get(key string) (*CacheContent, error) { - c.l.RLock() - defer c.l.RUnlock() - if i, ok := c.data[key]; ok { - if i == nil { - return nil, nil - } - - return &CacheContent{ - ReadSeekCloser: utils.NopCloser{ - bytes.NewReader(*i), - }, - Length: len(*i), - LastModified: c.lastModify[key], - }, nil - } - return nil, os.ErrNotExist -} - -func (c *CacheMemory) Delete(pattern string) error { - c.l.Lock() - defer c.l.Unlock() - nextOrder := make([]string, 0, len(c.ordered)) - for _, key := range c.ordered { - if strings.HasPrefix(key, pattern) { - c.current -= len(*c.data[key]) - delete(c.data, key) - delete(c.lastModify, key) - } else { - nextOrder = append(nextOrder, key) - } - } - clear(c.ordered) - c.ordered = nextOrder - return nil -} - -func (c *CacheMemory) Close() error { - c.l.Lock() - defer c.l.Unlock() - clear(c.ordered) - clear(c.data) - clear(c.lastModify) - c.current = 0 - return nil -} diff --git a/pkg/middleware/cache/cache_test.go b/pkg/middleware/cache/cache_test.go deleted file mode 100644 index a3e5be2..0000000 --- a/pkg/middleware/cache/cache_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package cache - -import ( - "fmt" - "io" - "os" - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestCacheGetPutDelete(t *testing.T) { - memory := NewCacheMemory(1024, 10240) - - require.NoError(t, memory.Put("hello", strings.NewReader("world"))) - - value, err := memory.Get("hello") - require.NoError(t, err) - all, err := io.ReadAll(value) - require.NoError(t, err) - require.Equal(t, "world", string(all)) - require.Equal(t, 5, memory.current) - - require.NoError(t, memory.Put("hello", strings.NewReader("kotlin"))) - - value, err = memory.Get("hello") - require.NoError(t, err) - all, err = io.ReadAll(value) - require.NoError(t, err) - require.Equal(t, "kotlin", string(all)) - require.Equal(t, 6, memory.current) - require.Equal(t, 1, len(memory.data)) - - require.NoError(t, memory.Put("data", strings.NewReader("kotlin"))) - require.Equal(t, 12, memory.current) - require.Equal(t, 2, len(memory.data)) - require.Equal(t, 2, len(memory.ordered)) - - require.NoError(t, memory.Delete("hello")) - value, err = memory.Get("hello") - require.Error(t, err) - require.Equal(t, 1, len(memory.data)) - require.Equal(t, 1, len(memory.ordered)) - - require.NoError(t, memory.Put("hello", nil)) - value, err = memory.Get("hello") - require.NoError(t, err) - require.Nil(t, value) -} - -func TestCacheLimit(t *testing.T) { - memory := NewCacheMemory(5, 5*5) - require.NoError(t, memory.Put("hello", strings.NewReader("world"))) - require.Equal(t, 5, memory.current) - require.ErrorIs(t, memory.Put("hello", strings.NewReader("world1")), ErrCacheOutOfMemory) - require.Equal(t, 5, memory.current) - for i := 0; i < 4; i++ { - require.NoError(t, memory.Put(fmt.Sprintf("hello-%d", i), strings.NewReader("govet"))) - } - value, err := memory.Get("hello") - require.NoError(t, err) - all, err := io.ReadAll(value) - require.NoError(t, err) - require.Equal(t, "world", string(all)) - - require.NoError(t, memory.Put("test", strings.NewReader("govet"))) - - value, err = memory.Get("hello") - require.ErrorIs(t, err, os.ErrNotExist) - - require.Equal(t, 5, len(memory.data)) - require.Equal(t, 5, len(memory.ordered)) - require.Equal(t, 5, len(memory.lastModify)) -} diff --git a/pkg/middleware/config/config.go b/pkg/middleware/config/config.go deleted file mode 100644 index eecaee9..0000000 --- a/pkg/middleware/config/config.go +++ /dev/null @@ -1,99 +0,0 @@ -package config - -import ( - "context" - "crypto/tls" - "crypto/x509" - "fmt" - "io" - "net/url" - "os" - "strconv" - "strings" - "time" -) - -const TtlKeep = -1 - -type KVConfig interface { - Put(ctx context.Context, key string, value string, ttl time.Duration) error - Get(ctx context.Context, key string) (string, error) - Delete(ctx context.Context, key string) error - io.Closer -} - -func NewAutoConfig(src string) (KVConfig, error) { - if src == "" || - strings.HasPrefix(src, "./") || - strings.HasPrefix(src, "/") || - strings.HasPrefix(src, "\\") || - strings.HasPrefix(src, ".\\") { - return NewConfigMemory(src) - } - parse, err := url.Parse(src) - if err != nil { - return nil, err - } - switch parse.Scheme { - case "local": - return NewConfigMemory(parse.Path) - case "redis": - query := parse.Query() - pass := query.Get("pass") - if pass == "" { - pass = query.Get("password") - } - db := strings.TrimPrefix(parse.Path, "/") - if db == "" { - db = "0" - } - dbi, err := strconv.Atoi(db) - if err != nil { - return nil, err - } - return NewConfigRedis(parse.Host, pass, dbi) - case "etcd": - query := parse.Query() - endpoints := []string{parse.Host} - - // 检查是否有多个端点 - if endpointsStr := query.Get("endpoints"); endpointsStr != "" { - endpoints = strings.Split(endpointsStr, ",") - } - - // 检查是否需要TLS认证 - var tlsConfig *tls.Config - caFile := query.Get("ca-file") - certFile := query.Get("cert-file") - keyFile := query.Get("key-file") - - if caFile != "" && certFile != "" && keyFile != "" { - caCert, err := os.ReadFile(caFile) - if err != nil { - return nil, fmt.Errorf("failed to read ca file: %v", err) - } - - caCertPool := x509.NewCertPool() - if !caCertPool.AppendCertsFromPEM(caCert) { - return nil, fmt.Errorf("failed to add ca certificate") - } - - cert, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - return nil, fmt.Errorf("failed to load client certificate: %v", err) - } - - tlsConfig = &tls.Config{ - Certificates: []tls.Certificate{cert}, - RootCAs: caCertPool, - } - } else if caFile != "" || certFile != "" || keyFile != "" { - // 部分TLS参数被提供,视为错误 - return nil, fmt.Errorf("incomplete tls configuration, need ca-file, cert-file and key-file") - } - - return NewConfigEtcd(endpoints, tlsConfig) - default: - return nil, fmt.Errorf("unsupported scheme: %s", parse.Scheme) - } -} diff --git a/pkg/middleware/config/config_etcd.go b/pkg/middleware/config/config_etcd.go deleted file mode 100644 index c5b9b7f..0000000 --- a/pkg/middleware/config/config_etcd.go +++ /dev/null @@ -1,97 +0,0 @@ -package config - -import ( - "context" - "crypto/tls" - "fmt" - "os" - "time" - - clientv3 "go.etcd.io/etcd/client/v3" - "go.uber.org/zap" -) - -// ConfigEtcd 实现了KVConfig接口的etcd配置存储 -type ConfigEtcd struct { - client *clientv3.Client - lease clientv3.Lease -} - -// NewConfigEtcd 创建一个新的etcd配置存储实例 -// endpoints是etcd集群的地址列表,如["localhost:2379"] -// tlsConfig是TLS配置,如果为nil则使用无认证模式 -func NewConfigEtcd(endpoints []string, tlsConfig *tls.Config) (*ConfigEtcd, error) { - if len(endpoints) == 0 { - return nil, fmt.Errorf("endpoints is empty") - } - - zap.L().Debug("connect etcd", zap.Strings("endpoints", endpoints)) - - // 创建etcd客户端配置 - cfg := clientv3.Config{ - Endpoints: endpoints, - DialTimeout: 5 * time.Second, - TLS: tlsConfig, - } - - // 建立连接 - client, err := clientv3.New(cfg) - if err != nil { - return nil, fmt.Errorf("failed to create etcd client: %v", err) - } - - // 创建lease用于处理TTL - lease := clientv3.NewLease(client) - - return &ConfigEtcd{ - client: client, - lease: lease, - }, nil -} - -// Put 存储键值对,可以设置过期时间 -func (e *ConfigEtcd) Put(ctx context.Context, key string, value string, ttl time.Duration) error { - if ttl == TtlKeep || ttl <= 0 { - // 无过期时间,直接存储 - _, err := e.client.Put(ctx, key, value) - return err - } - - // 创建lease - resp, err := e.lease.Grant(ctx, int64(ttl.Seconds())) - if err != nil { - return fmt.Errorf("failed to grant lease: %v", err) - } - - // 关联lease存储键值对 - _, err = e.client.Put(ctx, key, value, clientv3.WithLease(resp.ID)) - return err -} - -// Get 获取键对应的值 -func (e *ConfigEtcd) Get(ctx context.Context, key string) (string, error) { - resp, err := e.client.Get(ctx, key) - if err != nil { - return "", fmt.Errorf("failed to get key: %v", err) - } - - if len(resp.Kvs) == 0 { - return "", os.ErrNotExist - } - - return string(resp.Kvs[0].Value), nil -} - -// Delete 删除指定的键 -func (e *ConfigEtcd) Delete(ctx context.Context, key string) error { - _, err := e.client.Delete(ctx, key) - return err -} - -// Close 关闭etcd客户端连接 -func (e *ConfigEtcd) Close() error { - if err := e.lease.Close(); err != nil { - zap.L().Warn("failed to close etcd lease", zap.Error(err)) - } - return e.client.Close() -} diff --git a/pkg/middleware/config/config_memory.go b/pkg/middleware/config/config_memory.go deleted file mode 100644 index db4e749..0000000 --- a/pkg/middleware/config/config_memory.go +++ /dev/null @@ -1,106 +0,0 @@ -package config - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "sync" - "time" - - "go.uber.org/zap" -) - -// ConfigMemory 一个简单的内存配置归档,仅用于测试 -type ConfigMemory struct { - data sync.Map - store string -} - -func NewConfigMemory(store string) (KVConfig, error) { - ret := &ConfigMemory{ - store: store, - data: sync.Map{}, - } - if store != "" { - zap.L().Info("parse config from store", zap.String("store", store)) - if err := os.MkdirAll(filepath.Dir(store), 0o755); err != nil && !os.IsExist(err) { - return nil, err - } - item := make(map[string]ConfigContent) - data, err := os.ReadFile(store) - if err != nil && !os.IsNotExist(err) { - return nil, err - } - if err == nil { - err = json.Unmarshal(data, &item) - if err != nil { - return nil, err - } - } - for key, content := range item { - if content.Ttl == nil || time.Now().Before(*content.Ttl) { - ret.data.Store(key, content) - } - } - clear(item) - } - return ret, nil -} - -type ConfigContent struct { - Data string `json:"data"` - Ttl *time.Time `json:"ttl,omitempty"` -} - -func (m *ConfigMemory) Put(ctx context.Context, key string, value string, ttl time.Duration) error { - d := time.Now().Add(ttl) - td := &d - if ttl == -1 { - td = nil - } - m.data.Store(key, ConfigContent{ - Data: value, - Ttl: td, - }) - return nil -} - -func (m *ConfigMemory) Get(ctx context.Context, key string) (string, error) { - if value, ok := m.data.Load(key); ok { - content := value.(ConfigContent) - if content.Ttl != nil && time.Now().After(*content.Ttl) { - return "", os.ErrNotExist - } - return content.Data, nil - } - return "", os.ErrNotExist -} - -func (m *ConfigMemory) Delete(ctx context.Context, key string) error { - m.data.Delete(key) - return nil -} - -func (m *ConfigMemory) Close() error { - defer m.data.Clear() - if m.store != "" { - item := make(map[string]ConfigContent) - now := time.Now() - m.data.Range( - func(key, value interface{}) bool { - content := value.(ConfigContent) - if content.Ttl == nil || now.Before(*content.Ttl) { - item[key.(string)] = content - } - return true - }) - zap.L().Debug("回写内容到本地存储", zap.String("store", m.store), zap.Int("length", len(item))) - saved, err := json.Marshal(item) - if err != nil { - return err - } - return os.WriteFile(m.store, saved, 0o600) - } - return nil -} diff --git a/pkg/middleware/config/config_redis.go b/pkg/middleware/config/config_redis.go deleted file mode 100644 index 09d8497..0000000 --- a/pkg/middleware/config/config_redis.go +++ /dev/null @@ -1,60 +0,0 @@ -package config - -import ( - "context" - "fmt" - "os" - "time" - - "github.com/pkg/errors" - "github.com/valkey-io/valkey-go" - - "go.uber.org/zap" -) - -type ConfigRedis struct { - client valkey.Client -} - -func NewConfigRedis(addr string, password string, db int) (*ConfigRedis, error) { - if addr == "" { - return nil, fmt.Errorf("addr is empty") - } - zap.L().Debug("connect redis", zap.String("addr", addr)) - client, err := valkey.NewClient(valkey.ClientOption{ - InitAddress: []string{addr}, - Password: password, - SelectDB: db, - }) - if err != nil { - return nil, err - } - return &ConfigRedis{ - client: client, - }, nil -} - -func (r *ConfigRedis) Put(ctx context.Context, key string, value string, ttl time.Duration) error { - builder := r.client.B().Set().Key(key).Value(value) - if ttl != TtlKeep { - builder.Ex(ttl) - } - return r.client.Do(ctx, builder.Build()).Error() -} - -func (r *ConfigRedis) Get(ctx context.Context, key string) (string, error) { - v, err := r.client.Do(ctx, r.client.B().Get().Key(key).Build()).ToString() - if err != nil && errors.Is(err, valkey.Nil) { - return "", os.ErrNotExist - } - return v, err -} - -func (r *ConfigRedis) Delete(ctx context.Context, key string) error { - return r.client.Do(ctx, r.client.B().Del().Key(key).Build()).Error() -} - -func (r *ConfigRedis) Close() error { - r.client.Close() - return nil -} diff --git a/pkg/providers/gitea.go b/pkg/providers/gitea.go index 9cfcd58..6382598 100644 --- a/pkg/providers/gitea.go +++ b/pkg/providers/gitea.go @@ -109,7 +109,7 @@ func (g *ProviderGitea) Open(ctx context.Context, client *http.Client, owner, re return nil, err } giteaURL += "?ref=" + url.QueryEscape(commit) - req, err := http.NewRequest(http.MethodGet, giteaURL, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, giteaURL, nil) if err != nil { return nil, err } diff --git a/pkg/server.go b/pkg/server.go index 84c4aa4..9a8330d 100644 --- a/pkg/server.go +++ b/pkg/server.go @@ -12,18 +12,16 @@ import ( "path/filepath" "regexp" "slices" - "strconv" "strings" "time" "github.com/google/uuid" - "gopkg.d7z.net/gitea-pages/pkg/middleware/cache" - "gopkg.d7z.net/gitea-pages/pkg/middleware/config" + "gopkg.d7z.net/middleware/cache" + "gopkg.d7z.net/middleware/kv" "github.com/pkg/errors" "go.uber.org/zap" - "github.com/pbnjay/memory" "gopkg.d7z.net/gitea-pages/pkg/core" "gopkg.d7z.net/gitea-pages/pkg/utils" @@ -33,39 +31,48 @@ import ( var portExp = regexp.MustCompile(`:\d+$`) type ServerOptions struct { - Domain string - DefaultBranch string + Domain string //默认域名 + DefaultBranch string // 默认分支 - KVConfig config.KVConfig - Cache cache.Cache + Alias kv.KV // 配置映射关系 - MaxCacheSize int + CacheMeta kv.KV // 配置缓存 + CacheMetaTTL time.Duration // 配置缓存时长 - HttpClient *http.Client + CacheBlob cache.Cache // blob缓存 + CacheBlobTTL time.Duration // 配置缓存时长 + CacheBlobLimit uint64 // blob最大缓存大小 - MetaTTL time.Duration + HttpClient *http.Client //自定义客户端 - EnableRender bool - EnableProxy bool + EnableRender bool // 允许渲染 + EnableProxy bool // 允许代理 - StaticDir string + StaticDir string // 静态文件位置 DefaultErrorHandler func(w http.ResponseWriter, r *http.Request, err error) } func DefaultOptions(domain string) ServerOptions { - configMemory, _ := config.NewAutoConfig("") + configMemory, _ := kv.NewMemory("") + cacheMemory, _ := cache.NewMemoryCache(cache.MemoryCacheConfig{MaxCapacity: 4096, CleanupInt: time.Hour}) return ServerOptions{ Domain: domain, DefaultBranch: "gh-pages", - KVConfig: configMemory, - Cache: cache.NewCacheMemory(1024*1024*10, int(memory.FreeMemory()/3*2)), - MaxCacheSize: 1024 * 1024 * 10, - HttpClient: http.DefaultClient, - MetaTTL: time.Minute, - EnableRender: true, - EnableProxy: true, - StaticDir: "", + + Alias: configMemory, + CacheMeta: configMemory, + CacheMetaTTL: time.Minute, + + CacheBlob: cacheMemory, + CacheBlobTTL: time.Minute, + CacheBlobLimit: 1024 * 1024 * 10, + + HttpClient: http.DefaultClient, + + EnableRender: true, + EnableProxy: true, + StaticDir: "", DefaultErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { if errors.Is(err, os.ErrNotExist) { http.Error(w, "page not found.", http.StatusNotFound) @@ -87,10 +94,10 @@ type Server struct { var staticPrefix = "/.well-known/page-server/" func NewPageServer(backend core.Backend, options ServerOptions) *Server { - backend = core.NewCacheBackend(backend, options.KVConfig, options.MetaTTL) - svcMeta := core.NewServerMeta(options.HttpClient, backend, options.KVConfig, options.Domain, options.MetaTTL) - pageMeta := core.NewPageDomain(svcMeta, options.KVConfig, options.Domain, options.DefaultBranch) - reader := core.NewCacheBackendBlobReader(options.HttpClient, backend, options.Cache, options.MaxCacheSize) + backend = core.NewCacheBackend(backend, options.CacheMeta, options.CacheMetaTTL) + svcMeta := core.NewServerMeta(options.HttpClient, backend, options.CacheMeta, options.Domain, options.CacheMetaTTL) + pageMeta := core.NewPageDomain(svcMeta, options.Alias, options.Domain, options.DefaultBranch) + reader := core.NewCacheBackendBlobReader(options.HttpClient, backend, options.CacheBlob, options.CacheBlobLimit) var fs http.Handler if options.StaticDir != "" { fs = http.StripPrefix(staticPrefix, http.FileServer(http.Dir(options.StaticDir))) @@ -237,10 +244,10 @@ func (s *Server) Serve(writer http.ResponseWriter, request *http.Request) error render = nil } defer result.Close() - if reader, ok := result.(*cache.CacheContent); ok { - writer.Header().Add("X-Cache", "HIT") + if reader, ok := result.(*cache.Content); ok { + writer.Header().Add("X-CacheBlob", "HIT") writer.Header().Set("Content-Type", mime.TypeByExtension(filepath.Ext(fileName))) - writer.Header().Add("Cache-Control", "public, max-age=86400") + writer.Header().Add("CacheBlob-Control", "public, max-age=86400") if render != nil { if err = render.Render(writer, request, reader); err != nil { return err @@ -250,11 +257,11 @@ func (s *Server) Serve(writer http.ResponseWriter, request *http.Request) error } } else { if reader, ok := result.(*utils.SizeReadCloser); ok && render == nil { - writer.Header().Add("Content-Length", strconv.Itoa(reader.Size)) + writer.Header().Add("Content-Length", fmt.Sprintf("%d", reader.Size)) } // todo(bug) : 直连模式下告知数据长度 - writer.Header().Add("X-Cache", "MISS") - writer.Header().Add("Cache-Control", "public, max-age=86400") + writer.Header().Add("X-CacheBlob", "MISS") + writer.Header().Add("CacheBlob-Control", "public, max-age=86400") writer.Header().Set("Content-Type", mime.TypeByExtension(filepath.Ext(fileName))) writer.WriteHeader(http.StatusOK) if render != nil { @@ -269,10 +276,10 @@ func (s *Server) Serve(writer http.ResponseWriter, request *http.Request) error } func (s *Server) Close() error { - if err := s.options.KVConfig.Close(); err != nil { + if err := s.options.Alias.Close(); err != nil { return err } - if err := s.options.Cache.Close(); err != nil { + if err := s.options.CacheBlob.Close(); err != nil { return err } return s.backend.Close() diff --git a/pkg/utils/reader.go b/pkg/utils/reader.go index 54934af..7b5a786 100644 --- a/pkg/utils/reader.go +++ b/pkg/utils/reader.go @@ -4,5 +4,5 @@ import "io" type SizeReadCloser struct { io.ReadCloser - Size int + Size uint64 } diff --git a/tests/core/test.go b/tests/core/test.go index 0f621ff..edcce13 100644 --- a/tests/core/test.go +++ b/tests/core/test.go @@ -22,7 +22,7 @@ type SvcOpts func(options *pkg.ServerOptions) func NewDefaultTestServer() *TestServer { return NewTestServer("example.com", func(options *pkg.ServerOptions) { - options.MetaTTL = 0 + options.CacheMetaTTL = 0 }) }