完善 cache 相关内容
This commit is contained in:
dragon
12
go.mod
12
go.mod
@@ -2,16 +2,18 @@ module code.d7z.net/d7z-project/gitea-pages
|
||||
|
||||
go 1.23.2
|
||||
|
||||
require code.gitea.io/sdk/gitea v0.19.0
|
||||
require (
|
||||
code.gitea.io/sdk/gitea v0.19.0
|
||||
github.com/stretchr/testify v1.8.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/davidmz/go-pageant v1.0.2 // indirect
|
||||
github.com/go-fed/httpsig v1.1.0 // indirect
|
||||
github.com/hashicorp/go-version v1.6.0 // indirect
|
||||
github.com/ncruces/go-sqlite3 v0.21.3 // indirect
|
||||
github.com/ncruces/julianday v1.0.0 // indirect
|
||||
github.com/stretchr/testify v1.8.1 // indirect
|
||||
github.com/tetratelabs/wazero v1.8.2 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
golang.org/x/crypto v0.31.0 // indirect
|
||||
golang.org/x/sys v0.28.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
15
go.sum
15
go.sum
@@ -9,10 +9,6 @@ github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI=
|
||||
github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM=
|
||||
github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
|
||||
github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/ncruces/go-sqlite3 v0.21.3 h1:hHkfNQLcbnxPJZhC/RGw9SwP3bfkv/Y0xUHWsr1CdMQ=
|
||||
github.com/ncruces/go-sqlite3 v0.21.3/go.mod h1:zxMOaSG5kFYVFK4xQa0pdwIszqxqJ0W0BxBgwdrNjuA=
|
||||
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
|
||||
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
@@ -22,30 +18,25 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4=
|
||||
github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
|
||||
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
|
||||
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
|
||||
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
|
||||
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
|
||||
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
|
||||
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Cache interface {
|
||||
@@ -19,9 +21,8 @@ var ErrCacheOutOfMemory = errors.New("内容无法被缓存，超过最大限定
|
||||
type CacheMemory struct {
|
||||
l sync.RWMutex
|
||||
data map[string][]byte
|
||||
maxAge time.Duration
|
||||
sizeGl int
|
||||
sizeOne int
|
||||
sizeGlobal int
|
||||
sizeItem int
|
||||
|
||||
current int
|
||||
cache []byte
|
||||
@@ -32,8 +33,8 @@ func NewCacheMemory(maxUsage, maxGlobalUsage int) *CacheMemory {
|
||||
return &CacheMemory{
|
||||
data: make(map[string][]byte),
|
||||
l: sync.RWMutex{},
|
||||
sizeGl: maxGlobalUsage,
|
||||
sizeOne: maxUsage,
|
||||
sizeGlobal: maxGlobalUsage,
|
||||
sizeItem: maxUsage,
|
||||
|
||||
cache: make([]byte, maxUsage+1),
|
||||
ordered: make([]string, 0),
|
||||
@@ -43,32 +44,32 @@ func NewCacheMemory(maxUsage, maxGlobalUsage int) *CacheMemory {
|
||||
func (c *CacheMemory) Put(key string, reader io.Reader) error {
|
||||
c.l.Lock()
|
||||
defer c.l.Unlock()
|
||||
size, err := io.ReadAtLeast(reader, c.cache, 0)
|
||||
size, err := io.ReadAtLeast(reader, c.cache, 1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if size == len(c.cache) {
|
||||
return ErrCacheOutOfMemory
|
||||
}
|
||||
needed := c.sizeGl - c.current + size
|
||||
if needed < 0 {
|
||||
currentItemSize := len(c.data[key])
|
||||
available := c.sizeGlobal + currentItemSize - (c.current + size)
|
||||
if available < 0 {
|
||||
// 清理旧的内容
|
||||
count := 0
|
||||
for i, k := range c.ordered {
|
||||
needed += len(c.data[k])
|
||||
if needed > 0 {
|
||||
available += len(c.data[k])
|
||||
if available > 0 {
|
||||
break
|
||||
}
|
||||
count = i + 1
|
||||
}
|
||||
|
||||
if needed < 0 {
|
||||
if available < 0 {
|
||||
// 清理全部内容也无法留出空间
|
||||
return ErrCacheOutOfMemory
|
||||
}
|
||||
for _, s := range c.ordered[:count] {
|
||||
delete(c.data, s)
|
||||
c.current -= len(c.data)
|
||||
}
|
||||
c.ordered = c.ordered[count:]
|
||||
}
|
||||
@@ -76,22 +77,59 @@ func (c *CacheMemory) Put(key string, reader io.Reader) error {
|
||||
dest := make([]byte, size)
|
||||
copy(dest, c.cache[:size])
|
||||
c.data[key] = dest
|
||||
c.ordered = append(c.ordered, key)
|
||||
|
||||
c.current -= currentItemSize
|
||||
c.current += len(dest)
|
||||
|
||||
nextOrdered := make([]string, 0, len(c.ordered))
|
||||
for _, s := range c.ordered {
|
||||
if s != key {
|
||||
nextOrdered = append(nextOrdered, s)
|
||||
}
|
||||
}
|
||||
c.ordered = append(nextOrdered, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CacheMemory) Get(key string) (io.ReadSeekCloser, error) {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
c.l.RLock()
|
||||
defer c.l.RUnlock()
|
||||
if i, ok := c.data[key]; ok {
|
||||
return nopCloser{
|
||||
bytes.NewReader(i),
|
||||
}, nil
|
||||
}
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
|
||||
func (c *CacheMemory) Delete(key string) error {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
func (c *CacheMemory) Delete(pattern string) error {
|
||||
c.l.Lock()
|
||||
defer c.l.Unlock()
|
||||
nextOrder := make([]string, 0, len(c.ordered))
|
||||
for _, key := range c.ordered {
|
||||
if strings.HasPrefix(key, pattern) {
|
||||
c.current -= len(c.data[key])
|
||||
delete(c.data, key)
|
||||
} else {
|
||||
nextOrder = append(nextOrder, key)
|
||||
}
|
||||
}
|
||||
clear(c.ordered)
|
||||
c.ordered = nextOrder
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CacheMemory) Close() error {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
c.l.Lock()
|
||||
defer c.l.Unlock()
|
||||
clear(c.ordered)
|
||||
clear(c.data)
|
||||
c.current = 0
|
||||
return nil
|
||||
}
|
||||
|
||||
type nopCloser struct {
|
||||
io.ReadSeeker
|
||||
}
|
||||
|
||||
func (nopCloser) Close() error { return nil }
|
||||
|
||||
69
pkg/services/cache_test.go
Normal file
69
pkg/services/cache_test.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/stretchr/testify/require"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCacheGetPutDelete(t *testing.T) {
|
||||
memory := NewCacheMemory(1024, 10240)
|
||||
|
||||
require.NoError(t, memory.Put("hello", strings.NewReader("world")))
|
||||
|
||||
value, err := memory.Get("hello")
|
||||
require.NoError(t, err)
|
||||
all, err := io.ReadAll(value)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "world", string(all))
|
||||
require.Equal(t, 5, memory.current)
|
||||
|
||||
require.NoError(t, memory.Put("hello", strings.NewReader("kotlin")))
|
||||
|
||||
value, err = memory.Get("hello")
|
||||
require.NoError(t, err)
|
||||
all, err = io.ReadAll(value)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "kotlin", string(all))
|
||||
require.Equal(t, 6, memory.current)
|
||||
require.Equal(t, 1, len(memory.data))
|
||||
|
||||
require.NoError(t, memory.Put("data", strings.NewReader("kotlin")))
|
||||
require.Equal(t, 12, memory.current)
|
||||
require.Equal(t, 2, len(memory.data))
|
||||
require.Equal(t, 2, len(memory.ordered))
|
||||
|
||||
require.NoError(t, memory.Delete("hello"))
|
||||
value, err = memory.Get("hello")
|
||||
require.Error(t, err)
|
||||
require.Equal(t, 1, len(memory.data))
|
||||
require.Equal(t, 1, len(memory.ordered))
|
||||
|
||||
}
|
||||
|
||||
func TestCacheLimit(t *testing.T) {
|
||||
memory := NewCacheMemory(5, 5*5)
|
||||
require.NoError(t, memory.Put("hello", strings.NewReader("world")))
|
||||
require.Equal(t, 5, memory.current)
|
||||
require.ErrorIs(t, memory.Put("hello", strings.NewReader("world1")), ErrCacheOutOfMemory)
|
||||
require.Equal(t, 5, memory.current)
|
||||
for i := 0; i < 4; i++ {
|
||||
require.NoError(t, memory.Put(fmt.Sprintf("hello-%d", i), strings.NewReader("govet")))
|
||||
}
|
||||
value, err := memory.Get("hello")
|
||||
require.NoError(t, err)
|
||||
all, err := io.ReadAll(value)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "world", string(all))
|
||||
|
||||
require.NoError(t, memory.Put("test", strings.NewReader("govet")))
|
||||
|
||||
value, err = memory.Get("hello")
|
||||
require.ErrorIs(t, err, os.ErrNotExist)
|
||||
|
||||
require.Equal(t, 5, len(memory.data))
|
||||
require.Equal(t, 5, len(memory.ordered))
|
||||
}
|
||||
@@ -15,44 +15,33 @@ type Config interface {
|
||||
|
||||
func NewConfigMemory() Config {
|
||||
return &ConfigMemory{
|
||||
data: make(map[string]string),
|
||||
lock: sync.RWMutex{},
|
||||
data: sync.Map{},
|
||||
}
|
||||
}
|
||||
|
||||
// ConfigMemory 一个简单的内存配置归档，仅用于测试
|
||||
type ConfigMemory struct {
|
||||
data map[string]string
|
||||
lock sync.RWMutex
|
||||
data sync.Map
|
||||
}
|
||||
|
||||
func (m *ConfigMemory) Put(key string, value string) error {
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
m.data[key] = value
|
||||
m.data.Store(key, value)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *ConfigMemory) Get(key string) (string, error) {
|
||||
m.lock.RLock()
|
||||
defer m.lock.RUnlock()
|
||||
v, ok := m.data[key]
|
||||
if !ok {
|
||||
return "", os.ErrNotExist
|
||||
if value, ok := m.data.Load(key); ok {
|
||||
return value.(string), nil
|
||||
}
|
||||
return v, nil
|
||||
return "", os.ErrNotExist
|
||||
}
|
||||
|
||||
func (m *ConfigMemory) Delete(key string) error {
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
delete(m.data, key)
|
||||
m.data.Delete(key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *ConfigMemory) Close() error {
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
clear(m.data)
|
||||
m.data.Clear()
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user