org1/gitea-pages
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

补充实现细节

Browse Source
This commit is contained in:
dragon
2025-01-07 17:04:39 +08:00
parent bb8966521c
commit 0172dceaa8
9 changed files with 278 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@@ -2,13 +2,14 @@
> 新一代 Gitea Pages，替换之前的 caddy-gitea-proxy
注意，默认实现未考虑高并发高负载环境，
## Feature
- [x] 内容缓存
## TODO
- [ ] CNAME 自定义域名
- [ ] http01 自动签发证书 (CNAME)
- [ ] Web 钩子触发更新
- [ ] 内容缓存
- [ ] OAuth2 授权访问私有页面

85
pkg/core/backend.go
View File

@@ -1,23 +1,32 @@
package core
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"os"
"strconv"
"time"
"code.d7z.net/d7z-project/gitea-pages/pkg/utils"
)
type BranchInfo struct {
ID string `json:"id"`
LastModified time.Time `json:"last_modified"`
}
type Backend interface {
// Repos return repo name + default branch
Repos(owner string) (map[string]string, error)
// Branches return branch + commit id
Branches(owner, repo string) (map[string]string, error)
Branches(owner, repo string) (map[string]*BranchInfo, error)
// Open return file or error
Open(client *http.Client, owner, repo, commit, path string, headers map[string]string) (*http.Response, error)
Open(client *http.Client, owner, repo, commit, path string, headers http.Header) (*http.Response, error)
}
type CacheBackend struct {
@@ -60,8 +69,8 @@ func (c *CacheBackend) Repos(owner string) (map[string]string, error) {
return ret, nil
}
func (c *CacheBackend) Branches(owner, repo string) (map[string]string, error) {
ret := make(map[string]string)
func (c *CacheBackend) Branches(owner, repo string) (map[string]*BranchInfo, error) {
ret := make(map[string]*BranchInfo)
key := fmt.Sprintf("branches/%s/%s", owner, repo)
data, err := c.config.Get(key)
if err != nil {
@@ -90,6 +99,72 @@ func (c *CacheBackend) Branches(owner, repo string) (map[string]string, error) {
return ret, nil
}
func (c *CacheBackend) Open(client *http.Client, owner, repo, commit, path string, headers map[string]string) (*http.Response, error) {
func (c *CacheBackend) Open(client *http.Client, owner, repo, commit, path string, headers http.Header) (*http.Response, error) {
return c.backend.Open(client, owner, repo, commit, path, headers)
}
type CacheBackendBlobReader struct {
client *http.Client
cache utils.Cache
base Backend
maxSize int
}
func NewCacheBackendBlobReader(client *http.Client, base Backend, cache utils.Cache, maxCacheSize int) *CacheBackendBlobReader {
return &CacheBackendBlobReader{client: client, base: base, cache: cache, maxSize: maxCacheSize}
}
func (c *CacheBackendBlobReader) Open(owner, repo, commit, path string) (io.ReadCloser, error) {
key := fmt.Sprintf("%s/%s/%s%s", owner, repo, commit, path)
lastCache, err := c.cache.Get(key)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
} else if lastCache == nil && err == nil {
// 边界缓存
return nil, os.ErrNotExist
} else if lastCache != nil {
return lastCache, nil
}
open, err := c.base.Open(c.client, owner, repo, commit, path, http.Header{})
if err != nil {
if open != nil {
if open.StatusCode == http.StatusNotFound {
// 缓存 404 路由
_ = c.cache.Put(key, nil)
}
_ = open.Body.Close()
}
return nil, errors.Join(err, os.ErrNotExist)
}
lastMod, err := time.Parse(http.TimeFormat, open.Header.Get("Last-Modified"))
if err != nil {
// 无时间，跳过
return open.Body, nil
}
// 没法计算大小，跳过
lengthStr := open.Header.Get("Content-Length")
if lengthStr == "" {
return open.Body, nil
}
length, err := strconv.Atoi(lengthStr)
if err != nil || length > c.maxSize {
// 超过最大大小，跳过
return open.Body, err
}
defer open.Body.Close()
allBytes, err := io.ReadAll(open.Body)
if err != nil {
return nil, err
}
if err = c.cache.Put(key, bytes.NewBuffer(allBytes)); err != nil {
slog.Warn("缓存归档失败", "error", err)
}
return &utils.CacheContent{
ReadSeekCloser: utils.NopCloser{
ReadSeeker: bytes.NewReader(allBytes),
},
LastModified: lastMod,
Length: length,
}, nil
}

37
pkg/core/meta.go
View File

@@ -14,38 +14,39 @@ import (
)
type ServerMeta struct {
client *http.Client
Backend
cache utils.Config
ttl time.Duration
client *http.Client
cache utils.Config
ttl time.Duration
locker *utils.Locker
}
type PageMeta struct {
CommitID string `json:"id"` // 提交 COMMIT ID
IsPage bool `json:"pg"` // 是否为 Page
Domain string `json:"dm"` // 匹配的域名
HistoryRouteMode bool `json:"rt"` // 路由模式
CustomNotFound bool `json:"404"` // 注册了自定义 404 页面
type PageMetaContent struct {
CommitID string `json:"id"` // 提交 COMMIT ID
IsPage bool `json:"pg"` // 是否为 Page
Domain string `json:"dm"` // 匹配的域名
HistoryRouteMode bool `json:"rt"` // 路由模式
CustomNotFound bool `json:"404"` // 注册了自定义 404 页面
LastModified time.Time `json:"up"` // 上次更新时间
}
func (m *PageMeta) From(data string) error {
func (m *PageMetaContent) From(data string) error {
return json.Unmarshal([]byte(data), m)
}
func (m *PageMeta) String() string {
func (m *PageMetaContent) String() string {
marshal, _ := json.Marshal(m)
return string(marshal)
}
func NewServerMeta(client *http.Client, backend Backend, config utils.Config, ttl time.Duration) *ServerMeta {
return &ServerMeta{client, backend, config, ttl, utils.NewLocker()}
return &ServerMeta{backend, client, config, ttl, utils.NewLocker()}
}
func (s *ServerMeta) GetMeta(owner, repo, branch string) (*PageMeta, error) {
rel := &PageMeta{
func (s *ServerMeta) GetMeta(owner, repo, branch string) (*PageMetaContent, error) {
rel := &PageMetaContent{
IsPage: false,
}
if repos, err := s.Repos(owner); err != nil {
@@ -62,10 +63,12 @@ func (s *ServerMeta) GetMeta(owner, repo, branch string) (*PageMeta, error) {
if branches, err := s.Branches(owner, repo); err != nil {
return nil, err
} else {
rel.CommitID = branches[branch]
if rel.CommitID == "" {
info := branches[branch]
if info == nil {
return nil, os.ErrNotExist
}
rel.CommitID = info.ID
rel.LastModified = info.LastModified
}
key := fmt.Sprintf("meta/%s/%s/%s", owner, repo, branch)

77
pkg/core/page.go
View File

@@ -1,9 +1,78 @@
package core
type PageContent struct {
meta PageMeta
import (
"errors"
"fmt"
"os"
"strings"
)
type PageDomain struct {
*ServerMeta
baseDomain string
defaultBranch string
}
func (p *PageContent) GetMeta(domain, path string) (*PageMeta, error) {
func NewPageDomain(meta *ServerMeta, baseDomain, defaultBranch string) *PageDomain {
return &PageDomain{
baseDomain: baseDomain,
defaultBranch: defaultBranch,
ServerMeta: meta,
}
}
type PageDomainContent struct {
*PageMetaContent
Owner string
Repo string
Path string
}
func (m *PageDomainContent) CacheKey() string {
return fmt.Sprintf("%s/%s/%s%s", m.Owner, m.Repo, m.CommitID, m.Path)
}
func (p *PageDomain) ParseDomainMeta(domain, path, branch string) (*PageDomainContent, error) {
if branch == "" {
branch = p.defaultBranch
}
rel := &PageDomainContent{}
if !strings.HasSuffix(domain, "."+p.baseDomain) {
return nil, os.ErrNotExist
}
rel.Owner = strings.TrimSuffix(domain, "."+p.baseDomain)
pathS := strings.Split(strings.TrimPrefix(path, "/"), "/")
repo := pathS[0]
defaultRepo := rel.Owner + "." + p.baseDomain
if repo == "" {
// 回退到默认仓库
rel.Repo = defaultRepo
}
meta, err := p.GetMeta(rel.Owner, rel.Repo, branch)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
}
if err == nil {
rel.Path = "/" + strings.Join(pathS[1:], "/")
rel.PageMetaContent = meta
return rel, nil
}
if defaultRepo == rel.Repo {
return nil, os.ErrNotExist
}
if meta, err := p.GetMeta(rel.Owner, defaultRepo, branch); err == nil {
rel.PageMetaContent = meta
rel.Repo = defaultRepo
rel.Path = "/" + strings.Join(pathS, "/")
return rel, nil
}
if strings.HasSuffix(path, "/") {
rel.Path = rel.Path + "/index.html"
}
return nil, os.ErrNotExist
}

19
pkg/providers/gitea.go
View File

@@ -5,6 +5,8 @@ import (
"net/url"
"os"
"code.d7z.net/d7z-project/gitea-pages/pkg/core"
"code.gitea.io/sdk/gitea"
)
@@ -73,8 +75,8 @@ func (g *ProviderGitea) Repos(owner string) (map[string]string, error) {
return result, nil
}
func (g *ProviderGitea) Branches(owner, repo string) (map[string]string, error) {
result := make(map[string]string)
func (g *ProviderGitea) Branches(owner, repo string) (map[string]*core.BranchInfo, error) {
result := make(map[string]*core.BranchInfo)
if branches, resp, err := g.gitea.ListRepoBranches(owner, repo, gitea.ListRepoBranchesOptions{
ListOptions: gitea.ListOptions{
PageSize: GiteaMaxCount,
@@ -89,7 +91,10 @@ func (g *ProviderGitea) Branches(owner, repo string) (map[string]string, error)
_ = resp.Body.Close()
}
for _, branch := range branches {
result[branch.Name] = branch.Commit.ID
result[branch.Name] = &core.BranchInfo{
ID: branch.Commit.ID,
LastModified: branch.Commit.Timestamp,
}
}
}
if len(result) == 0 {
@@ -98,7 +103,7 @@ func (g *ProviderGitea) Branches(owner, repo string) (map[string]string, error)
return result, nil
}
func (g *ProviderGitea) Open(client *http.Client, owner, repo, commit, path string, headers map[string]string) (*http.Response, error) {
func (g *ProviderGitea) Open(client *http.Client, owner, repo, commit, path string, headers http.Header) (*http.Response, error) {
giteaURL, err := url.JoinPath(g.BaseUrl, "api/v1/repos", owner, repo, "media", path)
if err != nil {
return nil, err
@@ -109,8 +114,10 @@ func (g *ProviderGitea) Open(client *http.Client, owner, repo, commit, path stri
return nil, err
}
if headers != nil {
for key, value := range headers {
req.Header.Add(key, value)
for key, values := range headers {
for _, value := range values {
req.Header.Add(key, value)
}
}
}
req.Header.Add("Authorization", "token "+g.Token)

66
pkg/server.go
View File

@@ -1,7 +1,12 @@
package pkg
import (
"errors"
"io"
"mime"
"net/http"
"os"
"path/filepath"
"time"
"code.d7z.net/d7z-project/gitea-pages/pkg/core"
@@ -10,7 +15,9 @@ import (
)
type ServerOptions struct {
Domain string
Domain string
DefaultBranch string
Config utils.Config
Cache utils.Cache
@@ -22,29 +29,72 @@ type ServerOptions struct {
func DefaultOptions(domain string) ServerOptions {
configMemory, _ := utils.NewConfigMemory("")
return ServerOptions{
Domain: domain,
Config: configMemory,
Cache: utils.NewCacheMemory(1024*1024*10, int(memory.FreeMemory()/3*2)),
MaxCacheSize: 1024 * 1024 * 10,
HttpClient: http.DefaultClient,
Domain: domain,
DefaultBranch: "gh-pages",
Config: configMemory,
Cache: utils.NewCacheMemory(1024*1024*10, int(memory.FreeMemory()/3*2)),
MaxCacheSize: 1024 * 1024 * 10,
HttpClient: http.DefaultClient,
}
}
type Server struct {
meta *core.ServerMeta
meta *core.PageDomain
options *ServerOptions
reader *core.CacheBackendBlobReader
}
func NewPageServer(backend core.Backend, options ServerOptions) *Server {
backend = core.NewCacheBackend(backend, options.Config, time.Minute)
svcMeta := core.NewServerMeta(options.HttpClient, backend, options.Config, time.Minute)
pageMeta := core.NewPageDomain(svcMeta, options.Domain, options.DefaultBranch)
reader := core.NewCacheBackendBlobReader(options.HttpClient, backend, options.Cache, options.MaxCacheSize)
return &Server{
meta: core.NewServerMeta(options.HttpClient, backend, options.Config, time.Minute),
meta: pageMeta,
options: &options,
reader: reader,
}
}
func (s *Server) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
meta, err := s.meta.ParseDomainMeta(request.Method, request.RequestURI, request.URL.Query().Get("branch"))
if err != nil {
if errors.Is(err, os.ErrNotExist) {
s.writeError(writer, err)
} else {
s.writeNotfoundError(writer, request.RequestURI)
}
return
}
result, err := s.reader.Open(meta.Owner, meta.Repo, meta.CommitID, meta.Path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
s.writeError(writer, err)
} else {
s.writeNotfoundError(writer, request.RequestURI)
}
return
}
fileName := filepath.Base(meta.Path)
if reader, ok := result.(*utils.CacheContent); ok {
http.ServeContent(writer, request, fileName, reader.LastModified, reader)
_ = reader.Close()
return
} else {
writer.Header().Set("Content-Type", mime.TypeByExtension(meta.Path))
writer.WriteHeader(http.StatusOK)
_, _ = io.Copy(writer, reader)
_ = reader.Close()
return
}
}
func (s *Server) writeError(writer http.ResponseWriter, err error) {
http.Error(writer, err.Error(), http.StatusInternalServerError)
}
func (s *Server) writeNotfoundError(writer http.ResponseWriter, path string) {
http.Error(writer, "page not found.", http.StatusNotFound)
}
func (s *Server) Close() error {

36
pkg/utils/cache.go
View File

@@ -7,21 +7,31 @@ import (
"os"
"strings"
"sync"
"time"
)
type CacheContent struct {
io.ReadSeekCloser
Length int
LastModified time.Time
}
type Cache interface {
Put(key string, reader io.Reader) error
// Get return io.ReadSeekCloser or nil when put nil io.reader
Get(key string) (io.ReadSeekCloser, error)
// Get return CacheContent or nil when put nil io.reader
Get(key string) (*CacheContent, error)
Delete(pattern string) error
io.Closer
}
var ErrCacheOutOfMemory = errors.New("内容无法被缓存，超过最大限定值")
// TODO: 优化锁结构
type CacheMemory struct {
l sync.RWMutex
data map[string]*[]byte
lastModify map[string]time.Time
sizeGlobal int
sizeItem int
@@ -33,6 +43,7 @@ type CacheMemory struct {
func NewCacheMemory(maxUsage, maxGlobalUsage int) *CacheMemory {
return &CacheMemory{
data: make(map[string]*[]byte),
lastModify: make(map[string]time.Time),
l: sync.RWMutex{},
sizeGlobal: maxGlobalUsage,
sizeItem: maxUsage,
@@ -79,6 +90,7 @@ func (c *CacheMemory) Put(key string, reader io.Reader) error {
}
for _, s := range c.ordered[:count] {
delete(c.data, s)
delete(c.lastModify, s)
}
c.ordered = c.ordered[count:]
}
@@ -87,11 +99,14 @@ func (c *CacheMemory) Put(key string, reader io.Reader) error {
dest := make([]byte, size)
copy(dest, c.cache[:size])
c.data[key] = &dest
c.lastModify[key] = time.Now()
c.current -= currentItemSize
c.current += len(dest)
} else {
c.data[key] = nil
c.lastModify[key] = time.Now()
c.current -= currentItemSize
}
@@ -105,15 +120,20 @@ func (c *CacheMemory) Put(key string, reader io.Reader) error {
return nil
}
func (c *CacheMemory) Get(key string) (io.ReadSeekCloser, error) {
func (c *CacheMemory) Get(key string) (*CacheContent, error) {
c.l.RLock()
defer c.l.RUnlock()
if i, ok := c.data[key]; ok {
if i == nil {
return nil, nil
}
return nopCloser{
bytes.NewReader(*i),
return &CacheContent{
ReadSeekCloser: NopCloser{
bytes.NewReader(*i),
},
Length: len(*i),
LastModified: c.lastModify[key],
}, nil
}
return nil, os.ErrNotExist
@@ -127,6 +147,7 @@ func (c *CacheMemory) Delete(pattern string) error {
if strings.HasPrefix(key, pattern) {
c.current -= len(*c.data[key])
delete(c.data, key)
delete(c.lastModify, key)
} else {
nextOrder = append(nextOrder, key)
}
@@ -141,12 +162,13 @@ func (c *CacheMemory) Close() error {
defer c.l.Unlock()
clear(c.ordered)
clear(c.data)
clear(c.lastModify)
c.current = 0
return nil
}
type nopCloser struct {
type NopCloser struct {
io.ReadSeeker
}
func (nopCloser) Close() error { return nil }
func (NopCloser) Close() error { return nil }

1
pkg/utils/cache_test.go
View File

@@ -71,4 +71,5 @@ func TestCacheLimit(t *testing.T) {
require.Equal(t, 5, len(memory.data))
require.Equal(t, 5, len(memory.ordered))
require.Equal(t, 5, len(memory.lastModify))
}

1
pkg/utils/locker.go
View File

@@ -11,6 +11,7 @@ func NewLocker() *Locker {
sy: sync.Map{},
}
}
func (l *Locker) Open(key string) *sync.Mutex {
actual, _ := l.sy.LoadOrStore(key, &sync.Mutex{})
return actual.(*sync.Mutex)