清理代码 & 支持 ignore

This commit is contained in:
dragon
2025-05-09 14:50:10 +08:00
parent 9413188aa9
commit ef31433d91
14 changed files with 405 additions and 118 deletions

View File

@@ -8,4 +8,8 @@ gitea-pages: $(shell find . -type f -name "*.go" ) go.mod go.sum
.PHONY: debug
debug: gitea-pages
@./gitea-pages -conf config-local.yaml -debug
@./gitea-pages -conf config-local.yaml -debug
.PHONY: test
test:
@go test -v ./...

View File

@@ -30,37 +30,24 @@ make gitea-pages
在项目的默认分支创建 `.pages.yaml`,填入如下内容
```yaml
v-route: true
alias:
v-route: true # 虚拟路由
alias: # CNAME
- "example.com"
- "example2.com"
renders:
renders: # 渲染器
gotemplate: '**/*.tmpl,**/index.html'
proxy:
/api: https://github.com/api
/api: https://github.com/api
ignore: .git/**,.pages.yaml
```
### Render
说明: **不会**将文件系统 引入到渲染器中,复杂的渲染流程应该采用更加灵活轻便的方案
在项目的根目录创建 `.render` 文件,填入如下内容:
```sh
# parser match
gotemplate **/*.tmpl
```
其中,`gotemplate` 为解析器类型,`**/*.tmpl` 为匹配的路径,使用 `github.com/gobwas/glob`
## TODO
- [x] 内容缓存
- [x] CNAME 自定义域名
- [x] 模板渲染
- [ ] 反向代理请求
- [ ] HTTP
- [ ] Websocket
- [x] 反向代理请求
- [ ] OAuth2 授权访问私有页面
- [ ] ~~http01 自动签发证书~~: 交由 Caddy 完成
- [ ] ~~Web 钩子触发更新~~: 对实时性需求不大

46
pkg/core/config.go Normal file
View File

@@ -0,0 +1,46 @@
package core
import "strings"
type PageConfig struct {
Alias []string `yaml:"alias"` // 重定向地址
Render map[string]string `yaml:"templates"` // 渲染器地址
VirtualRoute bool `yaml:"v-route"` // 是否使用虚拟路由(任何路径均使用 /index.html 返回 200 响应)
ReverseProxy map[string]string `yaml:"proxy"` // 反向代理路由
Ignore string `yaml:"ignore"` // 跳过展示的内容
}
func (p *PageConfig) Ignores() []string {
i := make([]string, 0)
if p.Ignore == "" {
return i
}
for _, line := range strings.Split(p.Ignore, "\n") {
for _, item := range strings.Split(line, ",") {
item = strings.TrimSpace(item)
if item == "" {
continue
}
i = append(i, item)
}
}
return i
}
func (p *PageConfig) Renders() map[string]string {
result := make(map[string]string)
for sType, patterns := range p.Render {
for _, line := range strings.Split(patterns, "\n") {
for _, item := range strings.Split(line, ",") {
item = strings.TrimSpace(item)
if item == "" {
continue
}
result[sType] = item
}
}
}
return result
}

View File

@@ -4,9 +4,10 @@ import (
"os"
"strings"
"github.com/pkg/errors"
"gopkg.d7z.net/gitea-pages/pkg/utils"
"github.com/pkg/errors"
"go.uber.org/zap"
)
@@ -83,6 +84,10 @@ func (p *PageDomain) ReturnMeta(owner string, repo string, branch string, path [
return rel, nil
} else {
zap.L().Debug("查询错误", zap.Error(err))
if meta != nil {
// 解析错误汇报
return nil, errors.New(meta.ErrorMsg)
}
}
return nil, errors.Wrapf(os.ErrNotExist, strings.Join(path, "/"))
return nil, errors.Wrap(os.ErrNotExist, strings.Join(path, "/"))
}

View File

@@ -1,12 +1,12 @@
package core
import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
"strings"
"time"
@@ -35,75 +35,12 @@ type ServerMeta struct {
locker *utils.Locker
}
type renderCompiler struct {
regex glob.Glob
Render
}
// PageConfig 配置
type PageConfig struct {
Alias []string `yaml:"required"` // 重定向地址
Renders map[string]string `yaml:"templates"` // 渲染器地址
VirtualRoute bool `yaml:"v-route"` // 是否使用虚拟路由(任何路径均使用 /index.html 返回 200 响应)
ReverseProxy map[string]string `yaml:"proxy"` // 反向代理路由
}
type PageMetaContent struct {
CommitID string `json:"commit-id"` // 提交 COMMIT ID
LastModified time.Time `json:"last-modified"` // 上次更新时间
IsPage bool `json:"is-page"` // 是否为 Page
ErrorMsg string `json:"error"` // 错误消息
VRoute bool `yaml:"v-route"` // 虚拟路由
Alias []string `yaml:"aliases"` // 重定向
Proxy map[string]string `yaml:"proxy"` // 反向代理
Renders map[string][]string `json:"renders"` // 配置的渲染器
rendersL []*renderCompiler
}
func (m *PageMetaContent) From(data string) error {
err := json.Unmarshal([]byte(data), m)
clear(m.rendersL)
for key, gs := range m.Renders {
for _, g := range gs {
m.rendersL = append(m.rendersL, &renderCompiler{
regex: glob.MustCompile(g),
Render: GetRender(key),
})
}
}
return err
}
func (m *PageMetaContent) TryRender(path ...string) Render {
for _, s := range path {
for _, compiler := range m.rendersL {
if compiler.regex.Match(s) {
return compiler.Render
}
}
}
return nil
}
func (m *PageMetaContent) String() string {
marshal, _ := json.Marshal(m)
return string(marshal)
}
func NewServerMeta(client *http.Client, backend Backend, kv utils.KVConfig, domain string, ttl time.Duration) *ServerMeta {
return &ServerMeta{backend, domain, client, kv, ttl, utils.NewLocker()}
}
func (s *ServerMeta) GetMeta(owner, repo, branch string) (*PageMetaContent, error) {
rel := &PageMetaContent{
IsPage: false,
Proxy: make(map[string]string),
Alias: make([]string, 0),
Renders: make(map[string][]string),
}
rel := NewPageMetaContent()
if repos, err := s.Repos(owner); err != nil {
return nil, err
} else {
@@ -152,6 +89,7 @@ func (s *ServerMeta) GetMeta(owner, repo, branch string) (*PageMetaContent, erro
}
}
// 确定存在 index.html , 否则跳过
if find, _ := s.FileExists(owner, repo, rel.CommitID, "index.html"); !find {
rel.IsPage = false
_ = s.cache.Put(key, rel.String(), s.ttl)
@@ -166,13 +104,14 @@ func (s *ServerMeta) GetMeta(owner, repo, branch string) (*PageMetaContent, erro
return nil, err
}
// 解析配置
if data, err := s.ReadString(owner, repo, rel.CommitID, ".pages.yaml"); err == nil {
cfg := new(PageConfig)
if err = yaml.Unmarshal([]byte(data), cfg); err != nil {
return errFunc(err)
}
rel.Alias = cfg.Alias
rel.VRoute = cfg.VirtualRoute
// 处理 CNAME
for _, cname := range cfg.Alias {
cname = strings.TrimSpace(cname)
if regexpHostname.MatchString(cname) && !strings.HasSuffix(strings.ToLower(cname), strings.ToLower(s.Domain)) {
@@ -181,25 +120,34 @@ func (s *ServerMeta) GetMeta(owner, repo, branch string) (*PageMetaContent, erro
return errFunc(errors.New("invalid alias name " + cname))
}
}
for sType, patterns := range cfg.Renders {
if r := GetRender(sType); r != nil {
for _, pattern := range strings.Split(patterns, ",") {
rel.Renders[sType] = append(rel.Renders[sType], pattern)
if g, err := glob.Compile(strings.TrimSpace(pattern)); err == nil {
rel.rendersL = append(rel.rendersL, &renderCompiler{
regex: g,
Render: r,
})
} else {
return errFunc(err)
}
}
// 处理渲染器
for sType, pattern := range cfg.Renders() {
var r Render
if r = GetRender(sType); r == nil {
return errFunc(errors.Errorf("render not found %s", sType))
}
if g, err := glob.Compile(strings.TrimSpace(pattern)); err == nil {
rel.rendersL = append(rel.rendersL, &renderCompiler{
regex: g,
Render: r,
})
} else {
return errFunc(err)
}
rel.Renders[sType] = append(rel.Renders[sType], pattern)
}
// 处理跳过内容
for _, pattern := range cfg.Ignores() {
if g, err := glob.Compile(pattern); err == nil {
rel.ignoreL = append(rel.ignoreL, g)
} else {
return errFunc(err)
}
rel.Ignore = append(rel.Ignore, pattern)
}
// 处理反向代理 (清理内容,符合 /<item>)
for path, backend := range cfg.ReverseProxy {
path = strings.TrimSpace(path)
path = strings.ReplaceAll(path, "//", "/")
path = strings.ReplaceAll(path, "//", "/")
path = filepath.ToSlash(filepath.Clean(path))
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
@@ -220,6 +168,17 @@ func (s *ServerMeta) GetMeta(owner, repo, branch string) (*PageMetaContent, erro
zap.L().Debug("failed to read meta data", zap.String("error", err.Error()))
}
// 兼容 github 的 CNAME 模式
if cname, err := s.ReadString(owner, repo, rel.CommitID, "CNAME"); err == nil {
cname = strings.TrimSpace(cname)
if regexpHostname.MatchString(cname) && !strings.HasSuffix(strings.ToLower(cname), strings.ToLower(s.Domain)) {
rel.Alias = append(rel.Alias, cname)
} else {
zap.L().Debug("指定的 CNAME 不合法", zap.String("cname", cname))
}
}
rel.Alias = utils.ClearDuplicates(rel.Alias)
rel.Ignore = utils.ClearDuplicates(rel.Ignore)
_ = s.cache.Put(key, rel.String(), s.ttl)
return rel, nil
}

85
pkg/core/meta_content.go Normal file
View File

@@ -0,0 +1,85 @@
package core
import (
"encoding/json"
"time"
"github.com/gobwas/glob"
)
type renderCompiler struct {
regex glob.Glob
Render
}
// PageConfig 配置
type PageMetaContent struct {
CommitID string `json:"commit-id"` // 提交 COMMIT ID
LastModified time.Time `json:"last-modified"` // 上次更新时间
IsPage bool `json:"is-page"` // 是否为 Page
ErrorMsg string `json:"error"` // 错误消息
VRoute bool `yaml:"v-route"` // 虚拟路由
Proxy map[string]string `yaml:"proxy"` // 反向代理
Renders map[string][]string `json:"renders"` // 配置的渲染器
Alias []string `yaml:"aliases"` // 重定向
Ignore []string `yaml:"ignore"` // 跳过的内容
rendersL []*renderCompiler
ignoreL []glob.Glob
}
func NewPageMetaContent() *PageMetaContent {
return &PageMetaContent{
IsPage: false,
Proxy: make(map[string]string),
Alias: make([]string, 0),
Renders: make(map[string][]string),
Ignore: []string{".*", "**/.*"},
}
}
func (m *PageMetaContent) From(data string) error {
err := json.Unmarshal([]byte(data), m)
clear(m.rendersL)
for key, gs := range m.Renders {
for _, g := range gs {
m.rendersL = append(m.rendersL, &renderCompiler{
regex: glob.MustCompile(g),
Render: GetRender(key),
})
}
}
clear(m.ignoreL)
for _, g := range m.Ignore {
m.ignoreL = append(m.ignoreL, glob.MustCompile(g))
}
return err
}
func (m *PageMetaContent) IsIgnore(path string) bool {
for _, g := range m.ignoreL {
if g.Match(path) {
return true
}
}
return false
}
func (m *PageMetaContent) TryRender(path ...string) Render {
for _, s := range path {
for _, compiler := range m.rendersL {
if compiler.regex.Match(s) {
return compiler.Render
}
}
}
return nil
}
func (m *PageMetaContent) String() string {
marshal, _ := json.Marshal(m)
return string(marshal)
}

View File

@@ -118,18 +118,22 @@ func (s *Server) Serve(writer http.ResponseWriter, request *http.Request) error
http.Redirect(writer, request, fmt.Sprintf("https://%s/%s", meta.Alias[0], meta.Path), http.StatusFound)
return nil
}
for prefix, backend := range meta.Proxy {
if strings.HasPrefix(meta.Path, prefix) {
targetPath := strings.TrimPrefix(meta.Path, prefix)
proxyPath := "/" + meta.Path
if strings.HasPrefix(proxyPath, prefix) {
targetPath := strings.TrimPrefix(proxyPath, prefix)
if !strings.HasPrefix(targetPath, "/") {
targetPath = "/" + targetPath
}
zap.L().Debug("命中反向代理", zap.Any("prefix", prefix), zap.Any("backend", backend),
zap.Any("path", meta.Path), zap.Any("target", targetPath))
u, _ := url.Parse(backend)
request.URL.Path = targetPath
request.RequestURI = request.URL.RequestURI()
u, _ := url.Parse(backend)
httputil.NewSingleHostReverseProxy(u).ServeHTTP(writer, request)
proxy := httputil.NewSingleHostReverseProxy(u)
proxy.Transport = s.options.HttpClient.Transport
zap.L().Debug("命中反向代理", zap.Any("prefix", prefix), zap.Any("backend", backend),
zap.Any("path", proxyPath), zap.Any("target", fmt.Sprintf("%s%s", u, targetPath)))
proxy.ServeHTTP(writer, request)
return nil
}
}
@@ -137,6 +141,10 @@ func (s *Server) Serve(writer http.ResponseWriter, request *http.Request) error
if request.Method != "GET" {
return os.ErrNotExist
}
if meta.IsIgnore(meta.Path) {
zap.L().Debug("ignore path", zap.Any("request", request.RequestURI), zap.Any("meta.path", meta.Path))
return os.ErrNotExist
}
result, err := s.reader.Open(meta.Owner, meta.Repo, meta.CommitID, meta.Path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
@@ -222,6 +230,5 @@ func (s *Server) Close() error {
if err := s.options.Cache.Close(); err != nil {
return err
}
return nil
return s.backend.Close()
}

15
pkg/utils/array.go Normal file
View File

@@ -0,0 +1,15 @@
package utils
func ClearDuplicates[T comparable](slice []T) []T {
seen := make(map[T]bool)
for _, val := range slice {
seen[val] = true
}
var result []T
for key := range seen {
result = append(result, key)
}
return result
}

View File

@@ -19,7 +19,7 @@ type ProviderDummy struct {
}
func NewDummy() (*ProviderDummy, error) {
temp, err := os.MkdirTemp("", "dummy")
temp, err := os.MkdirTemp("", "dummy-*")
if err != nil {
return nil, err
}

View File

@@ -8,6 +8,7 @@ import (
"os"
"path/filepath"
"github.com/pkg/errors"
"go.uber.org/zap"
"gopkg.d7z.net/gitea-pages/pkg"
)
@@ -20,7 +21,9 @@ type TestServer struct {
type SvcOpts func(options *pkg.ServerOptions)
func NewDefaultTestServer() *TestServer {
return NewTestServer("example.com")
return NewTestServer("example.com", func(options *pkg.ServerOptions) {
options.MetaTTL = 0
})
}
func NewTestServer(domain string, opts ...SvcOpts) *TestServer {
@@ -47,13 +50,13 @@ func NewTestServer(domain string, opts ...SvcOpts) *TestServer {
}
}
func (t *TestServer) AddFile(path, data string) {
func (t *TestServer) AddFile(path, data string, args ...interface{}) {
join := filepath.Join(t.dummy.BaseDir, path)
err := os.MkdirAll(filepath.Dir(join), 0o755)
if err != nil {
panic(err)
}
err = os.WriteFile(join, []byte(data), 0o644)
err = os.WriteFile(join, []byte(fmt.Sprintf(data, args...)), 0o644)
if err != nil {
panic(err)
}
@@ -67,7 +70,7 @@ func (t *TestServer) OpenFile(url string) ([]byte, *http.Response, error) {
defer response.Body.Close()
}
if response.StatusCode != http.StatusOK {
return nil, response, fmt.Errorf(response.Status)
return nil, response, errors.New(response.Status)
}
all, _ := io.ReadAll(response.Body)
return all, response, nil

46
tests/core/vhserver.go Normal file
View File

@@ -0,0 +1,46 @@
package core
import (
"fmt"
"net"
"net/http"
"go.uber.org/zap"
)
type VServer struct {
URL string
mux *http.ServeMux
listener net.Listener
}
func NewServer() *VServer {
listener, err := net.Listen("tcp", ":0") // ":0" 表示让系统自动选择一个可用的端口
if err != nil {
panic(err)
}
port := listener.Addr().(*net.TCPAddr).Port
mux := http.NewServeMux()
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
zap.L().Debug("ServeHTTP", zap.String("url", r.URL.String()))
})
go func() {
_ = http.Serve(listener, mux)
}()
return &VServer{
listener: listener,
URL: fmt.Sprintf("http://127.0.0.1:%d", port),
mux: mux,
}
}
func (v *VServer) Add(path, data string) {
v.mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(data))
})
}
func (v *VServer) Close() error {
return v.listener.Close()
}

93
tests/pages_core_test.go Normal file
View File

@@ -0,0 +1,93 @@
package tests
import (
"testing"
"github.com/stretchr/testify/assert"
"gopkg.d7z.net/gitea-pages/tests/core"
)
func Test_get_simple_html(t *testing.T) {
server := core.NewDefaultTestServer()
defer server.Close()
server.AddFile("org1/repo1/gh-pages/index.html", "hello world")
data, _, err := server.OpenFile("https://org1.example.com/repo1/")
assert.NoError(t, err)
assert.Equal(t, "hello world", string(data))
_, resp, err := server.OpenFile("https://org1.example.com/repo1/404")
assert.NotNil(t, resp)
assert.Equal(t, resp.StatusCode, 404)
}
func Test_get_alias(t *testing.T) {
server := core.NewDefaultTestServer()
defer server.Close()
server.AddFile("org1/repo1/gh-pages/index.html", "hello world")
server.AddFile("org1/repo1/gh-pages/.pages.yaml", `
alias:
- www.example.org
`)
data, resp, err := server.OpenFile("https://www.example.org")
assert.Equal(t, resp.StatusCode, 404)
data, resp, err = server.OpenFile("https://org1.example.com/repo1/")
assert.Equal(t, resp.StatusCode, 302)
assert.Equal(t, resp.Header.Get("Location"), "https://www.example.org/index.html")
data, resp, err = server.OpenFile("https://www.example.org")
assert.NoError(t, err)
assert.Equal(t, "hello world", string(data))
server.AddFile("org1/repo1/gh-pages/.pages.yaml", `
alias:
- zzz.example.top
`)
data, resp, err = server.OpenFile("https://www.example.org")
assert.Equal(t, resp.StatusCode, 302)
assert.Equal(t, resp.Header.Get("Location"), "https://zzz.example.top/index.html")
data, resp, err = server.OpenFile("https://www.example.org")
assert.Equal(t, resp.StatusCode, 404)
data, resp, err = server.OpenFile("https://org1.example.com/repo1/")
assert.Equal(t, resp.StatusCode, 302)
assert.Equal(t, resp.Header.Get("Location"), "https://zzz.example.top/index.html")
}
func Test_get_v_route(t *testing.T) {
server := core.NewDefaultTestServer()
defer server.Close()
server.AddFile("org1/repo1/gh-pages/index.html", "hello world")
server.AddFile("org1/repo1/gh-pages/.pages.yaml", `
v-route: true
`)
data, _, err := server.OpenFile("https://org1.example.com/repo1/")
assert.NoError(t, err)
assert.Equal(t, "hello world", string(data))
data, _, err = server.OpenFile("https://org1.example.com/repo1/404")
assert.NoError(t, err)
assert.Equal(t, "hello world", string(data))
}
func Test_get_v_ignore(t *testing.T) {
server := core.NewDefaultTestServer()
defer server.Close()
server.AddFile("org1/repo1/gh-pages/index.html", "hello world")
server.AddFile("org1/repo1/gh-pages/bad.html", "hello world")
server.AddFile("org1/repo1/gh-pages/.pages.yaml", `
ignore: .pages.yaml
`)
data, _, err := server.OpenFile("https://org1.example.com/repo1/")
assert.NoError(t, err)
assert.Equal(t, "hello world", string(data))
data, _, err = server.OpenFile("https://org1.example.com/repo1/bad.html")
assert.NoError(t, err)
assert.Equal(t, "hello world", string(data))
server.AddFile("org1/repo1/gh-pages/.pages.yaml", `
ignore: bad.*
`)
_, resp, _ := server.OpenFile("https://org1.example.com/repo1/bad.html")
assert.Equal(t, 404, resp.StatusCode)
}

View File

@@ -7,12 +7,23 @@ import (
"gopkg.d7z.net/gitea-pages/tests/core"
)
func Test_get_simple_html(t *testing.T) {
func Test_proxy(t *testing.T) {
server := core.NewDefaultTestServer()
hs := core.NewServer()
defer server.Close()
defer hs.Close()
hs.Add("/test/", "hello proxy")
server.AddFile("org1/repo1/gh-pages/index.html", "hello world")
server.AddFile("org1/repo1/gh-pages/.pages.yaml", `
proxy:
/api: %s/test
`, hs.URL)
data, _, err := server.OpenFile("https://org1.example.com/repo1/")
assert.NoError(t, err)
assert.Equal(t, "hello world", string(data))
data, _, err = server.OpenFile("https://org1.example.com/repo1/api")
assert.NoError(t, err)
assert.Equal(t, "hello proxy", string(data))
}

View File

@@ -0,0 +1,26 @@
package tests
import (
"testing"
"github.com/stretchr/testify/assert"
"gopkg.d7z.net/gitea-pages/tests/core"
)
func Test_get_render(t *testing.T) {
server := core.NewDefaultTestServer()
defer server.Close()
server.AddFile("org1/repo1/gh-pages/index.html", "hello world")
server.AddFile("org1/repo1/gh-pages/tmpl/index.html", "hello world,{{ .Request.Host }}")
server.AddFile("org1/repo1/gh-pages/.pages.yaml", `
templates:
gotemplate: tmpl/*.html
`)
data, _, err := server.OpenFile("https://org1.example.com/repo1/")
assert.NoError(t, err)
assert.Equal(t, "hello world", string(data))
data, _, err = server.OpenFile("https://org1.example.com/repo1/tmpl/index.html")
assert.NoError(t, err)
assert.Equal(t, "hello world,org1.example.com", string(data))
}