Add R2 image uploads to admin
All checks were successful
ci/woodpecker/push/ci Pipeline was successful
All checks were successful
ci/woodpecker/push/ci Pipeline was successful
This commit is contained in:
parent
9186801c7f
commit
49a0d078da
16 changed files with 809 additions and 14 deletions
|
|
@ -27,6 +27,7 @@ type Config struct {
|
|||
LogMaxBackups int
|
||||
DeepSeek DeepSeekConfig
|
||||
LocalLLM LocalLLMConfig
|
||||
R2 R2Config
|
||||
SlugProvider string
|
||||
}
|
||||
|
||||
|
|
@ -44,6 +45,17 @@ type LocalLLMConfig struct {
|
|||
NumPredict int
|
||||
}
|
||||
|
||||
type R2Config struct {
|
||||
Endpoint string
|
||||
Bucket string
|
||||
Prefix string
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
Region string
|
||||
PublicBaseURL string
|
||||
MaxUploadBytes int64
|
||||
}
|
||||
|
||||
type localConfig struct {
|
||||
Database struct {
|
||||
PostgresDSN string `yaml:"postgres_dsn"`
|
||||
|
|
@ -64,6 +76,16 @@ type localConfig struct {
|
|||
TopP float64 `yaml:"top_p"`
|
||||
NumPredict int `yaml:"num_predict"`
|
||||
} `yaml:"local_llm"`
|
||||
R2 struct {
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Bucket string `yaml:"bucket"`
|
||||
Prefix string `yaml:"prefix"`
|
||||
AccessKeyID string `yaml:"accessKeyId"`
|
||||
SecretAccessKey string `yaml:"secretAccessKey"`
|
||||
Region string `yaml:"region"`
|
||||
PublicBaseURL string `yaml:"publicBaseUrl"`
|
||||
MaxUploadBytes int64 `yaml:"maxUploadBytes"`
|
||||
} `yaml:"r2"`
|
||||
}
|
||||
|
||||
func LoadConfig() Config {
|
||||
|
|
@ -144,6 +166,16 @@ func LoadConfig() Config {
|
|||
TopP: firstNonZeroFloat(envFloat("LOCAL_LLM_TOP_P"), local.LocalLLM.TopP, 0.8),
|
||||
NumPredict: firstNonZeroInt(envInt("LOCAL_LLM_NUM_PREDICT"), local.LocalLLM.NumPredict, 32),
|
||||
},
|
||||
R2: R2Config{
|
||||
Endpoint: firstNonEmpty(os.Getenv("OSAET_R2_ENDPOINT"), local.R2.Endpoint),
|
||||
Bucket: firstNonEmpty(os.Getenv("OSAET_R2_BUCKET"), local.R2.Bucket),
|
||||
Prefix: firstNonEmpty(os.Getenv("OSAET_R2_PREFIX"), local.R2.Prefix),
|
||||
AccessKeyID: firstNonEmpty(os.Getenv("OSAET_R2_ACCESS_KEY_ID"), local.R2.AccessKeyID),
|
||||
SecretAccessKey: firstNonEmpty(os.Getenv("OSAET_R2_SECRET_ACCESS_KEY"), local.R2.SecretAccessKey),
|
||||
Region: firstNonEmpty(os.Getenv("OSAET_R2_REGION"), local.R2.Region, "auto"),
|
||||
PublicBaseURL: firstNonEmpty(os.Getenv("OSAET_R2_PUBLIC_BASE_URL"), local.R2.PublicBaseURL),
|
||||
MaxUploadBytes: int64(firstNonZeroInt(envInt("OSAET_R2_MAX_UPLOAD_BYTES"), int(local.R2.MaxUploadBytes), 20*1024*1024)),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
248
backend/internal/admin/r2.go
Normal file
248
backend/internal/admin/r2.go
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
package admin
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
type ImageUploader struct {
|
||||
client *s3.Client
|
||||
bucket string
|
||||
prefix string
|
||||
publicBaseURL string
|
||||
maxUploadBytes int64
|
||||
}
|
||||
|
||||
type UploadedImage struct {
|
||||
Key string `json:"key"`
|
||||
URL string `json:"url"`
|
||||
Markdown string `json:"markdown"`
|
||||
Filename string `json:"filename"`
|
||||
Size int64 `json:"size"`
|
||||
ContentType string `json:"contentType"`
|
||||
}
|
||||
|
||||
func NewImageUploader(ctx context.Context, cfg R2Config) (*ImageUploader, error) {
|
||||
normalized, err := normalizeR2Config(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
awsConfig, err := config.LoadDefaultConfig(ctx,
|
||||
config.WithRegion(normalized.Region),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
|
||||
normalized.AccessKeyID,
|
||||
normalized.SecretAccessKey,
|
||||
"",
|
||||
)),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := s3.NewFromConfig(awsConfig, func(options *s3.Options) {
|
||||
options.BaseEndpoint = aws.String(normalized.Endpoint)
|
||||
options.UsePathStyle = true
|
||||
})
|
||||
|
||||
return &ImageUploader{
|
||||
client: client,
|
||||
bucket: normalized.Bucket,
|
||||
prefix: normalized.Prefix,
|
||||
publicBaseURL: normalized.PublicBaseURL,
|
||||
maxUploadBytes: normalized.MaxUploadBytes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (u *ImageUploader) Upload(ctx context.Context, filename string, body io.Reader, size int64) (UploadedImage, error) {
|
||||
if u == nil {
|
||||
return UploadedImage{}, errors.New("R2 image uploader is not configured")
|
||||
}
|
||||
if size <= 0 {
|
||||
return UploadedImage{}, errors.New("file is empty")
|
||||
}
|
||||
if u.maxUploadBytes > 0 && size > u.maxUploadBytes {
|
||||
return UploadedImage{}, fmt.Errorf("file is too large: max %d bytes", u.maxUploadBytes)
|
||||
}
|
||||
|
||||
limit := u.maxUploadBytes
|
||||
if limit <= 0 {
|
||||
limit = 20 * 1024 * 1024
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(body, limit+1))
|
||||
if err != nil {
|
||||
return UploadedImage{}, err
|
||||
}
|
||||
if int64(len(data)) > limit {
|
||||
return UploadedImage{}, fmt.Errorf("file is too large: max %d bytes", limit)
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return UploadedImage{}, errors.New("file is empty")
|
||||
}
|
||||
|
||||
head := data
|
||||
if len(head) > 512 {
|
||||
head = head[:512]
|
||||
}
|
||||
contentType := http.DetectContentType(head)
|
||||
if !strings.HasPrefix(contentType, "image/") {
|
||||
return UploadedImage{}, errors.New("only image files are supported")
|
||||
}
|
||||
|
||||
key, cleanFilename, err := u.objectKey(filename, contentType)
|
||||
if err != nil {
|
||||
return UploadedImage{}, err
|
||||
}
|
||||
|
||||
if _, err := u.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(u.bucket),
|
||||
Key: aws.String(key),
|
||||
Body: bytes.NewReader(data),
|
||||
ContentType: aws.String(contentType),
|
||||
}); err != nil {
|
||||
return UploadedImage{}, err
|
||||
}
|
||||
|
||||
publicURL := strings.TrimRight(u.publicBaseURL, "/") + "/" + escapePath(key)
|
||||
return UploadedImage{
|
||||
Key: key,
|
||||
URL: publicURL,
|
||||
Markdown: fmt.Sprintf("", cleanFilename, publicURL),
|
||||
Filename: cleanFilename,
|
||||
Size: size,
|
||||
ContentType: contentType,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (u *ImageUploader) objectKey(filename string, contentType string) (string, string, error) {
|
||||
cleanFilename := sanitizeFilename(filename)
|
||||
ext := strings.ToLower(filepath.Ext(cleanFilename))
|
||||
if ext == "" {
|
||||
extensions, _ := mime.ExtensionsByType(contentType)
|
||||
if len(extensions) > 0 {
|
||||
ext = extensions[0]
|
||||
cleanFilename += ext
|
||||
}
|
||||
}
|
||||
token, err := randomHex(4)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
stem := strings.TrimSuffix(cleanFilename, filepath.Ext(cleanFilename))
|
||||
keyName := fmt.Sprintf("%s-%s-%s%s", time.Now().Format("20060102-150405"), token, stem, ext)
|
||||
return joinObjectPath(u.prefix, keyName), cleanFilename, nil
|
||||
}
|
||||
|
||||
func normalizeR2Config(cfg R2Config) (R2Config, error) {
|
||||
cfg.Endpoint = strings.TrimSpace(cfg.Endpoint)
|
||||
cfg.Bucket = strings.Trim(strings.TrimSpace(cfg.Bucket), "/")
|
||||
cfg.Prefix = strings.Trim(strings.TrimSpace(cfg.Prefix), "/")
|
||||
cfg.AccessKeyID = strings.TrimSpace(cfg.AccessKeyID)
|
||||
cfg.SecretAccessKey = strings.TrimSpace(cfg.SecretAccessKey)
|
||||
cfg.Region = firstNonEmpty(cfg.Region, "auto")
|
||||
cfg.PublicBaseURL = strings.TrimSpace(cfg.PublicBaseURL)
|
||||
if cfg.MaxUploadBytes <= 0 {
|
||||
cfg.MaxUploadBytes = 20 * 1024 * 1024
|
||||
}
|
||||
|
||||
endpoint, err := url.Parse(cfg.Endpoint)
|
||||
if err != nil || endpoint.Scheme == "" || endpoint.Host == "" {
|
||||
return R2Config{}, errors.New("r2.endpoint must be a valid URL")
|
||||
}
|
||||
endpointPath := strings.Trim(endpoint.Path, "/")
|
||||
endpoint.Path = ""
|
||||
endpoint.RawPath = ""
|
||||
endpoint.RawQuery = ""
|
||||
endpoint.Fragment = ""
|
||||
cfg.Endpoint = endpoint.String()
|
||||
|
||||
if cfg.Bucket == "" {
|
||||
return R2Config{}, errors.New("r2.bucket is required")
|
||||
}
|
||||
bucketParts := strings.SplitN(cfg.Bucket, "/", 2)
|
||||
cfg.Bucket = bucketParts[0]
|
||||
if len(bucketParts) == 2 {
|
||||
cfg.Prefix = joinObjectPath(bucketParts[1], cfg.Prefix)
|
||||
}
|
||||
if endpointPath != "" && endpointPath != cfg.Bucket {
|
||||
cfg.Prefix = joinObjectPath(endpointPath, cfg.Prefix)
|
||||
}
|
||||
|
||||
if cfg.AccessKeyID == "" || cfg.SecretAccessKey == "" {
|
||||
return R2Config{}, errors.New("r2 access keys are required")
|
||||
}
|
||||
if cfg.PublicBaseURL == "" {
|
||||
return R2Config{}, errors.New("r2.publicBaseUrl is required")
|
||||
}
|
||||
if !strings.HasPrefix(cfg.PublicBaseURL, "http://") && !strings.HasPrefix(cfg.PublicBaseURL, "https://") {
|
||||
cfg.PublicBaseURL = "https://" + cfg.PublicBaseURL
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func sanitizeFilename(filename string) string {
|
||||
filename = strings.TrimSpace(filepath.Base(filename))
|
||||
if filename == "" || filename == "." {
|
||||
return "image"
|
||||
}
|
||||
var out strings.Builder
|
||||
for _, r := range filename {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
|
||||
out.WriteRune(r)
|
||||
case r == '.', r == '-', r == '_':
|
||||
out.WriteRune(r)
|
||||
default:
|
||||
out.WriteByte('-')
|
||||
}
|
||||
}
|
||||
cleaned := strings.Trim(out.String(), ".-") // keep keys readable and URL-safe
|
||||
if cleaned == "" {
|
||||
return "image"
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func joinObjectPath(parts ...string) string {
|
||||
clean := []string{}
|
||||
for _, part := range parts {
|
||||
part = strings.Trim(part, "/")
|
||||
if part != "" {
|
||||
clean = append(clean, part)
|
||||
}
|
||||
}
|
||||
return path.Join(clean...)
|
||||
}
|
||||
|
||||
func escapePath(value string) string {
|
||||
parts := strings.Split(value, "/")
|
||||
for index, part := range parts {
|
||||
parts[index] = url.PathEscape(part)
|
||||
}
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
func randomHex(bytes int) (string, error) {
|
||||
buffer := make([]byte, bytes)
|
||||
if _, err := rand.Read(buffer); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(buffer), nil
|
||||
}
|
||||
30
backend/internal/admin/r2_test.go
Normal file
30
backend/internal/admin/r2_test.go
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
package admin
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNormalizeR2ConfigSplitsBucketPath(t *testing.T) {
|
||||
cfg, err := normalizeR2Config(R2Config{
|
||||
Endpoint: "https://example.r2.cloudflarestorage.com/stroage",
|
||||
Bucket: "stroage/Image/",
|
||||
AccessKeyID: "access-key",
|
||||
SecretAccessKey: "secret-key",
|
||||
Region: "auto",
|
||||
PublicBaseURL: "r2.example.com",
|
||||
MaxUploadBytes: 20,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("normalizeR2Config returned error: %v", err)
|
||||
}
|
||||
if cfg.Endpoint != "https://example.r2.cloudflarestorage.com" {
|
||||
t.Fatalf("unexpected endpoint: %s", cfg.Endpoint)
|
||||
}
|
||||
if cfg.Bucket != "stroage" {
|
||||
t.Fatalf("unexpected bucket: %s", cfg.Bucket)
|
||||
}
|
||||
if cfg.Prefix != "Image" {
|
||||
t.Fatalf("unexpected prefix: %s", cfg.Prefix)
|
||||
}
|
||||
if cfg.PublicBaseURL != "https://r2.example.com" {
|
||||
t.Fatalf("unexpected public base URL: %s", cfg.PublicBaseURL)
|
||||
}
|
||||
}
|
||||
|
|
@ -22,6 +22,7 @@ type Server struct {
|
|||
builder *Builder
|
||||
deepSeek DeepSeekConfig
|
||||
localLLM LocalLLMConfig
|
||||
uploader *ImageUploader
|
||||
slugProvider string
|
||||
adminDir string
|
||||
staticDir string
|
||||
|
|
@ -46,12 +47,23 @@ func NewServerWithContext(ctx context.Context, db *pgxpool.Pool, cfg Config) *Se
|
|||
builder.Start(ctx)
|
||||
}
|
||||
}
|
||||
var uploader *ImageUploader
|
||||
if cfg.R2.Endpoint != "" || cfg.R2.Bucket != "" || cfg.R2.AccessKeyID != "" || cfg.R2.SecretAccessKey != "" {
|
||||
createdUploader, err := NewImageUploader(ctx, cfg.R2)
|
||||
if err != nil {
|
||||
log.Printf("R2 image uploader disabled: %v", err)
|
||||
} else {
|
||||
uploader = createdUploader
|
||||
}
|
||||
}
|
||||
|
||||
return &Server{
|
||||
db: db,
|
||||
store: store,
|
||||
builder: builder,
|
||||
deepSeek: cfg.DeepSeek,
|
||||
localLLM: cfg.LocalLLM,
|
||||
uploader: uploader,
|
||||
slugProvider: cfg.SlugProvider,
|
||||
adminDir: cfg.AdminDir,
|
||||
staticDir: cfg.StaticDir,
|
||||
|
|
@ -81,6 +93,7 @@ func (s *Server) Router() http.Handler {
|
|||
protected.GET("/me", s.me)
|
||||
protected.POST("/logout", s.logout)
|
||||
protected.POST("/slug", s.generateSlug)
|
||||
protected.POST("/images", s.uploadImage)
|
||||
protected.GET("/audit-logs", s.listAuditLogs)
|
||||
protected.GET("/posts", s.listPosts)
|
||||
protected.POST("/posts", s.createPost)
|
||||
|
|
@ -347,6 +360,39 @@ func (s *Server) generateSlug(c *gin.Context) {
|
|||
c.JSON(http.StatusOK, gin.H{"slug": slug})
|
||||
}
|
||||
|
||||
func (s *Server) uploadImage(c *gin.Context) {
|
||||
if s.uploader == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "R2 image uploader is not configured"})
|
||||
return
|
||||
}
|
||||
|
||||
file, err := c.FormFile("file")
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "file is required"})
|
||||
return
|
||||
}
|
||||
opened, err := file.Open()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
defer opened.Close()
|
||||
|
||||
uploaded, err := s.uploader.Upload(c.Request.Context(), file.Filename, opened, file.Size)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
s.auditCurrentUser(c, "image_upload", "image", uploaded.Key, gin.H{
|
||||
"filename": uploaded.Filename,
|
||||
"url": uploaded.URL,
|
||||
"size": uploaded.Size,
|
||||
"contentType": uploaded.ContentType,
|
||||
})
|
||||
c.JSON(http.StatusCreated, gin.H{"image": uploaded})
|
||||
}
|
||||
|
||||
func (s *Server) listAuditLogs(c *gin.Context) {
|
||||
if !s.requireStore(c) {
|
||||
return
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue