yarnom
f0b50d13ea
Add Go/Postgres admin APIs, Angular admin UI, manual build flow, asset uploads, markdown import/export, configurable slug generation, and the Yar reading theme. Exclude local docs and generated development artifacts from version control.
460 lines
11 KiB
Go
460 lines
11 KiB
Go
package postimport
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/rand"
|
|
"encoding/csv"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
"unicode"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
const defaultPostsDir = "content/posts"
|
|
|
|
type Options struct {
|
|
CSVPath string
|
|
PostsDir string
|
|
Overwrite bool
|
|
WorkingDir string
|
|
}
|
|
|
|
type Result struct {
|
|
Imported int
|
|
SkippedExisting int
|
|
SkippedNonPost int
|
|
}
|
|
|
|
type PostFile struct {
|
|
Path string
|
|
Frontmatter PostFrontmatter
|
|
Body string
|
|
}
|
|
|
|
type PostFrontmatter struct {
|
|
ID string `yaml:"id"`
|
|
Slug string `yaml:"slug"`
|
|
Title string `yaml:"title"`
|
|
Summary string `yaml:"summary"`
|
|
Status string `yaml:"status"`
|
|
Tags []string `yaml:"tags"`
|
|
Cover string `yaml:"cover"`
|
|
Version int `yaml:"version"`
|
|
SlugSource string `yaml:"slug_source"`
|
|
SlugLocked bool `yaml:"slug_locked"`
|
|
PublishedAt *string `yaml:"published_at"`
|
|
CreatedAt string `yaml:"created_at"`
|
|
UpdatedAt string `yaml:"updated_at"`
|
|
}
|
|
|
|
type csvArticle struct {
|
|
ID string
|
|
Slug string
|
|
Title string
|
|
BodyMD string
|
|
BodyHTML string
|
|
Status string
|
|
ArchiveID string
|
|
AuthorID string
|
|
PublishedAt string
|
|
CreatedAt string
|
|
UpdatedAt string
|
|
Type string
|
|
}
|
|
|
|
func Import(options Options) (Result, error) {
|
|
root, err := findProjectRoot(options.WorkingDir)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
|
|
csvPath := resolveRootPath(root, firstNonEmpty(options.CSVPath, "articles.csv"))
|
|
postsDir := resolveRootPath(root, firstNonEmpty(options.PostsDir, defaultPostsDir))
|
|
if err := os.MkdirAll(postsDir, 0o755); err != nil {
|
|
return Result{}, err
|
|
}
|
|
|
|
file, err := os.Open(csvPath)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
defer file.Close()
|
|
|
|
reader := csv.NewReader(file)
|
|
reader.FieldsPerRecord = -1
|
|
reader.LazyQuotes = false
|
|
|
|
headers, err := reader.Read()
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
index := map[string]int{}
|
|
for i, header := range headers {
|
|
index[strings.TrimSpace(header)] = i
|
|
}
|
|
|
|
var result Result
|
|
for rowNum := 2; ; rowNum++ {
|
|
record, err := reader.Read()
|
|
if err != nil {
|
|
if errors.Is(err, io.EOF) {
|
|
break
|
|
}
|
|
return result, fmt.Errorf("%s row %d: %w", csvPath, rowNum, err)
|
|
}
|
|
|
|
article := csvArticle{
|
|
ID: csvValue(record, index, "id"),
|
|
Slug: csvValue(record, index, "slug"),
|
|
Title: csvValue(record, index, "title"),
|
|
BodyMD: csvValue(record, index, "body_md"),
|
|
BodyHTML: csvValue(record, index, "body_html"),
|
|
Status: csvValue(record, index, "status"),
|
|
ArchiveID: csvValue(record, index, "archive_id"),
|
|
AuthorID: csvValue(record, index, "author_id"),
|
|
PublishedAt: csvValue(record, index, "published_at"),
|
|
CreatedAt: csvValue(record, index, "created_at"),
|
|
UpdatedAt: csvValue(record, index, "updated_at"),
|
|
Type: csvValue(record, index, "type"),
|
|
}
|
|
|
|
if strings.TrimSpace(article.Type) != "" && strings.TrimSpace(article.Type) != "post" {
|
|
result.SkippedNonPost++
|
|
continue
|
|
}
|
|
|
|
post, skippedExisting, err := articleToPost(root, postsDir, article, options.Overwrite)
|
|
if err != nil {
|
|
return result, fmt.Errorf("%s row %d (%s): %w", csvPath, rowNum, article.Slug, err)
|
|
}
|
|
if skippedExisting {
|
|
result.SkippedExisting++
|
|
continue
|
|
}
|
|
if err := writePostFile(post); err != nil {
|
|
return result, fmt.Errorf("%s row %d (%s): %w", csvPath, rowNum, article.Slug, err)
|
|
}
|
|
result.Imported++
|
|
}
|
|
|
|
return result, nil
|
|
}
|
|
|
|
func articleToPost(root string, postsDir string, article csvArticle, overwrite bool) (PostFile, bool, error) {
|
|
id := strings.TrimSpace(article.ID)
|
|
if id == "" {
|
|
id = randomID()
|
|
}
|
|
|
|
title := strings.TrimSpace(article.Title)
|
|
if title == "" {
|
|
return PostFile{}, false, errors.New("missing title")
|
|
}
|
|
|
|
slug := sanitizeSlug(article.Slug)
|
|
if slug == "" {
|
|
slug = fallbackSlug(title)
|
|
}
|
|
if slug == "" {
|
|
return PostFile{}, false, errors.New("missing slug")
|
|
}
|
|
|
|
status := strings.TrimSpace(article.Status)
|
|
if status != "published" && status != "draft" {
|
|
status = "draft"
|
|
}
|
|
|
|
createdAt, err := normalizeLegacyTime(article.CreatedAt)
|
|
if err != nil {
|
|
return PostFile{}, false, fmt.Errorf("invalid created_at: %w", err)
|
|
}
|
|
updatedAt, err := normalizeLegacyTime(article.UpdatedAt)
|
|
if err != nil {
|
|
return PostFile{}, false, fmt.Errorf("invalid updated_at: %w", err)
|
|
}
|
|
|
|
var publishedAt *string
|
|
if strings.TrimSpace(article.PublishedAt) != "" {
|
|
normalized, err := normalizeLegacyTime(article.PublishedAt)
|
|
if err != nil {
|
|
return PostFile{}, false, fmt.Errorf("invalid published_at: %w", err)
|
|
}
|
|
publishedAt = &normalized
|
|
}
|
|
|
|
path := filepath.Join(postsDir, slug+".md")
|
|
if _, err := os.Stat(path); err == nil {
|
|
if !overwrite {
|
|
return PostFile{}, true, nil
|
|
}
|
|
} else if !errors.Is(err, os.ErrNotExist) {
|
|
return PostFile{}, false, err
|
|
}
|
|
|
|
finalSlug := slug
|
|
if !overwrite {
|
|
finalSlug, err = uniqueSlug(root, slug)
|
|
if err != nil {
|
|
return PostFile{}, false, err
|
|
}
|
|
path, err = uniquePostPath(postsDir, finalSlug)
|
|
if err != nil {
|
|
return PostFile{}, false, err
|
|
}
|
|
}
|
|
|
|
body := strings.TrimLeft(article.BodyMD, "\n")
|
|
if strings.TrimSpace(body) == "" {
|
|
body = strings.TrimLeft(article.BodyHTML, "\n")
|
|
}
|
|
|
|
return PostFile{
|
|
Path: path,
|
|
Frontmatter: PostFrontmatter{
|
|
ID: id,
|
|
Slug: finalSlug,
|
|
Title: title,
|
|
Summary: "",
|
|
Status: status,
|
|
Tags: []string{},
|
|
Cover: "",
|
|
Version: 1,
|
|
SlugSource: "manual",
|
|
SlugLocked: true,
|
|
PublishedAt: publishedAt,
|
|
CreatedAt: createdAt,
|
|
UpdatedAt: updatedAt,
|
|
},
|
|
Body: body,
|
|
}, false, nil
|
|
}
|
|
|
|
func csvValue(record []string, index map[string]int, key string) string {
|
|
i, ok := index[key]
|
|
if !ok || i >= len(record) {
|
|
return ""
|
|
}
|
|
return record[i]
|
|
}
|
|
|
|
func normalizeLegacyTime(value string) (string, error) {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return "", errors.New("empty time")
|
|
}
|
|
layouts := []string{
|
|
"2006-01-02 15:04:05.999999999Z07",
|
|
"2006-01-02 15:04:05.999999Z07",
|
|
"2006-01-02 15:04:05Z07",
|
|
time.RFC3339,
|
|
}
|
|
for _, layout := range layouts {
|
|
parsed, err := time.Parse(layout, value)
|
|
if err == nil {
|
|
return parsed.Format(time.RFC3339), nil
|
|
}
|
|
}
|
|
return "", fmt.Errorf("unsupported time format %q", value)
|
|
}
|
|
|
|
func writePostFile(post PostFile) error {
|
|
var frontmatter bytes.Buffer
|
|
encoder := yaml.NewEncoder(&frontmatter)
|
|
encoder.SetIndent(2)
|
|
if err := encoder.Encode(post.Frontmatter); err != nil {
|
|
return err
|
|
}
|
|
if err := encoder.Close(); err != nil {
|
|
return err
|
|
}
|
|
|
|
var output bytes.Buffer
|
|
output.WriteString("---\n")
|
|
output.Write(frontmatter.Bytes())
|
|
output.WriteString("---\n\n")
|
|
output.WriteString(strings.TrimLeft(post.Body, "\n"))
|
|
|
|
tmp := post.Path + ".tmp"
|
|
if err := os.WriteFile(tmp, output.Bytes(), 0o644); err != nil {
|
|
return err
|
|
}
|
|
return os.Rename(tmp, post.Path)
|
|
}
|
|
|
|
func loadPosts(root string, postsDir string) ([]PostFile, error) {
|
|
dir := resolveRootPath(root, postsDir)
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return nil, nil
|
|
}
|
|
return nil, err
|
|
}
|
|
var posts []PostFile
|
|
for _, entry := range entries {
|
|
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".md") {
|
|
continue
|
|
}
|
|
posts = append(posts, PostFile{Path: filepath.Join(dir, entry.Name()), Frontmatter: PostFrontmatter{Slug: strings.TrimSuffix(entry.Name(), ".md")}})
|
|
}
|
|
return posts, nil
|
|
}
|
|
|
|
func slugExists(root string, slug string) bool {
|
|
posts, err := loadPosts(root, defaultPostsDir)
|
|
if err == nil {
|
|
for _, post := range posts {
|
|
fileSlug := strings.TrimSuffix(filepath.Base(post.Path), ".md")
|
|
if post.Frontmatter.Slug == slug || fileSlug == slug {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
|
|
_, err = os.Stat(filepath.Join(root, defaultPostsDir, slug+".md"))
|
|
return err == nil
|
|
}
|
|
|
|
func uniqueSlug(root string, slug string) (string, error) {
|
|
base := sanitizeSlug(slug)
|
|
if base == "" {
|
|
return "", errors.New("empty slug")
|
|
}
|
|
if !slugExists(root, base) {
|
|
return base, nil
|
|
}
|
|
for i := 2; i < 1000; i++ {
|
|
candidate := fmt.Sprintf("%s-%d", base, i)
|
|
if !slugExists(root, candidate) {
|
|
return candidate, nil
|
|
}
|
|
}
|
|
return "", fmt.Errorf("could not find available slug for %q", base)
|
|
}
|
|
|
|
func uniquePostPath(postsDir string, slug string) (string, error) {
|
|
candidate := filepath.Join(postsDir, slug+".md")
|
|
if _, err := os.Stat(candidate); errors.Is(err, os.ErrNotExist) {
|
|
return candidate, nil
|
|
} else if err != nil {
|
|
return "", err
|
|
}
|
|
for i := 2; i < 1000; i++ {
|
|
candidate = filepath.Join(postsDir, fmt.Sprintf("%s-%d.md", slug, i))
|
|
if _, err := os.Stat(candidate); errors.Is(err, os.ErrNotExist) {
|
|
return candidate, nil
|
|
} else if err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
return "", fmt.Errorf("could not find available filename for slug %q", slug)
|
|
}
|
|
|
|
func findProjectRoot(start string) (string, error) {
|
|
wd := start
|
|
if wd == "" {
|
|
var err error
|
|
wd, err = os.Getwd()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
for {
|
|
if isProjectRoot(wd) {
|
|
return wd, nil
|
|
}
|
|
parent := filepath.Dir(wd)
|
|
if parent == wd {
|
|
return "", errors.New("could not find project root")
|
|
}
|
|
wd = parent
|
|
}
|
|
}
|
|
|
|
func isProjectRoot(dir string) bool {
|
|
if _, err := os.Stat(filepath.Join(dir, ".git")); err == nil {
|
|
if _, err := os.Stat(filepath.Join(dir, "backend", "go.mod")); err == nil {
|
|
return true
|
|
}
|
|
}
|
|
if _, err := os.Stat(filepath.Join(dir, "backend", "cmd", "osaetctl")); err == nil {
|
|
if _, err := os.Stat(filepath.Join(dir, "frontend", "site", "package.json")); err == nil {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func resolveRootPath(root string, path string) string {
|
|
if filepath.IsAbs(path) {
|
|
return path
|
|
}
|
|
return filepath.Join(root, path)
|
|
}
|
|
|
|
func fallbackSlug(title string) string {
|
|
var words []string
|
|
var b strings.Builder
|
|
flush := func() {
|
|
if b.Len() > 0 {
|
|
words = append(words, b.String())
|
|
b.Reset()
|
|
}
|
|
}
|
|
for _, r := range strings.ToLower(title) {
|
|
switch {
|
|
case r >= 'a' && r <= 'z':
|
|
b.WriteRune(r)
|
|
case r >= '0' && r <= '9':
|
|
b.WriteRune(r)
|
|
case unicode.IsSpace(r) || r == '-' || r == '_' || r == '/':
|
|
flush()
|
|
default:
|
|
flush()
|
|
}
|
|
}
|
|
flush()
|
|
if len(words) == 0 {
|
|
return "post-" + time.Now().Format("20060102150405")
|
|
}
|
|
return sanitizeSlug(strings.Join(words, "-"))
|
|
}
|
|
|
|
func sanitizeSlug(slug string) string {
|
|
slug = strings.ToLower(strings.TrimSpace(slug))
|
|
re := regexp.MustCompile(`[^a-z0-9]+`)
|
|
slug = re.ReplaceAllString(slug, "-")
|
|
slug = strings.Trim(slug, "-")
|
|
for strings.Contains(slug, "--") {
|
|
slug = strings.ReplaceAll(slug, "--", "-")
|
|
}
|
|
if len(slug) > 80 {
|
|
slug = strings.Trim(slug[:80], "-")
|
|
}
|
|
return slug
|
|
}
|
|
|
|
func randomID() string {
|
|
var b [16]byte
|
|
if _, err := rand.Read(b[:]); err != nil {
|
|
return fmt.Sprintf("post-%d", time.Now().UnixNano())
|
|
}
|
|
return hex.EncodeToString(b[:])
|
|
}
|
|
|
|
func firstNonEmpty(values ...string) string {
|
|
for _, value := range values {
|
|
if strings.TrimSpace(value) != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|