feat: establish NekoNest Cloud control and relay

2026-08-12 23:25:43 +08:00
commit f27606b709
222 changed files with 71456 additions and 0 deletions
@@ -0,0 +1,593 @@
+package tenantbackup
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"database/sql"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/klarkxy/nekonest-cloud/relay/internal/tenantfs"
+	_ "modernc.org/sqlite"
+)
+
+const manifestVersion = 1
+
+type File struct {
+	Path   string `json:"path"`
+	Size   int64  `json:"size"`
+	SHA256 string `json:"sha256"`
+}
+
+type Manifest struct {
+	Version             int    `json:"version"`
+	TenantID            string `json:"tenant_id"`
+	PlacementGeneration int64  `json:"placement_generation"`
+	CreatedAt           string `json:"created_at"`
+	Files               []File `json:"files"`
+}
+
+type restoreReceipt struct {
+	Version             int    `json:"version"`
+	TenantID            string `json:"tenant_id"`
+	PlacementGeneration int64  `json:"placement_generation"`
+	ManifestSHA256      string `json:"manifest_sha256"`
+}
+
+type Result struct {
+	Path           string
+	BackupRef      string
+	Manifest       Manifest
+	ManifestSHA256 string
+}
+
+func validBackupRef(value string) bool {
+	parts := strings.Split(filepath.ToSlash(value), "/")
+	if len(parts) != 2 || len(parts[0]) != 32 || !strings.HasPrefix(parts[1], "g") || strings.Contains(parts[1], ".tmp") {
+		return false
+	}
+	for _, char := range parts[0] {
+		if !strings.ContainsRune("0123456789abcdef", char) {
+			return false
+		}
+	}
+	if len(parts[1]) != len("g00000000000000000000-20060102T150405Z-0000000000000000") {
+		return false
+	}
+	for index, char := range parts[1] {
+		switch index {
+		case 0:
+			if char != 'g' {
+				return false
+			}
+		case 21, 38:
+			if char != '-' {
+				return false
+			}
+		case 30:
+			if char != 'T' {
+				return false
+			}
+		case 37:
+			if char != 'Z' {
+				return false
+			}
+		default:
+			if index >= 39 {
+				if !strings.ContainsRune("0123456789abcdef", char) {
+					return false
+				}
+			} else if char < '0' || char > '9' {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+// ResolveReference turns an opaque control-plane backup reference into a
+// local immutable backup path without accepting an absolute path or symlink.
+func ResolveReference(backupRoot, reference string) (string, error) {
+	if !validBackupRef(reference) {
+		return "", fmt.Errorf("invalid backup reference")
+	}
+	root, err := filepath.Abs(strings.TrimSpace(backupRoot))
+	if err != nil || strings.TrimSpace(backupRoot) == "" {
+		return "", fmt.Errorf("invalid backup root")
+	}
+	if info, err := os.Lstat(root); err != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
+		return "", fmt.Errorf("backup root is not a real directory")
+	}
+	parts := strings.Split(filepath.ToSlash(reference), "/")
+	parent := filepath.Join(root, parts[0])
+	if info, err := os.Lstat(parent); err != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
+		return "", fmt.Errorf("backup tenant directory is not real")
+	}
+	resolved := filepath.Join(parent, parts[1])
+	if filepath.Dir(resolved) != parent {
+		return "", fmt.Errorf("backup reference escaped root")
+	}
+	if info, err := os.Lstat(resolved); err != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
+		return "", fmt.Errorf("backup reference is unavailable")
+	}
+	return resolved, nil
+}
+
+func randomHex(size int) (string, error) {
+	value := make([]byte, size)
+	if _, err := rand.Read(value); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(value), nil
+}
+
+func ensureRealDirectory(path string) error {
+	if err := os.MkdirAll(path, 0o700); err != nil {
+		return err
+	}
+	info, err := os.Lstat(path)
+	if err != nil {
+		return err
+	}
+	if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
+		return fmt.Errorf("%s is not a real directory", path)
+	}
+	return os.Chmod(path, 0o700)
+}
+
+func regularFile(path string) (os.FileInfo, error) {
+	info, err := os.Lstat(path)
+	if err != nil {
+		return nil, err
+	}
+	if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
+		return nil, fmt.Errorf("%s is not a regular file", path)
+	}
+	return info, nil
+}
+
+func validAttachmentName(name string) bool {
+	stem := ""
+	switch {
+	case strings.HasSuffix(name, ".bin"):
+		stem = strings.TrimSuffix(name, ".bin")
+	case strings.HasSuffix(name, ".json"):
+		stem = strings.TrimSuffix(name, ".json")
+	default:
+		return false
+	}
+	if len(stem) != 32 {
+		return false
+	}
+	for _, char := range stem {
+		if !strings.ContainsRune("0123456789abcdef", char) {
+			return false
+		}
+	}
+	return true
+}
+
+func checkpointAndVerify(path string) error {
+	if _, err := regularFile(path); err != nil {
+		return fmt.Errorf("inspect sqlite database: %w", err)
+	}
+	database, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)")
+	if err != nil {
+		return err
+	}
+	defer database.Close()
+	var busy, logFrames, checkpointed int
+	if err := database.QueryRow(`PRAGMA wal_checkpoint(TRUNCATE)`).Scan(&busy, &logFrames, &checkpointed); err != nil {
+		return fmt.Errorf("checkpoint sqlite: %w", err)
+	}
+	if busy != 0 {
+		return fmt.Errorf("sqlite checkpoint remained busy")
+	}
+	var result string
+	if err := database.QueryRow(`PRAGMA integrity_check`).Scan(&result); err != nil {
+		return fmt.Errorf("check sqlite integrity: %w", err)
+	}
+	if result != "ok" {
+		return fmt.Errorf("sqlite integrity check failed: %s", result)
+	}
+	return nil
+}
+
+func verifySQLiteReadOnly(path string) error {
+	if _, err := regularFile(path); err != nil {
+		return fmt.Errorf("inspect sqlite database: %w", err)
+	}
+	database, err := sql.Open("sqlite", "file:"+filepath.ToSlash(path)+"?mode=ro&immutable=1&_pragma=query_only(1)")
+	if err != nil {
+		return err
+	}
+	defer database.Close()
+	var result string
+	if err := database.QueryRow(`PRAGMA integrity_check`).Scan(&result); err != nil {
+		return fmt.Errorf("check restored sqlite integrity: %w", err)
+	}
+	if result != "ok" {
+		return fmt.Errorf("restored sqlite integrity check failed: %s", result)
+	}
+	return nil
+}
+
+func hashFile(ctx context.Context, path string) (int64, string, error) {
+	if _, err := regularFile(path); err != nil {
+		return 0, "", err
+	}
+	file, err := os.Open(path)
+	if err != nil {
+		return 0, "", err
+	}
+	defer file.Close()
+	hash := sha256.New()
+	buffer := make([]byte, 128<<10)
+	var total int64
+	for {
+		if err := ctx.Err(); err != nil {
+			return 0, "", err
+		}
+		count, readErr := file.Read(buffer)
+		if count > 0 {
+			total += int64(count)
+			_, _ = hash.Write(buffer[:count])
+		}
+		if errors.Is(readErr, io.EOF) {
+			break
+		}
+		if readErr != nil {
+			return 0, "", readErr
+		}
+	}
+	return total, hex.EncodeToString(hash.Sum(nil)), nil
+}
+
+func copyFile(ctx context.Context, source, destination string) (File, error) {
+	if _, err := regularFile(source); err != nil {
+		return File{}, err
+	}
+	input, err := os.Open(source)
+	if err != nil {
+		return File{}, err
+	}
+	defer input.Close()
+	if err := ensureRealDirectory(filepath.Dir(destination)); err != nil {
+		return File{}, err
+	}
+	output, err := os.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+	if err != nil {
+		return File{}, err
+	}
+	remove := true
+	defer func() {
+		_ = output.Close()
+		if remove {
+			_ = os.Remove(destination)
+		}
+	}()
+	hash := sha256.New()
+	written, err := io.Copy(io.MultiWriter(output, hash), &contextReader{ctx: ctx, reader: input})
+	if err != nil {
+		return File{}, err
+	}
+	if err := output.Sync(); err != nil {
+		return File{}, err
+	}
+	if err := output.Close(); err != nil {
+		return File{}, err
+	}
+	remove = false
+	return File{Size: written, SHA256: hex.EncodeToString(hash.Sum(nil))}, nil
+}
+
+type contextReader struct {
+	ctx    context.Context
+	reader io.Reader
+}
+
+func (reader *contextReader) Read(buffer []byte) (int, error) {
+	if err := reader.ctx.Err(); err != nil {
+		return 0, err
+	}
+	return reader.reader.Read(buffer)
+}
+
+func sourceFiles(paths tenantfs.Paths) ([]struct{ absolute, relative string }, error) {
+	if _, err := regularFile(paths.Database); err != nil {
+		return nil, err
+	}
+	attachmentInfo, err := os.Lstat(paths.Attachments)
+	if err != nil {
+		return nil, err
+	}
+	if attachmentInfo.Mode()&os.ModeSymlink != 0 || !attachmentInfo.IsDir() {
+		return nil, fmt.Errorf("attachment root is not a real directory")
+	}
+	result := []struct{ absolute, relative string }{{paths.Database, "relay.db"}}
+	entries, err := os.ReadDir(paths.Attachments)
+	if err != nil {
+		return nil, err
+	}
+	for _, entry := range entries {
+		if entry.IsDir() || entry.Type()&os.ModeSymlink != 0 || !validAttachmentName(entry.Name()) {
+			return nil, fmt.Errorf("invalid attachment artifact %q", entry.Name())
+		}
+		result = append(result, struct{ absolute, relative string }{
+			filepath.Join(paths.Attachments, entry.Name()), filepath.ToSlash(filepath.Join("attachments", entry.Name())),
+		})
+	}
+	sort.Slice(result, func(left, right int) bool { return result[left].relative < result[right].relative })
+	return result, nil
+}
+
+func Create(ctx context.Context, dataRoot, backupRoot, tenantID string, generation int64, now time.Time) (Result, error) {
+	if generation < 1 || now.IsZero() {
+		return Result{}, fmt.Errorf("invalid backup generation or timestamp")
+	}
+	paths, err := tenantfs.Derive(dataRoot, tenantID)
+	if err != nil {
+		return Result{}, err
+	}
+	if err := checkpointAndVerify(paths.Database); err != nil {
+		return Result{}, err
+	}
+	files, err := sourceFiles(paths)
+	if err != nil {
+		return Result{}, err
+	}
+	backupRoot, err = filepath.Abs(strings.TrimSpace(backupRoot))
+	if err != nil || strings.TrimSpace(backupRoot) == "" {
+		return Result{}, fmt.Errorf("invalid backup root")
+	}
+	if err := ensureRealDirectory(backupRoot); err != nil {
+		return Result{}, err
+	}
+	tenantBackupRoot := filepath.Join(backupRoot, filepath.Base(paths.Root))
+	if err := ensureRealDirectory(tenantBackupRoot); err != nil {
+		return Result{}, err
+	}
+	nonce, err := randomHex(8)
+	if err != nil {
+		return Result{}, err
+	}
+	name := fmt.Sprintf("g%020d-%s-%s", generation, now.UTC().Format("20060102T150405Z"), nonce)
+	finalPath := filepath.Join(tenantBackupRoot, name)
+	stagingPath := finalPath + ".tmp"
+	if err := os.Mkdir(stagingPath, 0o700); err != nil {
+		return Result{}, err
+	}
+	committed := false
+	defer func() {
+		if !committed {
+			_ = os.RemoveAll(stagingPath)
+		}
+	}()
+	manifest := Manifest{
+		Version: manifestVersion, TenantID: tenantID, PlacementGeneration: generation,
+		CreatedAt: now.UTC().Format(time.RFC3339Nano), Files: make([]File, 0, len(files)),
+	}
+	for _, item := range files {
+		copied, err := copyFile(ctx, item.absolute, filepath.Join(stagingPath, filepath.FromSlash(item.relative)))
+		if err != nil {
+			return Result{}, err
+		}
+		copied.Path = item.relative
+		manifest.Files = append(manifest.Files, copied)
+	}
+	manifestBytes, err := json.Marshal(manifest)
+	if err != nil {
+		return Result{}, err
+	}
+	manifestPath := filepath.Join(stagingPath, "manifest.json")
+	manifestFile, err := os.OpenFile(manifestPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+	if err != nil {
+		return Result{}, err
+	}
+	if _, err := manifestFile.Write(manifestBytes); err != nil {
+		_ = manifestFile.Close()
+		return Result{}, err
+	}
+	if err := manifestFile.Sync(); err != nil {
+		_ = manifestFile.Close()
+		return Result{}, err
+	}
+	if err := manifestFile.Close(); err != nil {
+		return Result{}, err
+	}
+	if err := os.Rename(stagingPath, finalPath); err != nil {
+		return Result{}, err
+	}
+	committed = true
+	digest := sha256.Sum256(manifestBytes)
+	backupRef := filepath.ToSlash(filepath.Join(filepath.Base(paths.Root), name))
+	if !validBackupRef(backupRef) {
+		return Result{}, fmt.Errorf("generated backup reference is invalid")
+	}
+	return Result{Path: finalPath, BackupRef: backupRef, Manifest: manifest, ManifestSHA256: hex.EncodeToString(digest[:])}, nil
+}
+
+func validManifestPath(path string) bool {
+	if path == "relay.db" {
+		return true
+	}
+	parts := strings.Split(path, "/")
+	return len(parts) == 2 && parts[0] == "attachments" && validAttachmentName(parts[1])
+}
+
+func Verify(ctx context.Context, backupPath, tenantID string, generation int64, expectedManifestSHA256 string) (Manifest, error) {
+	backupPath, err := filepath.Abs(strings.TrimSpace(backupPath))
+	if err != nil || strings.TrimSpace(backupPath) == "" {
+		return Manifest{}, fmt.Errorf("invalid backup path")
+	}
+	info, err := os.Lstat(backupPath)
+	if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
+		return Manifest{}, fmt.Errorf("backup path is not a real directory")
+	}
+	manifestPath := filepath.Join(backupPath, "manifest.json")
+	manifestInfo, err := regularFile(manifestPath)
+	if err != nil || manifestInfo.Size() > 1<<20 {
+		return Manifest{}, fmt.Errorf("invalid backup manifest")
+	}
+	manifestBytes, err := os.ReadFile(manifestPath)
+	if err != nil {
+		return Manifest{}, err
+	}
+	digest := sha256.Sum256(manifestBytes)
+	if expectedManifestSHA256 != hex.EncodeToString(digest[:]) {
+		return Manifest{}, fmt.Errorf("backup manifest digest mismatch")
+	}
+	decoder := json.NewDecoder(bytes.NewReader(manifestBytes))
+	decoder.DisallowUnknownFields()
+	var manifest Manifest
+	if err := decoder.Decode(&manifest); err != nil || decoder.Decode(&struct{}{}) != io.EOF {
+		return Manifest{}, fmt.Errorf("invalid backup manifest JSON")
+	}
+	if manifest.Version != manifestVersion || manifest.TenantID != tenantID || manifest.PlacementGeneration != generation || len(manifest.Files) == 0 {
+		return Manifest{}, fmt.Errorf("backup manifest fence mismatch")
+	}
+	expectedFiles := map[string]struct{}{"manifest.json": {}}
+	previous := ""
+	for _, item := range manifest.Files {
+		if !validManifestPath(item.Path) || item.Path <= previous || item.Size < 0 || len(item.SHA256) != 64 {
+			return Manifest{}, fmt.Errorf("invalid backup file manifest")
+		}
+		previous = item.Path
+		absolute := filepath.Join(backupPath, filepath.FromSlash(item.Path))
+		if !strings.HasPrefix(absolute, backupPath+string(os.PathSeparator)) {
+			return Manifest{}, fmt.Errorf("backup file escaped root")
+		}
+		size, checksum, err := hashFile(ctx, absolute)
+		if err != nil || size != item.Size || checksum != item.SHA256 {
+			return Manifest{}, fmt.Errorf("backup file verification failed for %s", item.Path)
+		}
+		expectedFiles[filepath.Clean(item.Path)] = struct{}{}
+	}
+	err = filepath.WalkDir(backupPath, func(path string, entry os.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return walkErr
+		}
+		if path == backupPath {
+			return nil
+		}
+		relative, err := filepath.Rel(backupPath, path)
+		if err != nil {
+			return err
+		}
+		if entry.Type()&os.ModeSymlink != 0 {
+			return fmt.Errorf("backup contains a symbolic link")
+		}
+		if entry.IsDir() {
+			if filepath.Clean(relative) != "attachments" {
+				return fmt.Errorf("backup contains an unexpected directory")
+			}
+			return nil
+		}
+		if _, ok := expectedFiles[filepath.Clean(relative)]; !ok {
+			return fmt.Errorf("backup contains an unexpected file %q", relative)
+		}
+		return nil
+	})
+	if err != nil {
+		return Manifest{}, err
+	}
+	if err := verifySQLiteReadOnly(filepath.Join(backupPath, "relay.db")); err != nil {
+		return Manifest{}, err
+	}
+	return manifest, nil
+}
+
+func Restore(ctx context.Context, backupPath, dataRoot, tenantID string, generation int64, manifestSHA256 string) (tenantfs.Paths, error) {
+	manifest, err := Verify(ctx, backupPath, tenantID, generation, manifestSHA256)
+	if err != nil {
+		return tenantfs.Paths{}, err
+	}
+	paths, err := tenantfs.Derive(dataRoot, tenantID)
+	if err != nil {
+		return tenantfs.Paths{}, err
+	}
+	dataRoot = filepath.Dir(filepath.Dir(paths.Root))
+	if err := ensureRealDirectory(dataRoot); err != nil {
+		return tenantfs.Paths{}, err
+	}
+	tenantsRoot := filepath.Dir(paths.Root)
+	if err := ensureRealDirectory(tenantsRoot); err != nil {
+		return tenantfs.Paths{}, err
+	}
+	if info, err := os.Lstat(paths.Root); err == nil {
+		if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
+			return tenantfs.Paths{}, fmt.Errorf("target tenant path is not a real directory")
+		}
+		receiptBytes, readErr := os.ReadFile(filepath.Join(paths.Root, ".restore-receipt.json"))
+		var receipt restoreReceipt
+		decoder := json.NewDecoder(bytes.NewReader(receiptBytes))
+		decoder.DisallowUnknownFields()
+		if readErr == nil && decoder.Decode(&receipt) == nil && decoder.Decode(&struct{}{}) == io.EOF &&
+			receipt.Version == manifestVersion && receipt.TenantID == tenantID &&
+			receipt.PlacementGeneration == generation && receipt.ManifestSHA256 == manifestSHA256 {
+			return paths, nil
+		}
+		return tenantfs.Paths{}, fmt.Errorf("target tenant directory already exists with another restore fence")
+	} else if !os.IsNotExist(err) {
+		return tenantfs.Paths{}, err
+	}
+	nonce, err := randomHex(8)
+	if err != nil {
+		return tenantfs.Paths{}, err
+	}
+	staging := filepath.Join(tenantsRoot, ".restore-"+filepath.Base(paths.Root)+"-"+nonce)
+	if err := os.Mkdir(staging, 0o700); err != nil {
+		return tenantfs.Paths{}, err
+	}
+	committed := false
+	defer func() {
+		if !committed {
+			_ = os.RemoveAll(staging)
+		}
+	}()
+	for _, item := range manifest.Files {
+		if _, err := copyFile(ctx, filepath.Join(backupPath, filepath.FromSlash(item.Path)), filepath.Join(staging, filepath.FromSlash(item.Path))); err != nil {
+			return tenantfs.Paths{}, err
+		}
+	}
+	if err := verifySQLiteReadOnly(filepath.Join(staging, "relay.db")); err != nil {
+		return tenantfs.Paths{}, err
+	}
+	receiptBytes, err := json.Marshal(restoreReceipt{
+		Version: manifestVersion, TenantID: tenantID,
+		PlacementGeneration: generation, ManifestSHA256: manifestSHA256,
+	})
+	if err != nil {
+		return tenantfs.Paths{}, err
+	}
+	receipt, err := os.OpenFile(filepath.Join(staging, ".restore-receipt.json"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+	if err != nil {
+		return tenantfs.Paths{}, err
+	}
+	if _, err := receipt.Write(receiptBytes); err != nil {
+		_ = receipt.Close()
+		return tenantfs.Paths{}, err
+	}
+	if err := receipt.Sync(); err != nil {
+		_ = receipt.Close()
+		return tenantfs.Paths{}, err
+	}
+	if err := receipt.Close(); err != nil {
+		return tenantfs.Paths{}, err
+	}
+	if err := os.Rename(staging, paths.Root); err != nil {
+		return tenantfs.Paths{}, err
+	}
+	committed = true
+	return paths, nil
+}