feat: establish NekoNest Cloud control and relay

This commit is contained in:
2026-08-12 23:25:43 +08:00
commit f27606b709
222 changed files with 71456 additions and 0 deletions
+852
View File
@@ -0,0 +1,852 @@
package registry
import (
"context"
"crypto/aes"
"crypto/cipher"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"encoding/base64"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"net/http"
"net/url"
"os"
"strings"
"sync"
"time"
"github.com/klarkxy/nekonest-cloud/relay/internal/attachmentstore"
"github.com/klarkxy/nekonest-cloud/relay/internal/authsnapshot"
"github.com/klarkxy/nekonest-cloud/relay/internal/controlplane"
"github.com/klarkxy/nekonest-cloud/relay/internal/tenantfs"
"github.com/klarkxy/nekonest-cloud/relay/internal/tenantstore"
"github.com/klarkxy/nekonest/relaycore"
"github.com/klarkxy/nekonest/relaycore/protocol"
)
type SnapshotSource interface {
AuthorizationSnapshot(context.Context, string, int64) (controlplane.SnapshotResponse, error)
AuthorizationDelta(context.Context, string, int64) (controlplane.DeltaResponse, error)
}
type Config struct {
DataRoot string
NodeID string
AllowedOrigins []string
RouteSecret []byte
AppVersion string
MaxTenants int
Keyring authsnapshot.Keyring
ControlPlane SnapshotSource
RefreshInterval time.Duration
DeltaInterval time.Duration
RequestTimeout time.Duration
PushSink relaycore.PushSink
AuditSink relaycore.AuditSink
Now func() time.Time
}
type Registry struct {
config Config
ctx context.Context
cancel context.CancelFunc
mu sync.RWMutex
slots map[string]*slot
routeHints map[string]*Tenant
phoneRoutes map[string]*Tenant
deviceOwners map[string]string
phoneOwners map[string]string
activeTenants int
}
type slot struct {
mu sync.Mutex
tenant *Tenant
reserved bool // guarded by Registry.mu; counts active or currently building Engines
}
type Tenant struct {
mu sync.RWMutex
engine *relaycore.Engine
store *tenantstore.DB
handler http.Handler
payload authsnapshot.Payload
routeHint string
cancel context.CancelFunc
closed bool
deviceIDs map[string]struct{}
phoneIDs map[string]struct{}
}
type attachmentURLBuilder struct{ routeHint string }
func (builder attachmentURLBuilder) BuildAttachmentURL(id, capabilityKey string) string {
query := make(url.Values)
query.Set("route", builder.routeHint)
query.Set("k", capabilityKey)
return "/api/attachments/" + url.PathEscape(id) + "?" + query.Encode()
}
func New(config Config) (*Registry, error) {
if config.DataRoot == "" || config.NodeID == "" || len(config.RouteSecret) < 32 {
return nil, errors.New("registry requires data root, node id, and a 32-byte route secret")
}
if config.MaxTenants <= 0 {
config.MaxTenants = 256
}
if config.RefreshInterval <= 0 {
config.RefreshInterval = 60 * time.Second
}
if config.DeltaInterval <= 0 {
// Poll more frequently than the public 15-second revocation objective:
// one healthy delta request plus one full snapshot refresh must still fit.
config.DeltaInterval = 5 * time.Second
}
if config.RequestTimeout <= 0 {
config.RequestTimeout = 3 * time.Second
}
if config.Now == nil {
config.Now = time.Now
}
if len(config.Keyring) == 0 || config.ControlPlane == nil {
return nil, errors.New("registry requires pinned snapshot keys and a control plane")
}
ctx, cancel := context.WithCancel(context.Background())
return &Registry{
config: config,
ctx: ctx,
cancel: cancel,
slots: make(map[string]*slot),
routeHints: make(map[string]*Tenant),
phoneRoutes: make(map[string]*Tenant),
deviceOwners: make(map[string]string),
phoneOwners: make(map[string]string),
}, nil
}
type principalReservation struct {
devices []string
phones []string
}
func (r *Registry) reservePrincipals(payload authsnapshot.Payload) (principalReservation, error) {
r.mu.Lock()
defer r.mu.Unlock()
for _, device := range payload.Devices {
if owner := r.deviceOwners[device.DeviceID]; owner != "" && owner != payload.TenantID {
return principalReservation{}, errors.New("device identity is already owned by another tenant")
}
}
for _, phone := range payload.Phones {
if owner := r.phoneOwners[phone.PhoneID]; owner != "" && owner != payload.TenantID {
return principalReservation{}, errors.New("phone identity is already owned by another tenant")
}
}
reservation := principalReservation{}
for _, device := range payload.Devices {
if r.deviceOwners[device.DeviceID] == "" {
r.deviceOwners[device.DeviceID] = payload.TenantID
reservation.devices = append(reservation.devices, device.DeviceID)
}
}
for _, phone := range payload.Phones {
if r.phoneOwners[phone.PhoneID] == "" {
r.phoneOwners[phone.PhoneID] = payload.TenantID
reservation.phones = append(reservation.phones, phone.PhoneID)
}
}
return reservation, nil
}
func (r *Registry) releaseReservation(tenantID string, reservation principalReservation) {
r.mu.Lock()
defer r.mu.Unlock()
for _, id := range reservation.devices {
if r.deviceOwners[id] == tenantID {
delete(r.deviceOwners, id)
}
}
for _, id := range reservation.phones {
if r.phoneOwners[id] == tenantID {
delete(r.phoneOwners, id)
}
}
}
func (r *Registry) releaseRemovedPrincipals(tenantID string, oldDevices, oldPhones map[string]struct{}, payload authsnapshot.Payload) {
newDevices := make(map[string]struct{}, len(payload.Devices))
for _, device := range payload.Devices {
newDevices[device.DeviceID] = struct{}{}
}
newPhones := make(map[string]struct{}, len(payload.Phones))
for _, phone := range payload.Phones {
newPhones[phone.PhoneID] = struct{}{}
}
r.mu.Lock()
defer r.mu.Unlock()
for id := range oldDevices {
if _, exists := newDevices[id]; !exists && r.deviceOwners[id] == tenantID {
delete(r.deviceOwners, id)
}
}
for id := range oldPhones {
if _, exists := newPhones[id]; !exists && r.phoneOwners[id] == tenantID {
delete(r.phoneOwners, id)
}
}
}
func cloneSet(source map[string]struct{}) map[string]struct{} {
result := make(map[string]struct{}, len(source))
for key := range source {
result[key] = struct{}{}
}
return result
}
func (r *Registry) routeHint(tenantID string, generation int64) string {
key := sha256.Sum256(append([]byte("nekonest-cloud/engine-route-key/v1\x00"), r.config.RouteSecret...))
block, err := aes.NewCipher(key[:])
if err != nil {
return ""
}
aead, err := cipher.NewGCM(block)
if err != nil {
return ""
}
nonce := make([]byte, aead.NonceSize())
if _, err := rand.Read(nonce); err != nil {
return ""
}
plaintext := make([]byte, len(tenantID)+8)
copy(plaintext, tenantID)
binary.BigEndian.PutUint64(plaintext[len(tenantID):], uint64(generation))
sealed := aead.Seal(nil, nonce, plaintext, []byte("nekonest-cloud/engine-route/v1"))
token := append([]byte{1}, nonce...)
token = append(token, sealed...)
return base64.RawURLEncoding.EncodeToString(token)
}
// DecodeRouteHint authenticates an opaque attachment routing token generated
// by any Relay node sharing the deployment route secret. It never accepts a
// client-supplied raw tenant id.
func (r *Registry) DecodeRouteHint(routeHint string) (string, int64, error) {
encoded, err := base64.RawURLEncoding.DecodeString(strings.TrimSpace(routeHint))
if err != nil || len(encoded) < 2 || encoded[0] != 1 {
return "", 0, errors.New("invalid route hint")
}
key := sha256.Sum256(append([]byte("nekonest-cloud/engine-route-key/v1\x00"), r.config.RouteSecret...))
block, err := aes.NewCipher(key[:])
if err != nil {
return "", 0, err
}
aead, err := cipher.NewGCM(block)
if err != nil {
return "", 0, err
}
if len(encoded) <= 1+aead.NonceSize() {
return "", 0, errors.New("invalid route hint")
}
nonce := encoded[1 : 1+aead.NonceSize()]
plaintext, err := aead.Open(nil, nonce, encoded[1+aead.NonceSize():], []byte("nekonest-cloud/engine-route/v1"))
if err != nil || len(plaintext) != len("tenant_")+32+8 {
return "", 0, errors.New("invalid route hint")
}
tenantID := string(plaintext[:len(plaintext)-8])
generation := int64(binary.BigEndian.Uint64(plaintext[len(plaintext)-8:]))
if !strings.HasPrefix(tenantID, "tenant_") || generation < 1 {
return "", 0, errors.New("invalid route hint")
}
for _, char := range strings.TrimPrefix(tenantID, "tenant_") {
if !strings.ContainsRune("0123456789abcdef", char) {
return "", 0, errors.New("invalid route hint")
}
}
return tenantID, generation, nil
}
func (r *Registry) engineAdminSecret(tenantID string, generation int64) string {
mac := hmac.New(sha256.New, r.config.RouteSecret)
_, _ = fmt.Fprintf(mac, "nekonest-cloud/engine-admin/v1\x00%s\x00%d", tenantID, generation)
return hex.EncodeToString(mac.Sum(nil))
}
func (r *Registry) verifyResponse(response controlplane.SnapshotResponse) (authsnapshot.Payload, error) {
key, ok := r.config.Keyring[response.Snapshot.KID]
if !ok {
return authsnapshot.Payload{}, errors.New("snapshot key id is not pinned")
}
responseKey, responseKID, err := authsnapshot.PublicKeyFromJWK(response.PublicKeyJWK)
if err != nil {
return authsnapshot.Payload{}, err
}
if responseKID != "" && responseKID != response.Snapshot.KID {
return authsnapshot.Payload{}, errors.New("snapshot JWK kid mismatch")
}
if len(responseKey) != len(key.PublicKey) || subtle.ConstantTimeCompare(responseKey, key.PublicKey) != 1 {
return authsnapshot.Payload{}, errors.New("control-plane key does not match pinned keyring")
}
payload, err := authsnapshot.Verify(response.Snapshot, r.config.Keyring, r.config.Now())
if err != nil {
return authsnapshot.Payload{}, err
}
if payload.RelayNodeID != r.config.NodeID {
return authsnapshot.Payload{}, errors.New("snapshot belongs to another relay node")
}
return payload, nil
}
func (r *Registry) getSlot(tenantID string) (*slot, error) {
r.mu.Lock()
defer r.mu.Unlock()
if existing := r.slots[tenantID]; existing != nil {
return existing, nil
}
created := &slot{}
r.slots[tenantID] = created
return created, nil
}
func (r *Registry) reserveSlot(slot *slot) error {
r.mu.Lock()
defer r.mu.Unlock()
if slot.reserved {
return nil
}
if r.activeTenants >= r.config.MaxTenants {
return errors.New("relay tenant capacity reached")
}
slot.reserved = true
r.activeTenants++
return nil
}
func (r *Registry) releaseSlot(slot *slot) {
r.mu.Lock()
defer r.mu.Unlock()
if !slot.reserved {
return
}
slot.reserved = false
if r.activeTenants > 0 {
r.activeTenants--
}
}
func (r *Registry) Accept(ctx context.Context, response controlplane.SnapshotResponse) (*Tenant, error) {
payload, err := r.verifyResponse(response)
if err != nil {
return nil, err
}
slot, err := r.getSlot(payload.TenantID)
if err != nil {
return nil, err
}
slot.mu.Lock()
defer slot.mu.Unlock()
if current := slot.tenant; current != nil {
current.mu.RLock()
generation := current.payload.PlacementGeneration
authorizationRevision := current.payload.AuthorizationRevision
closed := current.closed
oldDevices := cloneSet(current.deviceIDs)
oldPhones := cloneSet(current.phoneIDs)
current.mu.RUnlock()
if !closed && generation == payload.PlacementGeneration {
// Snapshot refreshes can complete out of order. Never let an older,
// still-valid signed snapshot re-authorize a principal removed by a
// newer revision.
if payload.AuthorizationRevision < authorizationRevision {
return nil, errors.New("stale authorization revision")
}
reservation, reserveErr := r.reservePrincipals(payload)
if reserveErr != nil {
return nil, reserveErr
}
if err := current.applyPayload(payload); err != nil {
r.releaseReservation(payload.TenantID, reservation)
r.expireLocked(slot, current)
r.releaseSlot(slot)
return nil, err
}
r.releaseRemovedPrincipals(payload.TenantID, oldDevices, oldPhones, payload)
return current, nil
}
if payload.PlacementGeneration < generation {
return nil, errors.New("stale placement generation")
}
r.expireLocked(slot, current)
}
if err := r.reserveSlot(slot); err != nil {
return nil, err
}
reservation, err := r.reservePrincipals(payload)
if err != nil {
r.releaseSlot(slot)
return nil, err
}
tenant, err := r.buildTenant(payload)
if err != nil {
r.releaseReservation(payload.TenantID, reservation)
r.releaseSlot(slot)
return nil, err
}
slot.tenant = tenant
r.mu.Lock()
r.routeHints[tenant.routeHint] = tenant
r.mu.Unlock()
watchContext, cancel := context.WithCancel(r.ctx)
tenant.cancel = cancel
go r.watch(watchContext, slot, tenant)
return tenant, nil
}
func (r *Registry) buildTenant(payload authsnapshot.Payload) (*Tenant, error) {
paths, err := tenantfs.Resolve(r.config.DataRoot, payload.TenantID)
if err != nil {
return nil, err
}
store, err := tenantstore.NewWithTransportMode(paths.Database, string(protocol.TransportSealed))
if err != nil {
return nil, err
}
attachments, err := attachmentstore.New(paths.Attachments)
if err != nil {
_ = store.Close()
return nil, err
}
routeHint := r.routeHint(payload.TenantID, payload.PlacementGeneration)
if routeHint == "" {
_ = store.Close()
return nil, errors.New("generate tenant route hint")
}
engine, err := relaycore.NewEngine(relaycore.Config{
Store: store,
Ports: relaycore.Ports{
PrincipalSynchronizer: store,
AttachmentStore: attachments,
AttachmentURLBuilder: attachmentURLBuilder{routeHint: routeHint},
PushSink: r.config.PushSink,
AuditSink: r.config.AuditSink,
},
PhoneSecret: r.engineAdminSecret(payload.TenantID, payload.PlacementGeneration),
AppVersion: r.config.AppVersion,
TransportMode: protocol.TransportSealed,
DuplicateDaemon: relaycore.RejectNew,
AllowedOrigins: r.config.AllowedOrigins,
})
if err != nil {
_ = store.Close()
return nil, err
}
mux := http.NewServeMux()
engine.RegisterDataPlaneRoutes(mux)
tenant := &Tenant{
engine: engine,
store: store,
handler: mux,
payload: payload,
routeHint: routeHint,
deviceIDs: make(map[string]struct{}),
phoneIDs: make(map[string]struct{}),
}
if err := tenant.applyPayload(payload); err != nil {
_ = engine.Close()
_ = store.Close()
return nil, err
}
return tenant, nil
}
func (tenant *Tenant) applyPayload(payload authsnapshot.Payload) error {
tenant.mu.Lock()
defer tenant.mu.Unlock()
if tenant.closed {
return errors.New("tenant engine is closed")
}
next := make(map[string]struct{}, len(payload.Devices))
for _, device := range payload.Devices {
if err := tenant.engine.SyncApprovedDevice(relaycore.ApprovedDevice{
ID: device.DeviceID,
Name: device.Name,
OS: device.OS,
Credential: relaycore.Credential{Kind: relaycore.CredentialSHA256, Value: device.CredentialHash},
}); err != nil {
return fmt.Errorf("sync device %s: %w", device.DeviceID, err)
}
if err := tenant.store.SetDevicePublicKeys(
device.DeviceID,
device.Ed25519Public,
device.X25519Public,
device.IdentityFingerprint,
); err != nil {
return fmt.Errorf("sync device identity %s: %w", device.DeviceID, err)
}
next[device.DeviceID] = struct{}{}
}
for deviceID := range tenant.deviceIDs {
if _, exists := next[deviceID]; !exists {
if err := tenant.engine.RevokeApprovedDevice(deviceID); err != nil {
return fmt.Errorf("revoke device %s: %w", deviceID, err)
}
}
}
nextPhones := make(map[string]struct{}, len(payload.Phones))
for _, phone := range payload.Phones {
if err := tenant.engine.SyncApprovedPhone(relaycore.ApprovedPhone{
ID: phone.PhoneID,
Name: phone.Name,
Credential: relaycore.Credential{Kind: relaycore.CredentialSHA256, Value: phone.CredentialHash},
}); err != nil {
return fmt.Errorf("sync phone %s: %w", phone.PhoneID, err)
}
if err := tenant.store.SetPhonePublicKeys(phone.PhoneID, phone.Ed25519Public, phone.X25519Public); err != nil {
return fmt.Errorf("sync phone identity %s: %w", phone.PhoneID, err)
}
nextPhones[phone.PhoneID] = struct{}{}
}
for phoneID := range tenant.phoneIDs {
if _, exists := nextPhones[phoneID]; !exists {
if err := tenant.engine.RevokeApprovedPhone(phoneID); err != nil {
return fmt.Errorf("revoke phone %s: %w", phoneID, err)
}
}
}
tenant.deviceIDs = next
tenant.phoneIDs = nextPhones
tenant.payload = payload
return nil
}
func (r *Registry) watch(ctx context.Context, slot *slot, tenant *Tenant) {
refresh := time.NewTicker(r.config.RefreshInterval)
delta := time.NewTicker(r.config.DeltaInterval)
defer refresh.Stop()
defer delta.Stop()
for {
tenant.mu.RLock()
payload := tenant.payload
closed := tenant.closed
tenant.mu.RUnlock()
if closed {
return
}
expiresAt, err := time.Parse(time.RFC3339Nano, payload.ExpiresAt)
if err != nil {
r.expire(slot, tenant)
return
}
untilExpiry := expiresAt.Sub(r.config.Now())
if untilExpiry <= 0 {
r.expire(slot, tenant)
return
}
expiry := time.NewTimer(untilExpiry)
select {
case <-ctx.Done():
expiry.Stop()
return
case <-expiry.C:
r.expire(slot, tenant)
return
case <-refresh.C:
expiry.Stop()
r.refresh(ctx, tenant, payload)
case <-delta.C:
expiry.Stop()
requestContext, cancel := context.WithTimeout(ctx, r.config.RequestTimeout)
change, err := r.config.ControlPlane.AuthorizationDelta(
requestContext, payload.TenantID, payload.AuthorizationRevision,
)
cancel()
if err == nil && change.TenantStatus != "active" {
r.expire(slot, tenant)
return
}
if err == nil && change.Changed {
r.refresh(ctx, tenant, payload)
}
}
}
}
func (r *Registry) refresh(ctx context.Context, tenant *Tenant, payload authsnapshot.Payload) {
requestContext, cancel := context.WithTimeout(ctx, r.config.RequestTimeout)
response, err := r.config.ControlPlane.AuthorizationSnapshot(
requestContext, payload.TenantID, payload.PlacementGeneration,
)
cancel()
if err != nil {
return // existing associations continue only until the current snapshot expires
}
_, _ = r.Accept(ctx, response)
}
func (r *Registry) expire(slot *slot, tenant *Tenant) {
slot.mu.Lock()
defer slot.mu.Unlock()
if slot.tenant == tenant {
r.expireLocked(slot, tenant)
r.releaseSlot(slot)
}
}
func (r *Registry) expireLocked(slot *slot, tenant *Tenant) {
tenant.mu.Lock()
if tenant.closed {
tenant.mu.Unlock()
if slot.tenant == tenant {
slot.tenant = nil
}
return
}
tenant.closed = true
cancel := tenant.cancel
tenantID := tenant.payload.TenantID
deviceIDs := cloneSet(tenant.deviceIDs)
phoneIDs := cloneSet(tenant.phoneIDs)
tenant.mu.Unlock()
if cancel != nil {
cancel()
}
_ = tenant.engine.Close()
_ = tenant.store.Close()
r.mu.Lock()
for id := range deviceIDs {
if r.deviceOwners[id] == tenantID {
delete(r.deviceOwners, id)
}
}
for id := range phoneIDs {
if r.phoneOwners[id] == tenantID {
delete(r.phoneOwners, id)
}
}
if r.routeHints[tenant.routeHint] == tenant {
delete(r.routeHints, tenant.routeHint)
}
for digest, routed := range r.phoneRoutes {
if routed == tenant {
delete(r.phoneRoutes, digest)
}
}
r.mu.Unlock()
if slot.tenant == tenant {
slot.tenant = nil
}
}
func (r *Registry) ByRouteHint(routeHint string) (*Tenant, bool) {
r.mu.RLock()
tenant := r.routeHints[routeHint]
r.mu.RUnlock()
if tenant == nil {
return nil, false
}
tenant.mu.RLock()
defer tenant.mu.RUnlock()
expiresAt, err := time.Parse(time.RFC3339Nano, tenant.payload.ExpiresAt)
return tenant, !tenant.closed && err == nil && r.config.Now().Before(expiresAt)
}
func phoneRouteDigest(routeHandle string) string {
digest := sha256.Sum256([]byte(routeHandle))
return hex.EncodeToString(digest[:])
}
// BindPhoneRoute caches only a one-way digest of the opaque handle. The handle
// is a routing hint and never authorizes a request without the phone token.
func (r *Registry) BindPhoneRoute(routeHandle string, tenant *Tenant) error {
if len(routeHandle) != 64 || tenant == nil {
return errors.New("invalid phone route handle")
}
for _, char := range routeHandle {
if !strings.ContainsRune("0123456789abcdef", char) {
return errors.New("invalid phone route handle")
}
}
if !tenant.Current(r.config.Now()) {
return errors.New("tenant authorization snapshot expired")
}
r.mu.Lock()
r.phoneRoutes[phoneRouteDigest(routeHandle)] = tenant
r.mu.Unlock()
return nil
}
func (r *Registry) ByPhoneRoute(routeHandle string) (*Tenant, bool) {
r.mu.RLock()
tenant := r.phoneRoutes[phoneRouteDigest(routeHandle)]
r.mu.RUnlock()
return tenant, tenant != nil && tenant.Current(r.config.Now())
}
func (r *Registry) ByDeviceID(deviceID string) (*Tenant, bool) {
r.mu.RLock()
slots := make([]*slot, 0, len(r.slots))
for _, item := range r.slots {
slots = append(slots, item)
}
r.mu.RUnlock()
for _, item := range slots {
item.mu.Lock()
tenant := item.tenant
item.mu.Unlock()
if tenant == nil || !tenant.Current(r.config.Now()) {
continue
}
tenant.mu.RLock()
_, exists := tenant.deviceIDs[deviceID]
tenant.mu.RUnlock()
if exists {
return tenant, true
}
}
return nil, false
}
func (tenant *Tenant) Handler() http.Handler { return tenant.handler }
func (tenant *Tenant) Engine() *relaycore.Engine { return tenant.engine }
func (tenant *Tenant) RouteHint() string { return tenant.routeHint }
func (tenant *Tenant) Current(now time.Time) bool {
tenant.mu.RLock()
defer tenant.mu.RUnlock()
expiresAt, err := time.Parse(time.RFC3339Nano, tenant.payload.ExpiresAt)
return !tenant.closed && err == nil && now.Before(expiresAt)
}
func (tenant *Tenant) Payload() authsnapshot.Payload {
tenant.mu.RLock()
defer tenant.mu.RUnlock()
return tenant.payload
}
func (tenant *Tenant) SyncPhone(phoneID, name, tokenHash, ed25519Public, x25519Public string) error {
if err := tenant.engine.SyncApprovedPhone(relaycore.ApprovedPhone{
ID: phoneID,
Name: name,
Credential: relaycore.Credential{Kind: relaycore.CredentialSHA256, Value: tokenHash},
}); err != nil {
return err
}
return tenant.store.SetPhonePublicKeys(phoneID, ed25519Public, x25519Public)
}
func (tenant *Tenant) RevokePhone(phoneID string) error {
return tenant.engine.RevokeApprovedPhone(phoneID)
}
// Quiesce closes the current generation and releases its in-process capacity
// before a backup is taken. The control plane must already have fenced the
// placement out of active state, so a new signed snapshot cannot reopen it.
func (r *Registry) Quiesce(tenantID string, generation int64) (tenantfs.Paths, error) {
paths, err := tenantfs.Derive(r.config.DataRoot, tenantID)
if err != nil {
return tenantfs.Paths{}, err
}
r.mu.RLock()
slot := r.slots[tenantID]
r.mu.RUnlock()
if slot == nil {
info, statErr := os.Lstat(paths.Database)
if statErr == nil && info.Mode()&os.ModeSymlink == 0 && info.Mode().IsRegular() {
return paths, nil // idempotent retry after the Engine was already closed
}
return tenantfs.Paths{}, errors.New("tenant engine is not active on this node")
}
slot.mu.Lock()
defer slot.mu.Unlock()
tenant := slot.tenant
if tenant == nil {
info, statErr := os.Lstat(paths.Database)
if statErr == nil && info.Mode()&os.ModeSymlink == 0 && info.Mode().IsRegular() {
return paths, nil
}
return tenantfs.Paths{}, errors.New("tenant engine is not active on this node")
}
tenant.mu.RLock()
currentGeneration := tenant.payload.PlacementGeneration
tenant.mu.RUnlock()
if generation < 1 || currentGeneration != generation {
return tenantfs.Paths{}, errors.New("tenant placement generation mismatch")
}
r.expireLocked(slot, tenant)
r.releaseSlot(slot)
return paths, nil
}
// QuiesceForPurge fences and closes an active generation, but unlike migration
// quiescing it also succeeds when a retry finds that the tenant data was
// already deleted. The exact generation is still enforced for a live Engine.
func (r *Registry) QuiesceForPurge(tenantID string, generation int64) error {
paths, err := tenantfs.Derive(r.config.DataRoot, tenantID)
if err != nil {
return err
}
if generation < 1 {
return errors.New("invalid tenant placement generation")
}
r.mu.RLock()
slot := r.slots[tenantID]
r.mu.RUnlock()
if slot == nil {
return validPurgeRoot(paths.Root)
}
slot.mu.Lock()
defer slot.mu.Unlock()
tenant := slot.tenant
if tenant == nil {
return validPurgeRoot(paths.Root)
}
tenant.mu.RLock()
currentGeneration := tenant.payload.PlacementGeneration
tenant.mu.RUnlock()
if currentGeneration != generation {
return errors.New("tenant placement generation mismatch")
}
r.expireLocked(slot, tenant)
r.releaseSlot(slot)
return nil
}
func validPurgeRoot(path string) error {
info, err := os.Lstat(path)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
return errors.New("tenant purge root is not a real directory")
}
return nil
}
func (r *Registry) Close() error {
r.cancel()
r.mu.RLock()
slots := make([]*slot, 0, len(r.slots))
for _, slot := range r.slots {
slots = append(slots, slot)
}
r.mu.RUnlock()
for _, slot := range slots {
slot.mu.Lock()
if slot.tenant != nil {
r.expireLocked(slot, slot.tenant)
r.releaseSlot(slot)
}
slot.mu.Unlock()
}
return nil
}