add some changes

2026-01-15 09:44:57 +00:00
parent 9b1f85479b
commit 1d9406c93a
19 changed files with 4922 additions and 887 deletions
--- a/backend/internal/common/database/migrations/012_add_snapshot_schedules_table.sql
+++ b/backend/internal/common/database/migrations/012_add_snapshot_schedules_table.sql
@@ -0,0 +1,23 @@
+-- Snapshot schedules table for automated snapshot creation
+CREATE TABLE IF NOT EXISTS snapshot_schedules (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    name VARCHAR(255) NOT NULL,
+    dataset VARCHAR(255) NOT NULL,
+    snapshot_name_template VARCHAR(255) NOT NULL, -- e.g., "auto-%Y-%m-%d-%H%M" or "daily-backup"
+    schedule_type VARCHAR(50) NOT NULL, -- 'hourly', 'daily', 'weekly', 'monthly', 'cron'
+    schedule_config JSONB NOT NULL, -- For cron: {"cron": "0 0 * * *"}, for others: {"time": "00:00", "day": 1, etc.}
+    recursive BOOLEAN NOT NULL DEFAULT false,
+    enabled BOOLEAN NOT NULL DEFAULT true,
+    retention_count INTEGER, -- Keep last N snapshots (null = unlimited)
+    retention_days INTEGER, -- Keep snapshots for N days (null = unlimited)
+    last_run_at TIMESTAMP,
+    next_run_at TIMESTAMP,
+    created_by UUID REFERENCES users(id),
+    created_at TIMESTAMP NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
+    UNIQUE(name)
+);
+
+CREATE INDEX IF NOT EXISTS idx_snapshot_schedules_enabled ON snapshot_schedules(enabled);
+CREATE INDEX IF NOT EXISTS idx_snapshot_schedules_next_run ON snapshot_schedules(next_run_at);
+CREATE INDEX IF NOT EXISTS idx_snapshot_schedules_dataset ON snapshot_schedules(dataset);
--- a/backend/internal/common/database/migrations/013_add_replication_tasks_table.sql
+++ b/backend/internal/common/database/migrations/013_add_replication_tasks_table.sql
@@ -0,0 +1,76 @@
+-- ZFS Replication Tasks Table
+-- Supports both outbound (sender) and inbound (receiver) replication
+CREATE TABLE IF NOT EXISTS replication_tasks (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    name VARCHAR(255) NOT NULL,
+    direction VARCHAR(20) NOT NULL, -- 'outbound' (sender) or 'inbound' (receiver)
+    
+    -- For outbound replication (sender)
+    source_dataset VARCHAR(512), -- Source dataset on this system (outbound) or remote system (inbound)
+    target_host VARCHAR(255), -- Target host IP or hostname (for outbound)
+    target_port INTEGER DEFAULT 22, -- SSH port (default 22, for outbound)
+    target_user VARCHAR(255) DEFAULT 'root', -- SSH user (for outbound)
+    target_dataset VARCHAR(512), -- Target dataset on remote system (for outbound)
+    target_ssh_key_path TEXT, -- Path to SSH private key (for outbound)
+    
+    -- For inbound replication (receiver)
+    source_host VARCHAR(255), -- Source host IP or hostname (for inbound)
+    source_port INTEGER DEFAULT 22, -- SSH port (for inbound)
+    source_user VARCHAR(255) DEFAULT 'root', -- SSH user (for inbound)
+    local_dataset VARCHAR(512), -- Local dataset to receive snapshots (for inbound)
+    
+    -- Common settings
+    schedule_type VARCHAR(50), -- 'manual', 'hourly', 'daily', 'weekly', 'monthly', 'cron'
+    schedule_config JSONB, -- Schedule configuration (similar to snapshot schedules)
+    compression VARCHAR(50) DEFAULT 'lz4', -- 'off', 'lz4', 'gzip', 'zstd'
+    encryption BOOLEAN DEFAULT false, -- Enable encryption during transfer
+    recursive BOOLEAN DEFAULT false, -- Replicate recursively
+    incremental BOOLEAN DEFAULT true, -- Use incremental replication
+    auto_snapshot BOOLEAN DEFAULT true, -- Auto-create snapshot before replication
+    
+    -- Status and tracking
+    enabled BOOLEAN NOT NULL DEFAULT true,
+    status VARCHAR(50) DEFAULT 'idle', -- 'idle', 'running', 'failed', 'paused'
+    last_run_at TIMESTAMP,
+    last_run_status VARCHAR(50), -- 'success', 'failed', 'partial'
+    last_run_error TEXT,
+    next_run_at TIMESTAMP,
+    last_snapshot_sent VARCHAR(512), -- Last snapshot successfully sent (for outbound)
+    last_snapshot_received VARCHAR(512), -- Last snapshot successfully received (for inbound)
+    
+    -- Statistics
+    total_runs INTEGER DEFAULT 0,
+    successful_runs INTEGER DEFAULT 0,
+    failed_runs INTEGER DEFAULT 0,
+    bytes_sent BIGINT DEFAULT 0, -- Total bytes sent (for outbound)
+    bytes_received BIGINT DEFAULT 0, -- Total bytes received (for inbound)
+    
+    created_by UUID REFERENCES users(id),
+    created_at TIMESTAMP NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
+    
+    -- Validation: ensure required fields based on direction
+    CONSTRAINT chk_direction CHECK (direction IN ('outbound', 'inbound')),
+    CONSTRAINT chk_outbound_fields CHECK (
+        direction != 'outbound' OR (
+            source_dataset IS NOT NULL AND
+            target_host IS NOT NULL AND
+            target_dataset IS NOT NULL
+        )
+    ),
+    CONSTRAINT chk_inbound_fields CHECK (
+        direction != 'inbound' OR (
+            source_host IS NOT NULL AND
+            source_dataset IS NOT NULL AND
+            local_dataset IS NOT NULL
+        )
+    )
+);
+
+-- Create indexes
+CREATE INDEX IF NOT EXISTS idx_replication_tasks_direction ON replication_tasks(direction);
+CREATE INDEX IF NOT EXISTS idx_replication_tasks_enabled ON replication_tasks(enabled);
+CREATE INDEX IF NOT EXISTS idx_replication_tasks_status ON replication_tasks(status);
+CREATE INDEX IF NOT EXISTS idx_replication_tasks_next_run ON replication_tasks(next_run_at);
+CREATE INDEX IF NOT EXISTS idx_replication_tasks_source_dataset ON replication_tasks(source_dataset);
+CREATE INDEX IF NOT EXISTS idx_replication_tasks_target_host ON replication_tasks(target_host);
--- a/backend/internal/common/database/migrations/014_add_client_categories_table.sql
+++ b/backend/internal/common/database/migrations/014_add_client_categories_table.sql
@@ -0,0 +1,30 @@
+-- AtlasOS - Calypso
+-- Client Categories Schema
+-- Version: 14.0
+-- Adds category support for backup clients (File, Database, Virtual)
+
+-- Client metadata table to store additional information about clients
+-- This extends the Bacula Client table with Calypso-specific metadata
+CREATE TABLE IF NOT EXISTS client_metadata (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    client_id INTEGER NOT NULL, -- Bacula Client.ClientId
+    client_name VARCHAR(255) NOT NULL, -- Bacula Client.Name (for reference)
+    category VARCHAR(50) NOT NULL DEFAULT 'File', -- 'File', 'Database', 'Virtual'
+    description TEXT,
+    tags JSONB, -- Additional tags/metadata as JSON
+    created_at TIMESTAMP NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
+    
+    -- Ensure one metadata entry per client
+    CONSTRAINT unique_client_id UNIQUE (client_id),
+    CONSTRAINT chk_category CHECK (category IN ('File', 'Database', 'Virtual'))
+);
+
+-- Indexes for performance
+CREATE INDEX IF NOT EXISTS idx_client_metadata_client_id ON client_metadata(client_id);
+CREATE INDEX IF NOT EXISTS idx_client_metadata_client_name ON client_metadata(client_name);
+CREATE INDEX IF NOT EXISTS idx_client_metadata_category ON client_metadata(category);
+
+-- Add comment
+COMMENT ON TABLE client_metadata IS 'Stores Calypso-specific metadata for backup clients, including category classification';
+COMMENT ON COLUMN client_metadata.category IS 'Client category: File (file system backups), Database (database backups), Virtual (virtual machine backups)';
--- a/backend/internal/common/router/router.go
+++ b/backend/internal/common/router/router.go
@@ -176,6 +176,11 @@ func NewRouter(cfg *config.Config, db *database.DB, log *logger.Logger) *gin.Eng
 			zfsPoolMonitor := storage.NewZFSPoolMonitor(db, log, 2*time.Minute)
 			go zfsPoolMonitor.Start(context.Background())

+			// Start snapshot schedule worker in background (checks schedules every 1 minute)
+			snapshotService := storage.NewSnapshotService(db, log)
+			snapshotScheduleWorker := storage.NewSnapshotScheduleWorker(db, log, snapshotService, 1*time.Minute)
+			go snapshotScheduleWorker.Start(context.Background())
+
 			storageGroup := protected.Group("/storage")
 			storageGroup.Use(requirePermission("storage", "read"))
 			{
@@ -196,6 +201,26 @@ func NewRouter(cfg *config.Config, db *database.DB, log *logger.Logger) *gin.Eng
 				storageGroup.GET("/zfs/pools/:id/datasets", storageHandler.ListZFSDatasets)
 				storageGroup.POST("/zfs/pools/:id/datasets", requirePermission("storage", "write"), storageHandler.CreateZFSDataset)
 				storageGroup.DELETE("/zfs/pools/:id/datasets/:dataset", requirePermission("storage", "write"), storageHandler.DeleteZFSDataset)
+				// ZFS Snapshots
+				storageGroup.GET("/zfs/snapshots", storageHandler.ListSnapshots)
+				storageGroup.POST("/zfs/snapshots", requirePermission("storage", "write"), storageHandler.CreateSnapshot)
+				storageGroup.DELETE("/zfs/snapshots/:name", requirePermission("storage", "write"), storageHandler.DeleteSnapshot)
+				storageGroup.POST("/zfs/snapshots/:name/rollback", requirePermission("storage", "write"), storageHandler.RollbackSnapshot)
+				storageGroup.POST("/zfs/snapshots/:name/clone", requirePermission("storage", "write"), storageHandler.CloneSnapshot)
+				// Snapshot Schedules
+				storageGroup.GET("/zfs/snapshot-schedules", storageHandler.ListSnapshotSchedules)
+				storageGroup.GET("/zfs/snapshot-schedules/:id", storageHandler.GetSnapshotSchedule)
+				storageGroup.POST("/zfs/snapshot-schedules", requirePermission("storage", "write"), storageHandler.CreateSnapshotSchedule)
+				storageGroup.PUT("/zfs/snapshot-schedules/:id", requirePermission("storage", "write"), storageHandler.UpdateSnapshotSchedule)
+				storageGroup.DELETE("/zfs/snapshot-schedules/:id", requirePermission("storage", "write"), storageHandler.DeleteSnapshotSchedule)
+				storageGroup.POST("/zfs/snapshot-schedules/:id/toggle", requirePermission("storage", "write"), storageHandler.ToggleSnapshotSchedule)
+
+				// Replication Tasks
+				storageGroup.GET("/zfs/replication-tasks", storageHandler.ListReplicationTasks)
+				storageGroup.GET("/zfs/replication-tasks/:id", storageHandler.GetReplicationTask)
+				storageGroup.POST("/zfs/replication-tasks", requirePermission("storage", "write"), storageHandler.CreateReplicationTask)
+				storageGroup.PUT("/zfs/replication-tasks/:id", requirePermission("storage", "write"), storageHandler.UpdateReplicationTask)
+				storageGroup.DELETE("/zfs/replication-tasks/:id", requirePermission("storage", "write"), storageHandler.DeleteReplicationTask)
 				// ZFS ARC Stats
 				storageGroup.GET("/zfs/arc/stats", storageHandler.GetARCStats)
 			}
@@ -228,26 +253,28 @@ func NewRouter(cfg *config.Config, db *database.DB, log *logger.Logger) *gin.Eng
 					objectStorageGroup := protected.Group("/object-storage")
 					objectStorageGroup.Use(requirePermission("storage", "read"))
 					{
-					// Setup endpoints
-					objectStorageGroup.GET("/setup/datasets", objectStorageHandler.GetAvailableDatasets)
-					objectStorageGroup.GET("/setup/current", objectStorageHandler.GetCurrentSetup)
-					objectStorageGroup.POST("/setup", requirePermission("storage", "write"), objectStorageHandler.SetupObjectStorage)
-					objectStorageGroup.PUT("/setup", requirePermission("storage", "write"), objectStorageHandler.UpdateObjectStorage)
-						
+						// Setup endpoints
+						objectStorageGroup.GET("/setup/datasets", objectStorageHandler.GetAvailableDatasets)
+						objectStorageGroup.GET("/setup/current", objectStorageHandler.GetCurrentSetup)
+						objectStorageGroup.POST("/setup", requirePermission("storage", "write"), objectStorageHandler.SetupObjectStorage)
+						objectStorageGroup.PUT("/setup", requirePermission("storage", "write"), objectStorageHandler.UpdateObjectStorage)
+
 						// Bucket endpoints
+						// IMPORTANT: More specific routes must come BEFORE less specific ones
 						objectStorageGroup.GET("/buckets", objectStorageHandler.ListBuckets)
+						objectStorageGroup.GET("/buckets/:name/objects", objectStorageHandler.ListObjects)
 						objectStorageGroup.GET("/buckets/:name", objectStorageHandler.GetBucket)
-					objectStorageGroup.POST("/buckets", requirePermission("storage", "write"), objectStorageHandler.CreateBucket)
-					objectStorageGroup.DELETE("/buckets/:name", requirePermission("storage", "write"), objectStorageHandler.DeleteBucket)
-					// User management routes
-					objectStorageGroup.GET("/users", objectStorageHandler.ListUsers)
-					objectStorageGroup.POST("/users", requirePermission("storage", "write"), objectStorageHandler.CreateUser)
-					objectStorageGroup.DELETE("/users/:access_key", requirePermission("storage", "write"), objectStorageHandler.DeleteUser)
-					// Service account (access key) management routes
-					objectStorageGroup.GET("/service-accounts", objectStorageHandler.ListServiceAccounts)
-					objectStorageGroup.POST("/service-accounts", requirePermission("storage", "write"), objectStorageHandler.CreateServiceAccount)
-					objectStorageGroup.DELETE("/service-accounts/:access_key", requirePermission("storage", "write"), objectStorageHandler.DeleteServiceAccount)
-				}
+						objectStorageGroup.POST("/buckets", requirePermission("storage", "write"), objectStorageHandler.CreateBucket)
+						objectStorageGroup.DELETE("/buckets/:name", requirePermission("storage", "write"), objectStorageHandler.DeleteBucket)
+						// User management routes
+						objectStorageGroup.GET("/users", objectStorageHandler.ListUsers)
+						objectStorageGroup.POST("/users", requirePermission("storage", "write"), objectStorageHandler.CreateUser)
+						objectStorageGroup.DELETE("/users/:access_key", requirePermission("storage", "write"), objectStorageHandler.DeleteUser)
+						// Service account (access key) management routes
+						objectStorageGroup.GET("/service-accounts", objectStorageHandler.ListServiceAccounts)
+						objectStorageGroup.POST("/service-accounts", requirePermission("storage", "write"), objectStorageHandler.CreateServiceAccount)
+						objectStorageGroup.DELETE("/service-accounts/:access_key", requirePermission("storage", "write"), objectStorageHandler.DeleteServiceAccount)
+					}
 				}
 			}

@@ -347,9 +374,9 @@ func NewRouter(cfg *config.Config, db *database.DB, log *logger.Logger) *gin.Eng
 				systemGroup.GET("/logs", systemHandler.GetSystemLogs)
 				systemGroup.GET("/network/throughput", systemHandler.GetNetworkThroughput)
 				systemGroup.POST("/support-bundle", systemHandler.GenerateSupportBundle)
-			systemGroup.GET("/interfaces", systemHandler.ListNetworkInterfaces)
-			systemGroup.GET("/management-ip", systemHandler.GetManagementIPAddress)
-			systemGroup.PUT("/interfaces/:name", systemHandler.UpdateNetworkInterface)
+				systemGroup.GET("/interfaces", systemHandler.ListNetworkInterfaces)
+				systemGroup.GET("/management-ip", systemHandler.GetManagementIPAddress)
+				systemGroup.PUT("/interfaces/:name", systemHandler.UpdateNetworkInterface)
 				systemGroup.GET("/ntp", systemHandler.GetNTPSettings)
 				systemGroup.POST("/ntp", systemHandler.SaveNTPSettings)
 				systemGroup.POST("/execute", requirePermission("system", "write"), systemHandler.ExecuteCommand)