othman.suseno/calypso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixing storage management dashboard

Browse Source
This commit is contained in:
Warp Agent
2025-12-25 20:02:59 +00:00
parent a5e6197bca
commit 419fcb7625
20 changed files with 3229 additions and 396 deletions
Download Patch File Download Diff File Expand all files Collapse all files

456
backend/internal/storage/zfs.go
View File

@@ -4,13 +4,15 @@ import (
"context"
"database/sql"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/lib/pq"
"github.com/atlasos/calypso/internal/common/database"
"github.com/atlasos/calypso/internal/common/logger"
"github.com/lib/pq"
)
// ZFSService handles ZFS pool management
@@ -29,23 +31,23 @@ func NewZFSService(db *database.DB, log *logger.Logger) *ZFSService {
// ZFSPool represents a ZFS pool
type ZFSPool struct {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
RaidLevel string `json:"raid_level"` // stripe, mirror, raidz, raidz2, raidz3
Disks []string `json:"disks"` // device paths
SpareDisks []string `json:"spare_disks"` // spare disk paths
SizeBytes int64 `json:"size_bytes"`
UsedBytes int64 `json:"used_bytes"`
Compression string `json:"compression"` // off, lz4, zstd, gzip
Deduplication bool `json:"deduplication"`
AutoExpand bool `json:"auto_expand"`
ScrubInterval int `json:"scrub_interval"` // days
IsActive bool `json:"is_active"`
HealthStatus string `json:"health_status"` // online, degraded, faulted, offline
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
CreatedBy string `json:"created_by"`
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
RaidLevel string `json:"raid_level"` // stripe, mirror, raidz, raidz2, raidz3
Disks []string `json:"disks"` // device paths
SpareDisks []string `json:"spare_disks"` // spare disk paths
SizeBytes int64 `json:"size_bytes"`
UsedBytes int64 `json:"used_bytes"`
Compression string `json:"compression"` // off, lz4, zstd, gzip
Deduplication bool `json:"deduplication"`
AutoExpand bool `json:"auto_expand"`
ScrubInterval int `json:"scrub_interval"` // days
IsActive bool `json:"is_active"`
HealthStatus string `json:"health_status"` // online, degraded, faulted, offline
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
CreatedBy string `json:"created_by"`
}
// CreatePool creates a new ZFS pool
@@ -559,122 +561,72 @@ func (s *ZFSService) AddSpareDisk(ctx context.Context, poolID string, diskPaths
// ZFSDataset represents a ZFS dataset
type ZFSDataset struct {
Name string `json:"name"`
Pool string `json:"pool"`
Type string `json:"type"` // filesystem, volume, snapshot
MountPoint string `json:"mount_point"`
UsedBytes int64 `json:"used_bytes"`
AvailableBytes int64 `json:"available_bytes"`
ReferencedBytes int64 `json:"referenced_bytes"`
Compression string `json:"compression"`
Deduplication string `json:"deduplication"`
Quota int64 `json:"quota"` // -1 for unlimited
Reservation int64 `json:"reservation"`
CreatedAt time.Time `json:"created_at"`
Name string `json:"name"`
Pool string `json:"pool"`
Type string `json:"type"` // filesystem, volume, snapshot
MountPoint string `json:"mount_point"`
UsedBytes int64 `json:"used_bytes"`
AvailableBytes int64 `json:"available_bytes"`
ReferencedBytes int64 `json:"referenced_bytes"`
Compression string `json:"compression"`
Deduplication string `json:"deduplication"`
Quota int64 `json:"quota"` // -1 for unlimited
Reservation int64 `json:"reservation"`
CreatedAt time.Time `json:"created_at"`
}
// ListDatasets lists all datasets in a ZFS pool
// ListDatasets lists all datasets in a ZFS pool from database
func (s *ZFSService) ListDatasets(ctx context.Context, poolName string) ([]*ZFSDataset, error) {
// Get all datasets in the pool using zfs list
cmd := exec.CommandContext(ctx, "zfs", "list", "-H", "-o", "name,used,avail,refer,compress,dedup,quota,reservation,mountpoint", "-r", poolName)
output, err := cmd.CombinedOutput()
// Get datasets from database
query := `
SELECT name, pool_name, type, mount_point,
used_bytes, available_bytes, referenced_bytes,
compression, deduplication, quota, reservation,
created_at
FROM zfs_datasets
WHERE pool_name = $1
ORDER BY name
`
rows, err := s.db.QueryContext(ctx, query, poolName)
if err != nil {
// If pool doesn't exist, return empty list
if strings.Contains(string(output), "does not exist") {
// If table doesn't exist, return empty list (migration not run yet)
if strings.Contains(err.Error(), "does not exist") {
s.logger.Warn("zfs_datasets table does not exist, returning empty list", "pool", poolName)
return []*ZFSDataset{}, nil
}
return nil, fmt.Errorf("failed to list datasets: %s: %w", string(output), err)
return nil, fmt.Errorf("failed to list datasets from database: %w", err)
}
defer rows.Close()
var datasets []*ZFSDataset
lines := strings.Split(strings.TrimSpace(string(output)), "\n")
for _, line := range lines {
if line == "" {
for rows.Next() {
var ds ZFSDataset
var mountPoint sql.NullString
err := rows.Scan(
&ds.Name, &ds.Pool, &ds.Type, &mountPoint,
&ds.UsedBytes, &ds.AvailableBytes, &ds.ReferencedBytes,
&ds.Compression, &ds.Deduplication, &ds.Quota, &ds.Reservation,
&ds.CreatedAt,
)
if err != nil {
s.logger.Error("Failed to scan dataset row", "error", err)
continue
}
fields := strings.Fields(line)
if len(fields) < 9 {
continue
// Handle nullable mount_point
if mountPoint.Valid {
ds.MountPoint = mountPoint.String
} else {
ds.MountPoint = "none"
}
datasetName := fields[0]
// Skip the pool itself (root dataset)
if datasetName == poolName {
continue
}
datasets = append(datasets, &ds)
}
// Extract pool name from dataset name (e.g., "pool/dataset" -> "pool")
poolFromName := strings.Split(datasetName, "/")[0]
if poolFromName != poolName {
continue
}
usedBytes, _ := parseZFSSize(fields[1])
availableBytes, _ := parseZFSSize(fields[2])
referencedBytes, _ := parseZFSSize(fields[3])
compression := fields[4]
deduplication := fields[5]
quotaStr := fields[6]
reservationStr := fields[7]
mountPoint := fields[8]
quota := int64(-1) // -1 means unlimited
if quotaStr != "-" && quotaStr != "none" {
if q, err := parseZFSSize(quotaStr); err == nil {
quota = q
}
}
reservation := int64(0)
if reservationStr != "-" && reservationStr != "none" {
if r, err := parseZFSSize(reservationStr); err == nil {
reservation = r
}
}
// Determine dataset type
datasetType := "filesystem"
volCmd := exec.CommandContext(ctx, "zfs", "get", "-H", "-o", "value", "type", datasetName)
volOutput, err := volCmd.Output()
if err == nil {
volType := strings.TrimSpace(string(volOutput))
if volType == "volume" {
datasetType = "volume"
} else if strings.Contains(volType, "snapshot") {
datasetType = "snapshot"
}
}
// Get creation time
createdAt := time.Now()
creationCmd := exec.CommandContext(ctx, "zfs", "get", "-H", "-o", "value", "creation", datasetName)
creationOutput, err := creationCmd.Output()
if err == nil {
creationStr := strings.TrimSpace(string(creationOutput))
// Try parsing different date formats
if t, err := time.Parse("Mon Jan 2 15:04:05 2006", creationStr); err == nil {
createdAt = t
} else if t, err := time.Parse(time.RFC3339, creationStr); err == nil {
createdAt = t
}
}
datasets = append(datasets, &ZFSDataset{
Name: datasetName,
Pool: poolName,
Type: datasetType,
MountPoint: mountPoint,
UsedBytes: usedBytes,
AvailableBytes: availableBytes,
ReferencedBytes: referencedBytes,
Compression: compression,
Deduplication: deduplication,
Quota: quota,
Reservation: reservation,
CreatedAt: createdAt,
})
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("error iterating dataset rows: %w", err)
}
return datasets, nil
@@ -682,12 +634,12 @@ func (s *ZFSService) ListDatasets(ctx context.Context, poolName string) ([]*ZFSD
// CreateDatasetRequest represents a request to create a ZFS dataset
type CreateDatasetRequest struct {
Name string `json:"name"` // Dataset name (e.g., "pool/dataset" or just "dataset")
Type string `json:"type"` // "filesystem" or "volume"
Compression string `json:"compression"` // off, lz4, zstd, gzip, etc.
Quota int64 `json:"quota"` // -1 for unlimited
Reservation int64 `json:"reservation"` // 0 for none
MountPoint string `json:"mount_point"` // Optional mount point
Name string `json:"name"` // Dataset name (e.g., "pool/dataset" or just "dataset")
Type string `json:"type"` // "filesystem" or "volume"
Compression string `json:"compression"` // off, lz4, zstd, gzip, etc.
Quota int64 `json:"quota"` // -1 for unlimited
Reservation int64 `json:"reservation"` // 0 for none
MountPoint string `json:"mount_point"` // Optional mount point
}
// CreateDataset creates a new ZFS dataset
@@ -695,9 +647,41 @@ func (s *ZFSService) CreateDataset(ctx context.Context, poolName string, req Cre
// Construct full dataset name
fullName := poolName + "/" + req.Name
// For filesystem datasets, create mount directory if mount point is provided
if req.Type == "filesystem" && req.MountPoint != "" {
// Clean and validate mount point path
mountPath := filepath.Clean(req.MountPoint)
// Check if directory already exists
if info, err := os.Stat(mountPath); err == nil {
if !info.IsDir() {
return nil, fmt.Errorf("mount point path exists but is not a directory: %s", mountPath)
}
// Directory exists, check if it's empty
dir, err := os.Open(mountPath)
if err == nil {
entries, err := dir.Readdirnames(1)
dir.Close()
if err == nil && len(entries) > 0 {
s.logger.Warn("Mount directory is not empty", "path", mountPath)
// Continue anyway, ZFS will mount over it
}
}
} else if os.IsNotExist(err) {
// Create directory with proper permissions (0755)
s.logger.Info("Creating mount directory", "path", mountPath)
if err := os.MkdirAll(mountPath, 0755); err != nil {
return nil, fmt.Errorf("failed to create mount directory %s: %w", mountPath, err)
}
s.logger.Info("Mount directory created successfully", "path", mountPath)
} else {
return nil, fmt.Errorf("failed to check mount directory %s: %w", mountPath, err)
}
}
// Build zfs create command
args := []string{"create"}
// Add type if volume
if req.Type == "volume" {
// For volumes, we need size (use quota as size)
@@ -715,8 +699,8 @@ func (s *ZFSService) CreateDataset(ctx context.Context, poolName string, req Cre
args = append(args, "-o", fmt.Sprintf("compression=%s", req.Compression))
}
// Set mount point if provided
if req.MountPoint != "" {
// Set mount point if provided (only for filesystems, not volumes)
if req.Type == "filesystem" && req.MountPoint != "" {
args = append(args, "-o", fmt.Sprintf("mountpoint=%s", req.MountPoint))
}
@@ -746,38 +730,196 @@ func (s *ZFSService) CreateDataset(ctx context.Context, poolName string, req Cre
}
}
// Get the created dataset info
datasets, err := s.ListDatasets(ctx, poolName)
// Get pool ID from pool name
var poolID string
err = s.db.QueryRowContext(ctx, "SELECT id FROM zfs_pools WHERE name = $1", poolName).Scan(&poolID)
if err != nil {
return nil, fmt.Errorf("failed to list datasets after creation: %w", err)
s.logger.Error("Failed to get pool ID", "pool", poolName, "error", err)
// Try to destroy the dataset if we can't save to database
exec.CommandContext(ctx, "zfs", "destroy", "-r", fullName).Run()
return nil, fmt.Errorf("failed to get pool ID: %w", err)
}
// Find the newly created dataset
for _, ds := range datasets {
if ds.Name == fullName {
s.logger.Info("ZFS dataset created successfully", "name", fullName)
return ds, nil
// Get dataset info from ZFS to save to database
cmd = exec.CommandContext(ctx, "zfs", "list", "-H", "-o", "name,used,avail,refer,compress,dedup,quota,reservation,mountpoint", fullName)
output, err = cmd.CombinedOutput()
if err != nil {
s.logger.Error("Failed to get dataset info", "name", fullName, "error", err)
// Try to destroy the dataset if we can't get info
exec.CommandContext(ctx, "zfs", "destroy", "-r", fullName).Run()
return nil, fmt.Errorf("failed to get dataset info: %w", err)
}
// Parse dataset info
lines := strings.TrimSpace(string(output))
if lines == "" {
exec.CommandContext(ctx, "zfs", "destroy", "-r", fullName).Run()
return nil, fmt.Errorf("dataset not found after creation")
}
fields := strings.Fields(lines)
if len(fields) < 9 {
exec.CommandContext(ctx, "zfs", "destroy", "-r", fullName).Run()
return nil, fmt.Errorf("invalid dataset info format")
}
usedBytes, _ := parseZFSSize(fields[1])
availableBytes, _ := parseZFSSize(fields[2])
referencedBytes, _ := parseZFSSize(fields[3])
compression := fields[4]
deduplication := fields[5]
quotaStr := fields[6]
reservationStr := fields[7]
mountPoint := fields[8]
// Determine dataset type
datasetType := req.Type
typeCmd := exec.CommandContext(ctx, "zfs", "get", "-H", "-o", "value", "type", fullName)
if typeOutput, err := typeCmd.Output(); err == nil {
volType := strings.TrimSpace(string(typeOutput))
if volType == "volume" {
datasetType = "volume"
} else if strings.Contains(volType, "snapshot") {
datasetType = "snapshot"
}
}
return nil, fmt.Errorf("dataset created but not found in list")
// Parse quota
quota := int64(-1)
if datasetType == "volume" {
// For volumes, get volsize
volsizeCmd := exec.CommandContext(ctx, "zfs", "get", "-H", "-o", "value", "volsize", fullName)
if volsizeOutput, err := volsizeCmd.Output(); err == nil {
volsizeStr := strings.TrimSpace(string(volsizeOutput))
if volsizeStr != "-" && volsizeStr != "none" {
if vs, err := parseZFSSize(volsizeStr); err == nil {
quota = vs
}
}
}
} else if quotaStr != "-" && quotaStr != "none" {
if q, err := parseZFSSize(quotaStr); err == nil {
quota = q
}
}
// Parse reservation
reservation := int64(0)
if reservationStr != "-" && reservationStr != "none" {
if r, err := parseZFSSize(reservationStr); err == nil {
reservation = r
}
}
// Normalize mount point for volumes
if datasetType == "volume" && mountPoint == "-" {
mountPoint = "none"
}
// Get creation time
createdAt := time.Now()
creationCmd := exec.CommandContext(ctx, "zfs", "get", "-H", "-o", "value", "creation", fullName)
if creationOutput, err := creationCmd.Output(); err == nil {
creationStr := strings.TrimSpace(string(creationOutput))
if t, err := time.Parse("Mon Jan 2 15:04:05 2006", creationStr); err == nil {
createdAt = t
} else if t, err := time.Parse(time.RFC3339, creationStr); err == nil {
createdAt = t
}
}
// Save to database (works for both filesystem and volume datasets)
// Volume datasets are stored in the same zfs_datasets table with type='volume'
insertQuery := `
INSERT INTO zfs_datasets (
name, pool_id, pool_name, type, mount_point,
used_bytes, available_bytes, referenced_bytes,
compression, deduplication, quota, reservation,
created_at, updated_at
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, NOW())
RETURNING id
`
var datasetID string
err = s.db.QueryRowContext(ctx, insertQuery,
fullName, poolID, poolName, datasetType, mountPoint,
usedBytes, availableBytes, referencedBytes,
compression, deduplication, quota, reservation,
createdAt,
).Scan(&datasetID)
if err != nil {
s.logger.Error("Failed to save dataset to database", "name", fullName, "error", err)
// Try to destroy the dataset if we can't save to database
exec.CommandContext(ctx, "zfs", "destroy", "-r", fullName).Run()
return nil, fmt.Errorf("failed to save dataset to database: %w", err)
}
// Return dataset info
dataset := &ZFSDataset{
Name: fullName,
Pool: poolName,
Type: datasetType,
MountPoint: mountPoint,
UsedBytes: usedBytes,
AvailableBytes: availableBytes,
ReferencedBytes: referencedBytes,
Compression: compression,
Deduplication: deduplication,
Quota: quota,
Reservation: reservation,
CreatedAt: createdAt,
}
s.logger.Info("ZFS dataset created and saved to database", "name", fullName, "id", datasetID)
return dataset, nil
}
// DeleteDataset deletes a ZFS dataset
func (s *ZFSService) DeleteDataset(ctx context.Context, datasetName string) error {
// Check if dataset exists
cmd := exec.CommandContext(ctx, "zfs", "list", "-H", "-o", "name", datasetName)
// Check if dataset exists and get its mount point before deletion
var mountPoint string
cmd := exec.CommandContext(ctx, "zfs", "list", "-H", "-o", "name,mountpoint", datasetName)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("dataset %s does not exist: %w", datasetName, err)
}
if strings.TrimSpace(string(output)) != datasetName {
lines := strings.TrimSpace(string(output))
if lines == "" {
return fmt.Errorf("dataset %s not found", datasetName)
}
// Delete the dataset (use -r for recursive to delete children)
s.logger.Info("Deleting ZFS dataset", "name", datasetName)
// Parse output to get mount point
fields := strings.Fields(lines)
if len(fields) >= 2 {
mountPoint = fields[1]
}
// Get dataset type to determine if we should clean up mount directory
var datasetType string
typeCmd := exec.CommandContext(ctx, "zfs", "get", "-H", "-o", "value", "type", datasetName)
typeOutput, err := typeCmd.Output()
if err == nil {
datasetType = strings.TrimSpace(string(typeOutput))
}
// Delete from database first (before ZFS deletion, so we have the record)
// This ensures we can clean up even if ZFS deletion partially fails
// Works for both filesystem and volume datasets
deleteQuery := "DELETE FROM zfs_datasets WHERE name = $1"
result, err := s.db.ExecContext(ctx, deleteQuery, datasetName)
if err != nil {
s.logger.Warn("Failed to delete dataset from database (may not exist)", "name", datasetName, "error", err)
// Continue with ZFS deletion anyway
} else {
rowsAffected, _ := result.RowsAffected()
if rowsAffected > 0 {
s.logger.Info("Dataset removed from database", "name", datasetName)
}
}
// Delete the dataset from ZFS (use -r for recursive to delete children)
s.logger.Info("Deleting ZFS dataset", "name", datasetName, "mountpoint", mountPoint)
cmd = exec.CommandContext(ctx, "zfs", "destroy", "-r", datasetName)
output, err = cmd.CombinedOutput()
if err != nil {
@@ -786,7 +928,35 @@ func (s *ZFSService) DeleteDataset(ctx context.Context, datasetName string) erro
return fmt.Errorf("failed to delete dataset: %s", errorMsg)
}
// Clean up mount directory if it exists and is a filesystem dataset
// Only remove if mount point is not "-" (volumes) and not "none" or "legacy"
if datasetType == "filesystem" && mountPoint != "" && mountPoint != "-" && mountPoint != "none" && mountPoint != "legacy" {
mountPath := filepath.Clean(mountPoint)
// Check if directory exists
if info, err := os.Stat(mountPath); err == nil && info.IsDir() {
// Check if directory is empty
dir, err := os.Open(mountPath)
if err == nil {
entries, err := dir.Readdirnames(1)
dir.Close()
// Only remove if directory is empty
if err == nil && len(entries) == 0 {
s.logger.Info("Removing empty mount directory", "path", mountPath)
if err := os.Remove(mountPath); err != nil {
s.logger.Warn("Failed to remove mount directory", "path", mountPath, "error", err)
// Don't fail the deletion if we can't remove the directory
} else {
s.logger.Info("Mount directory removed successfully", "path", mountPath)
}
} else {
s.logger.Info("Mount directory is not empty, keeping it", "path", mountPath)
}
}
}
}
s.logger.Info("ZFS dataset deleted successfully", "name", datasetName)
return nil
}