othman.suseno/calypso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixing storage management dashboard

Browse Source
This commit is contained in:
Warp Agent
2025-12-25 20:02:59 +00:00
parent a5e6197bca
commit 419fcb7625
20 changed files with 3229 additions and 396 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
backend/internal/common/database/migrations/005_add_zfs_datasets_table.sql Normal file
View File

@@ -0,0 +1,35 @@
-- AtlasOS - Calypso
-- Add ZFS Datasets Table
-- Version: 5.0
-- Description: Stores ZFS dataset metadata in database for faster queries and consistency
-- ZFS datasets table
CREATE TABLE IF NOT EXISTS zfs_datasets (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name VARCHAR(512) NOT NULL UNIQUE, -- Full dataset name (e.g., pool/dataset)
pool_id UUID NOT NULL REFERENCES zfs_pools(id) ON DELETE CASCADE,
pool_name VARCHAR(255) NOT NULL, -- Denormalized for faster queries
type VARCHAR(50) NOT NULL, -- filesystem, volume, snapshot
mount_point TEXT, -- Mount point path (null for volumes)
used_bytes BIGINT NOT NULL DEFAULT 0,
available_bytes BIGINT NOT NULL DEFAULT 0,
referenced_bytes BIGINT NOT NULL DEFAULT 0,
compression VARCHAR(50) NOT NULL DEFAULT 'lz4', -- off, lz4, zstd, gzip
deduplication VARCHAR(50) NOT NULL DEFAULT 'off', -- off, on, verify
quota BIGINT DEFAULT -1, -- -1 for unlimited, bytes otherwise
reservation BIGINT NOT NULL DEFAULT 0, -- Reserved space in bytes
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
created_by UUID REFERENCES users(id)
);
-- Create indexes for faster lookups
CREATE INDEX IF NOT EXISTS idx_zfs_datasets_pool_id ON zfs_datasets(pool_id);
CREATE INDEX IF NOT EXISTS idx_zfs_datasets_pool_name ON zfs_datasets(pool_name);
CREATE INDEX IF NOT EXISTS idx_zfs_datasets_name ON zfs_datasets(name);
CREATE INDEX IF NOT EXISTS idx_zfs_datasets_type ON zfs_datasets(type);
CREATE INDEX IF NOT EXISTS idx_zfs_datasets_created_by ON zfs_datasets(created_by);
-- Composite index for common queries (list datasets by pool)
CREATE INDEX IF NOT EXISTS idx_zfs_datasets_pool_type ON zfs_datasets(pool_id, type);

50
backend/internal/common/database/migrations/006_add_zfs_shares_and_iscsi.sql Normal file
View File

@@ -0,0 +1,50 @@
-- AtlasOS - Calypso
-- Add ZFS Shares and iSCSI Export Tables
-- Version: 6.0
-- Description: Separate tables for filesystem shares (NFS/SMB) and volume iSCSI exports
-- ZFS Filesystem Shares Table (for NFS/SMB)
CREATE TABLE IF NOT EXISTS zfs_shares (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
dataset_id UUID NOT NULL REFERENCES zfs_datasets(id) ON DELETE CASCADE,
share_type VARCHAR(50) NOT NULL, -- 'nfs', 'smb', 'both'
nfs_enabled BOOLEAN NOT NULL DEFAULT false,
nfs_options TEXT, -- e.g., "rw,sync,no_subtree_check"
nfs_clients TEXT[], -- Allowed client IPs/networks
smb_enabled BOOLEAN NOT NULL DEFAULT false,
smb_share_name VARCHAR(255), -- SMB share name
smb_path TEXT, -- SMB share path (usually same as mount_point)
smb_comment TEXT,
smb_guest_ok BOOLEAN NOT NULL DEFAULT false,
smb_read_only BOOLEAN NOT NULL DEFAULT false,
smb_browseable BOOLEAN NOT NULL DEFAULT true,
is_active BOOLEAN NOT NULL DEFAULT true,
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
created_by UUID REFERENCES users(id),
UNIQUE(dataset_id) -- One share config per dataset
);
-- ZFS Volume iSCSI Exports Table
CREATE TABLE IF NOT EXISTS zfs_iscsi_exports (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
dataset_id UUID NOT NULL REFERENCES zfs_datasets(id) ON DELETE CASCADE,
target_id UUID REFERENCES scst_targets(id) ON DELETE SET NULL, -- Link to SCST target
lun_number INTEGER, -- LUN number in the target
device_path TEXT, -- /dev/zvol/pool/volume path
is_active BOOLEAN NOT NULL DEFAULT true,
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
created_by UUID REFERENCES users(id),
UNIQUE(dataset_id) -- One iSCSI export per volume
);
-- Create indexes
CREATE INDEX IF NOT EXISTS idx_zfs_shares_dataset_id ON zfs_shares(dataset_id);
CREATE INDEX IF NOT EXISTS idx_zfs_shares_type ON zfs_shares(share_type);
CREATE INDEX IF NOT EXISTS idx_zfs_shares_active ON zfs_shares(is_active);
CREATE INDEX IF NOT EXISTS idx_zfs_iscsi_exports_dataset_id ON zfs_iscsi_exports(dataset_id);
CREATE INDEX IF NOT EXISTS idx_zfs_iscsi_exports_target_id ON zfs_iscsi_exports(target_id);
CREATE INDEX IF NOT EXISTS idx_zfs_iscsi_exports_active ON zfs_iscsi_exports(is_active);

17
backend/internal/common/router/cache.go
View File

@@ -6,6 +6,7 @@ import (
"encoding/hex"
"fmt"
"net/http"
"strings"
"time"
"github.com/atlasos/calypso/internal/common/cache"
@@ -18,21 +19,21 @@ func GenerateKey(prefix string, parts ...string) string {
for _, part := range parts {
key += ":" + part
}
// Hash long keys to keep them manageable
if len(key) > 200 {
hash := sha256.Sum256([]byte(key))
return prefix + ":" + hex.EncodeToString(hash[:])
}
return key
}
// CacheConfig holds cache configuration
type CacheConfig struct {
Enabled bool
DefaultTTL time.Duration
MaxAge int // seconds for Cache-Control header
Enabled bool
DefaultTTL time.Duration
MaxAge int // seconds for Cache-Control header
}
// cacheMiddleware creates a caching middleware
@@ -74,7 +75,7 @@ func cacheMiddleware(cfg CacheConfig, cache *cache.Cache) gin.HandlerFunc {
// Cache miss - capture response
writer := &responseWriter{
ResponseWriter: c.Writer,
body: &bytes.Buffer{},
body: &bytes.Buffer{},
}
c.Writer = writer
@@ -136,6 +137,9 @@ func cacheControlMiddleware() gin.HandlerFunc {
case path == "/api/v1/system/services":
// Service list can be cached briefly
c.Header("Cache-Control", "public, max-age=60")
case strings.HasPrefix(path, "/api/v1/storage/zfs/pools/") && strings.HasSuffix(path, "/datasets"):
// ZFS datasets should not be cached - they change frequently
c.Header("Cache-Control", "no-cache, no-store, must-revalidate")
default:
// Default: no cache for other endpoints
c.Header("Cache-Control", "no-cache, no-store, must-revalidate")
@@ -168,4 +172,3 @@ func InvalidateCachePattern(cache *cache.Cache, pattern string) {
cache.Clear()
}
}

19
backend/internal/common/router/router.go
View File

@@ -4,12 +4,12 @@ import (
"context"
"time"
"github.com/atlasos/calypso/internal/audit"
"github.com/atlasos/calypso/internal/auth"
"github.com/atlasos/calypso/internal/common/cache"
"github.com/atlasos/calypso/internal/common/config"
"github.com/atlasos/calypso/internal/common/database"
"github.com/atlasos/calypso/internal/common/logger"
"github.com/atlasos/calypso/internal/audit"
"github.com/atlasos/calypso/internal/auth"
"github.com/atlasos/calypso/internal/iam"
"github.com/atlasos/calypso/internal/monitoring"
"github.com/atlasos/calypso/internal/scst"
@@ -44,10 +44,10 @@ func NewRouter(cfg *config.Config, db *database.DB, log *logger.Logger) *gin.Eng
r.Use(securityHeadersMiddleware(cfg))
r.Use(rateLimitMiddleware(cfg, log))
r.Use(corsMiddleware(cfg))
// Cache control headers (always applied)
r.Use(cacheControlMiddleware())
// Response caching middleware (if enabled)
if cfg.Server.Cache.Enabled {
cacheConfig := CacheConfig{
@@ -84,7 +84,7 @@ func NewRouter(cfg *config.Config, db *database.DB, log *logger.Logger) *gin.Eng
// Initialize and start alert rule engine
alertRuleEngine := monitoring.NewAlertRuleEngine(db, log, alertService)
// Register default alert rules
alertRuleEngine.RegisterRule(monitoring.NewAlertRule(
"storage-capacity-warning",
@@ -160,6 +160,10 @@ func NewRouter(cfg *config.Config, db *database.DB, log *logger.Logger) *gin.Eng
// Storage
storageHandler := storage.NewHandler(db, log)
// Pass cache to storage handler for cache invalidation
if responseCache != nil {
storageHandler.SetCache(responseCache)
}
storageGroup := protected.Group("/storage")
storageGroup.Use(requirePermission("storage", "read"))
{
@@ -180,6 +184,8 @@ func NewRouter(cfg *config.Config, db *database.DB, log *logger.Logger) *gin.Eng
storageGroup.GET("/zfs/pools/:id/datasets", storageHandler.ListZFSDatasets)
storageGroup.POST("/zfs/pools/:id/datasets", requirePermission("storage", "write"), storageHandler.CreateZFSDataset)
storageGroup.DELETE("/zfs/pools/:id/datasets/:dataset", requirePermission("storage", "write"), storageHandler.DeleteZFSDataset)
// ZFS ARC Stats
storageGroup.GET("/zfs/arc/stats", storageHandler.GetARCStats)
}
// SCST
@@ -286,6 +292,3 @@ func ginLogger(log *logger.Logger) gin.HandlerFunc {
)
}
}