scrub operation + ZFS Pool CRUD

2025-12-15 01:19:44 +07:00
parent 9779b30a65
commit abd8cef10a
9 changed files with 1124 additions and 63 deletions
--- a/internal/httpapp/api_handlers.go
+++ b/internal/httpapp/api_handlers.go
@@ -100,35 +100,112 @@ func (a *App) handleGetPool(w http.ResponseWriter, r *http.Request) {
 func (a *App) handleDeletePool(w http.ResponseWriter, r *http.Request) {
 	name := pathParam(r, "/api/v1/pools/")
 	if name == "" {
-		writeJSON(w, http.StatusBadRequest, map[string]string{"error": "pool name required"})
+		writeError(w, errors.ErrBadRequest("pool name required"))
 		return
 	}

 	if err := a.zfs.DestroyPool(name); err != nil {
 		log.Printf("destroy pool error: %v", err)
-		writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
+		writeError(w, errors.ErrInternal("failed to destroy pool").WithDetails(err.Error()))
 		return
 	}

 	writeJSON(w, http.StatusOK, map[string]string{"message": "pool destroyed", "name": name})
 }

+func (a *App) handleImportPool(w http.ResponseWriter, r *http.Request) {
+	var req struct {
+		Name    string            `json:"name"`
+		Options map[string]string `json:"options,omitempty"`
+	}
+
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, errors.ErrBadRequest("invalid request body").WithDetails(err.Error()))
+		return
+	}
+
+	if req.Name == "" {
+		writeError(w, errors.ErrBadRequest("pool name required"))
+		return
+	}
+
+	if err := a.zfs.ImportPool(req.Name, req.Options); err != nil {
+		log.Printf("import pool error: %v", err)
+		writeError(w, errors.ErrInternal("failed to import pool").WithDetails(err.Error()))
+		return
+	}
+
+	writeJSON(w, http.StatusOK, map[string]string{"message": "pool imported", "name": req.Name})
+}
+
+func (a *App) handleExportPool(w http.ResponseWriter, r *http.Request) {
+	name := pathParam(r, "/api/v1/pools/")
+	if name == "" {
+		writeError(w, errors.ErrBadRequest("pool name required"))
+		return
+	}
+
+	var req struct {
+		Force bool `json:"force,omitempty"`
+	}
+	// Force is optional, decode if body exists
+	_ = json.NewDecoder(r.Body).Decode(&req)
+
+	if err := a.zfs.ExportPool(name, req.Force); err != nil {
+		log.Printf("export pool error: %v", err)
+		writeError(w, errors.ErrInternal("failed to export pool").WithDetails(err.Error()))
+		return
+	}
+
+	writeJSON(w, http.StatusOK, map[string]string{"message": "pool exported", "name": name})
+}
+
+func (a *App) handleListAvailablePools(w http.ResponseWriter, r *http.Request) {
+	pools, err := a.zfs.ListAvailablePools()
+	if err != nil {
+		log.Printf("list available pools error: %v", err)
+		writeError(w, errors.ErrInternal("failed to list available pools").WithDetails(err.Error()))
+		return
+	}
+
+	writeJSON(w, http.StatusOK, map[string]interface{}{
+		"pools": pools,
+	})
+}
+
 func (a *App) handleScrubPool(w http.ResponseWriter, r *http.Request) {
 	name := pathParam(r, "/api/v1/pools/")
 	if name == "" {
-		writeJSON(w, http.StatusBadRequest, map[string]string{"error": "pool name required"})
+		writeError(w, errors.ErrBadRequest("pool name required"))
 		return
 	}

 	if err := a.zfs.ScrubPool(name); err != nil {
 		log.Printf("scrub pool error: %v", err)
-		writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
+		writeError(w, errors.ErrInternal("failed to start scrub").WithDetails(err.Error()))
 		return
 	}

 	writeJSON(w, http.StatusOK, map[string]string{"message": "scrub started", "pool": name})
 }

+func (a *App) handleGetScrubStatus(w http.ResponseWriter, r *http.Request) {
+	name := pathParam(r, "/api/v1/pools/")
+	if name == "" {
+		writeError(w, errors.ErrBadRequest("pool name required"))
+		return
+	}
+
+	status, err := a.zfs.GetScrubStatus(name)
+	if err != nil {
+		log.Printf("get scrub status error: %v", err)
+		writeError(w, errors.ErrInternal("failed to get scrub status").WithDetails(err.Error()))
+		return
+	}
+
+	writeJSON(w, http.StatusOK, status)
+}
+
 // Dataset Handlers
 func (a *App) handleListDatasets(w http.ResponseWriter, r *http.Request) {
 	pool := r.URL.Query().Get("pool")
--- a/internal/httpapp/app.go
+++ b/internal/httpapp/app.go
@@ -18,6 +18,7 @@ import (
 	"gitea.avt.data-center.id/othman.suseno/atlas/internal/services"
 	"gitea.avt.data-center.id/othman.suseno/atlas/internal/snapshot"
 	"gitea.avt.data-center.id/othman.suseno/atlas/internal/storage"
+	"gitea.avt.data-center.id/othman.suseno/atlas/internal/tls"
 	"gitea.avt.data-center.id/othman.suseno/atlas/internal/zfs"
 )

@@ -50,6 +51,7 @@ type App struct {
 	startTime          time.Time
 	backupService      *backup.Service
 	maintenanceService *maintenance.Service
+	tlsConfig          *tls.Config
 }

 func New(cfg Config) (*App, error) {
@@ -112,27 +114,38 @@ func New(cfg Config) (*App, error) {
 		return nil, fmt.Errorf("init backup service: %w", err)
 	}

+	// Initialize maintenance service
+	maintenanceService := maintenance.NewService()
+
+	// Initialize TLS configuration
+	tlsConfig := tls.LoadConfig()
+	if err := tlsConfig.Validate(); err != nil {
+		return nil, fmt.Errorf("TLS configuration: %w", err)
+	}
+
 	a := &App{
-		cfg:              cfg,
-		tmpl:             tmpl,
-		mux:              http.NewServeMux(),
-		zfs:              zfsService,
-		snapshotPolicy:   policyStore,
-		jobManager:       jobMgr,
-		scheduler:        scheduler,
-		authService:      authService,
-		userStore:        userStore,
-		auditStore:       auditStore,
-		smbStore:         smbStore,
-		nfsStore:         nfsStore,
-		iscsiStore:       iscsiStore,
-		database:         database,
-		smbService:       smbService,
-		nfsService:       nfsService,
-		iscsiService:     iscsiService,
-		metricsCollector: metricsCollector,
-		startTime:        startTime,
-		backupService:    backupService,
+		cfg:                cfg,
+		tmpl:               tmpl,
+		mux:                http.NewServeMux(),
+		zfs:                zfsService,
+		snapshotPolicy:     policyStore,
+		jobManager:         jobMgr,
+		scheduler:          scheduler,
+		authService:        authService,
+		userStore:          userStore,
+		auditStore:         auditStore,
+		smbStore:           smbStore,
+		nfsStore:           nfsStore,
+		iscsiStore:         iscsiStore,
+		database:           database,
+		smbService:         smbService,
+		nfsService:         nfsService,
+		iscsiService:       iscsiService,
+		metricsCollector:   metricsCollector,
+		startTime:          startTime,
+		backupService:      backupService,
+		maintenanceService: maintenanceService,
+		tlsConfig:          tlsConfig,
 	}

 	// Start snapshot scheduler (runs every 15 minutes)
@@ -144,33 +157,36 @@ func New(cfg Config) (*App, error) {

 func (a *App) Router() http.Handler {
 	// Middleware chain order (outer to inner):
-	// 1. CORS (handles preflight)
-	// 2. Compression (gzip)
-	// 3. Security headers
-	// 4. Request size limit (10MB)
-	// 5. Content-Type validation
-	// 6. Rate limiting
-	// 7. Caching (for GET requests)
-	// 8. Error recovery
-	// 9. Request ID
-	// 10. Logging
-	// 11. Audit
-	// 12. Authentication
-	// 13. Maintenance mode (blocks operations during maintenance)
-	// 14. Routes
-	return a.corsMiddleware(
-		a.compressionMiddleware(
-			a.securityHeadersMiddleware(
-				a.requestSizeMiddleware(10 * 1024 * 1024)(
-					a.validateContentTypeMiddleware(
-						a.rateLimitMiddleware(
-							a.cacheMiddleware(
-								a.errorMiddleware(
-									requestID(
-										logging(
-											a.auditMiddleware(
-												a.maintenanceMiddleware(
-													a.authMiddleware(a.mux),
+	// 1. HTTPS enforcement (redirect HTTP to HTTPS)
+	// 2. CORS (handles preflight)
+	// 3. Compression (gzip)
+	// 4. Security headers
+	// 5. Request size limit (10MB)
+	// 6. Content-Type validation
+	// 7. Rate limiting
+	// 8. Caching (for GET requests)
+	// 9. Error recovery
+	// 10. Request ID
+	// 11. Logging
+	// 12. Audit
+	// 13. Authentication
+	// 14. Maintenance mode (blocks operations during maintenance)
+	// 15. Routes
+	return a.httpsEnforcementMiddleware(
+		a.corsMiddleware(
+			a.compressionMiddleware(
+				a.securityHeadersMiddleware(
+					a.requestSizeMiddleware(10 * 1024 * 1024)(
+						a.validateContentTypeMiddleware(
+							a.rateLimitMiddleware(
+								a.cacheMiddleware(
+									a.errorMiddleware(
+										requestID(
+											logging(
+												a.auditMiddleware(
+													a.maintenanceMiddleware(
+														a.authMiddleware(a.mux),
+													),
 												),
 											),
 										),
--- a/internal/httpapp/https_middleware.go
+++ b/internal/httpapp/https_middleware.go
@@ -0,0 +1,76 @@
+package httpapp
+
+import (
+	"net/http"
+	"strings"
+
+	"gitea.avt.data-center.id/othman.suseno/atlas/internal/errors"
+)
+
+// httpsEnforcementMiddleware enforces HTTPS connections
+func (a *App) httpsEnforcementMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Skip HTTPS enforcement for health checks and localhost
+		if a.isPublicEndpoint(r.URL.Path) || isLocalhost(r) {
+			next.ServeHTTP(w, r)
+			return
+		}
+
+		// If TLS is enabled, enforce HTTPS
+		if a.tlsConfig != nil && a.tlsConfig.Enabled {
+			// Check if request is already over HTTPS
+			if r.TLS != nil {
+				next.ServeHTTP(w, r)
+				return
+			}
+
+			// Check X-Forwarded-Proto header (for reverse proxies)
+			if r.Header.Get("X-Forwarded-Proto") == "https" {
+				next.ServeHTTP(w, r)
+				return
+			}
+
+			// Redirect HTTP to HTTPS
+			httpsURL := "https://" + r.Host + r.URL.RequestURI()
+			http.Redirect(w, r, httpsURL, http.StatusMovedPermanently)
+			return
+		}
+
+		next.ServeHTTP(w, r)
+	})
+}
+
+// isLocalhost checks if the request is from localhost
+func isLocalhost(r *http.Request) bool {
+	host := r.Host
+	if strings.Contains(host, ":") {
+		host = strings.Split(host, ":")[0]
+	}
+	return host == "localhost" || host == "127.0.0.1" || host == "::1"
+}
+
+// requireHTTPSMiddleware requires HTTPS for all requests (strict mode)
+func (a *App) requireHTTPSMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Skip for health checks
+		if a.isPublicEndpoint(r.URL.Path) {
+			next.ServeHTTP(w, r)
+			return
+		}
+
+		// If TLS is enabled, require HTTPS
+		if a.tlsConfig != nil && a.tlsConfig.Enabled {
+			// Check if request is over HTTPS
+			if r.TLS == nil && r.Header.Get("X-Forwarded-Proto") != "https" {
+				writeError(w, errors.NewAPIError(
+					errors.ErrCodeForbidden,
+					"HTTPS required",
+					http.StatusForbidden,
+				).WithDetails("this endpoint requires HTTPS"))
+				return
+			}
+		}
+
+		next.ServeHTTP(w, r)
+	})
+}
--- a/internal/httpapp/router_helpers.go
+++ b/internal/httpapp/router_helpers.go
@@ -60,12 +60,30 @@ func pathParam(r *http.Request, prefix string) string {

 // handlePoolOps routes pool operations by method
 func (a *App) handlePoolOps(w http.ResponseWriter, r *http.Request) {
+	// Extract pool name from path like /api/v1/pools/tank
+	name := pathParam(r, "/api/v1/pools/")
+	if name == "" {
+		writeError(w, errors.ErrBadRequest("pool name required"))
+		return
+	}
+
 	if strings.HasSuffix(r.URL.Path, "/scrub") {
 		if r.Method == http.MethodPost {
 			a.handleScrubPool(w, r)
-			return
+		} else if r.Method == http.MethodGet {
+			a.handleGetScrubStatus(w, r)
+		} else {
+			writeError(w, errors.NewAPIError(errors.ErrCodeBadRequest, "method not allowed", http.StatusMethodNotAllowed))
+		}
+		return
+	}
+
+	if strings.HasSuffix(r.URL.Path, "/export") {
+		if r.Method == http.MethodPost {
+			a.handleExportPool(w, r)
+		} else {
+			writeError(w, errors.NewAPIError(errors.ErrCodeBadRequest, "method not allowed", http.StatusMethodNotAllowed))
 		}
-		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
 		return
 	}

--- a/internal/httpapp/routes.go
+++ b/internal/httpapp/routes.go
@@ -80,6 +80,15 @@ func (a *App) routes() {
 		func(w http.ResponseWriter, r *http.Request) { a.handleCreatePool(w, r) },
 		nil, nil, nil,
 	))
+	a.mux.HandleFunc("/api/v1/pools/available", methodHandler(
+		func(w http.ResponseWriter, r *http.Request) { a.handleListAvailablePools(w, r) },
+		nil, nil, nil, nil,
+	))
+	a.mux.HandleFunc("/api/v1/pools/import", methodHandler(
+		nil,
+		func(w http.ResponseWriter, r *http.Request) { a.handleImportPool(w, r) },
+		nil, nil, nil,
+	))
 	a.mux.HandleFunc("/api/v1/pools/", a.handlePoolOps)

 	a.mux.HandleFunc("/api/v1/datasets", methodHandler(
--- a/internal/tls/config.go
+++ b/internal/tls/config.go
@@ -0,0 +1,104 @@
+package tls
+
+import (
+	"crypto/tls"
+	"fmt"
+	"os"
+)
+
+// Note: This package is named "tls" but provides configuration for crypto/tls
+
+// Config holds TLS configuration
+type Config struct {
+	CertFile   string
+	KeyFile    string
+	MinVersion uint16
+	MaxVersion uint16
+	Enabled    bool
+}
+
+// LoadConfig loads TLS configuration from environment variables
+func LoadConfig() *Config {
+	cfg := &Config{
+		CertFile:   os.Getenv("ATLAS_TLS_CERT"),
+		KeyFile:    os.Getenv("ATLAS_TLS_KEY"),
+		MinVersion: tls.VersionTLS12,
+		MaxVersion: tls.VersionTLS13,
+		Enabled:    false,
+	}
+
+	// Enable TLS if certificate and key are provided
+	if cfg.CertFile != "" && cfg.KeyFile != "" {
+		cfg.Enabled = true
+	}
+
+	// Check if TLS is explicitly enabled
+	if os.Getenv("ATLAS_TLS_ENABLED") == "true" {
+		cfg.Enabled = true
+	}
+
+	return cfg
+}
+
+// BuildTLSConfig builds a crypto/tls.Config from the configuration
+func (c *Config) BuildTLSConfig() (*tls.Config, error) {
+	if !c.Enabled {
+		return nil, nil
+	}
+
+	// Verify certificate and key files exist
+	if _, err := os.Stat(c.CertFile); os.IsNotExist(err) {
+		return nil, fmt.Errorf("TLS certificate file not found: %s", c.CertFile)
+	}
+
+	if _, err := os.Stat(c.KeyFile); os.IsNotExist(err) {
+		return nil, fmt.Errorf("TLS key file not found: %s", c.KeyFile)
+	}
+
+	// Load certificate
+	cert, err := tls.LoadX509KeyPair(c.CertFile, c.KeyFile)
+	if err != nil {
+		return nil, fmt.Errorf("load TLS certificate: %w", err)
+	}
+
+	config := &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		MinVersion:   c.MinVersion,
+		MaxVersion:   c.MaxVersion,
+		// Security best practices
+		PreferServerCipherSuites: true,
+		CurvePreferences: []tls.CurveID{
+			tls.CurveP256,
+			tls.CurveP384,
+			tls.CurveP521,
+			tls.X25519,
+		},
+		CipherSuites: []uint16{
+			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+		},
+	}
+
+	return config, nil
+}
+
+// Validate validates the TLS configuration
+func (c *Config) Validate() error {
+	if !c.Enabled {
+		return nil
+	}
+
+	if c.CertFile == "" {
+		return fmt.Errorf("TLS certificate file is required when TLS is enabled")
+	}
+
+	if c.KeyFile == "" {
+		return fmt.Errorf("TLS key file is required when TLS is enabled")
+	}
+
+	return nil
+}
--- a/internal/zfs/service.go
+++ b/internal/zfs/service.go
@@ -136,26 +136,184 @@ func (s *Service) DestroyPool(name string) error {
 	return err
 }

+// ImportPool imports a ZFS pool
+func (s *Service) ImportPool(name string, options map[string]string) error {
+	args := []string{"import"}
+
+	// Add options
+	for k, v := range options {
+		args = append(args, "-o", fmt.Sprintf("%s=%s", k, v))
+	}
+
+	args = append(args, name)
+	_, err := s.execCommand(s.zpoolPath, args...)
+	return err
+}
+
+// ExportPool exports a ZFS pool
+func (s *Service) ExportPool(name string, force bool) error {
+	args := []string{"export"}
+	if force {
+		args = append(args, "-f")
+	}
+	args = append(args, name)
+	_, err := s.execCommand(s.zpoolPath, args...)
+	return err
+}
+
+// ListAvailablePools returns pools that can be imported
+func (s *Service) ListAvailablePools() ([]string, error) {
+	output, err := s.execCommand(s.zpoolPath, "import")
+	if err != nil {
+		return nil, err
+	}
+
+	var pools []string
+	lines := strings.Split(output, "\n")
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		// Parse pool name from output like "pool: tank"
+		if strings.HasPrefix(line, "pool:") {
+			parts := strings.Fields(line)
+			if len(parts) >= 2 {
+				pools = append(pools, parts[1])
+			}
+		}
+	}
+
+	return pools, nil
+}
+
 // ScrubPool starts a scrub operation on a pool
 func (s *Service) ScrubPool(name string) error {
 	_, err := s.execCommand(s.zpoolPath, "scrub", name)
 	return err
 }

-// GetScrubStatus returns the current scrub status
-func (s *Service) GetScrubStatus(name string) (string, error) {
-	output, err := s.execCommand(s.zpoolPath, "status", name)
-	if err != nil {
-		return "", err
+// ScrubStatus represents detailed scrub operation status
+type ScrubStatus struct {
+	Status      string  `json:"status"`       // idle, in_progress, completed, error
+	Progress    float64 `json:"progress"`     // 0-100
+	TimeElapsed string  `json:"time_elapsed"` // e.g., "2h 15m"
+	TimeRemain  string  `json:"time_remain"`  // e.g., "30m"
+	Speed       string  `json:"speed"`        // e.g., "100M/s"
+	Errors      int     `json:"errors"`       // number of errors found
+	Repaired    int     `json:"repaired"`     // number of errors repaired
+	LastScrub   string  `json:"last_scrub"`   // timestamp of last completed scrub
+}
+
+// GetScrubStatus returns detailed scrub status with progress
+func (s *Service) GetScrubStatus(name string) (*ScrubStatus, error) {
+	status := &ScrubStatus{
+		Status: "idle",
 	}

-	if strings.Contains(output, "scrub in progress") {
-		return "in_progress", nil
+	// Get pool status
+	output, err := s.execCommand(s.zpoolPath, "status", name)
+	if err != nil {
+		return nil, err
 	}
-	if strings.Contains(output, "scrub repaired") {
-		return "completed", nil
+
+	// Parse scrub information
+	lines := strings.Split(output, "\n")
+	inScrubSection := false
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+
+		// Check if scrub is in progress
+		if strings.Contains(line, "scrub in progress") {
+			status.Status = "in_progress"
+			inScrubSection = true
+			continue
+		}
+
+		// Check if scrub completed
+		if strings.Contains(line, "scrub repaired") || strings.Contains(line, "scrub completed") {
+			status.Status = "completed"
+			status.Progress = 100.0
+			// Extract repair information
+			if strings.Contains(line, "repaired") {
+				// Try to extract number of repairs
+				parts := strings.Fields(line)
+				for i, part := range parts {
+					if part == "repaired" && i > 0 {
+						// Previous part might be the number
+						if repaired, err := strconv.Atoi(parts[i-1]); err == nil {
+							status.Repaired = repaired
+						}
+					}
+				}
+			}
+			continue
+		}
+
+		// Parse progress percentage
+		if strings.Contains(line, "%") && inScrubSection {
+			// Extract percentage from line like "scan: 45.2% done"
+			parts := strings.Fields(line)
+			for _, part := range parts {
+				if strings.HasSuffix(part, "%") {
+					if pct, err := strconv.ParseFloat(strings.TrimSuffix(part, "%"), 64); err == nil {
+						status.Progress = pct
+					}
+				}
+			}
+		}
+
+		// Parse time elapsed
+		if strings.Contains(line, "elapsed") && inScrubSection {
+			// Extract time like "elapsed: 2h15m"
+			parts := strings.Fields(line)
+			for i, part := range parts {
+				if part == "elapsed:" && i+1 < len(parts) {
+					status.TimeElapsed = parts[i+1]
+				}
+			}
+		}
+
+		// Parse time remaining
+		if strings.Contains(line, "remaining") && inScrubSection {
+			parts := strings.Fields(line)
+			for i, part := range parts {
+				if part == "remaining:" && i+1 < len(parts) {
+					status.TimeRemain = parts[i+1]
+				}
+			}
+		}
+
+		// Parse speed
+		if strings.Contains(line, "scan rate") && inScrubSection {
+			parts := strings.Fields(line)
+			for i, part := range parts {
+				if part == "rate" && i+1 < len(parts) {
+					status.Speed = parts[i+1]
+				}
+			}
+		}
+
+		// Parse errors
+		if strings.Contains(line, "errors:") && inScrubSection {
+			parts := strings.Fields(line)
+			for i, part := range parts {
+				if part == "errors:" && i+1 < len(parts) {
+					if errs, err := strconv.Atoi(parts[i+1]); err == nil {
+						status.Errors = errs
+					}
+				}
+			}
+		}
 	}
-	return "idle", nil
+
+	// Get last scrub time from pool properties
+	lastScrub, err := s.execCommand(s.zfsPath, "get", "-H", "-o", "value", "lastscrub", name)
+	if err == nil && lastScrub != "-" && lastScrub != "" {
+		status.LastScrub = strings.TrimSpace(lastScrub)
+	}
+
+	return status, nil
 }

 // ListDatasets returns all datasets in a pool (or all if pool is empty)