add function to s3
This commit is contained in:
@@ -10,11 +10,12 @@ import (
|
||||
|
||||
// Config represents the application configuration
|
||||
type Config struct {
|
||||
Server ServerConfig `yaml:"server"`
|
||||
Database DatabaseConfig `yaml:"database"`
|
||||
Auth AuthConfig `yaml:"auth"`
|
||||
Logging LoggingConfig `yaml:"logging"`
|
||||
Security SecurityConfig `yaml:"security"`
|
||||
Server ServerConfig `yaml:"server"`
|
||||
Database DatabaseConfig `yaml:"database"`
|
||||
Auth AuthConfig `yaml:"auth"`
|
||||
Logging LoggingConfig `yaml:"logging"`
|
||||
Security SecurityConfig `yaml:"security"`
|
||||
ObjectStorage ObjectStorageConfig `yaml:"object_storage"`
|
||||
}
|
||||
|
||||
// ServerConfig holds HTTP server configuration
|
||||
@@ -96,6 +97,14 @@ type SecurityHeadersConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
}
|
||||
|
||||
// ObjectStorageConfig holds MinIO configuration
|
||||
type ObjectStorageConfig struct {
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
AccessKey string `yaml:"access_key"`
|
||||
SecretKey string `yaml:"secret_key"`
|
||||
UseSSL bool `yaml:"use_ssl"`
|
||||
}
|
||||
|
||||
// Load reads configuration from file and environment variables
|
||||
func Load(path string) (*Config, error) {
|
||||
cfg := DefaultConfig()
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
-- Migration: Object Storage Configuration
|
||||
-- Description: Creates table for storing MinIO object storage configuration
|
||||
-- Date: 2026-01-09
|
||||
|
||||
CREATE TABLE IF NOT EXISTS object_storage_config (
|
||||
id SERIAL PRIMARY KEY,
|
||||
dataset_path VARCHAR(255) NOT NULL UNIQUE,
|
||||
mount_point VARCHAR(512) NOT NULL,
|
||||
pool_name VARCHAR(255) NOT NULL,
|
||||
dataset_name VARCHAR(255) NOT NULL,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_object_storage_config_pool_name ON object_storage_config(pool_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_object_storage_config_updated_at ON object_storage_config(updated_at);
|
||||
|
||||
COMMENT ON TABLE object_storage_config IS 'Stores MinIO object storage configuration, linking to ZFS datasets';
|
||||
COMMENT ON COLUMN object_storage_config.dataset_path IS 'Full ZFS dataset path (e.g., pool/dataset)';
|
||||
COMMENT ON COLUMN object_storage_config.mount_point IS 'Mount point path for the dataset';
|
||||
COMMENT ON COLUMN object_storage_config.pool_name IS 'ZFS pool name';
|
||||
COMMENT ON COLUMN object_storage_config.dataset_name IS 'ZFS dataset name';
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/atlasos/calypso/internal/common/logger"
|
||||
"github.com/atlasos/calypso/internal/iam"
|
||||
"github.com/atlasos/calypso/internal/monitoring"
|
||||
"github.com/atlasos/calypso/internal/object_storage"
|
||||
"github.com/atlasos/calypso/internal/scst"
|
||||
"github.com/atlasos/calypso/internal/shares"
|
||||
"github.com/atlasos/calypso/internal/storage"
|
||||
@@ -211,6 +212,45 @@ func NewRouter(cfg *config.Config, db *database.DB, log *logger.Logger) *gin.Eng
|
||||
sharesGroup.DELETE("/:id", requirePermission("storage", "write"), sharesHandler.DeleteShare)
|
||||
}
|
||||
|
||||
// Object Storage (MinIO)
|
||||
// Initialize MinIO service if configured
|
||||
if cfg.ObjectStorage.Endpoint != "" {
|
||||
objectStorageService, err := object_storage.NewService(
|
||||
cfg.ObjectStorage.Endpoint,
|
||||
cfg.ObjectStorage.AccessKey,
|
||||
cfg.ObjectStorage.SecretKey,
|
||||
log,
|
||||
)
|
||||
if err != nil {
|
||||
log.Error("Failed to initialize MinIO service", "error", err)
|
||||
} else {
|
||||
objectStorageHandler := object_storage.NewHandler(objectStorageService, db, log)
|
||||
objectStorageGroup := protected.Group("/object-storage")
|
||||
objectStorageGroup.Use(requirePermission("storage", "read"))
|
||||
{
|
||||
// Setup endpoints
|
||||
objectStorageGroup.GET("/setup/datasets", objectStorageHandler.GetAvailableDatasets)
|
||||
objectStorageGroup.GET("/setup/current", objectStorageHandler.GetCurrentSetup)
|
||||
objectStorageGroup.POST("/setup", requirePermission("storage", "write"), objectStorageHandler.SetupObjectStorage)
|
||||
objectStorageGroup.PUT("/setup", requirePermission("storage", "write"), objectStorageHandler.UpdateObjectStorage)
|
||||
|
||||
// Bucket endpoints
|
||||
objectStorageGroup.GET("/buckets", objectStorageHandler.ListBuckets)
|
||||
objectStorageGroup.GET("/buckets/:name", objectStorageHandler.GetBucket)
|
||||
objectStorageGroup.POST("/buckets", requirePermission("storage", "write"), objectStorageHandler.CreateBucket)
|
||||
objectStorageGroup.DELETE("/buckets/:name", requirePermission("storage", "write"), objectStorageHandler.DeleteBucket)
|
||||
// User management routes
|
||||
objectStorageGroup.GET("/users", objectStorageHandler.ListUsers)
|
||||
objectStorageGroup.POST("/users", requirePermission("storage", "write"), objectStorageHandler.CreateUser)
|
||||
objectStorageGroup.DELETE("/users/:access_key", requirePermission("storage", "write"), objectStorageHandler.DeleteUser)
|
||||
// Service account (access key) management routes
|
||||
objectStorageGroup.GET("/service-accounts", objectStorageHandler.ListServiceAccounts)
|
||||
objectStorageGroup.POST("/service-accounts", requirePermission("storage", "write"), objectStorageHandler.CreateServiceAccount)
|
||||
objectStorageGroup.DELETE("/service-accounts/:access_key", requirePermission("storage", "write"), objectStorageHandler.DeleteServiceAccount)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SCST
|
||||
scstHandler := scst.NewHandler(db, log)
|
||||
scstGroup := protected.Group("/scst")
|
||||
@@ -307,8 +347,9 @@ func NewRouter(cfg *config.Config, db *database.DB, log *logger.Logger) *gin.Eng
|
||||
systemGroup.GET("/logs", systemHandler.GetSystemLogs)
|
||||
systemGroup.GET("/network/throughput", systemHandler.GetNetworkThroughput)
|
||||
systemGroup.POST("/support-bundle", systemHandler.GenerateSupportBundle)
|
||||
systemGroup.GET("/interfaces", systemHandler.ListNetworkInterfaces)
|
||||
systemGroup.PUT("/interfaces/:name", systemHandler.UpdateNetworkInterface)
|
||||
systemGroup.GET("/interfaces", systemHandler.ListNetworkInterfaces)
|
||||
systemGroup.GET("/management-ip", systemHandler.GetManagementIPAddress)
|
||||
systemGroup.PUT("/interfaces/:name", systemHandler.UpdateNetworkInterface)
|
||||
systemGroup.GET("/ntp", systemHandler.GetNTPSettings)
|
||||
systemGroup.POST("/ntp", systemHandler.SaveNTPSettings)
|
||||
systemGroup.POST("/execute", requirePermission("system", "write"), systemHandler.ExecuteCommand)
|
||||
|
||||
285
backend/internal/object_storage/handler.go
Normal file
285
backend/internal/object_storage/handler.go
Normal file
@@ -0,0 +1,285 @@
|
||||
package object_storage
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/atlasos/calypso/internal/common/database"
|
||||
"github.com/atlasos/calypso/internal/common/logger"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// Handler handles HTTP requests for object storage
|
||||
type Handler struct {
|
||||
service *Service
|
||||
setupService *SetupService
|
||||
logger *logger.Logger
|
||||
}
|
||||
|
||||
// NewHandler creates a new object storage handler
|
||||
func NewHandler(service *Service, db *database.DB, log *logger.Logger) *Handler {
|
||||
return &Handler{
|
||||
service: service,
|
||||
setupService: NewSetupService(db, log),
|
||||
logger: log,
|
||||
}
|
||||
}
|
||||
|
||||
// ListBuckets lists all buckets
|
||||
func (h *Handler) ListBuckets(c *gin.Context) {
|
||||
buckets, err := h.service.ListBuckets(c.Request.Context())
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to list buckets", "error", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to list buckets: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"buckets": buckets})
|
||||
}
|
||||
|
||||
// GetBucket gets bucket information
|
||||
func (h *Handler) GetBucket(c *gin.Context) {
|
||||
bucketName := c.Param("name")
|
||||
|
||||
bucket, err := h.service.GetBucketStats(c.Request.Context(), bucketName)
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to get bucket", "bucket", bucketName, "error", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get bucket: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, bucket)
|
||||
}
|
||||
|
||||
// CreateBucketRequest represents a request to create a bucket
|
||||
type CreateBucketRequest struct {
|
||||
Name string `json:"name" binding:"required"`
|
||||
}
|
||||
|
||||
// CreateBucket creates a new bucket
|
||||
func (h *Handler) CreateBucket(c *gin.Context) {
|
||||
var req CreateBucketRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
h.logger.Error("Invalid create bucket request", "error", err)
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.service.CreateBucket(c.Request.Context(), req.Name); err != nil {
|
||||
h.logger.Error("Failed to create bucket", "bucket", req.Name, "error", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create bucket: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusCreated, gin.H{"message": "bucket created successfully", "name": req.Name})
|
||||
}
|
||||
|
||||
// DeleteBucket deletes a bucket
|
||||
func (h *Handler) DeleteBucket(c *gin.Context) {
|
||||
bucketName := c.Param("name")
|
||||
|
||||
if err := h.service.DeleteBucket(c.Request.Context(), bucketName); err != nil {
|
||||
h.logger.Error("Failed to delete bucket", "bucket", bucketName, "error", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to delete bucket: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"message": "bucket deleted successfully"})
|
||||
}
|
||||
|
||||
// GetAvailableDatasets gets all available pools and datasets for object storage setup
|
||||
func (h *Handler) GetAvailableDatasets(c *gin.Context) {
|
||||
datasets, err := h.setupService.GetAvailableDatasets(c.Request.Context())
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to get available datasets", "error", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get available datasets: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"pools": datasets})
|
||||
}
|
||||
|
||||
// SetupObjectStorageRequest represents a request to setup object storage
|
||||
type SetupObjectStorageRequest struct {
|
||||
PoolName string `json:"pool_name" binding:"required"`
|
||||
DatasetName string `json:"dataset_name" binding:"required"`
|
||||
CreateNew bool `json:"create_new"`
|
||||
}
|
||||
|
||||
// SetupObjectStorage configures object storage with a ZFS dataset
|
||||
func (h *Handler) SetupObjectStorage(c *gin.Context) {
|
||||
var req SetupObjectStorageRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
h.logger.Error("Invalid setup request", "error", err)
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
setupReq := SetupRequest{
|
||||
PoolName: req.PoolName,
|
||||
DatasetName: req.DatasetName,
|
||||
CreateNew: req.CreateNew,
|
||||
}
|
||||
|
||||
result, err := h.setupService.SetupObjectStorage(c.Request.Context(), setupReq)
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to setup object storage", "error", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to setup object storage: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, result)
|
||||
}
|
||||
|
||||
// GetCurrentSetup gets the current object storage configuration
|
||||
func (h *Handler) GetCurrentSetup(c *gin.Context) {
|
||||
setup, err := h.setupService.GetCurrentSetup(c.Request.Context())
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to get current setup", "error", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get current setup: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
if setup == nil {
|
||||
c.JSON(http.StatusOK, gin.H{"configured": false})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"configured": true, "setup": setup})
|
||||
}
|
||||
|
||||
// UpdateObjectStorage updates the object storage configuration
|
||||
func (h *Handler) UpdateObjectStorage(c *gin.Context) {
|
||||
var req SetupObjectStorageRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
h.logger.Error("Invalid update request", "error", err)
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
setupReq := SetupRequest{
|
||||
PoolName: req.PoolName,
|
||||
DatasetName: req.DatasetName,
|
||||
CreateNew: req.CreateNew,
|
||||
}
|
||||
|
||||
result, err := h.setupService.UpdateObjectStorage(c.Request.Context(), setupReq)
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to update object storage", "error", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to update object storage: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, result)
|
||||
}
|
||||
|
||||
// ListUsers lists all IAM users
|
||||
func (h *Handler) ListUsers(c *gin.Context) {
|
||||
users, err := h.service.ListUsers(c.Request.Context())
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to list users", "error", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to list users: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"users": users})
|
||||
}
|
||||
|
||||
// CreateUserRequest represents a request to create a user
|
||||
type CreateUserRequest struct {
|
||||
AccessKey string `json:"access_key" binding:"required"`
|
||||
SecretKey string `json:"secret_key" binding:"required"`
|
||||
}
|
||||
|
||||
// CreateUser creates a new IAM user
|
||||
func (h *Handler) CreateUser(c *gin.Context) {
|
||||
var req CreateUserRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
h.logger.Error("Invalid create user request", "error", err)
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.service.CreateUser(c.Request.Context(), req.AccessKey, req.SecretKey); err != nil {
|
||||
h.logger.Error("Failed to create user", "access_key", req.AccessKey, "error", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create user: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusCreated, gin.H{"message": "user created successfully", "access_key": req.AccessKey})
|
||||
}
|
||||
|
||||
// DeleteUser deletes an IAM user
|
||||
func (h *Handler) DeleteUser(c *gin.Context) {
|
||||
accessKey := c.Param("access_key")
|
||||
|
||||
if err := h.service.DeleteUser(c.Request.Context(), accessKey); err != nil {
|
||||
h.logger.Error("Failed to delete user", "access_key", accessKey, "error", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to delete user: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"message": "user deleted successfully"})
|
||||
}
|
||||
|
||||
// ListServiceAccounts lists all service accounts (access keys)
|
||||
func (h *Handler) ListServiceAccounts(c *gin.Context) {
|
||||
accounts, err := h.service.ListServiceAccounts(c.Request.Context())
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to list service accounts", "error", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to list service accounts: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"service_accounts": accounts})
|
||||
}
|
||||
|
||||
// CreateServiceAccountRequest represents a request to create a service account
|
||||
type CreateServiceAccountRequest struct {
|
||||
ParentUser string `json:"parent_user" binding:"required"`
|
||||
Policy string `json:"policy,omitempty"`
|
||||
Expiration *string `json:"expiration,omitempty"` // ISO 8601 format
|
||||
}
|
||||
|
||||
// CreateServiceAccount creates a new service account (access key)
|
||||
func (h *Handler) CreateServiceAccount(c *gin.Context) {
|
||||
var req CreateServiceAccountRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
h.logger.Error("Invalid create service account request", "error", err)
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
var expiration *time.Time
|
||||
if req.Expiration != nil {
|
||||
exp, err := time.Parse(time.RFC3339, *req.Expiration)
|
||||
if err != nil {
|
||||
h.logger.Error("Invalid expiration format", "error", err)
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid expiration format, use ISO 8601 (RFC3339)"})
|
||||
return
|
||||
}
|
||||
expiration = &exp
|
||||
}
|
||||
|
||||
account, err := h.service.CreateServiceAccount(c.Request.Context(), req.ParentUser, req.Policy, expiration)
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to create service account", "parent_user", req.ParentUser, "error", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create service account: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusCreated, account)
|
||||
}
|
||||
|
||||
// DeleteServiceAccount deletes a service account
|
||||
func (h *Handler) DeleteServiceAccount(c *gin.Context) {
|
||||
accessKey := c.Param("access_key")
|
||||
|
||||
if err := h.service.DeleteServiceAccount(c.Request.Context(), accessKey); err != nil {
|
||||
h.logger.Error("Failed to delete service account", "access_key", accessKey, "error", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to delete service account: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"message": "service account deleted successfully"})
|
||||
}
|
||||
297
backend/internal/object_storage/service.go
Normal file
297
backend/internal/object_storage/service.go
Normal file
@@ -0,0 +1,297 @@
|
||||
package object_storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/atlasos/calypso/internal/common/logger"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
madmin "github.com/minio/madmin-go/v3"
|
||||
)
|
||||
|
||||
// Service handles MinIO object storage operations
|
||||
type Service struct {
|
||||
client *minio.Client
|
||||
adminClient *madmin.AdminClient
|
||||
logger *logger.Logger
|
||||
endpoint string
|
||||
accessKey string
|
||||
secretKey string
|
||||
}
|
||||
|
||||
// NewService creates a new MinIO service
|
||||
func NewService(endpoint, accessKey, secretKey string, log *logger.Logger) (*Service, error) {
|
||||
// Create MinIO client
|
||||
minioClient, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: false, // Set to true if using HTTPS
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create MinIO client: %w", err)
|
||||
}
|
||||
|
||||
// Create MinIO Admin client
|
||||
adminClient, err := madmin.New(endpoint, accessKey, secretKey, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create MinIO admin client: %w", err)
|
||||
}
|
||||
|
||||
return &Service{
|
||||
client: minioClient,
|
||||
adminClient: adminClient,
|
||||
logger: log,
|
||||
endpoint: endpoint,
|
||||
accessKey: accessKey,
|
||||
secretKey: secretKey,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Bucket represents a MinIO bucket
|
||||
type Bucket struct {
|
||||
Name string `json:"name"`
|
||||
CreationDate time.Time `json:"creation_date"`
|
||||
Size int64 `json:"size"` // Total size in bytes
|
||||
Objects int64 `json:"objects"` // Number of objects
|
||||
AccessPolicy string `json:"access_policy"` // private, public-read, public-read-write
|
||||
}
|
||||
|
||||
// ListBuckets lists all buckets in MinIO
|
||||
func (s *Service) ListBuckets(ctx context.Context) ([]*Bucket, error) {
|
||||
buckets, err := s.client.ListBuckets(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list buckets: %w", err)
|
||||
}
|
||||
|
||||
result := make([]*Bucket, 0, len(buckets))
|
||||
for _, bucket := range buckets {
|
||||
bucketInfo, err := s.getBucketInfo(ctx, bucket.Name)
|
||||
if err != nil {
|
||||
s.logger.Warn("Failed to get bucket info", "bucket", bucket.Name, "error", err)
|
||||
// Continue with basic info
|
||||
result = append(result, &Bucket{
|
||||
Name: bucket.Name,
|
||||
CreationDate: bucket.CreationDate,
|
||||
Size: 0,
|
||||
Objects: 0,
|
||||
AccessPolicy: "private",
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
result = append(result, bucketInfo)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// getBucketInfo gets detailed information about a bucket
|
||||
func (s *Service) getBucketInfo(ctx context.Context, bucketName string) (*Bucket, error) {
|
||||
// Get bucket creation date
|
||||
buckets, err := s.client.ListBuckets(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var creationDate time.Time
|
||||
for _, b := range buckets {
|
||||
if b.Name == bucketName {
|
||||
creationDate = b.CreationDate
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Get bucket size and object count by listing objects
|
||||
var size int64
|
||||
var objects int64
|
||||
|
||||
// List objects in bucket to calculate size and count
|
||||
objectCh := s.client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{
|
||||
Recursive: true,
|
||||
})
|
||||
|
||||
for object := range objectCh {
|
||||
if object.Err != nil {
|
||||
s.logger.Warn("Error listing object", "bucket", bucketName, "error", object.Err)
|
||||
continue
|
||||
}
|
||||
objects++
|
||||
size += object.Size
|
||||
}
|
||||
|
||||
return &Bucket{
|
||||
Name: bucketName,
|
||||
CreationDate: creationDate,
|
||||
Size: size,
|
||||
Objects: objects,
|
||||
AccessPolicy: s.getBucketPolicy(ctx, bucketName),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// getBucketPolicy gets the access policy for a bucket
|
||||
func (s *Service) getBucketPolicy(ctx context.Context, bucketName string) string {
|
||||
policy, err := s.client.GetBucketPolicy(ctx, bucketName)
|
||||
if err != nil {
|
||||
return "private"
|
||||
}
|
||||
|
||||
// Parse policy JSON to determine access type
|
||||
// For simplicity, check if policy allows public read
|
||||
if policy != "" {
|
||||
// Check if policy contains public read access
|
||||
if strings.Contains(policy, "s3:GetObject") && strings.Contains(policy, "Principal") && strings.Contains(policy, "*") {
|
||||
if strings.Contains(policy, "s3:PutObject") {
|
||||
return "public-read-write"
|
||||
}
|
||||
return "public-read"
|
||||
}
|
||||
}
|
||||
|
||||
return "private"
|
||||
}
|
||||
|
||||
|
||||
// CreateBucket creates a new bucket
|
||||
func (s *Service) CreateBucket(ctx context.Context, bucketName string) error {
|
||||
err := s.client.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create bucket: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteBucket deletes a bucket
|
||||
func (s *Service) DeleteBucket(ctx context.Context, bucketName string) error {
|
||||
err := s.client.RemoveBucket(ctx, bucketName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete bucket: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetBucketStats gets statistics for a bucket
|
||||
func (s *Service) GetBucketStats(ctx context.Context, bucketName string) (*Bucket, error) {
|
||||
return s.getBucketInfo(ctx, bucketName)
|
||||
}
|
||||
|
||||
// User represents a MinIO IAM user
|
||||
type User struct {
|
||||
AccessKey string `json:"access_key"`
|
||||
Status string `json:"status"` // "enabled" or "disabled"
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// ListUsers lists all IAM users in MinIO
|
||||
func (s *Service) ListUsers(ctx context.Context) ([]*User, error) {
|
||||
users, err := s.adminClient.ListUsers(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list users: %w", err)
|
||||
}
|
||||
|
||||
result := make([]*User, 0, len(users))
|
||||
for accessKey, userInfo := range users {
|
||||
status := "enabled"
|
||||
if userInfo.Status == madmin.AccountDisabled {
|
||||
status = "disabled"
|
||||
}
|
||||
|
||||
// MinIO doesn't provide creation date, use current time
|
||||
result = append(result, &User{
|
||||
AccessKey: accessKey,
|
||||
Status: status,
|
||||
CreatedAt: time.Now(),
|
||||
})
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// CreateUser creates a new IAM user in MinIO
|
||||
func (s *Service) CreateUser(ctx context.Context, accessKey, secretKey string) error {
|
||||
err := s.adminClient.AddUser(ctx, accessKey, secretKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create user: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteUser deletes an IAM user from MinIO
|
||||
func (s *Service) DeleteUser(ctx context.Context, accessKey string) error {
|
||||
err := s.adminClient.RemoveUser(ctx, accessKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete user: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ServiceAccount represents a MinIO service account (access key)
|
||||
type ServiceAccount struct {
|
||||
AccessKey string `json:"access_key"`
|
||||
SecretKey string `json:"secret_key,omitempty"` // Only returned on creation
|
||||
ParentUser string `json:"parent_user"`
|
||||
Expiration time.Time `json:"expiration,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// ListServiceAccounts lists all service accounts in MinIO
|
||||
func (s *Service) ListServiceAccounts(ctx context.Context) ([]*ServiceAccount, error) {
|
||||
accounts, err := s.adminClient.ListServiceAccounts(ctx, "")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list service accounts: %w", err)
|
||||
}
|
||||
|
||||
result := make([]*ServiceAccount, 0, len(accounts.Accounts))
|
||||
for _, account := range accounts.Accounts {
|
||||
var expiration time.Time
|
||||
if account.Expiration != nil {
|
||||
expiration = *account.Expiration
|
||||
}
|
||||
|
||||
result = append(result, &ServiceAccount{
|
||||
AccessKey: account.AccessKey,
|
||||
ParentUser: account.ParentUser,
|
||||
Expiration: expiration,
|
||||
CreatedAt: time.Now(), // MinIO doesn't provide creation date
|
||||
})
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// CreateServiceAccount creates a new service account (access key) in MinIO
|
||||
func (s *Service) CreateServiceAccount(ctx context.Context, parentUser string, policy string, expiration *time.Time) (*ServiceAccount, error) {
|
||||
opts := madmin.AddServiceAccountReq{
|
||||
TargetUser: parentUser,
|
||||
}
|
||||
if policy != "" {
|
||||
opts.Policy = json.RawMessage(policy)
|
||||
}
|
||||
if expiration != nil {
|
||||
opts.Expiration = expiration
|
||||
}
|
||||
|
||||
creds, err := s.adminClient.AddServiceAccount(ctx, opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create service account: %w", err)
|
||||
}
|
||||
|
||||
return &ServiceAccount{
|
||||
AccessKey: creds.AccessKey,
|
||||
SecretKey: creds.SecretKey,
|
||||
ParentUser: parentUser,
|
||||
Expiration: creds.Expiration,
|
||||
CreatedAt: time.Now(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DeleteServiceAccount deletes a service account from MinIO
|
||||
func (s *Service) DeleteServiceAccount(ctx context.Context, accessKey string) error {
|
||||
err := s.adminClient.DeleteServiceAccount(ctx, accessKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete service account: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
511
backend/internal/object_storage/setup.go
Normal file
511
backend/internal/object_storage/setup.go
Normal file
@@ -0,0 +1,511 @@
|
||||
package object_storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/atlasos/calypso/internal/common/database"
|
||||
"github.com/atlasos/calypso/internal/common/logger"
|
||||
)
|
||||
|
||||
// SetupService handles object storage setup operations
|
||||
type SetupService struct {
|
||||
db *database.DB
|
||||
logger *logger.Logger
|
||||
}
|
||||
|
||||
// NewSetupService creates a new setup service
|
||||
func NewSetupService(db *database.DB, log *logger.Logger) *SetupService {
|
||||
return &SetupService{
|
||||
db: db,
|
||||
logger: log,
|
||||
}
|
||||
}
|
||||
|
||||
// PoolDatasetInfo represents a pool with its datasets
|
||||
type PoolDatasetInfo struct {
|
||||
PoolID string `json:"pool_id"`
|
||||
PoolName string `json:"pool_name"`
|
||||
Datasets []DatasetInfo `json:"datasets"`
|
||||
}
|
||||
|
||||
// DatasetInfo represents a dataset that can be used for object storage
|
||||
type DatasetInfo struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
FullName string `json:"full_name"` // pool/dataset
|
||||
MountPoint string `json:"mount_point"`
|
||||
Type string `json:"type"`
|
||||
UsedBytes int64 `json:"used_bytes"`
|
||||
AvailableBytes int64 `json:"available_bytes"`
|
||||
}
|
||||
|
||||
// GetAvailableDatasets returns all pools with their datasets that can be used for object storage
|
||||
func (s *SetupService) GetAvailableDatasets(ctx context.Context) ([]PoolDatasetInfo, error) {
|
||||
// Get all pools
|
||||
poolsQuery := `
|
||||
SELECT id, name
|
||||
FROM zfs_pools
|
||||
WHERE is_active = true
|
||||
ORDER BY name
|
||||
`
|
||||
|
||||
rows, err := s.db.QueryContext(ctx, poolsQuery)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query pools: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var pools []PoolDatasetInfo
|
||||
for rows.Next() {
|
||||
var pool PoolDatasetInfo
|
||||
if err := rows.Scan(&pool.PoolID, &pool.PoolName); err != nil {
|
||||
s.logger.Warn("Failed to scan pool", "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Get datasets for this pool
|
||||
datasetsQuery := `
|
||||
SELECT id, name, type, mount_point, used_bytes, available_bytes
|
||||
FROM zfs_datasets
|
||||
WHERE pool_name = $1 AND type = 'filesystem'
|
||||
ORDER BY name
|
||||
`
|
||||
|
||||
datasetRows, err := s.db.QueryContext(ctx, datasetsQuery, pool.PoolName)
|
||||
if err != nil {
|
||||
s.logger.Warn("Failed to query datasets", "pool", pool.PoolName, "error", err)
|
||||
pool.Datasets = []DatasetInfo{}
|
||||
pools = append(pools, pool)
|
||||
continue
|
||||
}
|
||||
|
||||
var datasets []DatasetInfo
|
||||
for datasetRows.Next() {
|
||||
var ds DatasetInfo
|
||||
var mountPoint sql.NullString
|
||||
|
||||
if err := datasetRows.Scan(&ds.ID, &ds.Name, &ds.Type, &mountPoint, &ds.UsedBytes, &ds.AvailableBytes); err != nil {
|
||||
s.logger.Warn("Failed to scan dataset", "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
ds.FullName = fmt.Sprintf("%s/%s", pool.PoolName, ds.Name)
|
||||
if mountPoint.Valid {
|
||||
ds.MountPoint = mountPoint.String
|
||||
} else {
|
||||
ds.MountPoint = ""
|
||||
}
|
||||
|
||||
datasets = append(datasets, ds)
|
||||
}
|
||||
datasetRows.Close()
|
||||
|
||||
pool.Datasets = datasets
|
||||
pools = append(pools, pool)
|
||||
}
|
||||
|
||||
return pools, nil
|
||||
}
|
||||
|
||||
// SetupRequest represents a request to setup object storage
|
||||
type SetupRequest struct {
|
||||
PoolName string `json:"pool_name" binding:"required"`
|
||||
DatasetName string `json:"dataset_name" binding:"required"`
|
||||
CreateNew bool `json:"create_new"` // If true, create new dataset instead of using existing
|
||||
}
|
||||
|
||||
// SetupResponse represents the response after setup
|
||||
type SetupResponse struct {
|
||||
DatasetPath string `json:"dataset_path"`
|
||||
MountPoint string `json:"mount_point"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// SetupObjectStorage configures MinIO to use a specific ZFS dataset
|
||||
func (s *SetupService) SetupObjectStorage(ctx context.Context, req SetupRequest) (*SetupResponse, error) {
|
||||
var datasetPath, mountPoint string
|
||||
|
||||
// Normalize dataset name - if it already contains pool name, use it as-is
|
||||
var fullDatasetName string
|
||||
if strings.HasPrefix(req.DatasetName, req.PoolName+"/") {
|
||||
// Dataset name already includes pool name (e.g., "pool/dataset")
|
||||
fullDatasetName = req.DatasetName
|
||||
} else {
|
||||
// Dataset name is just the name (e.g., "dataset"), combine with pool
|
||||
fullDatasetName = fmt.Sprintf("%s/%s", req.PoolName, req.DatasetName)
|
||||
}
|
||||
|
||||
if req.CreateNew {
|
||||
// Create new dataset for object storage
|
||||
|
||||
// Check if dataset already exists
|
||||
checkCmd := exec.CommandContext(ctx, "sudo", "zfs", "list", "-H", "-o", "name", fullDatasetName)
|
||||
if err := checkCmd.Run(); err == nil {
|
||||
return nil, fmt.Errorf("dataset %s already exists", fullDatasetName)
|
||||
}
|
||||
|
||||
// Create dataset
|
||||
createCmd := exec.CommandContext(ctx, "sudo", "zfs", "create", fullDatasetName)
|
||||
if output, err := createCmd.CombinedOutput(); err != nil {
|
||||
return nil, fmt.Errorf("failed to create dataset: %s - %w", string(output), err)
|
||||
}
|
||||
|
||||
// Get mount point
|
||||
getMountCmd := exec.CommandContext(ctx, "sudo", "zfs", "get", "-H", "-o", "value", "mountpoint", fullDatasetName)
|
||||
mountOutput, err := getMountCmd.Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get mount point: %w", err)
|
||||
}
|
||||
mountPoint = strings.TrimSpace(string(mountOutput))
|
||||
|
||||
datasetPath = fullDatasetName
|
||||
s.logger.Info("Created new dataset for object storage", "dataset", fullDatasetName, "mount_point", mountPoint)
|
||||
} else {
|
||||
// Use existing dataset
|
||||
// fullDatasetName already set above
|
||||
|
||||
// Verify dataset exists
|
||||
checkCmd := exec.CommandContext(ctx, "sudo", "zfs", "list", "-H", "-o", "name", fullDatasetName)
|
||||
if err := checkCmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("dataset %s does not exist", fullDatasetName)
|
||||
}
|
||||
|
||||
// Get mount point
|
||||
getMountCmd := exec.CommandContext(ctx, "sudo", "zfs", "get", "-H", "-o", "value", "mountpoint", fullDatasetName)
|
||||
mountOutput, err := getMountCmd.Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get mount point: %w", err)
|
||||
}
|
||||
mountPoint = strings.TrimSpace(string(mountOutput))
|
||||
|
||||
datasetPath = fullDatasetName
|
||||
s.logger.Info("Using existing dataset for object storage", "dataset", fullDatasetName, "mount_point", mountPoint)
|
||||
}
|
||||
|
||||
// Ensure mount point directory exists
|
||||
if mountPoint != "none" && mountPoint != "" {
|
||||
if err := os.MkdirAll(mountPoint, 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create mount point directory: %w", err)
|
||||
}
|
||||
} else {
|
||||
// If no mount point, use default path
|
||||
mountPoint = filepath.Join("/opt/calypso/data/pool", req.PoolName, req.DatasetName)
|
||||
if err := os.MkdirAll(mountPoint, 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create default directory: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Update MinIO configuration to use the selected dataset
|
||||
if err := s.updateMinIOConfig(ctx, mountPoint); err != nil {
|
||||
s.logger.Warn("Failed to update MinIO configuration", "error", err)
|
||||
// Continue anyway, configuration is saved to database
|
||||
}
|
||||
|
||||
// Save configuration to database
|
||||
_, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO object_storage_config (dataset_path, mount_point, pool_name, dataset_name, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, NOW(), NOW())
|
||||
ON CONFLICT (id) DO UPDATE
|
||||
SET dataset_path = $1, mount_point = $2, pool_name = $3, dataset_name = $4, updated_at = NOW()
|
||||
`, datasetPath, mountPoint, req.PoolName, req.DatasetName)
|
||||
|
||||
if err != nil {
|
||||
// If table doesn't exist, just log warning
|
||||
s.logger.Warn("Failed to save configuration to database (table may not exist)", "error", err)
|
||||
}
|
||||
|
||||
return &SetupResponse{
|
||||
DatasetPath: datasetPath,
|
||||
MountPoint: mountPoint,
|
||||
Message: fmt.Sprintf("Object storage configured to use dataset %s at %s. MinIO service needs to be restarted to use the new dataset.", datasetPath, mountPoint),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetCurrentSetup returns the current object storage configuration
|
||||
func (s *SetupService) GetCurrentSetup(ctx context.Context) (*SetupResponse, error) {
|
||||
// Check if table exists first
|
||||
var tableExists bool
|
||||
checkQuery := `
|
||||
SELECT EXISTS (
|
||||
SELECT FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'object_storage_config'
|
||||
)
|
||||
`
|
||||
err := s.db.QueryRowContext(ctx, checkQuery).Scan(&tableExists)
|
||||
if err != nil {
|
||||
s.logger.Warn("Failed to check if object_storage_config table exists", "error", err)
|
||||
return nil, nil // Return nil if can't check
|
||||
}
|
||||
|
||||
if !tableExists {
|
||||
s.logger.Debug("object_storage_config table does not exist")
|
||||
return nil, nil // No table, no configuration
|
||||
}
|
||||
|
||||
query := `
|
||||
SELECT dataset_path, mount_point, pool_name, dataset_name
|
||||
FROM object_storage_config
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 1
|
||||
`
|
||||
|
||||
var resp SetupResponse
|
||||
var poolName, datasetName string
|
||||
err = s.db.QueryRowContext(ctx, query).Scan(&resp.DatasetPath, &resp.MountPoint, &poolName, &datasetName)
|
||||
if err == sql.ErrNoRows {
|
||||
s.logger.Debug("No configuration found in database")
|
||||
return nil, nil // No configuration found
|
||||
}
|
||||
if err != nil {
|
||||
// Check if error is due to table not existing or permission denied
|
||||
errStr := err.Error()
|
||||
if strings.Contains(errStr, "does not exist") || strings.Contains(errStr, "permission denied") {
|
||||
s.logger.Debug("Table does not exist or permission denied, returning nil", "error", errStr)
|
||||
return nil, nil // Return nil instead of error
|
||||
}
|
||||
s.logger.Error("Failed to scan current setup", "error", err)
|
||||
return nil, fmt.Errorf("failed to get current setup: %w", err)
|
||||
}
|
||||
|
||||
s.logger.Debug("Found current setup", "dataset_path", resp.DatasetPath, "mount_point", resp.MountPoint, "pool", poolName, "dataset", datasetName)
|
||||
// Use dataset_path directly since it already contains the full path
|
||||
resp.Message = fmt.Sprintf("Using dataset %s at %s", resp.DatasetPath, resp.MountPoint)
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
// UpdateObjectStorage updates the object storage configuration to use a different dataset
|
||||
// This will update the configuration but won't migrate existing data
|
||||
func (s *SetupService) UpdateObjectStorage(ctx context.Context, req SetupRequest) (*SetupResponse, error) {
|
||||
// First check if there's existing configuration
|
||||
currentSetup, err := s.GetCurrentSetup(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to check current setup: %w", err)
|
||||
}
|
||||
|
||||
if currentSetup == nil {
|
||||
// No existing setup, just do normal setup
|
||||
return s.SetupObjectStorage(ctx, req)
|
||||
}
|
||||
|
||||
// There's existing setup, proceed with update
|
||||
var datasetPath, mountPoint string
|
||||
|
||||
// Normalize dataset name - if it already contains pool name, use it as-is
|
||||
var fullDatasetName string
|
||||
if strings.HasPrefix(req.DatasetName, req.PoolName+"/") {
|
||||
// Dataset name already includes pool name (e.g., "pool/dataset")
|
||||
fullDatasetName = req.DatasetName
|
||||
} else {
|
||||
// Dataset name is just the name (e.g., "dataset"), combine with pool
|
||||
fullDatasetName = fmt.Sprintf("%s/%s", req.PoolName, req.DatasetName)
|
||||
}
|
||||
|
||||
if req.CreateNew {
|
||||
// Create new dataset for object storage
|
||||
|
||||
// Check if dataset already exists
|
||||
checkCmd := exec.CommandContext(ctx, "sudo", "zfs", "list", "-H", "-o", "name", fullDatasetName)
|
||||
if err := checkCmd.Run(); err == nil {
|
||||
return nil, fmt.Errorf("dataset %s already exists", fullDatasetName)
|
||||
}
|
||||
|
||||
// Create dataset
|
||||
createCmd := exec.CommandContext(ctx, "sudo", "zfs", "create", fullDatasetName)
|
||||
if output, err := createCmd.CombinedOutput(); err != nil {
|
||||
return nil, fmt.Errorf("failed to create dataset: %s - %w", string(output), err)
|
||||
}
|
||||
|
||||
// Get mount point
|
||||
getMountCmd := exec.CommandContext(ctx, "sudo", "zfs", "get", "-H", "-o", "value", "mountpoint", fullDatasetName)
|
||||
mountOutput, err := getMountCmd.Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get mount point: %w", err)
|
||||
}
|
||||
mountPoint = strings.TrimSpace(string(mountOutput))
|
||||
|
||||
datasetPath = fullDatasetName
|
||||
s.logger.Info("Created new dataset for object storage update", "dataset", fullDatasetName, "mount_point", mountPoint)
|
||||
} else {
|
||||
// Use existing dataset
|
||||
// fullDatasetName already set above
|
||||
|
||||
// Verify dataset exists
|
||||
checkCmd := exec.CommandContext(ctx, "sudo", "zfs", "list", "-H", "-o", "name", fullDatasetName)
|
||||
if err := checkCmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("dataset %s does not exist", fullDatasetName)
|
||||
}
|
||||
|
||||
// Get mount point
|
||||
getMountCmd := exec.CommandContext(ctx, "sudo", "zfs", "get", "-H", "-o", "value", "mountpoint", fullDatasetName)
|
||||
mountOutput, err := getMountCmd.Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get mount point: %w", err)
|
||||
}
|
||||
mountPoint = strings.TrimSpace(string(mountOutput))
|
||||
|
||||
datasetPath = fullDatasetName
|
||||
s.logger.Info("Using existing dataset for object storage update", "dataset", fullDatasetName, "mount_point", mountPoint)
|
||||
}
|
||||
|
||||
// Ensure mount point directory exists
|
||||
if mountPoint != "none" && mountPoint != "" {
|
||||
if err := os.MkdirAll(mountPoint, 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create mount point directory: %w", err)
|
||||
}
|
||||
} else {
|
||||
// If no mount point, use default path
|
||||
mountPoint = filepath.Join("/opt/calypso/data/pool", req.PoolName, req.DatasetName)
|
||||
if err := os.MkdirAll(mountPoint, 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create default directory: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Update configuration in database
|
||||
_, err = s.db.ExecContext(ctx, `
|
||||
UPDATE object_storage_config
|
||||
SET dataset_path = $1, mount_point = $2, pool_name = $3, dataset_name = $4, updated_at = NOW()
|
||||
WHERE id = (SELECT id FROM object_storage_config ORDER BY updated_at DESC LIMIT 1)
|
||||
`, datasetPath, mountPoint, req.PoolName, req.DatasetName)
|
||||
|
||||
if err != nil {
|
||||
// If update fails, try insert
|
||||
_, err = s.db.ExecContext(ctx, `
|
||||
INSERT INTO object_storage_config (dataset_path, mount_point, pool_name, dataset_name, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, NOW(), NOW())
|
||||
ON CONFLICT (dataset_path) DO UPDATE
|
||||
SET mount_point = $2, pool_name = $3, dataset_name = $4, updated_at = NOW()
|
||||
`, datasetPath, mountPoint, req.PoolName, req.DatasetName)
|
||||
if err != nil {
|
||||
s.logger.Warn("Failed to update configuration in database", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Update MinIO configuration to use the selected dataset
|
||||
if err := s.updateMinIOConfig(ctx, mountPoint); err != nil {
|
||||
s.logger.Warn("Failed to update MinIO configuration", "error", err)
|
||||
// Continue anyway, configuration is saved to database
|
||||
} else {
|
||||
// Restart MinIO service to apply new configuration
|
||||
if err := s.restartMinIOService(ctx); err != nil {
|
||||
s.logger.Warn("Failed to restart MinIO service", "error", err)
|
||||
// Continue anyway, user can restart manually
|
||||
}
|
||||
}
|
||||
|
||||
return &SetupResponse{
|
||||
DatasetPath: datasetPath,
|
||||
MountPoint: mountPoint,
|
||||
Message: fmt.Sprintf("Object storage updated to use dataset %s at %s. Note: Existing data in previous dataset (%s) is not migrated automatically. MinIO service has been restarted.", datasetPath, mountPoint, currentSetup.DatasetPath),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// updateMinIOConfig updates MinIO configuration file to use dataset mount point directly
|
||||
// Note: MinIO erasure coding requires direct directory paths, not symlinks
|
||||
func (s *SetupService) updateMinIOConfig(ctx context.Context, datasetMountPoint string) error {
|
||||
configFile := "/opt/calypso/conf/minio/minio.conf"
|
||||
|
||||
// Ensure dataset mount point directory exists and has correct ownership
|
||||
if err := os.MkdirAll(datasetMountPoint, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create dataset mount point directory: %w", err)
|
||||
}
|
||||
|
||||
// Set ownership to minio-user so MinIO can write to it
|
||||
if err := exec.CommandContext(ctx, "sudo", "chown", "-R", "minio-user:minio-user", datasetMountPoint).Run(); err != nil {
|
||||
s.logger.Warn("Failed to set ownership on dataset mount point", "path", datasetMountPoint, "error", err)
|
||||
// Continue anyway, might already have correct ownership
|
||||
}
|
||||
|
||||
// Set permissions
|
||||
if err := exec.CommandContext(ctx, "sudo", "chmod", "755", datasetMountPoint).Run(); err != nil {
|
||||
s.logger.Warn("Failed to set permissions on dataset mount point", "path", datasetMountPoint, "error", err)
|
||||
}
|
||||
|
||||
s.logger.Info("Prepared dataset mount point for MinIO", "path", datasetMountPoint)
|
||||
|
||||
// Read current config file
|
||||
configContent, err := os.ReadFile(configFile)
|
||||
if err != nil {
|
||||
// If file doesn't exist, create it
|
||||
if os.IsNotExist(err) {
|
||||
configContent = []byte(fmt.Sprintf("MINIO_ROOT_USER=admin\nMINIO_ROOT_PASSWORD=HqBX1IINqFynkWFa\nMINIO_VOLUMES=%s\n", datasetMountPoint))
|
||||
} else {
|
||||
return fmt.Errorf("failed to read MinIO config file: %w", err)
|
||||
}
|
||||
} else {
|
||||
// Update MINIO_VOLUMES in config
|
||||
lines := strings.Split(string(configContent), "\n")
|
||||
updated := false
|
||||
for i, line := range lines {
|
||||
if strings.HasPrefix(strings.TrimSpace(line), "MINIO_VOLUMES=") {
|
||||
lines[i] = fmt.Sprintf("MINIO_VOLUMES=%s", datasetMountPoint)
|
||||
updated = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !updated {
|
||||
// Add MINIO_VOLUMES if not found
|
||||
lines = append(lines, fmt.Sprintf("MINIO_VOLUMES=%s", datasetMountPoint))
|
||||
}
|
||||
configContent = []byte(strings.Join(lines, "\n"))
|
||||
}
|
||||
|
||||
// Write updated config using sudo
|
||||
// Write temp file to a location we can write to
|
||||
userTempFile := fmt.Sprintf("/tmp/minio.conf.%d.tmp", os.Getpid())
|
||||
if err := os.WriteFile(userTempFile, configContent, 0644); err != nil {
|
||||
return fmt.Errorf("failed to write temp config file: %w", err)
|
||||
}
|
||||
defer os.Remove(userTempFile) // Cleanup
|
||||
|
||||
// Copy temp file to config location with sudo
|
||||
if err := exec.CommandContext(ctx, "sudo", "cp", userTempFile, configFile).Run(); err != nil {
|
||||
return fmt.Errorf("failed to update config file: %w", err)
|
||||
}
|
||||
|
||||
// Set proper ownership and permissions
|
||||
if err := exec.CommandContext(ctx, "sudo", "chown", "minio-user:minio-user", configFile).Run(); err != nil {
|
||||
s.logger.Warn("Failed to set config file ownership", "error", err)
|
||||
}
|
||||
if err := exec.CommandContext(ctx, "sudo", "chmod", "644", configFile).Run(); err != nil {
|
||||
s.logger.Warn("Failed to set config file permissions", "error", err)
|
||||
}
|
||||
|
||||
s.logger.Info("Updated MinIO configuration", "config_file", configFile, "volumes", datasetMountPoint)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// restartMinIOService restarts the MinIO service to apply new configuration
|
||||
func (s *SetupService) restartMinIOService(ctx context.Context) error {
|
||||
// Restart MinIO service using sudo
|
||||
cmd := exec.CommandContext(ctx, "sudo", "systemctl", "restart", "minio.service")
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("failed to restart MinIO service: %w", err)
|
||||
}
|
||||
|
||||
// Wait a moment for service to start
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
// Verify service is running
|
||||
checkCmd := exec.CommandContext(ctx, "sudo", "systemctl", "is-active", "minio.service")
|
||||
output, err := checkCmd.Output()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check MinIO service status: %w", err)
|
||||
}
|
||||
|
||||
status := strings.TrimSpace(string(output))
|
||||
if status != "active" {
|
||||
return fmt.Errorf("MinIO service is not active after restart, status: %s", status)
|
||||
}
|
||||
|
||||
s.logger.Info("MinIO service restarted successfully")
|
||||
return nil
|
||||
}
|
||||
@@ -730,10 +730,36 @@ func (s *ZFSService) CreateDataset(ctx context.Context, poolName string, req Cre
|
||||
// Construct full dataset name
|
||||
fullName := poolName + "/" + req.Name
|
||||
|
||||
// For filesystem datasets, create mount directory if mount point is provided
|
||||
if req.Type == "filesystem" && req.MountPoint != "" {
|
||||
// Clean and validate mount point path
|
||||
mountPath := filepath.Clean(req.MountPoint)
|
||||
// Get pool mount point to validate dataset mount point is within pool directory
|
||||
poolMountPoint := fmt.Sprintf("/opt/calypso/data/pool/%s", poolName)
|
||||
var mountPath string
|
||||
|
||||
// For filesystem datasets, validate and set mount point
|
||||
if req.Type == "filesystem" {
|
||||
if req.MountPoint != "" {
|
||||
// User provided mount point - validate it's within pool directory
|
||||
mountPath = filepath.Clean(req.MountPoint)
|
||||
|
||||
// Check if mount point is within pool mount point directory
|
||||
poolMountAbs, err := filepath.Abs(poolMountPoint)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve pool mount point: %w", err)
|
||||
}
|
||||
|
||||
mountPathAbs, err := filepath.Abs(mountPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve mount point: %w", err)
|
||||
}
|
||||
|
||||
// Check if mount path is within pool mount point directory
|
||||
relPath, err := filepath.Rel(poolMountAbs, mountPathAbs)
|
||||
if err != nil || strings.HasPrefix(relPath, "..") {
|
||||
return nil, fmt.Errorf("mount point must be within pool directory: %s (pool mount: %s)", mountPath, poolMountPoint)
|
||||
}
|
||||
} else {
|
||||
// No mount point provided - use default: /opt/calypso/data/pool/<pool-name>/<dataset-name>/
|
||||
mountPath = filepath.Join(poolMountPoint, req.Name)
|
||||
}
|
||||
|
||||
// Check if directory already exists
|
||||
if info, err := os.Stat(mountPath); err == nil {
|
||||
@@ -782,9 +808,9 @@ func (s *ZFSService) CreateDataset(ctx context.Context, poolName string, req Cre
|
||||
args = append(args, "-o", fmt.Sprintf("compression=%s", req.Compression))
|
||||
}
|
||||
|
||||
// Set mount point if provided (only for filesystems, not volumes)
|
||||
if req.Type == "filesystem" && req.MountPoint != "" {
|
||||
args = append(args, "-o", fmt.Sprintf("mountpoint=%s", req.MountPoint))
|
||||
// Set mount point for filesystems (always set, either user-provided or default)
|
||||
if req.Type == "filesystem" {
|
||||
args = append(args, "-o", fmt.Sprintf("mountpoint=%s", mountPath))
|
||||
}
|
||||
|
||||
// Execute zfs create
|
||||
|
||||
@@ -133,6 +133,18 @@ func (h *Handler) ListNetworkInterfaces(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, gin.H{"interfaces": interfaces})
|
||||
}
|
||||
|
||||
// GetManagementIPAddress returns the management IP address
|
||||
func (h *Handler) GetManagementIPAddress(c *gin.Context) {
|
||||
ip, err := h.service.GetManagementIPAddress(c.Request.Context())
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to get management IP address", "error", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get management IP address"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"ip_address": ip})
|
||||
}
|
||||
|
||||
// SaveNTPSettings saves NTP configuration to the OS
|
||||
func (h *Handler) SaveNTPSettings(c *gin.Context) {
|
||||
var settings NTPSettings
|
||||
|
||||
@@ -648,6 +648,40 @@ func (s *Service) ListNetworkInterfaces(ctx context.Context) ([]NetworkInterface
|
||||
return interfaces, nil
|
||||
}
|
||||
|
||||
// GetManagementIPAddress returns the IP address of the management interface
|
||||
func (s *Service) GetManagementIPAddress(ctx context.Context) (string, error) {
|
||||
interfaces, err := s.ListNetworkInterfaces(ctx)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to list network interfaces: %w", err)
|
||||
}
|
||||
|
||||
// First, try to find interface with Role "Management"
|
||||
for _, iface := range interfaces {
|
||||
if iface.Role == "Management" && iface.IPAddress != "" && iface.Status == "Connected" {
|
||||
s.logger.Info("Found management interface", "interface", iface.Name, "ip", iface.IPAddress)
|
||||
return iface.IPAddress, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: use interface with default route (primary interface)
|
||||
for _, iface := range interfaces {
|
||||
if iface.Gateway != "" && iface.IPAddress != "" && iface.Status == "Connected" {
|
||||
s.logger.Info("Using primary interface as management", "interface", iface.Name, "ip", iface.IPAddress)
|
||||
return iface.IPAddress, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Final fallback: use first connected interface with IP
|
||||
for _, iface := range interfaces {
|
||||
if iface.IPAddress != "" && iface.Status == "Connected" && iface.Name != "lo" {
|
||||
s.logger.Info("Using first connected interface as management", "interface", iface.Name, "ip", iface.IPAddress)
|
||||
return iface.IPAddress, nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("no management interface found")
|
||||
}
|
||||
|
||||
// UpdateNetworkInterfaceRequest represents the request to update a network interface
|
||||
type UpdateNetworkInterfaceRequest struct {
|
||||
IPAddress string `json:"ip_address"`
|
||||
|
||||
Reference in New Issue
Block a user