initial commit
This commit is contained in:
Othman Hendy Suseno
210
cmd/migrate.go
Normal file
210
cmd/migrate.go
Normal file
@@ -0,0 +1,210 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/cloudmigration/drive-migrator/internal/tracker"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
sourceName string
|
||||
sourceType string
|
||||
destName string
|
||||
destType string
|
||||
trackFile string
|
||||
resume bool
|
||||
include []string
|
||||
exclude []string
|
||||
|
||||
// Source remote config flags
|
||||
sourceURL string
|
||||
sourceUser string
|
||||
sourcePass string
|
||||
sourceSAFile string
|
||||
|
||||
// Dest remote config flags
|
||||
destURL string
|
||||
destUser string
|
||||
destPass string
|
||||
destSAFile string
|
||||
)
|
||||
|
||||
func init() {
|
||||
migrateCmd := &cobra.Command{
|
||||
Use: "migrate",
|
||||
Short: "Run migration between two cloud drives (auto-creates remotes if needed)",
|
||||
RunE: runMigrate,
|
||||
}
|
||||
|
||||
migrateCmd.Flags().StringVar(&sourceName, "source-name", "", "Nama remote/source di rclone (akan dibuat otomatis jika belum ada)")
|
||||
migrateCmd.Flags().StringVar(&sourceType, "source-type", "", "Tipe cloud drive sumber (nextcloud, webdav, gdrive, onedrive, dll)")
|
||||
migrateCmd.Flags().StringVar(&destName, "dest-name", "", "Nama remote/tujuan di rclone (akan dibuat otomatis jika belum ada)")
|
||||
migrateCmd.Flags().StringVar(&destType, "dest-type", "", "Tipe cloud drive tujuan")
|
||||
migrateCmd.Flags().StringVar(&trackFile, "track-file", "migration.db", "Lokasi file tracking (bolt)")
|
||||
migrateCmd.Flags().BoolVar(&resume, "resume", false, "Lanjutkan migrasi yang terhenti")
|
||||
migrateCmd.Flags().StringSliceVar(&include, "include", nil, "Pola file/folder yang akan dimigrasikan (bisa diulang)")
|
||||
migrateCmd.Flags().StringSliceVar(&exclude, "exclude", nil, "Pola file/folder yang akan dikecualikan (bisa diulang)")
|
||||
|
||||
// Source remote extra flags
|
||||
migrateCmd.Flags().StringVar(&sourceURL, "source-url", "", "URL untuk remote sumber (Nextcloud/WebDAV)")
|
||||
migrateCmd.Flags().StringVar(&sourceUser, "source-user", "", "Username remote sumber")
|
||||
migrateCmd.Flags().StringVar(&sourcePass, "source-pass", "", "Password remote sumber")
|
||||
migrateCmd.Flags().StringVar(&sourceSAFile, "source-sa-file", "", "Service account JSON untuk GDrive sumber")
|
||||
|
||||
// Dest remote extra flags
|
||||
migrateCmd.Flags().StringVar(&destURL, "dest-url", "", "URL untuk remote tujuan (Nextcloud/WebDAV)")
|
||||
migrateCmd.Flags().StringVar(&destUser, "dest-user", "", "Username remote tujuan")
|
||||
migrateCmd.Flags().StringVar(&destPass, "dest-pass", "", "Password remote tujuan")
|
||||
migrateCmd.Flags().StringVar(&destSAFile, "dest-sa-file", "", "Service account JSON untuk GDrive tujuan")
|
||||
|
||||
migrateCmd.MarkFlagRequired("source-name")
|
||||
migrateCmd.MarkFlagRequired("dest-name")
|
||||
|
||||
rootCmd.AddCommand(migrateCmd)
|
||||
}
|
||||
|
||||
// ensureRemoteExists checks if remote exists and if not, creates it using provided parameters.
|
||||
func ensureRemoteExists(ctx context.Context, name, rtype, url, user, pass, saFile string) error {
|
||||
// Check if remote exists using rclone listremotes
|
||||
listCmd := exec.CommandContext(ctx, "rclone", "listremotes")
|
||||
var out bytes.Buffer
|
||||
listCmd.Stdout = &out
|
||||
if err := listCmd.Run(); err != nil {
|
||||
return fmt.Errorf("failed to list rclone remotes: %w", err)
|
||||
}
|
||||
remotes := strings.Split(out.String(), "\n")
|
||||
for _, r := range remotes {
|
||||
if strings.TrimSuffix(r, ":") == name {
|
||||
logrus.Debugf("Remote %s already exists", name)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
logrus.Infof("Remote %s not found, creating...", name)
|
||||
|
||||
switch strings.ToLower(rtype) {
|
||||
case "nextcloud", "webdav":
|
||||
if url == "" {
|
||||
return fmt.Errorf("url is required to create webdav/nextcloud remote")
|
||||
}
|
||||
// If pass provided, obscure it first
|
||||
encPass := pass
|
||||
if pass != "" {
|
||||
obscCmd := exec.CommandContext(ctx, "rclone", "obscure", pass)
|
||||
var ob bytes.Buffer
|
||||
obscCmd.Stdout = &ob
|
||||
if err := obscCmd.Run(); err != nil {
|
||||
return fmt.Errorf("failed to obscure password: %w", err)
|
||||
}
|
||||
encPass = strings.TrimSpace(ob.String())
|
||||
}
|
||||
args := []string{"config", "create", name, "webdav", "vendor", "nextcloud", "url", url}
|
||||
if user != "" {
|
||||
args = append(args, "user", user)
|
||||
}
|
||||
if encPass != "" {
|
||||
args = append(args, "pass", encPass)
|
||||
}
|
||||
createCmd := exec.CommandContext(ctx, "rclone", args...)
|
||||
createCmd.Stdout = logrus.StandardLogger().Writer()
|
||||
createCmd.Stderr = logrus.StandardLogger().Writer()
|
||||
if err := createCmd.Run(); err != nil {
|
||||
return fmt.Errorf("failed to create remote: %w", err)
|
||||
}
|
||||
case "drive", "gdrive":
|
||||
if saFile == "" {
|
||||
return fmt.Errorf("sa-file is required to create gdrive remote")
|
||||
}
|
||||
args := []string{"config", "create", name, "drive", "service_account_file", saFile}
|
||||
createCmd := exec.CommandContext(ctx, "rclone", args...)
|
||||
createCmd.Stdout = logrus.StandardLogger().Writer()
|
||||
createCmd.Stderr = logrus.StandardLogger().Writer()
|
||||
if err := createCmd.Run(); err != nil {
|
||||
return fmt.Errorf("failed to create gdrive remote: %w", err)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("auto create not supported for remote type: %s", rtype)
|
||||
}
|
||||
|
||||
logrus.Infof("Remote %s created successfully", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runMigrate(cmd *cobra.Command, args []string) error {
|
||||
ctx := context.Background()
|
||||
|
||||
// Ensure remotes exist or create them automatically
|
||||
if err := ensureRemoteExists(ctx, sourceName, sourceType, sourceURL, sourceUser, sourcePass, sourceSAFile); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureRemoteExists(ctx, destName, destType, destURL, destUser, destPass, destSAFile); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Initialize tracker
|
||||
t, err := tracker.NewTracker(trackFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize tracker: %w", err)
|
||||
}
|
||||
defer t.Close()
|
||||
|
||||
// Create job ID
|
||||
jobID := fmt.Sprintf("%s-to-%s-%d", sourceName, destName, time.Now().Unix())
|
||||
|
||||
job := &tracker.MigrationJob{
|
||||
ID: jobID,
|
||||
SourceName: sourceName,
|
||||
DestName: destName,
|
||||
Status: tracker.StatusInProgress,
|
||||
StartTime: time.Now(),
|
||||
}
|
||||
if err := t.CreateJob(job); err != nil {
|
||||
return fmt.Errorf("failed to create job: %w", err)
|
||||
}
|
||||
|
||||
logrus.Infof("Starting migration from %s to %s (Job ID: %s)", sourceName, destName, jobID)
|
||||
|
||||
// Build rclone arguments
|
||||
rcloneArgs := []string{"copy", sourceName + ":", destName + ":", "--progress", "--stats", "10s"}
|
||||
|
||||
for _, pattern := range include {
|
||||
rcloneArgs = append(rcloneArgs, "--include", pattern)
|
||||
}
|
||||
for _, pattern := range exclude {
|
||||
rcloneArgs = append(rcloneArgs, "--exclude", pattern)
|
||||
}
|
||||
if resume {
|
||||
rcloneArgs = append(rcloneArgs, "--ignore-existing")
|
||||
logrus.Info("Resume mode: ignoring existing files")
|
||||
}
|
||||
|
||||
cmdRclone := exec.CommandContext(ctx, "rclone", rcloneArgs...)
|
||||
cmdRclone.Stdout = logrus.StandardLogger().Writer()
|
||||
cmdRclone.Stderr = logrus.StandardLogger().Writer()
|
||||
|
||||
start := time.Now()
|
||||
|
||||
if err := cmdRclone.Run(); err != nil {
|
||||
job.Status = tracker.StatusFailed
|
||||
job.EndTime = time.Now()
|
||||
job.ErrorMessage = err.Error()
|
||||
t.UpdateJob(job)
|
||||
return fmt.Errorf("rclone finished with error: %w", err)
|
||||
}
|
||||
|
||||
job.Status = tracker.StatusCompleted
|
||||
job.EndTime = time.Now()
|
||||
t.UpdateJob(job)
|
||||
|
||||
elapsed := time.Since(start)
|
||||
logrus.Infof("Migration completed in %s (Job ID: %s)", elapsed, jobID)
|
||||
t.LogProgress(jobID)
|
||||
return nil
|
||||
}
|
||||
109
cmd/remote.go
Normal file
109
cmd/remote.go
Normal file
@@ -0,0 +1,109 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
remoteName string
|
||||
remoteType string
|
||||
remoteURL string
|
||||
remoteUser string
|
||||
remotePass string
|
||||
saFile string
|
||||
)
|
||||
|
||||
func init() {
|
||||
remoteCmd := &cobra.Command{
|
||||
Use: "remote",
|
||||
Short: "Manage rclone remotes via drive-migrator",
|
||||
}
|
||||
|
||||
createCmd := &cobra.Command{
|
||||
Use: "create",
|
||||
Short: "Create new rclone remote",
|
||||
RunE: runRemoteCreate,
|
||||
}
|
||||
|
||||
// Common flags
|
||||
createCmd.Flags().StringVar(&remoteName, "name", "", "Remote name (required)")
|
||||
createCmd.Flags().StringVar(&remoteType, "type", "", "Remote type (nextcloud|gdrive|webdav|onedrive|...) (required)")
|
||||
|
||||
// WebDAV / Nextcloud
|
||||
createCmd.Flags().StringVar(&remoteURL, "url", "", "URL for WebDAV/Nextcloud remote")
|
||||
createCmd.Flags().StringVar(&remoteUser, "user", "", "Username for remote")
|
||||
createCmd.Flags().StringVar(&remotePass, "pass", "", "Password for remote")
|
||||
|
||||
// Google Drive
|
||||
createCmd.Flags().StringVar(&saFile, "sa-file", "", "Service account JSON file for Google Drive")
|
||||
|
||||
createCmd.MarkFlagRequired("name")
|
||||
createCmd.MarkFlagRequired("type")
|
||||
|
||||
remoteCmd.AddCommand(createCmd)
|
||||
rootCmd.AddCommand(remoteCmd)
|
||||
}
|
||||
|
||||
func runRemoteCreate(cmd *cobra.Command, args []string) error {
|
||||
ctx := context.Background()
|
||||
|
||||
if remoteName == "" {
|
||||
return fmt.Errorf("remote name is required")
|
||||
}
|
||||
|
||||
if remoteType == "" {
|
||||
return fmt.Errorf("remote type is required")
|
||||
}
|
||||
|
||||
var rcloneArgs []string
|
||||
|
||||
switch remoteType {
|
||||
case "nextcloud":
|
||||
if remoteURL == "" || remoteUser == "" || remotePass == "" {
|
||||
return fmt.Errorf("url, user, and pass are required for nextcloud remote")
|
||||
}
|
||||
rcloneArgs = []string{"config", "create", remoteName, "webdav", "vendor", "nextcloud", "url", remoteURL, "user", remoteUser, "pass", remotePass}
|
||||
case "webdav":
|
||||
if remoteURL == "" {
|
||||
return fmt.Errorf("url is required for webdav remote")
|
||||
}
|
||||
// If user/pass provided, include them
|
||||
rcloneArgs = []string{"config", "create", remoteName, "webdav", "url", remoteURL}
|
||||
if remoteUser != "" {
|
||||
rcloneArgs = append(rcloneArgs, "user", remoteUser)
|
||||
}
|
||||
if remotePass != "" {
|
||||
rcloneArgs = append(rcloneArgs, "pass", remotePass)
|
||||
}
|
||||
case "gdrive", "drive":
|
||||
if saFile == "" {
|
||||
return fmt.Errorf("sa-file is required for gdrive remote (service account)")
|
||||
}
|
||||
// Ensure file exists
|
||||
if _, err := os.Stat(saFile); err != nil {
|
||||
return fmt.Errorf("service account file not found: %v", err)
|
||||
}
|
||||
rcloneArgs = []string{"config", "create", remoteName, "drive", "service_account_file", saFile}
|
||||
default:
|
||||
return fmt.Errorf("unsupported remote type: %s", remoteType)
|
||||
}
|
||||
|
||||
logrus.Infof("Creating rclone remote '%s' of type '%s'", remoteName, remoteType)
|
||||
|
||||
cmdRclone := exec.CommandContext(ctx, "rclone", rcloneArgs...)
|
||||
cmdRclone.Stdout = logrus.StandardLogger().Writer()
|
||||
cmdRclone.Stderr = logrus.StandardLogger().Writer()
|
||||
|
||||
if err := cmdRclone.Run(); err != nil {
|
||||
return fmt.Errorf("failed to create remote: %w", err)
|
||||
}
|
||||
|
||||
logrus.Infof("Remote '%s' created successfully", remoteName)
|
||||
return nil
|
||||
}
|
||||
99
cmd/root.go
Normal file
99
cmd/root.go
Normal file
@@ -0,0 +1,99 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
var cfgFile string
|
||||
|
||||
// rootCmd is the base command for drive-migrator CLI.
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "drive-migrator",
|
||||
Short: "Simple cloud drive migration tool powered by rclone",
|
||||
Long: `Drive Migrator adalah alat baris perintah sederhana untuk memigrasi data antar berbagai cloud drive
|
||||
(misal Google Drive, OneDrive, NextCloud, dll) menggunakan rclone di belakang layar.
|
||||
|
||||
Fitur:
|
||||
- Migrasi antar berbagai cloud storage
|
||||
- Tracking progress dengan database lokal
|
||||
- Resume migrasi yang terputus
|
||||
- Filter file/folder dengan include/exclude patterns
|
||||
- Logging detail untuk monitoring`,
|
||||
}
|
||||
|
||||
// Execute runs the root command.
|
||||
func Execute() {
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
cobra.OnInitialize(initConfig, initLogger)
|
||||
|
||||
// Persistent Flags
|
||||
rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.drive-migrator.yaml)")
|
||||
rootCmd.PersistentFlags().Bool("debug", false, "enable debug logging")
|
||||
rootCmd.PersistentFlags().String("log-file", "", "log file path (default: logs to stdout)")
|
||||
}
|
||||
|
||||
func initConfig() {
|
||||
if cfgFile != "" {
|
||||
viper.SetConfigFile(cfgFile)
|
||||
} else {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
viper.AddConfigPath(home)
|
||||
viper.SetConfigName(".drive-migrator")
|
||||
}
|
||||
|
||||
viper.SetEnvPrefix("DRIVE_MIG")
|
||||
viper.AutomaticEnv()
|
||||
|
||||
if err := viper.ReadInConfig(); err == nil {
|
||||
fmt.Println("Using config file:", viper.ConfigFileUsed())
|
||||
}
|
||||
}
|
||||
|
||||
func initLogger() {
|
||||
logrus.SetFormatter(&logrus.TextFormatter{
|
||||
FullTimestamp: true,
|
||||
ForceColors: true,
|
||||
})
|
||||
|
||||
// Set log level
|
||||
logrus.SetLevel(logrus.InfoLevel)
|
||||
if viper.GetBool("debug") {
|
||||
logrus.SetLevel(logrus.DebugLevel)
|
||||
logrus.Debug("Debug logging enabled")
|
||||
}
|
||||
|
||||
// Set log file if specified
|
||||
logFile := viper.GetString("log-file")
|
||||
if logFile != "" {
|
||||
// Create log directory if it doesn't exist
|
||||
logDir := filepath.Dir(logFile)
|
||||
if err := os.MkdirAll(logDir, 0755); err != nil {
|
||||
logrus.Fatalf("Failed to create log directory: %v", err)
|
||||
}
|
||||
|
||||
file, err := os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
|
||||
if err != nil {
|
||||
logrus.Fatalf("Failed to open log file: %v", err)
|
||||
}
|
||||
|
||||
logrus.SetOutput(file)
|
||||
logrus.Infof("Logging to file: %s", logFile)
|
||||
}
|
||||
}
|
||||
74
cmd/status.go
Normal file
74
cmd/status.go
Normal file
@@ -0,0 +1,74 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/cloudmigration/drive-migrator/internal/tracker"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
statusCmd := &cobra.Command{
|
||||
Use: "status [job-id]",
|
||||
Short: "Show migration status",
|
||||
Long: "Show status of migration jobs. If no job-id provided, shows all recent jobs.",
|
||||
RunE: runStatus,
|
||||
}
|
||||
|
||||
listCmd := &cobra.Command{
|
||||
Use: "list",
|
||||
Short: "List all migration jobs",
|
||||
RunE: runList,
|
||||
}
|
||||
|
||||
rootCmd.AddCommand(statusCmd)
|
||||
rootCmd.AddCommand(listCmd)
|
||||
}
|
||||
|
||||
func runStatus(cmd *cobra.Command, args []string) error {
|
||||
trackFile := "migration.db" // Default, could be made configurable
|
||||
|
||||
t, err := tracker.NewTracker(trackFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize tracker: %w", err)
|
||||
}
|
||||
defer t.Close()
|
||||
|
||||
if len(args) > 0 {
|
||||
jobID := args[0]
|
||||
job, err := t.GetJob(jobID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("job not found: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Job ID: %s\n", job.ID)
|
||||
fmt.Printf("Source: %s\n", job.SourceName)
|
||||
fmt.Printf("Destination: %s\n", job.DestName)
|
||||
fmt.Printf("Status: %s\n", job.Status)
|
||||
fmt.Printf("Start Time: %s\n", job.StartTime.Format("2006-01-02 15:04:05"))
|
||||
if !job.EndTime.IsZero() {
|
||||
fmt.Printf("End Time: %s\n", job.EndTime.Format("2006-01-02 15:04:05"))
|
||||
fmt.Printf("Duration: %s\n", job.EndTime.Sub(job.StartTime))
|
||||
}
|
||||
if job.ErrorMessage != "" {
|
||||
fmt.Printf("Error: %s\n", job.ErrorMessage)
|
||||
}
|
||||
|
||||
completed := t.GetCompletedFiles(jobID)
|
||||
fmt.Printf("Completed Files: %d\n", completed)
|
||||
if job.TotalFiles > 0 {
|
||||
percentage := float64(completed) / float64(job.TotalFiles) * 100
|
||||
fmt.Printf("Progress: %.2f%%\n", percentage)
|
||||
}
|
||||
} else {
|
||||
fmt.Println("Please provide a job ID. Use 'list' command to see all jobs.")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func runList(cmd *cobra.Command, args []string) error {
|
||||
fmt.Println("Job listing feature will be implemented soon.")
|
||||
fmt.Println("For now, check the migration.db file or use status with specific job ID.")
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user