feat: implement initial GoDump backup service with management web UI and MySQL support

2026-06-04 16:59:53 +00:00
parent 2dc6ee723f
commit 318f6c9b46
16 changed files with 1935 additions and 1 deletions
@@ -0,0 +1,50 @@
+package backup
+
+import (
+	"database/sql"
+	"fmt"
+
+	"godump/config"
+
+	_ "github.com/go-sql-driver/mysql"
+)
+
+func discoverDatabases(cfg config.InstanceConfig) ([]string, error) {
+	dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/?parseTime=true", cfg.User, cfg.Password, cfg.Host, cfg.Port)
+	db, err := sql.Open("mysql", dsn)
+	if err != nil {
+		return nil, err
+	}
+	defer db.Close()
+
+	// Ensure the connection is actually valid
+	if err := db.Ping(); err != nil {
+		return nil, err
+	}
+
+	query := `
+		SELECT SCHEMA_NAME 
+		FROM INFORMATION_SCHEMA.SCHEMATA 
+		WHERE SCHEMA_NAME NOT IN ('information_schema', 'performance_schema', 'mysql', 'sys')
+	`
+	rows, err := db.Query(query)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var databases []string
+	for rows.Next() {
+		var name string
+		if err := rows.Scan(&name); err != nil {
+			return nil, err
+		}
+		databases = append(databases, name)
+	}
+
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
+	return databases, nil
+}
@@ -0,0 +1,311 @@
+package backup
+
+import (
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"godump/config"
+	"godump/logger"
+
+	"github.com/robfig/cron/v3"
+)
+
+type DBStatus struct {
+	Name            string    `json:"name"`
+	FirstDiscovered time.Time `json:"first_discovered"`
+	LastBackupTime  time.Time `json:"last_backup_time"`
+	LastBackupSize  int64     `json:"last_backup_size"`
+	LastBackupResult string   `json:"last_backup_result"` // success, skipped, failed
+}
+
+type InstanceStatus struct {
+	Config          config.InstanceConfig
+	LastRunTime     time.Time
+	NextRunTime     time.Time
+	OverallResult   string // success, partial, failed, running
+	Databases       map[string]*DBStatus
+	IsRunning       bool
+	CronEntryID     cron.EntryID
+	mu              sync.RWMutex
+}
+
+type DBStatusSnapshot struct {
+	Name            string    `json:"name"`
+	FirstDiscovered time.Time `json:"first_discovered"`
+	LastBackupTime  time.Time `json:"last_backup_time"`
+	LastBackupSize  int64     `json:"last_backup_size"`
+	LastBackupResult string   `json:"last_backup_result"`
+}
+
+type InstanceSnapshot struct {
+	Name          string
+	Host          string
+	LastRunTime   time.Time
+	NextRunTime   time.Time
+	OverallResult string
+	IsRunning     bool
+	Databases     []DBStatusSnapshot
+}
+
+func (s *InstanceStatus) Snapshot() InstanceSnapshot {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	snap := InstanceSnapshot{
+		Name:          s.Config.Name,
+		Host:          s.Config.Host,
+		LastRunTime:   s.LastRunTime,
+		NextRunTime:   s.NextRunTime,
+		OverallResult: s.OverallResult,
+		IsRunning:     s.IsRunning,
+		Databases:     make([]DBStatusSnapshot, 0, len(s.Databases)),
+	}
+
+	for _, db := range s.Databases {
+		snap.Databases = append(snap.Databases, DBStatusSnapshot{
+			Name:             db.Name,
+			FirstDiscovered:  db.FirstDiscovered,
+			LastBackupTime:   db.LastBackupTime,
+			LastBackupSize:   db.LastBackupSize,
+			LastBackupResult: db.LastBackupResult,
+		})
+	}
+
+	return snap
+}
+
+type Manager struct {
+	instances map[string]*InstanceStatus
+	cron      *cron.Cron
+	mu        sync.RWMutex
+}
+
+func NewManager(cfg *config.Config) *Manager {
+	c := cron.New()
+	c.Start()
+
+	m := &Manager{
+		instances: make(map[string]*InstanceStatus),
+		cron:      c,
+	}
+
+	for _, instCfg := range cfg.Instances {
+		status := &InstanceStatus{
+			Config:    instCfg,
+			Databases: make(map[string]*DBStatus),
+		}
+		m.instances[instCfg.Name] = status
+
+		// Schedule cron job
+		if instCfg.Schedule != "" {
+			var id cron.EntryID
+			id, err := c.AddFunc(instCfg.Schedule, func(name string) func() {
+				return func() {
+					m.RunInstance(name)
+				}
+			}(instCfg.Name))
+			
+			if err != nil {
+				logger.Error(instCfg.Name, "Failed to schedule cron job: %v", err)
+			} else {
+				status.CronEntryID = id
+				status.NextRunTime = c.Entry(id).Next
+				logger.Info(instCfg.Name, "Scheduled backups, next run at %v", status.NextRunTime)
+			}
+		}
+	}
+
+	return m
+}
+
+func (m *Manager) GetInstances() []*InstanceStatus {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	var result []*InstanceStatus
+	for _, status := range m.instances {
+		result = append(result, status)
+	}
+	return result
+}
+
+func (m *Manager) GetInstance(name string) *InstanceStatus {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.instances[name]
+}
+
+func (m *Manager) DiscoverInitial() {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	for name, inst := range m.instances {
+		dbs, err := discoverDatabases(inst.Config)
+		if err != nil {
+			logger.Error(name, "Initial database discovery failed: %v", err)
+			inst.mu.Lock()
+			inst.OverallResult = "failed"
+			inst.mu.Unlock()
+			continue
+		}
+
+		inst.mu.Lock()
+		var latestInstanceTime time.Time
+		var hasAnyBackup bool
+
+		for _, db := range dbs {
+			if _, exists := inst.Databases[db]; !exists {
+				var lastTime time.Time
+				var lastSize int64
+				var lastResult string
+
+				dbPath := filepath.Join(inst.Config.BackupDir, db)
+				if entries, err := os.ReadDir(dbPath); err == nil {
+					for _, e := range entries {
+						if e.IsDir() {
+							continue
+						}
+						if info, err := e.Info(); err == nil {
+							if info.ModTime().After(lastTime) {
+								lastTime = info.ModTime()
+								lastSize = info.Size()
+								lastResult = "success"
+							}
+						}
+					}
+				}
+
+				inst.Databases[db] = &DBStatus{
+					Name:             db,
+					FirstDiscovered:  time.Now(),
+					LastBackupTime:   lastTime,
+					LastBackupSize:   lastSize,
+					LastBackupResult: lastResult,
+				}
+				if lastTime.After(latestInstanceTime) {
+					latestInstanceTime = lastTime
+				}
+				if lastResult != "" {
+					hasAnyBackup = true
+				}
+				logger.Info(name, "Discovered initial database: %s", db)
+			}
+		}
+		
+		if latestInstanceTime.After(inst.LastRunTime) {
+			inst.LastRunTime = latestInstanceTime
+		}
+		if hasAnyBackup && inst.OverallResult == "" {
+			inst.OverallResult = "success"
+		}
+		inst.mu.Unlock()
+	}
+}
+
+func (m *Manager) RunAll() {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	for name := range m.instances {
+		go m.RunInstance(name)
+	}
+}
+
+func (m *Manager) RunInstance(name string) {
+	inst := m.GetInstance(name)
+	if inst == nil {
+		return
+	}
+
+	inst.mu.Lock()
+	if inst.IsRunning {
+		inst.mu.Unlock()
+		logger.Warn(name, "Backup job already running, skipping")
+		return
+	}
+	inst.IsRunning = true
+	inst.OverallResult = "running"
+	inst.mu.Unlock()
+
+	defer func() {
+		inst.mu.Lock()
+		inst.IsRunning = false
+		inst.LastRunTime = time.Now()
+		if inst.CronEntryID != 0 {
+			inst.NextRunTime = m.cron.Entry(inst.CronEntryID).Next
+		}
+		inst.mu.Unlock()
+	}()
+
+	logger.Info(name, "Starting backup job")
+
+	// 1. Discovery
+	dbs, err := discoverDatabases(inst.Config)
+	if err != nil {
+		logger.Error(name, "Database discovery failed: %v", err)
+		inst.mu.Lock()
+		inst.OverallResult = "failed"
+		inst.mu.Unlock()
+		return
+	}
+
+	inst.mu.Lock()
+	for _, db := range dbs {
+		if _, exists := inst.Databases[db]; !exists {
+			inst.Databases[db] = &DBStatus{
+				Name:            db,
+				FirstDiscovered: time.Now(),
+			}
+			logger.Info(name, "Discovered new database: %s", db)
+		}
+	}
+	inst.mu.Unlock()
+
+	// 2. Backup Execution
+	successCount := 0
+	failedCount := 0
+
+	// We only backup the discovered databases. If a DB disappeared, it will not be in `dbs`.
+	for _, db := range dbs {
+		logger.Info(name, "Starting backup for database %s", db)
+		start := time.Now()
+		size, err := backupDatabase(inst.Config, db)
+		duration := time.Since(start)
+
+		inst.mu.Lock()
+		dbStatus := inst.Databases[db]
+		dbStatus.LastBackupTime = time.Now()
+		if err != nil {
+			logger.Error(name, "Failed backup for database %s: %v", db, err)
+			dbStatus.LastBackupResult = "failed"
+			failedCount++
+		} else {
+			logger.Info(name, "Completed backup for database %s in %v, size %d bytes", db, duration, size)
+			dbStatus.LastBackupResult = "success"
+			dbStatus.LastBackupSize = size
+			successCount++
+		}
+		inst.mu.Unlock()
+	}
+
+	// 3. Retention Enforcement
+	logger.Info(name, "Running retention policy cleanup (keep %d days)", inst.Config.RetentionDays)
+	deleted, err := enforceRetention(inst.Config)
+	if err != nil {
+		logger.Error(name, "Retention cleanup encountered errors: %v", err)
+	} else {
+		logger.Info(name, "Retention cleanup finished, deleted %d files", deleted)
+	}
+
+	inst.mu.Lock()
+	if failedCount == 0 {
+		inst.OverallResult = "success"
+	} else if successCount == 0 {
+		inst.OverallResult = "failed"
+	} else {
+		inst.OverallResult = "partial"
+	}
+	inst.mu.Unlock()
+	
+	logger.Info(name, "Backup job completed. Result: %s", inst.OverallResult)
+}
@@ -0,0 +1,87 @@
+package backup
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+
+	"godump/config"
+	"godump/logger"
+)
+
+func backupDatabase(cfg config.InstanceConfig, dbName string) (int64, error) {
+	// Create backup directory for this database
+	targetDir := filepath.Join(cfg.BackupDir, dbName)
+	if err := os.MkdirAll(targetDir, 0755); err != nil {
+		return 0, fmt.Errorf("failed to create backup directory %s: %w", targetDir, err)
+	}
+
+	// Filename format: instancename_dbname_2006-01-02_15-04-05.sql.gz
+	timestamp := time.Now().Format("2006-01-02_15-04-05")
+	filename := fmt.Sprintf("%s_%s_%s.sql.gz", cfg.Name, dbName, timestamp)
+	targetFile := filepath.Join(targetDir, filename)
+
+	f, err := os.Create(targetFile)
+	if err != nil {
+		return 0, fmt.Errorf("failed to create backup file %s: %w", targetFile, err)
+	}
+	defer f.Close()
+
+	// Using sh -c allows us to pipe mysqldump output to gzip easily.
+	// Since we are taking dbName from INFORMATION_SCHEMA, we should still be careful with quotes,
+	// but mysqldump arguments can be passed via command directly, and we can pipe in go, or use sh.
+	// Piping in Go is safer.
+
+	cmdDump := exec.Command("mysqldump",
+		fmt.Sprintf("-h%s", cfg.Host),
+		fmt.Sprintf("-P%d", cfg.Port),
+		fmt.Sprintf("-u%s", cfg.User),
+		fmt.Sprintf("-p%s", cfg.Password),
+		"--single-transaction",
+		"--routines",
+		"--triggers",
+		dbName,
+	)
+
+	cmdGzip := exec.Command("gzip", "-c")
+
+	// Create pipe between mysqldump and gzip
+	dumpOut, err := cmdDump.StdoutPipe()
+	if err != nil {
+		return 0, fmt.Errorf("failed to create dump stdout pipe: %w", err)
+	}
+	cmdDump.Stderr = os.Stderr // or capture it
+
+	cmdGzip.Stdin = dumpOut
+	cmdGzip.Stdout = f
+	cmdGzip.Stderr = os.Stderr
+
+	if err := cmdDump.Start(); err != nil {
+		return 0, fmt.Errorf("failed to start mysqldump: %w", err)
+	}
+
+	if err := cmdGzip.Start(); err != nil {
+		cmdDump.Process.Kill()
+		return 0, fmt.Errorf("failed to start gzip: %w", err)
+	}
+
+	if err := cmdDump.Wait(); err != nil {
+		cmdGzip.Process.Kill()
+		return 0, fmt.Errorf("mysqldump failed: %w", err)
+	}
+
+	if err := cmdGzip.Wait(); err != nil {
+		return 0, fmt.Errorf("gzip failed: %w", err)
+	}
+
+	// Get file size
+	stat, err := f.Stat()
+	if err != nil {
+		logger.Warn(cfg.Name, "Could not stat file %s to get size: %v", targetFile, err)
+		return 0, nil
+	}
+
+	return stat.Size(), nil
+}
@@ -0,0 +1,60 @@
+package backup
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"time"
+
+	"godump/config"
+	"godump/logger"
+)
+
+func enforceRetention(cfg config.InstanceConfig) (int, error) {
+	if cfg.RetentionDays <= 0 {
+		return 0, nil
+	}
+
+	cutoff := time.Now().AddDate(0, 0, -cfg.RetentionDays)
+	deletedCount := 0
+
+	err := filepath.WalkDir(cfg.BackupDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			if os.IsNotExist(err) {
+				return nil
+			}
+			return err
+		}
+
+		if d.IsDir() {
+			return nil
+		}
+
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+
+		if info.ModTime().Before(cutoff) {
+			ageDays := int(time.Since(info.ModTime()).Hours() / 24)
+			if err := os.Remove(path); err != nil {
+				logger.Error(cfg.Name, "Failed to delete old backup %s: %v", path, err)
+			} else {
+				logger.Info(cfg.Name, "Deleted old backup %s (age: %d days)", path, ageDays)
+				deletedCount++
+			}
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			return 0, nil
+		}
+		return deletedCount, fmt.Errorf("error walking backup directory: %w", err)
+	}
+
+	return deletedCount, nil
+}